diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..9e95c0a Binary files /dev/null and b/.DS_Store differ diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 43385f8..fa295ef 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -93,38 +93,97 @@ jobs: sleep 150 # Logstash is very slow to start up [ "401" = "$(curl -w '%{http_code}' --output /dev/null --silent https://logstash-stage-datagov.app.cloud.gov)" ] + drain-apps-in-management-staging: + name: drain apps in management-staging space + environment: management-staging + runs-on: ubuntu-latest + needs: + - deploy-management-staging + steps: + - name: checkout + uses: actions/checkout@v2 + - name: drain-management-staging-space + uses: cloud-gov/cg-cli-tools@37d58fc10abf00a45926e8886879c81a498f7ea8 + with: + command: | + ./create-space-drain.sh management-staging + cf_org: gsa-datagov + cf_space: management-staging + cf_username: ${{secrets.CF_SERVICE_USER}} + cf_password: ${{secrets.CF_SERVICE_AUTH}} + drain-apps-in-staging: - name: drain everything in staging space - environment: management + name: drain apps in staging space + environment: staging runs-on: ubuntu-latest needs: - - deploy-management + - deploy-management-staging steps: - name: checkout uses: actions/checkout@v2 - name: drain-staging-space - uses: cloud-gov/cg-cli-tools@main + uses: cloud-gov/cg-cli-tools@37d58fc10abf00a45926e8886879c81a498f7ea8 with: - command: apt-get install -y jq && cf install-plugin -f -r CF-Community "drains" && mkdir -p /root/.cf/ && touch /root/.cf/config.json && ./create-space-drain.sh + command: | + ./create-space-drain.sh management-staging cf_org: gsa-datagov cf_space: staging cf_username: ${{secrets.CF_SERVICE_USER}} cf_password: ${{secrets.CF_SERVICE_AUTH}} - drain-apps-in-prod: - name: drain everything in prod space + drain-apps-in-management: + name: drain apps in management space environment: management runs-on: ubuntu-latest + needs: + - deploy-management + steps: + - name: checkout + uses: actions/checkout@v2 + - name: drain-management-space + uses: cloud-gov/cg-cli-tools@37d58fc10abf00a45926e8886879c81a498f7ea8 + with: + command: | + ./create-space-drain.sh management + cf_org: gsa-datagov + cf_space: management + cf_username: ${{secrets.CF_SERVICE_USER}} + cf_password: ${{secrets.CF_SERVICE_AUTH}} + + drain-apps-in-prod: + name: drain apps in prod space + environment: prod + runs-on: ubuntu-latest needs: - deploy-management steps: - name: checkout uses: actions/checkout@v2 - name: drain-prod-space - uses: cloud-gov/cg-cli-tools@main + uses: cloud-gov/cg-cli-tools@37d58fc10abf00a45926e8886879c81a498f7ea8 with: - command: apt-get install -y jq && cf install-plugin -f -r CF-Community "drains" && mkdir -p /root/.cf/ && touch /root/.cf/config.json && ./create-space-drain.sh + command: | + ./create-space-drain.sh management cf_org: gsa-datagov cf_space: prod cf_username: ${{secrets.CF_SERVICE_USER}} cf_password: ${{secrets.CF_SERVICE_AUTH}} + + drain-apps-in-prod-egress: + name: drain apps in prod-egress space + environment: prod-egress + runs-on: ubuntu-latest + needs: + - deploy-management + steps: + - name: checkout + uses: actions/checkout@v2 + - name: drain-prod-egress-space + uses: cloud-gov/cg-cli-tools@37d58fc10abf00a45926e8886879c81a498f7ea8 + with: + command: | + ./create-space-drain.sh management + cf_org: gsa-datagov + cf_space: prod-egress + cf_username: ${{secrets.CF_SERVICE_USER}} + cf_password: ${{secrets.CF_SERVICE_AUTH}} diff --git a/.github/workflows/dev-deploy.yml b/.github/workflows/dev-deploy.yml new file mode 100644 index 0000000..d68105c --- /dev/null +++ b/.github/workflows/dev-deploy.yml @@ -0,0 +1,89 @@ +--- +name: dev-deploy + +on: + push: + branches: + - development + workflow_dispatch: + +jobs: + create-cloudgov-services-development-ssb: + name: create services (development-ssb) + environment: development-ssb + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v2 + - name: create services + uses: cloud-gov/cg-cli-tools@main + with: + command: ./create-cloudgov-services.sh + cf_org: gsa-datagov + cf_space: development-ssb + cf_username: ${{secrets.CF_SERVICE_USER}} + cf_password: ${{secrets.CF_SERVICE_AUTH}} + + deploy-development-ssb: + name: deploy (development-ssb) + environment: development-ssb + runs-on: ubuntu-latest + needs: + - create-cloudgov-services-development-ssb + steps: + - name: checkout + uses: actions/checkout@v2 + - name: build dependencies + run: | + chmod -R 777 logstash + PWD=`pwd` make logstash-installation + - name: deploy-logstash + uses: cloud-gov/cg-cli-tools@main + with: + command: cf push --vars-file vars.development-ssb.yml --strategy rolling + cf_org: gsa-datagov + cf_space: development-ssb + cf_username: ${{secrets.CF_SERVICE_USER}} + cf_password: ${{secrets.CF_SERVICE_AUTH}} + - name: smoke test logstash + run: | + sleep 150 # Logstash is very slow to start up + [ "401" = "$(curl -w '%{http_code}' --output /dev/null --silent https://logstash-development-datagov.app.cloud.gov)" ] + + drain-apps-in-development-ssb: + name: drain apps in development-ssb space + environment: development-ssb + runs-on: ubuntu-latest + needs: + - deploy-development-ssb + steps: + - name: checkout + uses: actions/checkout@v2 + - name: drain-development-ssb-space + uses: cloud-gov/cg-cli-tools@37d58fc10abf00a45926e8886879c81a498f7ea8 + with: + command: | + ./create-space-drain.sh development-ssb + cf_org: gsa-datagov + cf_space: development-ssb + cf_username: ${{secrets.CF_SERVICE_USER}} + cf_password: ${{secrets.CF_SERVICE_AUTH}} + + drain-apps-in-development: + name: drain apps in development space + environment: development + runs-on: ubuntu-latest + needs: + - deploy-development-ssb + steps: + - name: checkout + uses: actions/checkout@v2 + - name: drain-development-space + uses: cloud-gov/cg-cli-tools@37d58fc10abf00a45926e8886879c81a498f7ea8 + with: + command: | + ./create-space-drain.sh development-ssb + cf_org: gsa-datagov + cf_space: development + cf_username: ${{secrets.CF_SERVICE_USER}} + cf_password: ${{secrets.CF_SERVICE_AUTH}} diff --git a/.gitignore b/.gitignore index a03129a..13920ff 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ logstash/logstash-oss-*-linux-x86_64.tar.gz logstash/plugins.zip logstash/awscliv2.zip +.DS_Store diff --git a/Pipfile.lock b/Pipfile.lock index e2b6589..4fc02f1 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -30,6 +30,7 @@ "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082", "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9" ], + "index": "pypi", "markers": "python_version >= '3.6'", "version": "==2023.7.22" }, diff --git a/README.md b/README.md index 1a1e308..56ceab8 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,11 @@ Alternatively, you can auto-drain all apps in a given space by targeting that sp cf target -s prod ./create-space-drain.sh +Note: at development space, it will use the logstack-shipper in development-ssb space: + + cf target -s development + ./create-space-drain.sh development-ssb + After a short delay, logs should begin to flow automatically. ## Setup diff --git a/create-cloudgov-services.sh b/create-cloudgov-services.sh index f97018a..31b3fcd 100755 --- a/create-cloudgov-services.sh +++ b/create-cloudgov-services.sh @@ -13,7 +13,7 @@ space=$(cf target | grep space | cut -d : -f 2 | xargs) randpw(){ openssl rand -base64 40 | tr -dc _A-Z-a-z-0-9 | head -c${1:-32};echo; } # Only create stuff in production and staging spaces -if [ "$space" = "management" ] || [ "$space" = "management-staging" ]; then +if [ "$space" = "management" ] || [ "$space" = "management-staging" ] || [ "$space" = "development-ssb" ]; then cf service "${app_name}-s3" > /dev/null 2>&1 || cf create-service s3 basic "${app_name}-s3" --wait& cf service "${app_name}-secrets" > /dev/null 2>&1 || cf create-user-provided-service "${app_name}-secrets" -p '{"DRAIN_USER":"'$(randpw)'","DRAIN_PASSWORD":"'$(randpw)'"}' & diff --git a/create-space-drain.sh b/create-space-drain.sh index 609f397..a909598 100755 --- a/create-space-drain.sh +++ b/create-space-drain.sh @@ -18,8 +18,18 @@ prefix=${3:-logstack} # If the app already exists, exit early/successfully cf app "${prefix}-space-drain" > /dev/null 2>&1 && echo "Drain already exists." && exit 0 -# If the drain plugin isn't already installed, we can't proceed! -cf drains --help > /dev/null 2>&1 || ( printf "cf_drain_cli plugin not found!\nInstall it with:\n cf install-plugin -r CF-Community drains\n\n" && exit 1 ) +# install drain plugin if it isn't installed +if ! cf plugins | grep -q drain; then + echo "cf-drain-cli plugin not found. Installing..." + apt install jq curl -y && + curl -L -o drain-plugin https://github.com/cloudfoundry/cf-drain-cli/releases/download/v2.0.0/cf-drain-cli-linux --insecure && + cf install-plugin -f drain-plugin && + rm -f drain-plugin && + mkdir -p /root/.cf/ && touch /root/.cf/config.json && + echo "cf-drain-cli plugin installed successfully." +else + echo "cf-drain-cli plugin already exists." +fi space=$(cf target | grep space: | cut -d : -f 2 | sed s/\ //g) @@ -44,4 +54,4 @@ applications: no-route: true EOF -cf drain-space --drain-name "${prefix}-space-drain" "$drain_url" +cf drain-space --drain-name "${prefix}-space-drain" "$drain_url" \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 1a34921..eb6efa4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,21 +15,14 @@ services: timeout: 10s start_period: 10s ports: - - "127.0.0.1:4566:4566" - - "127.0.0.1:4571:4571" + - "127.0.0.1:4566:4566" # LocalStack Gateway + - "127.0.0.1:4510-4559:4510-4559" # external services port range environment: - - SERVICES=${SERVICES-} - DEBUG=${DEBUG-} - - DATA_DIR=${DATA_DIR-} - - LAMBDA_EXECUTOR=${LAMBDA_EXECUTOR-} - - HOST_TMP_FOLDER=${TMPDIR:-/tmp/}localstack - DOCKER_HOST=unix:///var/run/docker.sock - - AWS_DEFAULT_REGION=us-east-1 volumes: - - "${TMPDIR:-/tmp}/localstack:/tmp/localstack" + - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" - "/var/run/docker.sock:/var/run/docker.sock" - - ./localstack.d:/docker-entrypoint-initaws.d - logstash: depends_on: diff --git a/logstash/logstash.conf b/logstash/logstash.conf old mode 100644 new mode 100755 index 3521a6b..9d9b597 --- a/logstash/logstash.conf +++ b/logstash/logstash.conf @@ -63,11 +63,52 @@ filter{ # } # mutate { add_field => { "[@metadata][target_index]" => "index-%{+YYYY.MM}" } } + grok { + match => { "message" => "%{GREEDYDATA:raw_message_content}" } + } + grok { + match => { + "message" => '<%{GREEDYDATA:skip} %{TIMESTAMP_ISO8601:skip} %{DATA:host} %{UUID:proc_id} \[%{DATA:instance_info}\] - \[tags@%{GREEDYDATA:skip} instance_id="%{INT:instance_id}" ip="%{IP:ip}" job="%{GREEDYDATA:skip} space_name="%{DATA:space_name}"\] %{GREEDYDATA:log_data}' + tag_on_failure => [] + overwrite => ["message"] + break_on_match => false + } + } + # for message like: catalog-dev.data.gov - [2023-12-15T21:11:49.508628938Z] "GET /0000000 HTTP/1.1" 404 0 21445 "-" "Mozilla/5.0 ..." "127.0.x.x:xxxxx" "10.xx.2.10:xxxxx" x_forwarded_for:"108.xx.xxx.xxx, 64.252.66.xxx, 127.0.x.x" x_forwarded_proto:"https" ... ... b3:xxx + grok { + match => { + "log_data" => '%{HOSTNAME:hostname} - \[%{TIMESTAMP_ISO8601:timestamp}\] "%{WORD:http_method} %{GREEDYDATA:request} %{DATA:http_version}" %{NUMBER:status} %{NUMBER:bytes_sent} %{NUMBER:bytes_received} "%{DATA:http_referer}" "%{DATA:http_user_agent}" %{GREEDYDATA:skip} x_forwarded_for:"%{IP:real_ip}(?:, %{GREEDYDATA:forwarded_ips})*" x_forwarded_proto:%{GREEDYDATA:skip}' + tag_on_failure => [] + overwrite => ["message"] + break_on_match => false + } + } + # for message like: 2023-12-14 20:31:53,839 INFO [ckan.config.middleware.flask_app] 404 /dataset/xxxx render time 0.023 seconds + grok { + match => { + "log_data" => '%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:log_level} \[%{DATA:logger}\] %{NUMBER:status:int} %{URIPATH:request_path} render time %{NUMBER:render_time:float} seconds' + tag_on_failure => [] + overwrite => ["message"] + break_on_match => false + } + } + if ( + ([log_data] and [log_data] =~ /^NginxLog/) + or + ([hostname] and [status] and [hostname] =~ /^logstash-/ and [status] == "200") + ) { + drop { } + } mutate { add_field => { "[@metadata][NEWRELIC_KEY]" => "${NEWRELIC_LICENSE_KEY:notpresent}" } add_field => { "[@metadata][AWS_S3_PROXY]" => "${AWS_S3_PROXY:notpresent}" } + remove_field => ["skip"] + } + if [space_name] and [space_name] !~ /development/ { + mutate { + remove_field => ["log_data", "raw_message_content"] + } } - } output { diff --git a/manifest.yml b/manifest.yml index f1d39ab..9e9f52a 100644 --- a/manifest.yml +++ b/manifest.yml @@ -17,7 +17,4 @@ applications: - route: ((hostname)).((domain)) services: - ((app_name))-s3 - - ((app_name))-secrets -env: - NEWRELIC_LICENSE_KEY: ((logstash_newrelic_license_key)) - NEWRELIC_LOG_URI: ((logstash_newrelic_log_uri)) + - ((app_name))-secrets \ No newline at end of file diff --git a/vars.development-ssb.yml b/vars.development-ssb.yml new file mode 100644 index 0000000..68391b8 --- /dev/null +++ b/vars.development-ssb.yml @@ -0,0 +1,20 @@ +--- +# Name prefix for applications in the logstack. +app_name: logstack + +# Options to pass the JVM +logstash_java_options: -Xms512m -Xmx512m +# Number of instances of the Logstash to run. +logstash_instances: 1 +# Amount of memory to provide to the Logstash (Java) application. +logstash_memory: 1024 +# Assign route for the log shipper. This will be used as the logdrain for other +# Cloud Foundry applications. +hostname: logstash-development-datagov +domain: app.cloud.gov +https_proxy: + +# Optional license key for delivering logs to New Relic +# See https://github.com/newrelic/logstash-output-plugin +# logstash_newrelic_license_key: +# logstash_newrelic_log_uri: https://gov-log-api.newrelic.com/log/v1 diff --git a/vars.example.yml b/vars.example.yml index c35cbfc..a978056 100644 --- a/vars.example.yml +++ b/vars.example.yml @@ -16,4 +16,4 @@ https_proxy: # Optional license key for delivering logs to New Relic # See https://github.com/newrelic/logstash-output-plugin -logstash_newrelic_license_key: +# logstash_newrelic_license_key: diff --git a/vars.management-staging.yml b/vars.management-staging.yml index 560f1e9..c4929b5 100644 --- a/vars.management-staging.yml +++ b/vars.management-staging.yml @@ -16,5 +16,5 @@ https_proxy: # Optional license key for delivering logs to New Relic # See https://github.com/newrelic/logstash-output-plugin -logstash_newrelic_license_key: -logstash_newrelic_log_uri: https://gov-log-api.newrelic.com/log/v1 +# logstash_newrelic_license_key: +# logstash_newrelic_log_uri: https://gov-log-api.newrelic.com/log/v1 diff --git a/vars.management.yml b/vars.management.yml index 733c357..f2d48dd 100644 --- a/vars.management.yml +++ b/vars.management.yml @@ -16,5 +16,5 @@ https_proxy: # Optional license key for delivering logs to New Relic # See https://github.com/newrelic/logstash-output-plugin -logstash_newrelic_license_key: -logstash_newrelic_log_uri: https://gov-log-api.newrelic.com/log/v1 +# logstash_newrelic_license_key: +# logstash_newrelic_log_uri: https://gov-log-api.newrelic.com/log/v1