Merge branch 'main' into dependabot/pip/requests-2.31.0

.github/workflows/deploy.yml

@@ -93,38 +93,97 @@ jobs:
           sleep 150  # Logstash is very slow to start up
           [ "401" = "$(curl -w '%{http_code}' --output /dev/null --silent https://logstash-stage-datagov.app.cloud.gov)" ]
 
+  drain-apps-in-management-staging:
+    name: drain apps in management-staging space
+    environment: management-staging
+    runs-on: ubuntu-latest
+    needs:
+      - deploy-management-staging
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: drain-management-staging-space
+        uses: cloud-gov/cg-cli-tools@37d58fc10abf00a45926e8886879c81a498f7ea8
+        with:
+          command: |
+            ./create-space-drain.sh management-staging
+          cf_org: gsa-datagov
+          cf_space: management-staging
+          cf_username: ${{secrets.CF_SERVICE_USER}}
+          cf_password: ${{secrets.CF_SERVICE_AUTH}}
+
   drain-apps-in-staging:
-    name: drain everything in staging space
-    environment: management
+    name: drain apps in staging space
+    environment: staging
     runs-on: ubuntu-latest
     needs:
-      - deploy-management
+      - deploy-management-staging
     steps:
       - name: checkout
         uses: actions/checkout@v2
       - name: drain-staging-space
-        uses: cloud-gov/cg-cli-tools@main
+        uses: cloud-gov/cg-cli-tools@37d58fc10abf00a45926e8886879c81a498f7ea8
         with:
-          command: apt-get install -y jq && cf install-plugin -f -r CF-Community "drains" && mkdir -p /root/.cf/ && touch /root/.cf/config.json && ./create-space-drain.sh
+          command: |
+            ./create-space-drain.sh management-staging
           cf_org: gsa-datagov
           cf_space: staging
           cf_username: ${{secrets.CF_SERVICE_USER}}
           cf_password: ${{secrets.CF_SERVICE_AUTH}}
 
-  drain-apps-in-prod:
-    name: drain everything in prod space
+  drain-apps-in-management:
+    name: drain apps in management space
     environment: management
     runs-on: ubuntu-latest
+    needs:
+      - deploy-management
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: drain-management-space
+        uses: cloud-gov/cg-cli-tools@37d58fc10abf00a45926e8886879c81a498f7ea8
+        with:
+          command: |
+            ./create-space-drain.sh management
+          cf_org: gsa-datagov
+          cf_space: management
+          cf_username: ${{secrets.CF_SERVICE_USER}}
+          cf_password: ${{secrets.CF_SERVICE_AUTH}}
+
+  drain-apps-in-prod:
+    name: drain apps in prod space
+    environment: prod
+    runs-on: ubuntu-latest
     needs:
       - deploy-management
     steps:
       - name: checkout
         uses: actions/checkout@v2
       - name: drain-prod-space
-        uses: cloud-gov/cg-cli-tools@main
+        uses: cloud-gov/cg-cli-tools@37d58fc10abf00a45926e8886879c81a498f7ea8
         with:
-          command: apt-get install -y jq && cf install-plugin -f -r CF-Community "drains" && mkdir -p /root/.cf/ && touch /root/.cf/config.json && ./create-space-drain.sh
+          command: |
+            ./create-space-drain.sh management
           cf_org: gsa-datagov
           cf_space: prod
           cf_username: ${{secrets.CF_SERVICE_USER}}
           cf_password: ${{secrets.CF_SERVICE_AUTH}}
+
+  drain-apps-in-prod-egress:
+    name: drain apps in prod-egress space
+    environment: prod-egress
+    runs-on: ubuntu-latest
+    needs:
+      - deploy-management
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: drain-prod-egress-space
+        uses: cloud-gov/cg-cli-tools@37d58fc10abf00a45926e8886879c81a498f7ea8
+        with:
+          command: |
+            ./create-space-drain.sh management
+          cf_org: gsa-datagov
+          cf_space: prod-egress
+          cf_username: ${{secrets.CF_SERVICE_USER}}
+          cf_password: ${{secrets.CF_SERVICE_AUTH}}

.github/workflows/dev-deploy.yml

@@ -0,0 +1,89 @@
+---
+name: dev-deploy
+
+on:
+  push:
+    branches:
+      - development
+  workflow_dispatch:
+
+jobs:
+  create-cloudgov-services-development-ssb:
+    name: create services (development-ssb)
+    environment: development-ssb
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: create services
+        uses: cloud-gov/cg-cli-tools@main
+        with:
+          command: ./create-cloudgov-services.sh
+          cf_org: gsa-datagov
+          cf_space: development-ssb
+          cf_username: ${{secrets.CF_SERVICE_USER}}
+          cf_password: ${{secrets.CF_SERVICE_AUTH}}
+
+  deploy-development-ssb:
+    name: deploy (development-ssb)
+    environment: development-ssb
+    runs-on: ubuntu-latest
+    needs:
+      - create-cloudgov-services-development-ssb
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: build dependencies
+        run: |
+          chmod -R 777 logstash
+          PWD=`pwd` make logstash-installation
+      - name: deploy-logstash
+        uses: cloud-gov/cg-cli-tools@main
+        with:
+          command: cf push --vars-file vars.development-ssb.yml --strategy rolling
+          cf_org: gsa-datagov
+          cf_space: development-ssb
+          cf_username: ${{secrets.CF_SERVICE_USER}}
+          cf_password: ${{secrets.CF_SERVICE_AUTH}}
+      - name: smoke test logstash
+        run: |
+          sleep 150  # Logstash is very slow to start up
+          [ "401" = "$(curl -w '%{http_code}' --output /dev/null --silent https://logstash-development-datagov.app.cloud.gov)" ]
+
+  drain-apps-in-development-ssb:
+    name: drain apps in development-ssb space
+    environment: development-ssb
+    runs-on: ubuntu-latest
+    needs:
+      - deploy-development-ssb
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: drain-development-ssb-space
+        uses: cloud-gov/cg-cli-tools@37d58fc10abf00a45926e8886879c81a498f7ea8
+        with:
+          command: |
+            ./create-space-drain.sh development-ssb
+          cf_org: gsa-datagov
+          cf_space: development-ssb
+          cf_username: ${{secrets.CF_SERVICE_USER}}
+          cf_password: ${{secrets.CF_SERVICE_AUTH}}   
+
+  drain-apps-in-development:
+    name: drain apps in development space
+    environment: development
+    runs-on: ubuntu-latest
+    needs:
+      - deploy-development-ssb
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: drain-development-space
+        uses: cloud-gov/cg-cli-tools@37d58fc10abf00a45926e8886879c81a498f7ea8
+        with:
+          command: |
+            ./create-space-drain.sh development-ssb
+          cf_org: gsa-datagov
+          cf_space: development
+          cf_username: ${{secrets.CF_SERVICE_USER}}
+          cf_password: ${{secrets.CF_SERVICE_AUTH}}

.gitignore

@@ -2,3 +2,4 @@
 logstash/logstash-oss-*-linux-x86_64.tar.gz
 logstash/plugins.zip
 logstash/awscliv2.zip
+.DS_Store

README.md

@@ -30,6 +30,11 @@ Alternatively, you can auto-drain all apps in a given space by targeting that sp
     cf target -s prod
     ./create-space-drain.sh
 
+Note: at development space, it will use the logstack-shipper in development-ssb space:
+
+    cf target -s development
+    ./create-space-drain.sh development-ssb
+
 After a short delay, logs should begin to flow automatically.
 
 ## Setup

create-cloudgov-services.sh

@@ -13,7 +13,7 @@ space=$(cf target | grep space | cut -d : -f 2 | xargs)
 randpw(){ openssl rand -base64 40 | tr -dc _A-Z-a-z-0-9 | head -c${1:-32};echo; }
 
 # Only create stuff in production and staging spaces
-if [ "$space" = "management" ] || [ "$space" = "management-staging" ]; then
+if [ "$space" = "management" ] || [ "$space" = "management-staging" ] || [ "$space" = "development-ssb" ]; then
     cf service "${app_name}-s3"      > /dev/null 2>&1 || cf create-service s3 basic "${app_name}-s3" --wait&
     cf service "${app_name}-secrets" > /dev/null 2>&1 || 
         cf create-user-provided-service "${app_name}-secrets" -p '{"DRAIN_USER":"'$(randpw)'","DRAIN_PASSWORD":"'$(randpw)'"}' &

create-space-drain.sh

@@ -18,8 +18,18 @@ prefix=${3:-logstack}
 # If the app already exists, exit early/successfully
 cf app "${prefix}-space-drain" > /dev/null 2>&1 && echo "Drain already exists." && exit 0
 
-# If the drain plugin isn't already installed, we can't proceed!
-cf drains --help > /dev/null 2>&1 || ( printf "cf_drain_cli plugin not found!\nInstall it with:\n    cf install-plugin -r CF-Community drains\n\n" && exit 1 )
+# install drain plugin if it isn't installed
+if ! cf plugins | grep -q drain; then
+    echo "cf-drain-cli plugin not found. Installing..."
+    apt install jq curl -y &&
+    curl -L -o drain-plugin https://github.com/cloudfoundry/cf-drain-cli/releases/download/v2.0.0/cf-drain-cli-linux --insecure &&
+    cf install-plugin -f  drain-plugin &&
+    rm -f drain-plugin &&
+    mkdir -p /root/.cf/ && touch /root/.cf/config.json && 
+    echo "cf-drain-cli plugin installed successfully."
+else
+    echo "cf-drain-cli plugin already exists."
+fi
 
 space=$(cf target | grep space: | cut -d : -f 2 | sed s/\ //g)
 
@@ -44,4 +54,4 @@ applications:
   no-route: true
 EOF
 
-cf drain-space --drain-name "${prefix}-space-drain" "$drain_url"
+cf drain-space --drain-name "${prefix}-space-drain" "$drain_url"

docker-compose.yml

@@ -15,21 +15,14 @@ services:
       timeout: 10s
       start_period: 10s
     ports:
-      - "127.0.0.1:4566:4566"
-      - "127.0.0.1:4571:4571"
+      - "127.0.0.1:4566:4566"            # LocalStack Gateway
+      - "127.0.0.1:4510-4559:4510-4559"  # external services port range
     environment:
-      - SERVICES=${SERVICES-}
       - DEBUG=${DEBUG-}
-      - DATA_DIR=${DATA_DIR-}
-      - LAMBDA_EXECUTOR=${LAMBDA_EXECUTOR-}
-      - HOST_TMP_FOLDER=${TMPDIR:-/tmp/}localstack
       - DOCKER_HOST=unix:///var/run/docker.sock
-      - AWS_DEFAULT_REGION=us-east-1
     volumes:
-      - "${TMPDIR:-/tmp}/localstack:/tmp/localstack"
+      - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack"
       - "/var/run/docker.sock:/var/run/docker.sock"
-      - ./localstack.d:/docker-entrypoint-initaws.d
-
 
   logstash:
     depends_on:

logstash/logstash.conf

@@ -63,11 +63,52 @@ filter{
   # }
   # mutate { add_field => { "[@metadata][target_index]" => "index-%{+YYYY.MM}" } }
 
+  grok {
+      match => { "message" => "%{GREEDYDATA:raw_message_content}" }
+  }
+  grok {
+      match => {
+        "message" => '<%{GREEDYDATA:skip} %{TIMESTAMP_ISO8601:skip} %{DATA:host} %{UUID:proc_id} \[%{DATA:instance_info}\] - \[tags@%{GREEDYDATA:skip} instance_id="%{INT:instance_id}" ip="%{IP:ip}" job="%{GREEDYDATA:skip} space_name="%{DATA:space_name}"\] %{GREEDYDATA:log_data}'
+        tag_on_failure => []
+        overwrite => ["message"]
+        break_on_match => false
+      }
+  }
+  # for message like: catalog-dev.data.gov - [2023-12-15T21:11:49.508628938Z] "GET /0000000 HTTP/1.1" 404 0 21445 "-" "Mozilla/5.0 ..." "127.0.x.x:xxxxx" "10.xx.2.10:xxxxx" x_forwarded_for:"108.xx.xxx.xxx, 64.252.66.xxx, 127.0.x.x" x_forwarded_proto:"https" ... ... b3:xxx
+  grok {
+      match => {
+        "log_data" => '%{HOSTNAME:hostname} - \[%{TIMESTAMP_ISO8601:timestamp}\] "%{WORD:http_method} %{GREEDYDATA:request} %{DATA:http_version}" %{NUMBER:status} %{NUMBER:bytes_sent} %{NUMBER:bytes_received} "%{DATA:http_referer}" "%{DATA:http_user_agent}" %{GREEDYDATA:skip} x_forwarded_for:"%{IP:real_ip}(?:, %{GREEDYDATA:forwarded_ips})*" x_forwarded_proto:%{GREEDYDATA:skip}'
+        tag_on_failure => []
+        overwrite => ["message"]
+        break_on_match => false
+      }
+  }
+  # for message like: 2023-12-14 20:31:53,839 INFO  [ckan.config.middleware.flask_app]  404 /dataset/xxxx render time 0.023 seconds
+  grok {
+      match => {
+        "log_data" => '%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:log_level}  \[%{DATA:logger}\]  %{NUMBER:status:int} %{URIPATH:request_path} render time %{NUMBER:render_time:float} seconds'
+        tag_on_failure => []
+        overwrite => ["message"]
+        break_on_match => false
+      }
+  }
+  if (
+      ([log_data] and [log_data] =~ /^NginxLog/)
+      or
+      ([hostname] and [status] and [hostname] =~ /^logstash-/ and [status] == "200")
+  ) {
+      drop { }
+  }
   mutate {
     add_field => { "[@metadata][NEWRELIC_KEY]" => "${NEWRELIC_LICENSE_KEY:notpresent}" }
     add_field => { "[@metadata][AWS_S3_PROXY]" => "${AWS_S3_PROXY:notpresent}" }  
+    remove_field => ["skip"]
+  }
+  if [space_name] and [space_name] !~ /development/  {
+    mutate {
+      remove_field => ["log_data", "raw_message_content"]
+    }
   }
-
 }
 
 output {

manifest.yml

@@ -17,7 +17,4 @@ applications:
     - route: ((hostname)).((domain))
   services:
     - ((app_name))-s3
-    - ((app_name))-secrets
-env:
-  NEWRELIC_LICENSE_KEY: ((logstash_newrelic_license_key))
-  NEWRELIC_LOG_URI: ((logstash_newrelic_log_uri))
+    - ((app_name))-secrets

vars.development-ssb.yml

@@ -0,0 +1,20 @@
+---
+# Name prefix for applications in the logstack.
+app_name: logstack
+
+# Options to pass the JVM
+logstash_java_options: -Xms512m -Xmx512m
+# Number of instances of the Logstash to run.
+logstash_instances: 1
+# Amount of memory to provide to the Logstash (Java) application.
+logstash_memory: 1024
+# Assign route for the log shipper. This will be used as the logdrain for other
+# Cloud Foundry applications.
+hostname: logstash-development-datagov
+domain: app.cloud.gov
+https_proxy:
+
+# Optional license key for delivering logs to New Relic
+# See https://github.com/newrelic/logstash-output-plugin
+# logstash_newrelic_license_key:
+# logstash_newrelic_log_uri: https://gov-log-api.newrelic.com/log/v1

vars.example.yml

@@ -16,4 +16,4 @@ https_proxy:
 
 # Optional license key for delivering logs to New Relic
 # See https://github.com/newrelic/logstash-output-plugin
-logstash_newrelic_license_key: 
+# logstash_newrelic_license_key: 

vars.management-staging.yml

@@ -16,5 +16,5 @@ https_proxy:
 
 # Optional license key for delivering logs to New Relic
 # See https://github.com/newrelic/logstash-output-plugin
-logstash_newrelic_license_key: 
-logstash_newrelic_log_uri: https://gov-log-api.newrelic.com/log/v1
+# logstash_newrelic_license_key: 
+# logstash_newrelic_log_uri: https://gov-log-api.newrelic.com/log/v1

vars.management.yml

@@ -16,5 +16,5 @@ https_proxy:
 
 # Optional license key for delivering logs to New Relic
 # See https://github.com/newrelic/logstash-output-plugin
-logstash_newrelic_license_key: 
-logstash_newrelic_log_uri: https://gov-log-api.newrelic.com/log/v1
+# logstash_newrelic_license_key: 
+# logstash_newrelic_log_uri: https://gov-log-api.newrelic.com/log/v1