Splunk direct export of events using sidecar

.env

@@ -28,6 +28,11 @@ NEXTAUTH_SECRET=secret
 RETRACED_HOST_URL=http://retraced-api:3000/auditlog
 RETRACED_EXTERNAL_URL=http://localhost:3000/auditlog
 
+# Export Logs
+# EXPORT_WEBHOOK_URL=http://vector:9000
+# EXPORT_WEBHOOK_USERNAME=admin
+# EXPORT_WEBHOOK_PASSWORD=admin
+
 # OpenTelemetry 
 # https://opentelemetry.io/docs/concepts/sdk-configuration/otlp-exporter-configuration/
 # If you have any issues with using the otel exporter and want to enable debug logs
@@ -41,4 +46,4 @@ OTEL_EXPORTER_OTLP_METRICS_HEADERS=
 GEOIPUPDATE_LICENSE_KEY=
 GEOIPUPDATE_ACCOUNT_ID=
 GEOIPUPDATE_USE_MMDB=
-GEOIPUPDATE_DB_DIR=/etc/mmdb
+GEOIPUPDATE_DB_DIR=/etc/mmdb

.github/workflows/main.yml

@@ -154,14 +154,18 @@ jobs:
         run: |
           echo "SHA7=$(echo ${GITHUB_SHA} | cut -c1-7)" >> $GITHUB_OUTPUT
           imagePath="${{ secrets.DOCKER_HUB_USERNAME }}/retraced"
+          sidecarPath="${{ secrets.DOCKER_HUB_USERNAME }}/vector"
 
           if [[ "$GITHUB_REF" != *\/release ]]
           then
             imagePath="${{ secrets.DOCKER_HUB_USERNAME }}/retraced-beta"
+            sidecarPath="${{ secrets.DOCKER_HUB_USERNAME }}/vector-beta"
           fi
           echo "${imagePath}"
+          echo "${sidecarPath}"
 
-          echo "IMAGE_PATH=${imagePath}" >> $GITHUB_OUTPUT
+          echo "RETRACED_IMAGE_PATH=${imagePath}" >> $GITHUB_OUTPUT
+          echo "SIDECAR_IMAGE_PATH=${sidecarPath}" >> $GITHUB_OUTPUT
 
       - name: Set up Docker Buildx
         if: github.ref == 'refs/heads/release'
@@ -179,20 +183,35 @@ jobs:
           username: boxyhq
           password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
 
-      - name: Build and push
+      - name: Build and push Retraced
         if: github.ref == 'refs/heads/release'
-        id: docker_build
+        id: docker_build_retraced
         uses: docker/build-push-action@v5
         with:
           context: ./
           file: ./deploy/Dockerfile-slim
           platforms: linux/amd64,linux/arm64
           push: true
-          tags: ${{ steps.slug.outputs.IMAGE_PATH }}:${{ needs.ci.outputs.PUBLISH_TAG }},${{ steps.slug.outputs.IMAGE_PATH }}:${{ steps.slug.outputs.SHA7 }},${{ steps.slug.outputs.IMAGE_PATH }}:${{ needs.ci.outputs.NPM_VERSION }}
+          tags: ${{ steps.slug.outputs.RETRACED_IMAGE_PATH }}:${{ needs.ci.outputs.PUBLISH_TAG }},${{ steps.slug.outputs.RETRACED_IMAGE_PATH }}:${{ steps.slug.outputs.SHA7 }},${{ steps.slug.outputs.RETRACED_IMAGE_PATH }}:${{ needs.ci.outputs.NPM_VERSION }}
 
-      - name: Image digest
+      - name: Build and push Sidecar
         if: github.ref == 'refs/heads/release'
-        run: echo ${{ steps.docker_build.outputs.digest }}
+        id: docker_build_sidecar
+        uses: docker/build-push-action@v5
+        with:
+          context: ./
+          file: ./deploy/sidecar/Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.slug.outputs.SIDECAR_IMAGE_PATH }}:${{ needs.ci.outputs.PUBLISH_TAG }},${{ steps.slug.outputs.SIDECAR_IMAGE_PATH }}:${{ steps.slug.outputs.SHA7 }},${{ steps.slug.outputs.SIDECAR_IMAGE_PATH }}:${{ needs.ci.outputs.NPM_VERSION }}
+
+      - name: Image digest Retraced
+        if: github.ref == 'refs/heads/release'
+        run: echo ${{ steps.docker_build_retraced.outputs.digest }}
+
+      - name: Image digest Sidecar
+        if: github.ref == 'refs/heads/release'
+        run: echo ${{ steps.docker_build_sidecar.outputs.digest }}
 
       - name: Login to GitHub Container Registry
         if: github.ref == 'refs/heads/release'
@@ -214,9 +233,15 @@ jobs:
         env:
           COSIGN_KEY: ${{secrets.COSIGN_KEY}}
 
-      - name: Sign the image
+      - name: Sign the image [Retraced]
+        if: github.ref == 'refs/heads/release'
+        run: cosign sign --key /tmp/cosign.key -y ${{ steps.slug.outputs.RETRACED_IMAGE_PATH }}@${{ steps.docker_build_retraced.outputs.digest }}
+        env:
+          COSIGN_PASSWORD: ${{secrets.COSIGN_PASSWORD}}
+
+      - name: Sign the image [Sidecar]
         if: github.ref == 'refs/heads/release'
-        run: cosign sign --key /tmp/cosign.key -y ${{ steps.slug.outputs.IMAGE_PATH }}@${{ steps.docker_build.outputs.digest }}
+        run: cosign sign --key /tmp/cosign.key -y ${{ steps.slug.outputs.SIDECAR_IMAGE_PATH }}@${{ steps.docker_build_sidecar.outputs.digest }}
         env:
           COSIGN_PASSWORD: ${{secrets.COSIGN_PASSWORD}}
 
@@ -226,85 +251,132 @@ jobs:
           format: spdx
           artifact-name: retraced_sbom.spdx
           upload-artifact-retention: 1
+
       - name: Publish report [SPDX]
         uses: anchore/sbom-action/publish-sbom@v0
         with:
           sbom-artifact-match: ".*\\.spdx$"
+
       - name: Create SBOM Report [CycloneDx]
         uses: anchore/sbom-action@v0
         with:
           format: cyclonedx
           artifact-name: retraced_sbom.cyclonedx
           upload-artifact-retention: 1
+
       - name: Publish report [CycloneDx]
         uses: anchore/sbom-action/publish-sbom@v0
         with:
           sbom-artifact-match: ".*\\.cyclonedx$"
+
       - name: Download artifact for SPDX Report
         if: github.ref == 'refs/heads/release'
         uses: actions/download-artifact@v3
         with:
           name: retraced_sbom.spdx
+
       - name: Download artifact for CycloneDx Report
         if: github.ref == 'refs/heads/release'
         uses: actions/download-artifact@v3
         with:
           name: retraced_sbom.cyclonedx
+
       - name: Remove older SBOMs
         if: github.ref == 'refs/heads/release'
         run: rm -rf ./sbom*.* || true
+
       - name: Move SPDX Report
         if: github.ref == 'refs/heads/release'
         run: mv retraced_sbom.spdx "./sbom.spdx"
+
       - name: Move CycloneDx Report
         if: github.ref == 'refs/heads/release'
         run: mv retraced_sbom.cyclonedx "./sbom.cyclonedx"
 
-      - name: Create SBOM Report [Docker][SPDX]
+      - name: Create SBOM Report [Docker][SPDX][Retraced]
+        if: github.ref == 'refs/heads/release'
+        uses: anchore/sbom-action@v0
+        with:
+          image: ${{ steps.slug.outputs.RETRACED_IMAGE_PATH }}:${{ needs.ci.outputs.PUBLISH_TAG }}
+          format: spdx
+          artifact-name: retraced_docker_sbom.spdx
+          upload-artifact-retention: 1
+
+      - name: Create SBOM Report [Docker][SPDX][Sidecar]
         if: github.ref == 'refs/heads/release'
         uses: anchore/sbom-action@v0
         with:
-          image: ${{ steps.slug.outputs.IMAGE_PATH }}:${{ needs.ci.outputs.PUBLISH_TAG }}
+          image: ${{ steps.slug.outputs.SIDECAR_IMAGE_PATH }}:${{ needs.ci.outputs.PUBLISH_TAG }}
           format: spdx
-          artifact-name: docker_sbom.spdx
+          artifact-name: sidecar_docker_sbom.spdx
           upload-artifact-retention: 1
+
       - name: Publish report [Docker][SPDX]
         if: github.ref == 'refs/heads/release'
         uses: anchore/sbom-action/publish-sbom@v0
         with:
           sbom-artifact-match: ".*\\.spdx$"
-      - name: Create SBOM Report [Docker][CycloneDx]
+
+      - name: Create SBOM Report [Docker][CycloneDx][Retraced]
+        if: github.ref == 'refs/heads/release'
+        uses: anchore/sbom-action@v0
+        with:
+          image: ${{ steps.slug.outputs.RETRACED_IMAGE_PATH }}:${{ needs.ci.outputs.PUBLISH_TAG }}
+          format: cyclonedx
+          artifact-name: retraced_docker_sbom.cyclonedx
+          upload-artifact-retention: 1
+
+      - name: Create SBOM Report [Docker][CycloneDx][Sidecar]
         if: github.ref == 'refs/heads/release'
         uses: anchore/sbom-action@v0
         with:
-          image: ${{ steps.slug.outputs.IMAGE_PATH }}:${{ needs.ci.outputs.PUBLISH_TAG }}
+          image: ${{ steps.slug.outputs.SIDECAR_IMAGE_PATH }}:${{ needs.ci.outputs.PUBLISH_TAG }}
           format: cyclonedx
-          artifact-name: docker_sbom.cyclonedx
+          artifact-name: sidecar_docker_sbom.cyclonedx
           upload-artifact-retention: 1
+
       - name: Publish report [Docker][CycloneDx]
         if: github.ref == 'refs/heads/release'
         uses: anchore/sbom-action/publish-sbom@v0
         with:
           sbom-artifact-match: ".*\\.cyclonedx$"
-      - name: Download artifact for SPDX Report [Docker]
+
+      - name: Download artifact for SPDX Report [Docker][Retraced]
+        if: github.ref == 'refs/heads/release'
+        uses: actions/download-artifact@v3
+        with:
+          name: retraced_docker_sbom.spdx
+
+      - name: Download artifact for SPDX Report [Docker][Sidecar]
+        if: github.ref == 'refs/heads/release'
+        uses: actions/download-artifact@v3
+        with:
+          name: sidecar_docker_sbom.spdx
+
+      - name: Download artifact for CycloneDx Report [Docker][Retraced]
         if: github.ref == 'refs/heads/release'
         uses: actions/download-artifact@v3
         with:
-          name: docker_sbom.spdx
-      - name: Download artifact for CycloneDx Report [Docker]
+          name: retraced_docker_sbom.cyclonedx
+
+      - name: Download artifact for CycloneDx Report [Docker][Sidecar]
         if: github.ref == 'refs/heads/release'
         uses: actions/download-artifact@v3
         with:
-          name: docker_sbom.cyclonedx
+          name: sidecar_docker_sbom.cyclonedx
+
       - name: Create/Clear folder [Docker]
         if: github.ref == 'refs/heads/release'
         run: mkdir -p ./_docker/ && rm -rf ./_docker/*.* || true
 
       - name: Move Report & cleanup [Docker]
         if: github.ref == 'refs/heads/release'
         run: |
-          mv docker_sbom.spdx "./_docker/sbom.spdx" || true
-          mv docker_sbom.cyclonedx ./_docker/sbom.cyclonedx || true
+          mv retraced_docker_sbom.spdx "./_docker/retraced_sbom.spdx" || true
+          mv retraced_docker_sbom.cyclonedx ./_docker/retraced_sbom.cyclonedx || true
+          mv sidecar_docker_sbom.spdx "./_docker/sidecar_sbom.spdx" || true
+          mv sidecar_docker_sbom.cyclonedx ./_docker/sidecar_sbom.cyclonedx || true
+
       - name: ORAS Setup
         if: github.ref == 'refs/heads/release'
         run: |
@@ -325,7 +397,7 @@ jobs:
           fi
           cosign sign -y --key /tmp/cosign.key ghcr.io/${{ github.repository }}/sbom${{ needs.ci.outputs.IMAGE_SUFFIX }}@${ORAS_DIGEST}
           cd _docker || true
-          result=$(../oras_install/oras push ghcr.io/${{ github.repository }}/sbom${{ needs.ci.outputs.IMAGE_SUFFIX }}:docker-${{ needs.ci.outputs.NPM_VERSION }} ./sbom.*)
+          result=$(../oras_install/oras push ghcr.io/${{ github.repository }}/sbom${{ needs.ci.outputs.IMAGE_SUFFIX }}:docker-${{ needs.ci.outputs.NPM_VERSION }} ./*sbom.*)
           ORAS_DIGEST=$(echo $result | grep -oE 'sha256:[a-f0-9]{64}')
           if [ -z "$ORAS_DIGEST" ]; then
             echo "Error: ORAS_DIGEST is empty"

.gitignore

@@ -107,4 +107,4 @@ test-results.xml
 .env.test.local
 .env.production.local
 mmdb/**/**
-GeoIP.conf
+GeoIP.conf

deploy/crontab

@@ -4,3 +4,4 @@
 * * * * *		curl -d "{}" http://${NSQD_HOST}:${NSQD_HTTP_PORT}/pub?topic=every_minute
 0 12 1-8 * *		test -z "$RETRACED_DISABLE_GEOSYNC" && test $(date +\%u) -eq 3 && curl -d "{}" http://${NSQD_HOST}:${NSQD_HTTP_PORT}/pub?topic=first_wed_of_month
 * * * * * * *		curl -d "{}" http://${NSQD_HOST}:${NSQD_HTTP_PORT}/pub?topic=every_second
+*/5 * * * *		curl -d "{}" http://${NSQD_HOST}:${NSQD_HTTP_PORT}/pub?topic=pull_events_for_export

deploy/sidecar/Dockerfile

@@ -0,0 +1,14 @@
+FROM node:20.10.0-alpine3.19
+
+WORKDIR /src
+ADD package.json /src
+ADD package-lock.json /src
+RUN npm install
+# Copy the Node.js application code
+ADD . /src
+RUN npm run build
+ENV PORT 3002
+# Expose the port used by the Node.js application
+EXPOSE 3002
+EXPOSE 9229
+CMD  ["node", "--inspect=0.0.0.0", "--enable-source-maps", "/src/build/src/ee/sidecar/index.js"]

docker-compose.yaml

@@ -29,6 +29,7 @@ x-common-variables: &common-variables
   GEOIPUPDATE_USE_MMDB: ${GEOIPUPDATE_USE_MMDB}
   GEOIPUPDATE_ACCOUNT_ID: ${GEOIPUPDATE_ACCOUNT_ID}
   PG_SEARCH: ${PG_SEARCH}
+  SIDECAR_PORT: 3002
 
 services:
   retraced-api:
@@ -199,7 +200,7 @@ services:
     restart: "on-failure"
 
   admin-portal:
-    image: boxyhq/jackson:1.19.1
+    image: boxyhq/jackson:1.21.4
     ports:
       - "5225:5225"
     networks:
@@ -218,8 +219,29 @@ services:
       - RETRACED_HOST_URL=http://retraced-api:3000/auditlog
       - RETRACED_EXTERNAL_URL=http://localhost:3000/auditlog
       - RETRACED_ADMIN_ROOT_TOKEN=dev
+      - BOXYHQ_HOSTED=${BOXYHQ_HOSTED}
+      - BOXYHQ_LICENSE_KEY=${BOXYHQ_LICENSE_KEY}
     depends_on:
       - "retraced-api"
     restart: "always"
+
+  sidecar:
+    build:
+      context: .
+      dockerfile: ./deploy/sidecar/Dockerfile
+    environment: *common-variables
+    ports:
+      - "9393:9229"
+      - "3002:3002"
+    depends_on:
+      elasticsearch:
+        condition: service_started
+      postgres:
+        condition: service_started
+      nsqd:
+        condition: service_started
+    networks:
+      - retraced
 volumes:
   mmdb:
+  postgres-data:

kustomize/base/kustomization.yaml

@@ -8,5 +8,6 @@ resources:
   - ./nsqd-service.yaml
   - ./processor-deployment.yaml
   - ./geoipupdate.yaml
+  - ./sidecar-deployment.yaml
 
 namespace: default

kustomize/base/sidecar-deployment.yaml

@@ -0,0 +1,46 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: retraced-sidecar
+  namespace: '{{repl ConfigOption "namespace"}}'
+spec:
+  selector:
+    matchLabels:
+      tier: sidecar
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: auditlog
+        tier: sidecar
+    spec:
+      containers:
+        - name: sidecar
+          image: retracedhq/vector-local
+          imagePullPolicy: IfNotPresent
+          startupProbe:
+            httpGet:
+              port: 3002
+              path: /api/v1/health
+            periodSeconds: 10
+            timeoutSeconds: 10
+            failureThreshold: 5
+          readinessProbe:
+            httpGet:
+              port: 3002
+              path: /api/v1/health
+            periodSeconds: 30
+            timeoutSeconds: 10
+            failureThreshold: 5
+            successThreshold: 2
+          ports:
+            - containerPort: 3002
+            - containerPort: 8686
+          envFrom:
+            - secretRef:
+                name: auditlog
+          resources:
+            requests:
+              cpu: 100m
+            limits:
+              cpu: 1000m

kustomize/overlays/demo/kustomization.yaml

@@ -13,7 +13,10 @@ patches:
   - ./migratees-job.yaml
   - ./migratepg-job.yaml
   - ./processor-deployment.yaml
+  - ./sidecar-deployment.yaml
 
 images:
   - name: retracedhq/retraced
     newTag: 1.8.4
+  - name: retracedhq/vector
+    newTag: 0.0.1