feat(deployments): separate db, storage, and reports deployments (#192)

Signed-off-by: Andrew Azores <me@andrewazor.es> Signed-off-by: Thuan Vo <thuan.votann@gmail.com> Co-authored-by: Thuan Vo <thuan.votann@gmail.com>
cryostatio · Oct 25, 2024 · 5bda844 · 5bda844
1 parent 786beee
commit 5bda844
Show file tree

Hide file tree

Showing 41 changed files with 1,823 additions and 397 deletions.
diff --git a/.github/workflows/chart-test.yml b/.github/workflows/chart-test.yml
@@ -24,13 +24,21 @@ on:
       - v[0-9]+
       - v[0-9]+.[0-9]+
       - cryostat-v[0-9]+.[0-9]+
+      # TODO: Remove before merging https://github.com/cryostatio/cryostat-helm/pull/192
+      - separate-db-storage
 
 jobs:
   helm-test:
     runs-on: ubuntu-latest
     env:
       TARGET_BRANCH: ${{ github.event.pull_request.base.ref || github.ref_name }}
       TEST_NAMESPACE: helm-test
+    strategy:
+      matrix:
+        # Supported strategies
+        upgrade-strategy:
+          - reset-values
+          - reset-then-reuse-values
     steps:
       - name: Fail if safe-to-test label NOT applied
         if: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'safe-to-test') }}
@@ -57,7 +65,7 @@ jobs:
 
           HELM_LOCATION="$(which helm)"
           sudo mv $HELM_LOCATION "$(dirname $HELM_LOCATION)/.helm"
-          cat <(echo '#!/usr/bin/env bash') <(echo 'exec .helm "${@//--reuse-values/--reset-then-reuse-values}"') | sudo tee $HELM_LOCATION
+          cat <(echo '#!/usr/bin/env bash') <(echo 'exec .helm "${@//--reuse-values/--${{ matrix.upgrade-strategy }}}"') | sudo tee $HELM_LOCATION
           sudo chmod +x $HELM_LOCATION
 
           bash ct.bash --upgrade

diff --git a/charts/cryostat/README.md b/charts/cryostat/README.md
diff --git a/charts/cryostat/templates/NOTES.txt b/charts/cryostat/templates/NOTES.txt
@@ -27,7 +27,7 @@
 {{- end }}
 
 {{- if not (empty $envVars) }}
-  kubectl -n {{ .Release.Namespace }} set env deploy --containers={{ .Chart.Name }} {{ include "cryostat.fullname" . }} {{ join " " $envVars }}
+  kubectl -n {{ .Release.Namespace }} set env deploy --containers={{ .Chart.Name }} {{ include "cryostat.deploymentName" . }} {{ join " " $envVars }}
 {{- end }}
   ```
 {{- end }}
@@ -36,11 +36,10 @@
 
 {{ $listNum }}. Forward local port to the application's pod:
   ```
-  kubectl -n {{ .Release.Namespace }} wait --for=condition=available --timeout=60s deploy/{{ include "cryostat.fullname" . }}
+  kubectl -n {{ .Release.Namespace }} wait --for=condition=available --timeout=60s deploy/{{ include "cryostat.deploymentName" . }}
 
-  export POD_NAME=$(kubectl get pods -n {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "cryostat.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" --sort-by=.metadata.creationTimestamp -o jsonpath="{.items[-1:].metadata.name}")
-  export CONTAINER_PORT=$(kubectl get pod -n {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
-  kubectl -n {{ .Release.Namespace }} port-forward $POD_NAME "8080:$CONTAINER_PORT"
+  export POD_NAME=$(kubectl get pods -n {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "cryostat.name" . }},app.kubernetes.io/instance={{ .Release.Name }},app.kubernetes.io/component=cryostat" --sort-by=.metadata.creationTimestamp -o jsonpath="{.items[-1:].metadata.name}")
+  kubectl -n {{ .Release.Namespace }} port-forward $POD_NAME 8080:http
   ```
   {{- $listNum = add1 $listNum }}
 {{- end }}

diff --git a/charts/cryostat/templates/_helpers.tpl b/charts/cryostat/templates/_helpers.tpl
@@ -101,12 +101,12 @@ Get or generate a default encryption key for database.
 Get or generate a default secret key for object storage.
 */}}
 {{- define "cryostat.objectStorageSecretKey" -}}
-{{- $secret := (lookup "v1" "Secret" .Release.Namespace (printf "%s-storage" .Release.Name)) -}}
+{{- $secret := (lookup "v1" "Secret" .Release.Namespace (printf "%s-storage-secret" .Release.Name)) -}}
 {{- if $secret -}}
 {{/*
    Use current secret. Do not regenerate.
 */}}
-{{- $secret.data.SECRET_KEY -}}
+{{- $secret.data.STORAGE_ACCESS_KEY -}}
 {{- else -}}
 {{/*
     Generate new secret
@@ -145,3 +145,11 @@ Get or generate a default secret key for auth proxy cookies.
 {{- end -}}
 {{- join "," (default list $l | compact | uniq) | quote -}}
 {{- end -}}
+
+{{/*
+Get the name for managed deployments.
+*/}}
+{{- define "cryostat.deploymentName" -}}
+{{- $version := semver .Chart.AppVersion -}}
+{{- printf "%s-v%d" (include "cryostat.fullname" .) $version.Major -}}
+{{- end -}}
diff --git a/charts/cryostat/templates/_oauth2Proxy.tpl b/charts/cryostat/templates/_oauth2Proxy.tpl
@@ -15,7 +15,7 @@ Create OAuth2 Proxy container. Configurations defined in alpha_config.yaml
     - name: OAUTH2_PROXY_COOKIE_SECRET
       valueFrom:
         secretKeyRef:
-          name: {{ .Release.Name }}-cookie-secret
+          name: {{ default (printf "%s-cookie-secret" .Release.Name) .Values.authentication.cookieSecretName }}
           key: COOKIE_SECRET
           optional: false
     - name: OAUTH2_PROXY_EMAIL_DOMAINS
@@ -35,7 +35,10 @@ Create OAuth2 Proxy container. Configurations defined in alpha_config.yaml
     {{- end }}
   ports:
     - containerPort: 4180
+      name: http
       protocol: TCP
+  resources:
+    {{- toYaml .Values.oauth2Proxy.resources | nindent 4 }}
   volumeMounts:
     - name: alpha-config
       mountPath: /etc/oauth2_proxy/alpha_config

diff --git a/charts/cryostat/templates/_openshiftOauthProxy.tpl b/charts/cryostat/templates/_openshiftOauthProxy.tpl
@@ -10,7 +10,7 @@ Create OpenShift OAuth Proxy container.
     - name: COOKIE_SECRET
       valueFrom:
         secretKeyRef:
-          name: {{ .Release.Name }}-cookie-secret
+          name: {{ default (printf "%s-cookie-secret" .Release.Name) .Values.authentication.cookieSecretName }}
           key: COOKIE_SECRET
           optional: false
   args:
@@ -20,7 +20,6 @@ Create OpenShift OAuth Proxy container.
     - --pass-basic-auth=false
     - --upstream=http://localhost:8181/
     - --upstream=http://localhost:3000/grafana/
-    - --upstream=http://localhost:8333/storage/
     - --cookie-secret="$(COOKIE_SECRET)"
     - --openshift-service-account={{ include "cryostat.serviceAccountName" . }}
     - --proxy-websockets=true
@@ -40,7 +39,13 @@ Create OpenShift OAuth Proxy container.
   imagePullPolicy: {{ .Values.openshiftOauthProxy.image.pullPolicy }}
   ports:
     - containerPort: 4180
+      name: http
       protocol: TCP
+    - containerPort: 8443
+      name: https
+      protocol: TCP
+  resources:
+    {{- toYaml .Values.openshiftOauthProxy.resources | nindent 4 }}
   volumeMounts:
     {{- if .Values.authentication.basicAuth.enabled }}
     - name: {{ .Release.Name }}-htpasswd
@@ -49,7 +54,6 @@ Create OpenShift OAuth Proxy container.
     {{- end }}
     - name: {{ .Release.Name }}-proxy-tls
       mountPath: /etc/tls/private
-  resources: {}
   terminationMessagePath: /dev/termination-log
   terminationMessagePolicy: File
 {{- end}}
diff --git a/charts/cryostat/templates/alpha_config.yaml b/charts/cryostat/templates/alpha_config.yaml
@@ -21,12 +21,6 @@ data:
         - id: grafana
           path: /grafana/
           uri: http://localhost:3000
-        - id: storage
-          path: ^/storage/(.*)$
-          rewriteTarget: /$1
-          uri: http://localhost:8333
-          passHostHeader: false
-          proxyWebSockets: false
     providers:
       - id: dummy
         name: Unused - Sign In Below

diff --git a/charts/cryostat/templates/cookie_secret.yaml b/charts/cryostat/templates/cookie_secret.yaml
@@ -1,7 +1,11 @@
+{{- if empty .Values.authentication.cookieSecretName -}}
 apiVersion: v1
 kind: Secret
 metadata:
   name: {{ .Release.Name }}-cookie-secret
+  labels:
+    {{- include "cryostat.labels" $ | nindent 4 }}
 type: Opaque
 data:
   COOKIE_SECRET: {{ include "cryostat.cookieSecret" . }}
+{{- end -}}
diff --git a/charts/cryostat/templates/deployment.yaml → ...yostat/templates/cryostat_deployment.yaml b/charts/cryostat/templates/deployment.yaml → ...yostat/templates/cryostat_deployment.yaml
@@ -3,16 +3,18 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: {{ include "cryostat.fullname" . }}
+  name: {{ include "cryostat.deploymentName" . }}
   labels:
     {{- include "cryostat.labels" . | nindent 4 }}
+    app.kubernetes.io/component: cryostat
 spec:
   replicas: 1
   strategy:
     type: Recreate
   selector:
     matchLabels:
       {{- include "cryostat.selectorLabels" . | nindent 6 }}
+      app.kubernetes.io/component: cryostat
   template:
     metadata:
       {{- with .Values.podAnnotations }}
@@ -21,6 +23,7 @@ spec:
       {{- end }}
       labels:
         {{- include "cryostat.selectorLabels" . | nindent 8 }}
+        app.kubernetes.io/component: cryostat
     spec:
       {{- with .Values.imagePullSecrets }}
       imagePullSecrets:
@@ -43,6 +46,8 @@ spec:
           env:
           - name: QUARKUS_HTTP_HOST
             value: localhost
+          - name: QUARKUS_LOG_LEVEL
+            value: {{ .Values.core.debug.log.level }}
           - name: QUARKUS_HTTP_PORT
             value: "8181"
           - name: QUARKUS_HTTP_PROXY_PROXY_ADDRESS_FORWARDING
@@ -57,6 +62,10 @@ spec:
             value: none
           - name: QUARKUS_HIBERNATE_ORM_SQL_LOAD_SCRIPT
             value: no-file
+          {{- if gt (int (.Values.reports).replicas) 0 }}
+          - name: QUARKUS_REST_CLIENT_REPORTS_URL
+            value: {{ printf "http://%s-reports:%d" $fullName (int .Values.reports.service.httpPort) }}
+          {{- end }}
           - name: QUARKUS_DATASOURCE_USERNAME
             value: cryostat
           - name: QUARKUS_DATASOURCE_PASSWORD
@@ -66,11 +75,11 @@ spec:
                 key: CONNECTION_KEY
                 optional: false
           - name: QUARKUS_DATASOURCE_JDBC_URL
-            value: jdbc:postgresql://localhost:5432/cryostat
+            value: jdbc:postgresql://{{ $fullName }}-db:5432/cryostat
           - name: STORAGE_BUCKETS_ARCHIVES_NAME
             value: archivedrecordings
           - name: QUARKUS_S3_ENDPOINT_OVERRIDE
-            value: http://localhost:8333
+            value: http://{{ $fullName }}-storage:8333
           - name: QUARKUS_S3_PATH_STYLE_ACCESS
             value: "true"
           - name: QUARKUS_S3_AWS_REGION
@@ -84,8 +93,8 @@ spec:
           - name: QUARKUS_S3_AWS_CREDENTIALS_STATIC_PROVIDER_SECRET_ACCESS_KEY
             valueFrom:
               secretKeyRef:
-                name: {{ printf "%s-storage" .Release.Name }}
-                key: SECRET_KEY
+                name: {{ default (printf "%s-storage-secret" .Release.Name) .Values.storage.storageSecretName }}
+                key: STORAGE_ACCESS_KEY
                 optional: false
           - name: AWS_SECRET_ACCESS_KEY
             value: $(QUARKUS_S3_AWS_CREDENTIALS_STATIC_PROVIDER_SECRET_ACCESS_KEY)
@@ -121,84 +130,6 @@ spec:
             failureThreshold: 18
           resources:
             {{- toYaml .Values.core.resources | nindent 12 }}
-        - name: {{ printf "%s-%s" .Chart.Name "db" }}
-          securityContext:
-            {{- toYaml (.Values.db).securityContext | nindent 12 }}
-          image: "{{ (.Values.db).image.repository }}:{{ (.Values.db).image.tag }}"
-          imagePullPolicy: {{ (.Values.db).image.pullPolicy }}
-          env:
-          - name: POSTGRESQL_USER
-            value: cryostat
-          - name: POSTGRESQL_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: {{ default (printf "%s-db" .Release.Name) .Values.core.databaseSecretName }}
-                key: CONNECTION_KEY
-                optional: false
-          - name: POSTGRESQL_DATABASE
-            value: cryostat
-          - name: PG_ENCRYPT_KEY
-            valueFrom:
-              secretKeyRef:
-                name: {{ default (printf "%s-db" .Release.Name) .Values.core.databaseSecretName }}
-                key: ENCRYPTION_KEY
-                optional: false
-          ports:
-            - containerPort: 5432
-              protocol: TCP
-          volumeMounts:
-          - mountPath: /var/lib/pgsql/data
-            name: {{ .Chart.Name }}
-            subPath: postgres
-          readinessProbe:
-            exec:
-              command:
-              - pg_isready
-              - -U
-              - cryostat
-              - -d
-              - cryostat
-        - name: {{ printf "%s-%s" .Chart.Name "storage" }}
-          securityContext:
-            {{- toYaml (.Values.storage).securityContext | nindent 12 }}
-          image: "{{ (.Values.storage).image.repository }}:{{ (.Values.storage).image.tag }}"
-          imagePullPolicy: {{ (.Values.storage).image.pullPolicy }}
-          env:
-          - name: CRYOSTAT_BUCKETS
-            value: archivedrecordings,archivedreports,eventtemplates,probes
-          - name: CRYOSTAT_ACCESS_KEY
-            value: cryostat
-          - name: CRYOSTAT_SECRET_KEY
-            valueFrom:
-              secretKeyRef:
-                name: {{ printf "%s-storage" .Release.Name }}
-                key: SECRET_KEY
-                optional: false
-          - name: DATA_DIR
-            value: /data
-          - name: IP_BIND
-            value: 0.0.0.0
-          ports:
-            - containerPort: 8333
-              protocol: TCP
-          volumeMounts:
-          - mountPath: /data
-            name: {{ .Chart.Name }}
-            subPath: seaweed
-          livenessProbe:
-            httpGet:
-              path: "/status"
-              port: 8333
-            periodSeconds: 10
-            failureThreshold: 2
-          startupProbe:
-            httpGet:
-              path: "/status"
-              port: 8333
-            periodSeconds: 10
-            failureThreshold: 9
-          resources:
-            {{- toYaml (.Values.storage).resources | nindent 12 }}
         - name: {{ printf "%s-%s" .Chart.Name "grafana" }}
           securityContext:
             {{- toYaml .Values.grafana.securityContext | nindent 12 }}
@@ -258,15 +189,6 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       volumes:
-      {{- if ((.Values.pvc).enabled) }}
-      - name: {{ .Chart.Name }}
-        persistentVolumeClaim:
-          claimName: {{ .Release.Name }}
-      {{- end }}
-      {{- if not ((.Values.pvc).enabled) }}
-      - name: {{ .Chart.Name }}
-        emptyDir: {}
-      {{- end }}
       {{- if not (.Values.authentication.openshift).enabled }}
       - name: alpha-config
         configMap:

diff --git a/charts/cryostat/templates/service.yaml → .../cryostat/templates/cryostat_service.yaml b/charts/cryostat/templates/service.yaml → .../cryostat/templates/cryostat_service.yaml
@@ -6,6 +6,7 @@ metadata:
   name: {{ $fullName }}
   labels:
     {{- include "cryostat.labels" $ | nindent 4 }}
+    app.kubernetes.io/component: cryostat
   {{- if (.Values.authentication.openshift).enabled }}
   annotations:
     service.alpha.openshift.io/serving-cert-secret-name: {{ .Release.Name }}-proxy-tls
@@ -25,3 +26,4 @@ spec:
     {{- end }}
   selector:
     {{- include "cryostat.selectorLabels" $ | nindent 4 }}
+    app.kubernetes.io/component: cryostat