Disaster recovery feature via mongodb replica set (#1311)

## Description The primary and standby cluster MongoDB members are in the same MongoDB replica set. The standby cluster members are configured as hidden, delayed, and with zero priority. When the primary cluster goes down, the standby cluster MongoDB members are promoted to standby state—one of them will become primary by administrator. After the primary is back online, the primary cluster members will be demoted to hidden. For switching back, the primary cluster members will be promoted to secondary MongoDB members and standby cluster members will be demoted. ## Another changes - Upgrade mongodb to v7
plgd-dev · Jun 27, 2024 · 5e5424d · 5e5424d
1 parent 791822e
commit 5e5424d
Show file tree

Hide file tree

Showing 43 changed files with 1,954 additions and 170 deletions.
diff --git a/.github/workflows/build-publish.yaml b/.github/workflows/build-publish.yaml
@@ -109,6 +109,12 @@ jobs:
           - name: cert-tool
             directory: tools/cert-tool
             file: tools/cert-tool/Dockerfile
+          - name: mongodb-standby-tool
+            directory: tools/mongodb/standby-tool
+            file: tools/mongodb/standby-tool/Dockerfile
+          - name: mongodb-admin-tool
+            directory: tools/mongodb/admin-tool
+            file: tools/mongodb/admin-tool/Dockerfile
     uses: ./.github/workflows/build-publish-cfg.yaml
     with:
       name: ${{ matrix.name }}

diff --git a/charts/plgd-hub/Chart.lock b/charts/plgd-hub/Chart.lock
@@ -4,9 +4,9 @@ dependencies:
   version: 1.1.9
 - name: mongodb
   repository: https://charts.bitnami.com/bitnami
-  version: 13.18.3
+  version: 15.4.4
 - name: scylla
   repository: https://scylla-operator-charts.storage.googleapis.com/stable
   version: v1.10.0
-digest: sha256:3ed14035e169597cbd91e8521414c8bbae13879d27bda6726ce45b1e74572050
-generated: "2024-03-05T10:43:45.524423289Z"
+digest: sha256:2c3e908c6556633c6fd5467498177029023aa737b9436d1001907ecd9ba282ec
+generated: "2024-05-20T14:26:43.823740553Z"
diff --git a/charts/plgd-hub/Chart.yaml b/charts/plgd-hub/Chart.yaml
@@ -15,7 +15,7 @@ dependencies:
     repository: "https://nats-io.github.io/k8s/helm/charts/"
     condition: nats.enabled
   - name: "mongodb"
-    version: "13.18.3"
+    version: "15.4.4"
     repository: "https://charts.bitnami.com/bitnami"
     condition: mongodb.enabled
   - name: "scylla"

diff --git a/charts/plgd-hub/templates/_helpers.tpl b/charts/plgd-hub/templates/_helpers.tpl
@@ -2,7 +2,7 @@
 Expand the name of the chart.
 */}}
 {{- define "plgd-hub.name" -}}
-{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- default .Chart.Name .Values.nameOverride | replace "+" "_" | trunc 63 | trimSuffix "-" }}
 {{- end }}
 
 {{/*
@@ -76,15 +76,24 @@ If release name contains chart name it will be used as a full name.
 {{- end }}
 {{- end }}
 
-{{- define "plgd-hub.certificateConfig" }}
+
+{{- define "plgd-hub.certificateConfigWithExtraCAPool" }}
   {{- $ := index . 0 }}
   {{- $certDefinition := index . 1 }}
   {{- $certPath := index . 2 }}
+  {{- $useCAPool := index . 3 }}
+  {{- $caPool := "" }}
   {{- if $certDefinition.caPool }}
-  caPool: {{ printf "%s" $certDefinition.caPool | quote }}
+  {{- $caPool = printf "%s" $certDefinition.caPool | quote }}
   {{- else if $.Values.certmanager.enabled }}
-  caPool: {{ printf "%s/ca.crt" $certPath | quote  }}
+  {{- $caPool = printf "%s/ca.crt" $certPath | quote }}
   {{- end }}
+  {{- $extraCAPool := include "plgd-hub.extraCAPoolConfig" (list $ $useCAPool) }}
+  caPool:
+    - {{ $caPool }}
+    {{- if $extraCAPool }}
+    {{ $extraCAPool }}
+    {{- end }}
   {{- if $certDefinition.keyFile }}
   keyFile: {{ printf "%s" $certDefinition.keyFile | quote }}
   {{- else if $.Values.certmanager.enabled }}
@@ -97,23 +106,36 @@ If release name contains chart name it will be used as a full name.
   {{- end }}
 {{- end }}
 
-{{- define "plgd-hub.authorizationCaCertificateConfig" }}
+{{- define "plgd-hub.internalCertificateConfig" }}
+{{- $ := index . 0 }}
+{{- $certDefinition := index . 1 }}
+{{- $certPath := index . 2 }}
+{{- include "plgd-hub.certificateConfigWithExtraCAPool" (list $ $certDefinition $certPath $.Values.extraCAPool.internal) }}
+{{- end }}
+
+{{- define "plgd-hub.coapCertificateConfig" }}
+{{- $ := index . 0 }}
+{{- $certDefinition := index . 1 }}
+{{- $certPath := index . 2 }}
+{{- include "plgd-hub.certificateConfigWithExtraCAPool" (list $ $certDefinition $certPath $.Values.extraCAPool.coap) }}
+{{- end }}
+
+{{- define "plgd-hub.storageCertificateConfig" }}
+{{- $ := index . 0 }}
+{{- $certDefinition := index . 1 }}
+{{- $certPath := index . 2 }}
+{{- include "plgd-hub.certificateConfigWithExtraCAPool" (list $ $certDefinition $certPath $.Values.extraCAPool.storage) }}
+{{- end }}
+
+{{- define "plgd-hub.certificateConfig" }}
   {{- $ := index . 0 }}
   {{- $certDefinition := index . 1 }}
   {{- $certPath := index . 2 }}
-  {{- $caPool := list (printf "%s/%s" $.Values.extraAuthorizationCAPool.mountPath $.Values.extraAuthorizationCAPool.fileName | quote) (printf "%s/ca.crt" $certPath | quote) }}
   {{- if $certDefinition.caPool }}
   caPool: {{ printf "%s" $certDefinition.caPool | quote }}
   {{- else if $.Values.certmanager.enabled }}
-  {{- if $.Values.global.authorizationCAPool }}
-  caPool:
-    {{- range $caPool }}
-    - {{ printf "%s" . }}
-    {{- end }}
-  {{- else }}
   caPool: {{ printf "%s/ca.crt" $certPath | quote  }}
   {{- end }}
-  {{- end }}
   {{- if $certDefinition.keyFile }}
   keyFile: {{ printf "%s" $certDefinition.keyFile | quote }}
   {{- else if $.Values.certmanager.enabled }}
@@ -126,6 +148,12 @@ If release name contains chart name it will be used as a full name.
   {{- end }}
 {{- end }}
 
+{{- define "plgd-hub.authorizationCaCertificateConfig" }}
+{{- $ := index . 0 }}
+{{- $certDefinition := index . 1 }}
+{{- $certPath := index . 2 }}
+{{- include "plgd-hub.certificateConfigWithExtraCAPool" (list $ $certDefinition $certPath $.Values.extraCAPool.authorization) }}
+{{- end }}
 
 {{- define "plgd-hub.authorizationConfig" }}
   {{- $ := index . 0 }}
@@ -154,8 +182,6 @@ If release name contains chart name it will be used as a full name.
   {{- end }}
 {{- end }}
 
-
-
 {{- define "plgd-hub.createInternalCertByCm" }}
     {{- $natsTls := .Values.coapgateway.clients.eventBus.nats.tls.certFile }}
     {{- $authClientTls := .Values.coapgateway.clients.identityStore.grpc.tls.certFile }}
@@ -416,4 +442,148 @@ openTelemetryCollector:
   {{- else }}
   {{- printf "mongoDB" }}
   {{- end }}
-{{- end }}
+{{- end }}
+
+{{- define "plgd-hub.replicas" -}}
+  {{- $ := index . 0 -}}
+  {{- $useReplicas := index . 1 }}
+  {{- if $.Values.global.standby -}}
+  0
+  {{- else -}}
+  {{- $useReplicas -}}
+  {{- end -}}
+{{- end -}}
+
+{{- define "plgd-hub.extraCAPoolAuthorizationEnabled" -}}
+{{- $ := . }}
+{{- if include "plgd-hub.resolveTemplateString" (list . $.Values.global.extraCAPool.authorization) -}}
+true
+{{- else -}}
+{{- printf "" }}
+{{- end -}}
+{{- end -}}
+
+{{- define "plgd-hub.extraCAPoolInternalEnabled" -}}
+{{- $ := . }}
+{{- if include "plgd-hub.resolveTemplateString" (list . $.Values.global.extraCAPool.internal) -}}
+true
+{{- else -}}
+{{- printf "" }}
+{{- end -}}
+{{- end -}}
+
+{{- define "plgd-hub.extraCAPoolStorageEnabled" -}}
+{{- $ := . }}
+{{- if include "plgd-hub.resolveTemplateString" (list . $.Values.global.extraCAPool.storage) -}}
+true
+{{- else -}}
+{{- printf "" }}
+{{- end -}}
+{{- end -}}
+
+{{- define "plgd-hub.extraCAPoolCoapEnabled" -}}
+{{- $ := . }}
+{{- if include "plgd-hub.resolveTemplateString" (list . $.Values.global.extraCAPool.coap) -}}
+true
+{{- else -}}
+{{- printf "" }}
+{{- end -}}
+{{- end -}}
+
+{{- define "plgd-hub.extraCAPoolVolume" -}}
+{{- $ := index . 0 -}}
+{{- $useCAPool := index . 1 }}
+{{- with $useCAPool -}}
+{{- $enabled := include "plgd-hub.resolveTemplateString" (list $ .enabled) -}}
+{{- if and $enabled (or .configMapName .secretName) -}}
+- name: {{ .name | quote }}
+{{- if .configMapName }}
+  configMap:
+    name: {{ include "plgd-hub.resolveTemplateString" (list $ .configMapName) }}
+{{- else if .secretName }}
+  secret:
+    secretName: {{ include "plgd-hub.resolveTemplateString" (list $ .secretName) }}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{- define "plgd-hub.extraCAPoolMount" -}}
+{{- $ := index . 0 -}}
+{{- $useCAPool := index . 1 -}}
+{{- with $useCAPool -}}
+{{- $enabled := include "plgd-hub.resolveTemplateString" (list $ .enabled) -}}
+{{- if and $enabled (or .configMapName .secretName) -}}
+- name: {{ .name | quote }}
+  mountPath: {{ .mountPath | quote }}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{- define "plgd-hub.extraCAPoolConfig" }}
+{{- $ := index . 0 -}}
+{{- $useCAPool := index . 1 }}
+{{- with $useCAPool -}}
+{{- $enabled := include "plgd-hub.resolveTemplateString" (list $ .enabled) -}}
+{{- if and $enabled (or .configMapName .secretName) -}}
+- {{ printf "%s/%s" .mountPath (include "plgd-hub.resolveTemplateString" (list $ .key) ) | quote }}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{- define "plgd-hub.isTemplateString" }}
+{{- $ret := "" }}
+{{- if typeIs "string" . -}}
+{{- if and (hasPrefix "{{" .) (hasSuffix "}}" .) -}}
+{{- $ret = "true" }}
+{{- end }}
+{{- end }}
+{{- printf $ret }}
+{{- end }}
+
+{{- define "plgd-hub.resolveTemplateString" }}
+{{- $ := index . 0 -}}
+{{- $string := index . 1 }}
+{{- $ret := "" }}
+{{- if include "plgd-hub.isTemplateString" $string -}}
+{{- $ret = tpl $string $ -}}
+{{- else -}}
+{{- $ret = $string -}}
+{{- end }}
+{{- if $ret }}
+{{- $ret }}
+{{- else }}
+{{- printf "" }}
+{{- end }}
+{{- end }}
+
+{{- define "plgd-hub.oldExtraCAPoolAuthorizationFileName" }}
+{{- $ := . -}}
+{{- $fileName := "ca.crt" -}}
+{{- if $.Values.extraAuthorizationCAPool -}}
+{{- if $.Values.extraAuthorizationCAPool.fileName -}}
+{{- $fileName = $.Values.extraAuthorizationCAPool.fileName -}}
+{{- end -}}
+{{- end -}}
+{{- printf "%s" $fileName }}
+{{- end -}}
+
+{{- define "plgd-hub.oldExtraCAPoolAuthorizationSecretName" }}
+{{- $ := . -}}
+{{- $secretName := "authorization-ca-pool" -}}
+{{- if $.Values.extraAuthorizationCAPool -}}
+{{- if $.Values.extraAuthorizationCAPool.name -}}
+{{- $secretName = $.Values.extraAuthorizationCAPool.name -}}
+{{- end -}}
+{{- end -}}
+{{- printf "%s" $secretName }}
+{{- end -}}
+
+{{- define "plgd-hub.oldGlobalAuthorizationCAPool" }}
+{{- $ := . -}}
+{{- $ca := "" -}}
+{{- if $.Values.global.authorizationCAPool -}}
+{{- $ca = $.Values.global.authorizationCAPool -}}
+{{- end -}}
+{{- printf "%s" $ca }}
+{{- end -}}
diff --git a/charts/plgd-hub/templates/certificate-authority/config.yaml b/charts/plgd-hub/templates/certificate-authority/config.yaml
@@ -37,7 +37,7 @@ data:
           timeout: {{ .apis.grpc.keepAlive.maxConnectionIdle }}
         tls:
           {{- $tls := .apis.grpc.tls }}
-          {{- include "plgd-hub.certificateConfig" (list $ $tls $cert ) | indent 8 }}
+          {{- include "plgd-hub.internalCertificateConfig" (list $ $tls $cert ) | indent 8 }}
           clientCertificateRequired: {{ .apis.grpc.tls.clientCertificateRequired }}
         authorization:
           {{- $authorization := .apis.grpc.authorization }}
@@ -69,7 +69,7 @@ data:
           maxConnIdleTime: {{ .clients.storage.mongoDB.maxConnIdleTime }}
           tls:
             {{- $mongoDbTls := .clients.storage.mongoDB.tls }}
-            {{- include "plgd-hub.certificateConfig" (list $ $mongoDbTls $cert ) | indent 10 }}
+            {{- include "plgd-hub.internalCertificateConfig" (list $ $mongoDbTls $cert ) | indent 10 }}
             useSystemCAPool: {{ .clients.storage.mongoDB.tls.useSystemCAPool }}
           bulkWrite:
             timeout: {{ .clients.storage.mongoDB.bulkWrite.timeout  | quote }}
@@ -94,7 +94,7 @@ data:
               {{- toYaml .clients.storage.cqlDB.keyspace.replication | nindent 14 }}
           tls:
             {{- $cqlDbTls := .clients.storage.cqlDB.tls }}
-            {{- include "plgd-hub.certificateConfig" (list $ $cqlDbTls $cert ) | indent 10 }}
+            {{- include "plgd-hub.internalCertificateConfig" (list $ $cqlDbTls $cert ) | indent 10 }}
             useSystemCAPool: {{ .clients.storage.cqlDB.tls.useSystemCAPool }}
       {{- include "plgd-hub.openTelemetryExporterConfig" (list $ $cert ) | nindent 6 }}
     signer:

diff --git a/charts/plgd-hub/templates/certificate-authority/deployment.yaml b/charts/plgd-hub/templates/certificate-authority/deployment.yaml
@@ -16,7 +16,7 @@ metadata:
   {{- . | toYaml | nindent 4 }}
   {{- end }}
 spec:
-  replicas: {{ .Values.certificateauthority.replicas }}
+  replicas: {{ include "plgd-hub.replicas" (list $ .Values.certificateauthority.replicas) }}
   selector:
     matchLabels:
       {{- include "plgd-hub.certificateauthority.selectorLabels" . | nindent 6 }}
@@ -92,10 +92,8 @@ spec:
             - name: {{ .Values.certificateauthority.ca.volume.name }}
               mountPath: {{ .Values.certificateauthority.ca.volume.mountPath }}
             {{- end }}
-            {{- if .Values.global.authorizationCAPool }}
-            - name: {{ .Values.extraAuthorizationCAPool.name }}
-              mountPath: {{ .Values.extraAuthorizationCAPool.mountPath }}
-            {{- end }}
+            {{- include "plgd-hub.extraCAPoolMount" (list . .Values.extraCAPool.authorization) | nindent 12 }}
+            {{- include "plgd-hub.extraCAPoolMount" (list . .Values.extraCAPool.internal) | nindent 12 }}
             {{- with .Values.certificateauthority.extraVolumeMounts }}
             {{- toYaml . | nindent 12 }}
             {{- end }}
@@ -116,11 +114,8 @@ spec:
           secret:
             secretName: {{ .Values.certificateauthority.ca.secret.name | default $.Values.certmanager.default.ca.secret.name }}
         {{- end }}
-        {{- if .Values.global.authorizationCAPool }}
-        - name: {{ .Values.extraAuthorizationCAPool.name }}
-          secret:
-            secretName: {{ .Values.extraAuthorizationCAPool.name }}
-        {{- end }}
+        {{- include "plgd-hub.extraCAPoolVolume" (list . .Values.extraCAPool.authorization) | nindent 8 }}
+        {{- include "plgd-hub.extraCAPoolVolume" (list . .Values.extraCAPool.internal) | nindent 8 }}
         {{- with .Values.certificateauthority.extraVolumes }}
         {{- toYaml . | nindent 8 }}
         {{- end }}

diff --git a/charts/plgd-hub/templates/certs/authorization-ca-pool.yaml b/charts/plgd-hub/templates/certs/authorization-ca-pool.yaml
@@ -1,11 +1,12 @@
-{{- if .Values.global.authorizationCAPool }}
+{{- $secretData := include "plgd-hub.resolveTemplateString" (list . .Values.global.extraCAPool.authorization) }}
+{{- if $secretData }}
 apiVersion: v1
 kind: Secret
 metadata:
-  name: {{ .Values.extraAuthorizationCAPool.name }}
+  name: {{ include "plgd-hub.resolveTemplateString" (list . .Values.extraCAPool.authorization.secretName) }}
   namespace: {{ $.Release.Namespace }}
   labels:
     {{- include "plgd-hub.labels" $ | nindent 4 }}
 data:
-  {{ .Values.extraAuthorizationCAPool.fileName }}: {{ .Values.global.authorizationCAPool | b64enc }}
+  {{ include "plgd-hub.resolveTemplateString" (list . .Values.extraCAPool.authorization.key) }}: {{ $secretData | b64enc }}
 {{- end }}
diff --git a/charts/plgd-hub/templates/certs/coap/ca-pool.yaml b/charts/plgd-hub/templates/certs/coap/ca-pool.yaml
@@ -0,0 +1,12 @@
+{{- $secretData := include "plgd-hub.resolveTemplateString" (list . .Values.global.extraCAPool.coap) }}
+{{- if $secretData }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "plgd-hub.resolveTemplateString" (list . .Values.extraCAPool.coap.secretName) }}
+  namespace: {{ $.Release.Namespace }}
+  labels:
+    {{- include "plgd-hub.labels" $ | nindent 4 }}
+data:
+  {{ include "plgd-hub.resolveTemplateString" (list . .Values.extraCAPool.coap.key) }}: {{ $secretData | b64enc }}
+{{- end }}
diff --git a/charts/plgd-hub/templates/certs/internal/ca-pool.yaml b/charts/plgd-hub/templates/certs/internal/ca-pool.yaml
@@ -0,0 +1,12 @@
+{{- $secretData := include "plgd-hub.resolveTemplateString" (list . .Values.global.extraCAPool.internal) }}
+{{- if $secretData }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "plgd-hub.resolveTemplateString" (list . .Values.extraCAPool.internal.secretName) }}
+  namespace: {{ $.Release.Namespace }}
+  labels:
+    {{- include "plgd-hub.labels" $ | nindent 4 }}
+data:
+  {{ include "plgd-hub.resolveTemplateString" (list . .Values.extraCAPool.internal.key) }}: {{ $secretData | b64enc }}
+{{- end }}