From 3575473cb59ee509cca4b0d76650014c9aeb09e1 Mon Sep 17 00:00:00 2001 From: William Hearn Date: Tue, 5 Nov 2024 12:15:44 -0500 Subject: [PATCH] feat(ckan): Add backup logic --- stable/ckan/conf/ckan/overlay-production.ini | 9 ++++- stable/ckan/templates/cm/ckan-crontab.yaml | 13 +++++++ stable/ckan/templates/deploy/ckan.yaml | 36 ++++++++++++++++--- .../templates/secret/ckan-backup-secret.yaml | 10 ++++++ stable/ckan/values.yaml | 6 ++++ 5 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 stable/ckan/templates/cm/ckan-crontab.yaml create mode 100644 stable/ckan/templates/secret/ckan-backup-secret.yaml diff --git a/stable/ckan/conf/ckan/overlay-production.ini b/stable/ckan/conf/ckan/overlay-production.ini index 81847e58..35389971 100644 --- a/stable/ckan/conf/ckan/overlay-production.ini +++ b/stable/ckan/conf/ckan/overlay-production.ini @@ -18,9 +18,16 @@ # [app:main] -ckan.site_title = CKAN-SCHEMING-FDI +ckan.locales_offered = en fr ckan.site_logo = /base/images/ckan-logo.png ckan.site_description = +ckan.auth.user_create_organizations = False +ckan.auth.user_create_groups = False +ckan.auth.user_delete_groups = false +ckan.auth.user_delete_organizations = false +ckan.auth.create_dataset_if_not_in_organization = False +ckan.auth.create_unowned_dataset = False +ckan.auth.public_activity_stream_detail = true ckan.favicon = /base/images/ckan.ico ckan.gravatar_default = identicon ckan.preview.direct = png jpg gif diff --git a/stable/ckan/templates/cm/ckan-crontab.yaml b/stable/ckan/templates/cm/ckan-crontab.yaml new file mode 100644 index 00000000..24cbc685 --- /dev/null +++ b/stable/ckan/templates/cm/ckan-crontab.yaml @@ -0,0 +1,13 @@ +{{- if .Values.ckan.backup.enabled }} +{{- $fullName := include "ckan.fullname" . -}} +{{- $adminuser := .Values.ckan.backup.adminuser -}} +kind: ConfigMap +apiVersion: v1 +metadata: + name: {{ include "ckan.fullname" . }}-crontabs +data: +{{- range $key, $value := .Values.ckan.cronjobs }} + {{ $key }}.sh: | + {{ $value.command }} -c /srv/app/production.ini -O /app/backup/{{ $fullName }}-{{ $key }}-$(date +%Y-%m-%d).json1.gz -z -u {{ $adminuser }} >> /app/backup/{{ $fullName }}-{{ $key }}-$(date +%Y-%m-%d)-log.txt ; +{{- end }} +{{- end }} diff --git a/stable/ckan/templates/deploy/ckan.yaml b/stable/ckan/templates/deploy/ckan.yaml index 535d0c26..7662c5bf 100644 --- a/stable/ckan/templates/deploy/ckan.yaml +++ b/stable/ckan/templates/deploy/ckan.yaml @@ -33,6 +33,16 @@ spec: securityContext: {{- toYaml .Values.ckan.podSecurityContext | nindent 8 }} volumes: + {{- if .Values.ckan.backup.enabled }} + - name: "ckan-backup" + azureFile: + secretName: {{ include "ckan.fullname" . }}-ckan-backup-secret + shareName: {{ .Values.ckan.backup.shareName }} + - name: {{ include "ckan.fullname" . }}-crontabs + configMap: + name: {{ include "ckan.fullname" . }}-crontabs + defaultMode: 0755 + {{- end }} - name: {{ include "ckan.fullname" . }}-ini-overlay-configmap configMap: name: {{ include "ckan.fullname" . }}-ini-overlay-configmap @@ -66,7 +76,7 @@ spec: {{ end }} initContainers: - name: create-production-ini - image: "{{ .Values.initContainer.image.repository }}:{{ .Values.initContainer.image.tag }}" + image: busybox command: - "sh" - "-c" @@ -82,7 +92,7 @@ spec: readOnly: false {{- if .Values.ckan.persistence.enabled }} - name: set-volume-ownership - image: "{{ .Values.initContainer.image.repository }}:{{ .Values.initContainer.image.tag }}" + image: busybox command: - "sh" - "-c" @@ -97,7 +107,7 @@ spec: containers: {{ if .Values.pgbouncer.enabled }} - name: pgbouncer - image: "{{ .Values.pgbouncer.image.repository }}:{{ .Values.pgbouncer.image.tag }}" + image: mcr.microsoft.com/azure-oss-db-tools/pgbouncer-sidecar:latest imagePullPolicy: Always ports: - containerPort: 5432 @@ -241,6 +251,14 @@ spec: - | set -e source $CKAN_VENV/bin/activate + sudo service cron start + {{- if .Values.ckan.backup.enabled }} + echo "#!/usr/bin/bash" | crontab - + (crontab -l ; echo "PATH=/usr/lib/ckan/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin") | crontab - + {{- range $key, $value := .Values.ckan.cronjobs }} + (crontab -l ; echo {{ $value.crontab | quote }}) | crontab - + {{- end }} + {{- end }} ckan generate config /srv/app/production.ini python3 < /tmp/enviroment + sudo bash -c "cat /tmp/enviroment > /etc/environment" ckan -c /srv/app/production.ini run --host 0.0.0.0 ports: - name: http @@ -305,6 +325,8 @@ spec: secretKeyRef: name: {{ include "ckan.fullname" . }} key: sysadminPassword + - name: BEAKER_SESSION_SECRET + value: {{ .Values.ckan.beaker_session_secret }} - name: CKAN_SYSADMIN_EMAIL value: {{ .Values.ckan.sysadminEmail }} - name: CKAN_SITE_TITLE @@ -364,7 +386,7 @@ spec: {{- end }} {{- if .Values.redis.enabled }} - name: CKAN_REDIS_URL - value: redis://ckan-redis-headless:6379/0 + value: redis://{{ include "ckan.fullname" . }}-redis-headless:6379/0 {{- end }} - name: CKANEXT_SPATIAL_SEARCH_BACKEND value: {{ .Values.ckan.spatialBackend }} @@ -410,6 +432,12 @@ spec: failureThreshold: {{ .Values.ckan.liveness.failureThreshold }} timeoutSeconds: {{ .Values.ckan.liveness.timeoutSeconds }} volumeMounts: + {{- if .Values.ckan.backup.enabled}} + - mountPath: "/app/backup" + name: "ckan-backup" + - name: {{ include "ckan.fullname" . }}-crontabs + mountPath: /app/ + {{- end }} - name: {{ include "ckan.fullname" . }}-prerun-configmap mountPath: /srv/prerun - name: {{ include "ckan.fullname" . }}-ini-overlay-configmap diff --git a/stable/ckan/templates/secret/ckan-backup-secret.yaml b/stable/ckan/templates/secret/ckan-backup-secret.yaml new file mode 100644 index 00000000..2f66bb9e --- /dev/null +++ b/stable/ckan/templates/secret/ckan-backup-secret.yaml @@ -0,0 +1,10 @@ +{{- if .Values.ckan.backup.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "ckan.fullname" . }}-ckan-backup-secret +type: Opaque +data: + azurestorageaccountname: {{ .Values.ckan.backup.azurestorageaccountname | b64enc }} + azurestorageaccountkey: {{ .Values.ckan.backup.azurestorageaccountkey | b64enc }} +{{- end }} diff --git a/stable/ckan/values.yaml b/stable/ckan/values.yaml index 274dc69a..9a135aee 100644 --- a/stable/ckan/values.yaml +++ b/stable/ckan/values.yaml @@ -69,6 +69,12 @@ ckan: activityStreamsEmailNotifications_jobs: "false" debug: "false" maintenanceMode: "false" + backup: + adminuser: "" + enabled: true + shareName: "" + azurestorageaccountname: "" + azurestorageaccountkey: "" psql: initialize: true db: