NEXUS-36465 - Use dynamic provisioning and stateful sets

nxrm-aws-resiliency/templates/external-dns-rbac.yml

@@ -63,4 +63,4 @@ spec:
             - --txt-owner-id=external-dns
           env:
             - name: AWS_DEFAULT_REGION
-              value: {{ .Values.deployment.clusterRegion }}
+              value: {{ .Values.statefulset.clusterRegion }}

nxrm-aws-resiliency/templates/fluent-bit.yaml

@@ -39,12 +39,12 @@ metadata:
   name: fluent-bit-cluster-info
   namespace: {{ .Values.namespaces.cloudwatchNs }}
 data:
-  cluster.name: {{ .Values.deployment.clusterName }}
+  cluster.name: {{ .Values.statefulset.clusterName }}
   http.server: "On"
   http.port: "2020"
   read.head: "Off"
   read.tail: "On"
-  logs.region: {{ .Values.deployment.logsRegion }}
+  logs.region: {{ .Values.statefulset.logsRegion }}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -77,7 +77,7 @@ data:
     [INPUT]
         Name                tail
         Tag                 nexus.nexus-log
-        Path                /var/log/containers/{{ .Chart.Name }}-{{ .Chart.Version }}.{{ .Release.Name }}-nxrm.deployment*{{ .Values.namespaces.nexusNs }}_nxrm-app-*.log
+        Path                /var/log/containers/{{ .Chart.Name }}-{{ .Chart.Version  | replace "." "-" }}-{{ .Release.Name }}-{{ .Values.statefulset.name }}*{{ .Values.namespaces.nexusNs }}_nxrm-app-*.log
         Parser              docker
         DB                  /var/fluent-bit/state/flb_container.db
         Mem_Buf_Limit       5MB
@@ -112,7 +112,7 @@ data:
     [INPUT]
         Name                tail
         Tag                 nexus.request-log
-        Path                /var/log/containers/{{ .Chart.Name }}-{{ .Chart.Version }}.{{ .Release.Name }}-nxrm.deployment*{{ .Values.namespaces.nexusNs }}_request-log-*.log
+        Path                /var/log/containers/{{ .Chart.Name }}-{{ .Chart.Version  | replace "." "-" }}-{{ .Release.Name }}-{{ .Values.statefulset.name }}*{{ .Values.namespaces.nexusNs }}_request-log-*.log
         Parser              docker
         DB                  /var/fluent-bit/state/flb_container.db
         Mem_Buf_Limit       5MB
@@ -147,7 +147,7 @@ data:
     [INPUT]
         Name                tail
         Tag                 nexus.audit-log
-        Path                /var/log/containers/{{ .Chart.Name }}-{{ .Chart.Version }}.{{ .Release.Name }}-nxrm.deployment*{{ .Values.namespaces.nexusNs }}_audit-log-*.log
+        Path                /var/log/containers/{{ .Chart.Name }}-{{ .Chart.Version  | replace "." "-" }}-{{ .Release.Name }}-{{ .Values.statefulset.name }}*{{ .Values.namespaces.nexusNs }}_audit-log-*.log
         Parser              docker
         DB                  /var/fluent-bit/state/flb_container.db
         Mem_Buf_Limit       5MB
@@ -182,7 +182,7 @@ data:
     [INPUT]
         Name                tail
         Tag                 nexus.tasks-log
-        Path                /var/log/containers/{{ .Chart.Name }}-{{ .Chart.Version }}.{{ .Release.Name }}-nxrm.deployment*{{ .Values.namespaces.nexusNs }}_tasks-log-*.log
+        Path                /var/log/containers/{{ .Chart.Name }}-{{ .Chart.Version  | replace "." "-" }}-{{ .Release.Name }}-{{ .Values.statefulset.name }}*{{ .Values.namespaces.nexusNs }}_tasks-log-*.log
         Parser              docker
         DB                  /var/fluent-bit/state/flb_container.db
         Mem_Buf_Limit       5MB
@@ -263,7 +263,7 @@ spec:
     spec:
       containers:
         - name: fluent-bit
-          image: amazon/aws-for-fluent-bit:{{ .Values.deployment.fluentBitVersion }}
+          image: amazon/aws-for-fluent-bit:{{ .Values.statefulset.fluentBitVersion }}
           imagePullPolicy: Always
           env:
             - name: AWS_REGION

nxrm-aws-resiliency/templates/deployment.yaml → nxrm-aws-resiliency/templates/statefulset.yaml

@@ -1,12 +1,13 @@
 apiVersion: apps/v1
-kind: Deployment
+kind: StatefulSet
 metadata:
-  name: {{ .Chart.Name }}-{{ .Chart.Version }}.{{ .Release.Name }}-{{ .Values.deployment.name }}
+  name: {{ .Chart.Name }}-{{ .Chart.Version  | replace "." "-"}}-{{ .Release.Name }}-{{ .Values.statefulset.name }}
   namespace: {{ .Values.namespaces.nexusNs }}
   labels:
     app: nxrm
 spec:
   replicas: 1
+  serviceName: "{{ .Chart.Name }}-{{ .Chart.Version  | replace "." "-"}}-{{ .Release.Name }}-{{ .Values.statefulset.name }}"
   selector:
     matchLabels:
       app: nxrm
@@ -21,7 +22,7 @@ spec:
         # otherwise the side car containers will crash a couple of times and backoff whilst waiting
         # for nxrm-app to start and this increases the total start up time.
         - name: chown-nexusdata-owner-to-nexus-and-init-log-dir
-          image: {{ .Values.deployment.initContainer.image.repository }}:{{ .Values.deployment.initContainer.image.tag }}
+          image: {{ .Values.statefulset.initContainer.image.repository }}:{{ .Values.statefulset.initContainer.image.tag }}
           command: [/bin/sh]
           args:
             - -c
@@ -34,19 +35,20 @@ spec:
               touch -a /nexus-data/log/request.log &&
               chown -R '200:200' /nexus-data
           volumeMounts:
-            - name: nexusdata
+            - name: nexus-data
               mountPath: /nexus-data
+      terminationGracePeriodSeconds: 20
       containers:
         - name: nxrm-app
-          image: {{ .Values.deployment.container.image.repository }}:{{ .Values.deployment.container.image.tag }}
+          image: {{ .Values.statefulset.container.image.repository }}:{{ .Values.statefulset.container.image.tag }}
           securityContext:
             runAsUser: 200
-          imagePullPolicy: {{ .Values.deployment.container.pullPolicy }}
+          imagePullPolicy: {{ .Values.statefulset.container.pullPolicy }}
           ports:
-            - containerPort: {{ .Values.deployment.container.containerPort }}
+            - containerPort: {{ .Values.statefulset.container.containerPort }}
           env:
             - name: DB_NAME
-              value: "{{ .Values.deployment.container.env.nexusDBName }}"
+              value: "{{ .Values.statefulset.container.env.nexusDBName }}"
             - name: DB_PASSWORD
               valueFrom:
                 secretKeyRef:
@@ -70,41 +72,38 @@ spec:
             - name: NEXUS_SECURITY_RANDOMPASSWORD
               value: "false"
             - name: INSTALL4J_ADD_VM_PARAMS
-              value: "{{ .Values.deployment.container.env.install4jAddVmParams }} -Dnexus.licenseFile=/nxrm-secrets/{{ .Values.secret.license.alias }} \
+              value: "{{ .Values.statefulset.container.env.install4jAddVmParams }} -Dnexus.licenseFile=/nxrm-secrets/{{ .Values.secret.license.alias }} \
           -Dnexus.datastore.enabled=true -Djava.util.prefs.userRoot=${NEXUS_DATA}/javaprefs \
-          -Dnexus.datastore.nexus.jdbcUrl=jdbc:postgresql://${DB_HOST}:{{ .Values.deployment.container.env.nexusDBPort }}/${DB_NAME} \
+          -Dnexus.datastore.nexus.jdbcUrl=jdbc:postgresql://${DB_HOST}:{{ .Values.statefulset.container.env.nexusDBPort }}/${DB_NAME} \
           -Dnexus.datastore.nexus.username=${DB_USER} \
           -Dnexus.datastore.nexus.password=${DB_PASSWORD}"
           volumeMounts:
             - mountPath: /nxrm-secrets
               name: nxrm-secrets
-            - name: nexusdata
+            - name: nexus-data
               mountPath: /nexus-data
             - name: logback-tasklogfile-override
               mountPath: /nexus-data/etc/logback/logback-tasklogfile-appender-override.xml
               subPath: logback-tasklogfile-appender-override.xml
         - name: request-log
-          image: {{ .Values.deployment.requestLogContainer.image.repository }}:{{ .Values.deployment.requestLogContainer.image.tag }}
+          image: {{ .Values.statefulset.requestLogContainer.image.repository }}:{{ .Values.statefulset.requestLogContainer.image.tag }}
           args: [/bin/sh, -c, 'tail -n+1 -F /nexus-data/log/request.log']
           volumeMounts:
-            - name: nexusdata
+            - name: nexus-data
               mountPath: /nexus-data
         - name: audit-log
-          image: {{ .Values.deployment.auditLogContainer.image.repository }}:{{ .Values.deployment.auditLogContainer.image.tag }}
+          image: {{ .Values.statefulset.auditLogContainer.image.repository }}:{{ .Values.statefulset.auditLogContainer.image.tag }}
           args: [/bin/sh, -c, 'tail -n+1 -F /nexus-data/log/audit/audit.log']
           volumeMounts:
-            - name: nexusdata
+            - name: nexus-data
               mountPath: /nexus-data
         - name: tasks-log
-          image: {{ .Values.deployment.taskLogContainer.image.repository }}:{{ .Values.deployment.taskLogContainer.image.tag }}
+          image: {{ .Values.statefulset.taskLogContainer.image.repository }}:{{ .Values.statefulset.taskLogContainer.image.tag }}
           args: [/bin/sh, -c, 'tail -n+1 -F /nexus-data/log/tasks/allTasks.log']
           volumeMounts:
-            - name: nexusdata
+            - name: nexus-data
               mountPath: /nexus-data
       volumes:
-        - name: nexusdata
-          persistentVolumeClaim:
-            claimName: {{ .Chart.Name }}-{{ .Chart.Version }}.{{ .Release.Name }}-ebs-claim
         - name: nxrm-secrets
           csi:
             driver: secrets-store.csi.k8s.io
@@ -118,3 +117,12 @@ spec:
             items:
               - key: logback-tasklogfile-appender-override.xml
                 path: logback-tasklogfile-appender-override.xml
+  volumeClaimTemplates:
+    - metadata:
+        name: nexus-data
+      spec:
+        accessModes: [ "{{.Values.pvc.accessModes }}" ]
+        storageClassName: "{{ .Chart.Name }}-{{ .Chart.Version}}-{{ .Release.Name }}-ebs-storage"
+        resources:
+          requests:
+            storage: {{.Values.pvc.storage }}

nxrm-aws-resiliency/templates/storageclass.yaml

@@ -1,7 +1,11 @@
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:
-  name: {{ .Chart.Name }}-{{ .Chart.Version }}.{{ .Release.Name }}-local-storage
+  name: "{{ .Chart.Name }}-{{ .Chart.Version}}-{{ .Release.Name }}-ebs-storage"
   namespace: {{ .Values.namespaces.nexusNs }}
-provisioner: kubernetes.io/no-provisioner
-volumeBindingMode: WaitForFirstConsumer
+provisioner: kubernetes.io/aws-ebs
+parameters:
+  type: io1
+  fsType: "ext4"
+  iopsPerGB: "{{ .Values.storageClass.iopsPerGB }}"
+volumeBindingMode: WaitForFirstConsumer

nxrm-aws-resiliency/values.yaml

@@ -6,9 +6,9 @@ namespaces:
 externaldns:
   domainFilter: example.com #your root domain e.g example.com
   awsZoneType: private # hosted zone to look at (valid values are public, private or no value for both)
-deployment:
+statefulset:
   clusterRegion: us-east-1
-  name: nxrm.deployment
+  name: nxrm-statefulset
   clusterName: nxrm-nexus
   logsRegion: us-east-1
   fluentBitVersion: 2.28.0
@@ -19,7 +19,7 @@ deployment:
   container:
     image:
       repository: sonatype/nexus3
-      tag: 3.41.1
+      tag: 3.44.0
     containerPort: 8081
     pullPolicy: IfNotPresent
     env:
@@ -43,32 +43,29 @@ serviceAccount:
   role: arn:aws:iam::000000000000:role/nxrm-nexus-role #Role with secretsmanager permissions
   externaldns:
     name: external-dns
-    role: arn:aws:iam::000000000000:role/nexusrepo-external-dns-irsa-role #Role with route53 permissions needed by external-dns     
+    role: arn:aws:iam::000000000000:role/nexusrepo-external-dns-irsa-role #Role with route53 permissions needed by external-dns
 ingress:
   #host: "example.com" #host to apply this ingress rule to. Uncomment this in your values.yaml and set it as you wish
   annotations:
     kubernetes.io/ingress.class: alb
     alb.ingress.kubernetes.io/scheme: internal # scheme
     alb.ingress.kubernetes.io/subnets: subnet-1,subnet-2 #comma separated list of subnet ids
     alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":443}]'
-    alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-1:0000000000000:certificate/00000000-1111-2222-3333-444444444444  # The AWS Certificate Manager ARN for your HTTPS certificate    
+    alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-1:0000000000000:certificate/00000000-1111-2222-3333-444444444444  # The AWS Certificate Manager ARN for your HTTPS certificate
   dockerIngress:  #Ingress for Docker Connector - comment out if you don't use docker repositories
     annotations:
       kubernetes.io/ingress.class: alb # comment out if you don't use docker repositories
       alb.ingress.kubernetes.io/scheme: internal # scheme comment out if you don't use docker repositories
       alb.ingress.kubernetes.io/subnets: subnet-1,subnet-2 #comma separated list of subnet ids, comment out if you don't use docker repositories
       alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":443}]' #comment out if you don't use docker repositories
       alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-1:0000000000000:certificate/00000000-1111-2222-3333-444444444444  # Comment out if you don't use docker repositories - The AWS Certificate Manager ARN for your HTTPS certificate
-      external-dns.alpha.kubernetes.io/hostname: dockerrepo1.example.com, dockerrepo2.example.com, dockerrepo3.example.com # Add more docker subdomains using dockerrepoName.example.com othereise comment out if you don't use docker repositories
-pv:
-  storage: 120Gi
-  volumeMode: Filesystem
-  accessModes: ReadWriteOnce
-  reclaimPolicy: Retain
-  path: /mnt
+      external-dns.alpha.kubernetes.io/hostname: dockerrepo1.example.com, dockerrepo2.example.com, dockerrepo3.example.com # Add more docker subdomains using dockerrepoName.example.com otherwise comment out if you don't use docker repositories
+storageClass:
   zones:
-    zone1: us-east-1a
-    zone2: us-east-1b
+    zone1: zone1
+    zone2: zone2
+    zone3: zone3
+  iopsPerGB: "10"
 pvc:
   accessModes: ReadWriteOnce
   storage: 100Gi