Merge pull request #314 from abdasgupta/rework

Updated configmap mountpoints based on ETCD replicas.
gardener · Mar 30, 2022 · d204052 · d204052
2 parents 0e26179 + 8382e61
commit d204052
Show file tree

Hide file tree

Showing 27 changed files with 1,461 additions and 667 deletions.
diff --git a/api/v1alpha1/etcd_types.go b/api/v1alpha1/etcd_types.go
@@ -88,11 +88,11 @@ type StoreSpec struct {
 // TLSConfig hold the TLS configuration details.
 type TLSConfig struct {
 	// +required
-	ServerTLSSecretRef corev1.SecretReference `json:"serverTLSSecretRef"`
+	TLSCASecretRef SecretReference `json:"tlsCASecretRef"`
 	// +required
+	ServerTLSSecretRef corev1.SecretReference `json:"serverTLSSecretRef"`
+	// +optional
 	ClientTLSSecretRef corev1.SecretReference `json:"clientTLSSecretRef"`
-	// +required
-	TLSCASecretRef SecretReference `json:"tlsCASecretRef"`
 }
 
 // SecretReference defines a reference to a secret.
@@ -217,8 +217,13 @@ type EtcdConfig struct {
 	// More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/
 	// +optional
 	Resources *corev1.ResourceRequirements `json:"resources,omitempty"`
+	// ClientUrlTLS contains the ca, server TLS and client TLS secrets for client communication to ETCD cluster
 	// +optional
-	TLS *TLSConfig `json:"tls,omitempty"`
+	ClientUrlTLS *TLSConfig `json:"clientUrlTls,omitempty"`
+	// PeerUrlTLS contains the ca and server TLS secrets for peer communication within ETCD cluster
+	// Currently, PeerUrlTLS does not require client TLS secrets for gardener implementation of ETCD cluster.
+	// +optional
+	PeerUrlTLS *TLSConfig `json:"peerUrlTls,omitempty"`
 	// EtcdDefragTimeout defines the timeout duration for etcd defrag call
 	// +optional
 	EtcdDefragTimeout *metav1.Duration `json:"etcdDefragTimeout,omitempty"`

diff --git a/api/v1alpha1/etcd_types_test.go b/api/v1alpha1/etcd_types_test.go
@@ -104,18 +104,29 @@ func getEtcd(name, namespace string) *Etcd {
 	prefix := "etcd-test"
 	garbageCollectionPolicy := GarbageCollectionPolicy(GarbageCollectionPolicyExponential)
 
-	tlsConfig := &TLSConfig{
+	clientTlsConfig := &TLSConfig{
+		TLSCASecretRef: SecretReference{
+			SecretReference: corev1.SecretReference{
+				Name: "client-url-ca-etcd",
+			},
+		},
 		ClientTLSSecretRef: corev1.SecretReference{
-			Name: "etcd-client-tls",
+			Name: "client-url-etcd-client-tls",
 		},
 		ServerTLSSecretRef: corev1.SecretReference{
-			Name: "etcd-server-tls",
+			Name: "client-url-etcd-server-tls",
 		},
+	}
+
+	peerTlsConfig := &TLSConfig{
 		TLSCASecretRef: SecretReference{
 			SecretReference: corev1.SecretReference{
-				Name: "ca-etcd",
+				Name: "peer-url-ca-etcd",
 			},
 		},
+		ServerTLSSecretRef: corev1.SecretReference{
+			Name: "peer-url-etcd-server-tls",
+		},
 	}
 
 	instance := &Etcd{
@@ -146,6 +157,7 @@ func getEtcd(name, namespace string) *Etcd {
 			Backup: BackupSpec{
 				Image:                    &imageBR,
 				Port:                     &backupPort,
+				TLS:                      clientTlsConfig,
 				FullSnapshotSchedule:     &snapshotSchedule,
 				GarbageCollectionPolicy:  &garbageCollectionPolicy,
 				GarbageCollectionPeriod:  &garbageCollectionPeriod,
@@ -186,9 +198,10 @@ func getEtcd(name, namespace string) *Etcd {
 						"memory": parseQuantity("1000Mi"),
 					},
 				},
-				ClientPort: &clientPort,
-				ServerPort: &serverPort,
-				TLS:        tlsConfig,
+				ClientPort:   &clientPort,
+				ServerPort:   &serverPort,
+				ClientUrlTLS: clientTlsConfig,
+				PeerUrlTLS:   peerTlsConfig,
 			},
 		},
 	}

diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/charts/etcd/templates/etcd-configmap.yaml b/charts/etcd/templates/etcd-configmap.yaml
diff --git a/charts/etcd/templates/etcd-statefulset.yaml b/charts/etcd/templates/etcd-statefulset.yaml
@@ -16,6 +16,7 @@ metadata:
 {{ toYaml .Values.labels | indent 4 }}
 {{- end }}
 spec:
+  podManagementPolicy: Parallel
   updateStrategy:
     type: RollingUpdate
   serviceName: {{ .Values.serviceName }}
@@ -27,7 +28,6 @@ spec:
   template:
     metadata:
       annotations:
-        checksum/etcd-configmap: {{ include (print $.Template.BasePath "/etcd-configmap.yaml") . | sha256sum }}
 {{- if .Values.annotations }}
 {{ toYaml .Values.annotations | indent 8 }}
 {{- end }}
@@ -53,12 +53,20 @@ spec:
         command:
         - /var/etcd/bin/bootstrap.sh
         readinessProbe:
-          httpGet:
-{{- if .Values.etcd.enableTLS }}
-            scheme: HTTPS
-{{- end }}
-            path: /healthz
-            port: {{ .Values.backup.port }}
+          exec:
+            command:
+            - /usr/bin/curl
+{{- if .Values.etcd.enableClientTLS }}
+            - --cert
+            - /var/etcd/ssl/client/client/tls.crt
+            - --key
+            - /var/etcd/ssl/client/client/tls.key
+            - --cacert
+            - /var/etcd/ssl/client/ca/{{ .Values.clientTlsCASecretKey }}
+            - https://{{ .Values.name }}-local:{{- if eq .Values.replicas 1.0 }}{{ .Values.backup.port }}{{ else }}{{ .Values.etcd.clientPort }}{{ end }}/health{{- if eq .Values.replicas 1.0 }}z{{ end }}
+{{ else }}
+            - http://{{ .Values.name }}-local:{{- if eq .Values.replicas 1.0 }}{{ .Values.backup.port }}{{ else }}{{ .Values.etcd.clientPort }}{{ end }}/health{{- if eq .Values.replicas 1.0 }}z{{ end }}
+{{ end }}
           initialDelaySeconds: 15
           periodSeconds: 5
         livenessProbe:
@@ -68,10 +76,10 @@ spec:
             - -ec
             - ETCDCTL_API=3
             - etcdctl
-{{- if .Values.etcd.enableTLS }}
-            - --cert=/var/etcd/ssl/client/tls.crt
-            - --key=/var/etcd/ssl/client/tls.key
-            - --cacert=/var/etcd/ssl/ca/{{ .Values.tlsCASecretKey }}
+{{- if .Values.etcd.enableClientTLS }}
+            - --cert=/var/etcd/ssl/client/client/tls.crt
+            - --key=/var/etcd/ssl/client/client/tls.key
+            - --cacert=/var/etcd/ssl/client/ca/{{ .Values.clientTlsCASecretKey }}
             - --endpoints=https://{{ .Values.name }}-local:{{ .Values.etcd.clientPort }}
 {{ else }}
             - --endpoints=http://{{ .Values.name }}-local:{{ .Values.etcd.clientPort }}
@@ -94,23 +102,27 @@ spec:
 {{ toYaml .Values.etcd.resources | indent 10 }}
         env:
         - name: ENABLE_TLS
-          value: {{ .Values.etcd.enableTLS }}
+          value: {{ .Values.etcd.enableClientTLS }}
         - name: BACKUP_ENDPOINT
-          value: "http{{ if .Values.etcd.enableTLS }}s{{ end }}://{{ .Values.name }}-local:{{ .Values.backup.port }}"
+          value: "http{{ if .Values.backup.enableTLS }}s{{ end }}://{{ .Values.name }}-local:{{ .Values.backup.port }}"
         - name: FAIL_BELOW_REVISION_PARAMETER
           value: "{{ if .Values.backup.failBelowRevision }}&failbelowrevision={{ int $.Values.backup.failBelowRevision }}{{ end }}"
         volumeMounts:
         - name: {{ .Values.volumeClaimTemplateName }}
           mountPath: /var/etcd/data/
-        - name: etcd-config-file
-          mountPath: /var/etcd/config/
-{{- if .Values.etcd.enableTLS }}
-        - name: ca-etcd
-          mountPath: /var/etcd/ssl/ca
-        - name: etcd-server-tls
-          mountPath: /var/etcd/ssl/server
-        - name: etcd-client-tls
-          mountPath: /var/etcd/ssl/client
+{{- if .Values.etcd.enableClientTLS }}
+        - name: client-url-ca-etcd
+          mountPath: /var/etcd/ssl/client/ca
+        - name: client-url-etcd-server-tls
+          mountPath: /var/etcd/ssl/client/server
+        - name: client-url-etcd-client-tls
+          mountPath: /var/etcd/ssl/client/client
+{{- end }}
+{{- if .Values.etcd.enablePeerTLS }}
+        - name: peer-url-ca-etcd
+          mountPath: /var/etcd/ssl/peer/ca
+        - name: peer-url-etcd-server-tls
+          mountPath: /var/etcd/ssl/peer/server
 {{- end }}
       - name: backup-restore
         command:
@@ -144,16 +156,16 @@ spec:
 {{- if .Values.backup.enableProfiling }}
         - --enable-profiling={{ .Values.backup.enableProfiling }}
 {{- end }}
-{{- if .Values.etcd.enableTLS }}
-        - --cert=/var/etcd/ssl/client/tls.crt
-        - --key=/var/etcd/ssl/client/tls.key
-        - --cacert=/var/etcd/ssl/ca/{{ .Values.tlsCASecretKey }}
+{{- if .Values.backup.enableTLS }}
+        - --cert=/var/etcd/ssl/client/client/tls.crt
+        - --key=/var/etcd/ssl/client/client/tls.key
+        - --cacert=/var/etcd/ssl/client/ca/{{ .Values.clientTlsCASecretKey }}
         - --insecure-transport=false
         - --insecure-skip-tls-verify=false
         - --endpoints=https://{{ .Values.name }}-local:{{ .Values.etcd.clientPort }}
         # enable TLS on backup-restore server reusing etcd cert bundle
-        - --server-cert=/var/etcd/ssl/server/tls.crt
-        - --server-key=/var/etcd/ssl/server/tls.key
+        - --server-cert=/var/etcd/ssl/client/server/tls.crt
+        - --server-key=/var/etcd/ssl/client/server/tls.key
 {{ else }}
         - --insecure-transport=true
         - --insecure-skip-tls-verify=true
@@ -177,7 +189,7 @@ spec:
         {{- if .Values.backup.compression.enabled }}
         - --compress-snapshots={{ .Values.backup.compression.enabled }}
         {{- end }}
-        {{- if .Values.backup.compression.policy }}        
+        {{- if .Values.backup.compression.policy }}
         - --compression-policy={{ .Values.backup.compression.policy }}
         {{- end }}
 {{- end }}
@@ -315,13 +327,13 @@ spec:
           mountPath: /var/etcd/data
         - name: etcd-config-file
           mountPath: /var/etcd/config/
-{{- if .Values.etcd.enableTLS }}
-        - name: ca-etcd
-          mountPath: /var/etcd/ssl/ca
-        - name: etcd-server-tls
-          mountPath: /var/etcd/ssl/server
-        - name: etcd-client-tls
-          mountPath: /var/etcd/ssl/client
+{{- if .Values.backup.enableTLS }}
+        - name: client-url-ca-etcd
+          mountPath: /var/etcd/ssl/client/ca
+        - name: client-url-etcd-server-tls
+          mountPath: /var/etcd/ssl/client/server
+        - name: client-url-etcd-client-tls
+          mountPath: /var/etcd/ssl/client/client
 {{- end }}
 {{- if eq .Values.store.storageProvider "GCS" }}
         - name: etcd-backup
@@ -362,16 +374,24 @@ spec:
           items:
           - key: etcd.conf.yaml
             path: etcd.conf.yaml
-{{- if .Values.etcd.enableTLS }}
-      - name: etcd-server-tls
+{{- if .Values.etcd.enableClientTLS }}
+      - name: client-url-ca-etcd
+        secret:
+          secretName: {{ .Values.clientUrlTlsCASecret }}
+      - name: client-url-etcd-server-tls
         secret:
-          secretName: {{ .Values.tlsServerSecret }}
-      - name: etcd-client-tls
+          secretName: {{ .Values.clientUrlTlsServerSecret }}
+      - name: client-url-etcd-client-tls
+        secret:
+          secretName: {{ .Values.clientUrlTlsClientSecret }}
+{{- end }}
+{{- if .Values.etcd.enablePeerTLS }}
+      - name: peer-url-ca-etcd
         secret:
-          secretName: {{ .Values.tlsClientSecret }}
-      - name: ca-etcd
+          secretName: {{ .Values.peerUrlTlsCASecret }}
+      - name: peer-url-etcd-server-tls
         secret:
-          secretName: {{ .Values.tlsCASecret }}
+          secretName: {{ .Values.peerUrlTlsServerSecret }}
 {{- end }}
 {{- if eq .Values.store.storageProvider "GCS" }}
       - name: etcd-backup