From 657935b368d7e7050a32934e7a915031087f7373 Mon Sep 17 00:00:00 2001 From: Abhishek Dasgupta Date: Wed, 16 Jun 2021 23:11:45 +0530 Subject: [PATCH] Updated configmap mountpoints based on ETCD replicas. --- api/v1alpha1/etcd_types.go | 8 +- api/v1alpha1/etcd_types_test.go | 25 +- api/v1alpha1/zz_generated.deepcopy.go | 11 +- charts/etcd/templates/etcd-configmap.yaml | 84 --- charts/etcd/templates/etcd-statefulset.yaml | 106 ++-- charts/etcd/values.yaml | 15 +- .../10-crd-druid.gardener.cloud_etcds.yaml | 123 +++-- config/default/manager_image_patch.yaml | 2 +- config/rbac/leader_election_role.yaml | 2 +- controllers/compaction_lease_controller.go | 20 +- .../compaction_lease_controller_test.go | 8 +- controllers/controller_ref_manager.go | 57 -- controllers/etcd_controller.go | 332 ++++-------- controllers/etcd_controller_test.go | 501 +++++++++++++----- pkg/component/etcd/configmap/configmap.go | 242 +++++++++ .../etcd/configmap/configmap_suite_test.go | 27 + .../etcd/configmap/configmap_test.go | 255 +++++++++ pkg/component/etcd/configmap/values.go | 62 +++ pkg/component/etcd/configmap/values_helper.go | 75 +++ pkg/component/etcd/lease/lease.go | 2 + pkg/component/etcd/service/values_helper.go | 7 +- pkg/component/etcd/values.go | 6 +- pkg/utils/names.go | 56 ++ 23 files changed, 1402 insertions(+), 624 deletions(-) delete mode 100644 charts/etcd/templates/etcd-configmap.yaml create mode 100644 pkg/component/etcd/configmap/configmap.go create mode 100644 pkg/component/etcd/configmap/configmap_suite_test.go create mode 100644 pkg/component/etcd/configmap/configmap_test.go create mode 100644 pkg/component/etcd/configmap/values.go create mode 100644 pkg/component/etcd/configmap/values_helper.go create mode 100644 pkg/utils/names.go diff --git a/api/v1alpha1/etcd_types.go b/api/v1alpha1/etcd_types.go index 5fad2c7ab..38b33c6dd 100644 --- a/api/v1alpha1/etcd_types.go +++ b/api/v1alpha1/etcd_types.go @@ -87,12 +87,12 @@ type StoreSpec struct { // TLSConfig hold the TLS configuration details. type TLSConfig struct { + // +required + TLSCASecretRef corev1.SecretReference `json:"tlsCASecretRef"` // +required ServerTLSSecretRef corev1.SecretReference `json:"serverTLSSecretRef"` // +required ClientTLSSecretRef corev1.SecretReference `json:"clientTLSSecretRef"` - // +required - TLSCASecretRef corev1.SecretReference `json:"tlsCASecretRef"` } // CompressionSpec defines parameters related to compression of Snapshots(full as well as delta). @@ -210,7 +210,9 @@ type EtcdConfig struct { // +optional Resources *corev1.ResourceRequirements `json:"resources,omitempty"` // +optional - TLS *TLSConfig `json:"tls,omitempty"` + ClientUrlTLS *TLSConfig `json:"clientUrlTls,omitempty"` + // +optional + PeerUrlTLS *TLSConfig `json:"peerUrlTls,omitempty"` // EtcdDefragTimeout defines the timeout duration for etcd defrag call // +optional EtcdDefragTimeout *metav1.Duration `json:"etcdDefragTimeout,omitempty"` diff --git a/api/v1alpha1/etcd_types_test.go b/api/v1alpha1/etcd_types_test.go index 8d26e6de7..cbb465c53 100644 --- a/api/v1alpha1/etcd_types_test.go +++ b/api/v1alpha1/etcd_types_test.go @@ -104,15 +104,24 @@ func getEtcd(name, namespace string) *Etcd { prefix := "etcd-test" garbageCollectionPolicy := GarbageCollectionPolicy(GarbageCollectionPolicyExponential) - tlsConfig := &TLSConfig{ + clientTlsConfig := &TLSConfig{ + TLSCASecretRef: corev1.SecretReference{ + Name: "client-url-ca-etcd", + }, ClientTLSSecretRef: corev1.SecretReference{ - Name: "etcd-client-tls", + Name: "client-url-etcd-client-tls", }, ServerTLSSecretRef: corev1.SecretReference{ - Name: "etcd-server-tls", + Name: "client-url-etcd-server-tls", }, + } + + peerTlsConfig := &TLSConfig{ TLSCASecretRef: corev1.SecretReference{ - Name: "ca-etcd", + Name: "peer-url-ca-etcd", + }, + ServerTLSSecretRef: corev1.SecretReference{ + Name: "peer-url-etcd-server-tls", }, } @@ -144,6 +153,7 @@ func getEtcd(name, namespace string) *Etcd { Backup: BackupSpec{ Image: &imageBR, Port: &backupPort, + TLS: clientTlsConfig, FullSnapshotSchedule: &snapshotSchedule, GarbageCollectionPolicy: &garbageCollectionPolicy, GarbageCollectionPeriod: &garbageCollectionPeriod, @@ -184,9 +194,10 @@ func getEtcd(name, namespace string) *Etcd { "memory": parseQuantity("1000Mi"), }, }, - ClientPort: &clientPort, - ServerPort: &serverPort, - TLS: tlsConfig, + ClientPort: &clientPort, + ServerPort: &serverPort, + ClientUrlTLS: clientTlsConfig, + PeerUrlTLS: peerTlsConfig, }, }, } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index b6ee43795..a60e8e8a0 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -247,8 +247,13 @@ func (in *EtcdConfig) DeepCopyInto(out *EtcdConfig) { *out = new(v1.ResourceRequirements) (*in).DeepCopyInto(*out) } - if in.TLS != nil { - in, out := &in.TLS, &out.TLS + if in.ClientUrlTLS != nil { + in, out := &in.ClientUrlTLS, &out.ClientUrlTLS + *out = new(TLSConfig) + **out = **in + } + if in.PeerUrlTLS != nil { + in, out := &in.PeerUrlTLS, &out.PeerUrlTLS *out = new(TLSConfig) **out = **in } @@ -689,9 +694,9 @@ func (in *StoreSpec) DeepCopy() *StoreSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TLSConfig) DeepCopyInto(out *TLSConfig) { *out = *in + out.TLSCASecretRef = in.TLSCASecretRef out.ServerTLSSecretRef = in.ServerTLSSecretRef out.ClientTLSSecretRef = in.ClientTLSSecretRef - out.TLSCASecretRef = in.TLSCASecretRef } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSConfig. diff --git a/charts/etcd/templates/etcd-configmap.yaml b/charts/etcd/templates/etcd-configmap.yaml deleted file mode 100644 index ce8c0b0c8..000000000 --- a/charts/etcd/templates/etcd-configmap.yaml +++ /dev/null @@ -1,84 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Values.configMapName }} - namespace: {{ .Release.Namespace }} - labels: - name: etcd - instance: {{ .Values.name }} -{{- if .Values.labels }} -{{ toYaml .Values.labels | indent 4 }} -{{- end }} - ownerReferences: - - apiVersion: druid.gardener.cloud/v1alpha1 - blockOwnerDeletion: true - controller: true - kind: Etcd - name: {{ .Values.name }} - uid: {{ .Values.uid }} -data: - etcd.conf.yaml: |- - # Human-readable name for this member. - name: etcd-{{ printf "%.6s" .Values.uid }} - - # Path to the data directory. - data-dir: /var/etcd/data/new.etcd - - # metrics configuration - metrics: {{ .Values.etcd.metrics }} - - # Number of committed transactions to trigger a snapshot to disk. - snapshot-count: 75000 - - # Accept etcd V2 client requests - enable-v2: false - - # Raise alarms when backend size exceeds the given quota. 0 means use the - # default quota. - {{- if .Values.backup.etcdQuotaBytes }} - quota-backend-bytes: {{ int $.Values.backup.etcdQuotaBytes }} - {{- end }} - - # List of comma separated URLs to listen on for client traffic. - listen-client-urls: {{ if .Values.etcd.enableTLS }}https{{ else }}http{{ end }}://0.0.0.0:{{ .Values.etcd.clientPort }} - - # List of this member's client URLs to advertise to the public. - # The URLs needed to be a comma-separated list. - advertise-client-urls: {{ if .Values.etcd.enableTLS }}https{{ else }}http{{ end }}://0.0.0.0:{{ .Values.etcd.clientPort }} - - # Initial cluster token for the etcd cluster during bootstrap. - initial-cluster-token: {{ .Values.etcd.initialClusterToken }} - - # Initial cluster state ('new' or 'existing'). - initial-cluster-state: {{ .Values.etcd.initialClusterState }} - - {{- if .Values.sharedConfig }} - # auto-compaction-mode ("periodic" or "revision"). - {{- if .Values.sharedConfig.autoCompactionMode }} - auto-compaction-mode: {{ .Values.sharedConfig.autoCompactionMode }} - {{- end }} - - # auto-compaction-retention defines Auto compaction retention length for etcd. - {{- if .Values.sharedConfig.autoCompactionRetention }} - auto-compaction-retention: {{ .Values.sharedConfig.autoCompactionRetention }} - {{- end }} - {{- end }} - -{{- if .Values.etcd.enableTLS }} - client-transport-security: - # Path to the client server TLS cert file. - cert-file: /var/etcd/ssl/server/tls.crt - - # Path to the client server TLS key file. - key-file: /var/etcd/ssl/server/tls.key - - # Enable client cert authentication. - client-cert-auth: true - - # Path to the client server TLS trusted CA cert file. - trusted-ca-file: /var/etcd/ssl/ca/ca.crt - - # Client TLS using generated certificates - auto-tls: false -{{- end }} diff --git a/charts/etcd/templates/etcd-statefulset.yaml b/charts/etcd/templates/etcd-statefulset.yaml index f0c15d564..c277233a2 100644 --- a/charts/etcd/templates/etcd-statefulset.yaml +++ b/charts/etcd/templates/etcd-statefulset.yaml @@ -16,6 +16,7 @@ metadata: {{ toYaml .Values.labels | indent 4 }} {{- end }} spec: + podManagementPolicy: Parallel updateStrategy: type: RollingUpdate serviceName: {{ .Values.serviceName }} @@ -27,7 +28,6 @@ spec: template: metadata: annotations: - checksum/etcd-configmap: {{ include (print $.Template.BasePath "/etcd-configmap.yaml") . | sha256sum }} {{- if .Values.annotations }} {{ toYaml .Values.annotations | indent 8 }} {{- end }} @@ -53,12 +53,20 @@ spec: command: - /var/etcd/bin/bootstrap.sh readinessProbe: - httpGet: -{{- if .Values.etcd.enableTLS }} - scheme: HTTPS -{{- end }} - path: /healthz - port: {{ .Values.backup.port }} + exec: + command: + - /usr/bin/curl +{{- if .Values.etcd.enableClientTLS }} + - --cert + - /var/etcd/ssl/client/client/tls.crt + - --key + - /var/etcd/ssl/client/client/tls.key + - --cacert + - /var/etcd/ssl/client/ca/ca.crt + - https://{{ .Values.name }}-local:{{- if eq .Values.replicas 1.0 }}{{ .Values.backup.port }}{{ else }}{{ .Values.etcd.clientPort }}{{ end }}/health{{- if eq .Values.replicas 1.0 }}z{{ end }} +{{ else }} + - http://{{ .Values.name }}-local:{{- if eq .Values.replicas 1.0 }}{{ .Values.backup.port }}{{ else }}{{ .Values.etcd.clientPort }}{{ end }}/health{{- if eq .Values.replicas 1.0 }}z{{ end }} +{{ end }} initialDelaySeconds: 15 periodSeconds: 5 livenessProbe: @@ -68,10 +76,10 @@ spec: - -ec - ETCDCTL_API=3 - etcdctl -{{- if .Values.etcd.enableTLS }} - - --cert=/var/etcd/ssl/client/tls.crt - - --key=/var/etcd/ssl/client/tls.key - - --cacert=/var/etcd/ssl/ca/ca.crt +{{- if .Values.etcd.enableClientTLS }} + - --cert=/var/etcd/ssl/client/client/tls.crt + - --key=/var/etcd/ssl/client/client/tls.key + - --cacert=/var/etcd/ssl/client/ca/ca.crt - --endpoints=https://{{ .Values.name }}-local:{{ .Values.etcd.clientPort }} {{ else }} - --endpoints=http://{{ .Values.name }}-local:{{ .Values.etcd.clientPort }} @@ -94,23 +102,27 @@ spec: {{ toYaml .Values.etcd.resources | indent 10 }} env: - name: ENABLE_TLS - value: {{ .Values.etcd.enableTLS }} + value: {{ .Values.etcd.enableClientTLS }} - name: BACKUP_ENDPOINT - value: "http{{ if .Values.etcd.enableTLS }}s{{ end }}://{{ .Values.name }}-local:{{ .Values.backup.port }}" + value: "http{{ if .Values.backup.enableTLS }}s{{ end }}://{{ .Values.name }}-local:{{ .Values.backup.port }}" - name: FAIL_BELOW_REVISION_PARAMETER value: "{{ if .Values.backup.failBelowRevision }}&failbelowrevision={{ int $.Values.backup.failBelowRevision }}{{ end }}" volumeMounts: - name: {{ .Values.volumeClaimTemplateName }} mountPath: /var/etcd/data/ - - name: etcd-config-file - mountPath: /var/etcd/config/ -{{- if .Values.etcd.enableTLS }} - - name: ca-etcd - mountPath: /var/etcd/ssl/ca - - name: etcd-server-tls - mountPath: /var/etcd/ssl/server - - name: etcd-client-tls - mountPath: /var/etcd/ssl/client +{{- if .Values.etcd.enableClientTLS }} + - name: client-url-ca-etcd + mountPath: /var/etcd/ssl/client/ca + - name: client-url-etcd-server-tls + mountPath: /var/etcd/ssl/client/server + - name: client-url-etcd-client-tls + mountPath: /var/etcd/ssl/client/client +{{- end }} +{{- if .Values.etcd.enablePeerTLS }} + - name: peer-url-ca-etcd + mountPath: /var/etcd/ssl/peer/ca + - name: peer-url-etcd-server-tls + mountPath: /var/etcd/ssl/peer/server {{- end }} - name: backup-restore command: @@ -144,16 +156,16 @@ spec: {{- if .Values.backup.enableProfiling }} - --enable-profiling={{ .Values.backup.enableProfiling }} {{- end }} -{{- if .Values.etcd.enableTLS }} - - --cert=/var/etcd/ssl/client/tls.crt - - --key=/var/etcd/ssl/client/tls.key - - --cacert=/var/etcd/ssl/ca/ca.crt +{{- if .Values.backup.enableTLS }} + - --cert=/var/etcd/ssl/client/client/tls.crt + - --key=/var/etcd/ssl/client/client/tls.key + - --cacert=/var/etcd/ssl/client/ca/ca.crt - --insecure-transport=false - --insecure-skip-tls-verify=false - --endpoints=https://{{ .Values.name }}-local:{{ .Values.etcd.clientPort }} # enable TLS on backup-restore server reusing etcd cert bundle - - --server-cert=/var/etcd/ssl/server/tls.crt - - --server-key=/var/etcd/ssl/server/tls.key + - --server-cert=/var/etcd/ssl/client/server/tls.crt + - --server-key=/var/etcd/ssl/client/server/tls.key {{ else }} - --insecure-transport=true - --insecure-skip-tls-verify=true @@ -177,7 +189,7 @@ spec: {{- if .Values.backup.compression.enabled }} - --compress-snapshots={{ .Values.backup.compression.enabled }} {{- end }} - {{- if .Values.backup.compression.policy }} + {{- if .Values.backup.compression.policy }} - --compression-policy={{ .Values.backup.compression.policy }} {{- end }} {{- end }} @@ -315,13 +327,13 @@ spec: mountPath: /var/etcd/data - name: etcd-config-file mountPath: /var/etcd/config/ -{{- if .Values.etcd.enableTLS }} - - name: ca-etcd - mountPath: /var/etcd/ssl/ca - - name: etcd-server-tls - mountPath: /var/etcd/ssl/server - - name: etcd-client-tls - mountPath: /var/etcd/ssl/client +{{- if .Values.backup.enableTLS }} + - name: client-url-ca-etcd + mountPath: /var/etcd/ssl/client/ca + - name: client-url-etcd-server-tls + mountPath: /var/etcd/ssl/client/server + - name: client-url-etcd-client-tls + mountPath: /var/etcd/ssl/client/client {{- end }} {{- if eq .Values.store.storageProvider "GCS" }} - name: etcd-backup @@ -362,16 +374,24 @@ spec: items: - key: etcd.conf.yaml path: etcd.conf.yaml -{{- if .Values.etcd.enableTLS }} - - name: etcd-server-tls +{{- if .Values.etcd.enableClientTLS }} + - name: client-url-ca-etcd + secret: + secretName: {{ .Values.clientUrlTlsCASecret }} + - name: client-url-etcd-server-tls secret: - secretName: {{ .Values.tlsServerSecret }} - - name: etcd-client-tls + secretName: {{ .Values.clientUrlTlsServerSecret }} + - name: client-url-etcd-client-tls + secret: + secretName: {{ .Values.clientUrlTlsClientSecret }} +{{- end }} +{{- if .Values.etcd.enablePeerTLS }} + - name: peer-url-ca-etcd secret: - secretName: {{ .Values.tlsClientSecret }} - - name: ca-etcd + secretName: {{ .Values.peerUrlTlsCASecret }} + - name: peer-url-etcd-server-tls secret: - secretName: {{ .Values.tlsCASecret }} + secretName: {{ .Values.peerUrlTlsServerSecret }} {{- end }} {{- if eq .Values.store.storageProvider "GCS" }} - name: etcd-backup diff --git a/charts/etcd/values.yaml b/charts/etcd/values.yaml index d76671f89..7f3ac773d 100644 --- a/charts/etcd/values.yaml +++ b/charts/etcd/values.yaml @@ -9,16 +9,21 @@ disableEtcdServiceAccountAutomount: false replicas: 1 #priorityClassName: foo -tlsServerSecret: etcd-server-tls -tlsClientSecret: etcd-client-tls -tlsCASecret: ca-etcd +clientUrlTlsCASecret: etcd-client-ca +clientUrlTlsServerSecret: etcd-client-server +clientUrlTlsClientSecret: etcd-client-client + +peerUrlTlsCASecret: etcd-peer-ca +peerUrlTlsServerSecret: etcd-peer-server + annotations: {} labels: {} etcd: initialClusterToken: initial initialClusterState: new - enableTLS: false + enableClientTLS: false + enablePeerTLS: false pullPolicy: IfNotPresent metrics: basic etcdDefragTimeout: 8m @@ -34,6 +39,8 @@ etcd: heartbeatDuration: 10s backup: + port: 8080 + enableTLS: false pullPolicy: IfNotPresent snapstoreTempDir: "/var/etcd/data/temp" etcdConnectionTimeout: 5m diff --git a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml index 85f7916d0..137ef3f02 100644 --- a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml +++ b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml @@ -298,6 +298,53 @@ spec: clientPort: format: int32 type: integer + clientUrlTls: + description: TLSConfig hold the TLS configuration details. + properties: + clientTLSSecretRef: + description: SecretReference represents a Secret Reference. + It has enough information to retrieve secret in any namespace + properties: + name: + description: Name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: Namespace defines the space within which + the secret name must be unique. + type: string + type: object + serverTLSSecretRef: + description: SecretReference represents a Secret Reference. + It has enough information to retrieve secret in any namespace + properties: + name: + description: Name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: Namespace defines the space within which + the secret name must be unique. + type: string + type: object + tlsCASecretRef: + description: SecretReference represents a Secret Reference. + It has enough information to retrieve secret in any namespace + properties: + name: + description: Name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: Namespace defines the space within which + the secret name must be unique. + type: string + type: object + required: + - clientTLSSecretRef + - serverTLSSecretRef + - tlsCASecretRef + type: object defragmentationSchedule: description: DefragmentationSchedule defines the cron standard schedule for defragmentation of etcd. @@ -320,44 +367,7 @@ spec: - basic - extensive type: string - quota: - anyOf: - - type: integer - - type: string - description: Quota defines the etcd DB quota. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resources: - description: 'Resources defines the compute Resources required - by etcd container. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute - resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount of compute - resources required. If Requests is omitted for a container, - it defaults to Limits if that is explicitly specified, otherwise - to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - type: object - serverPort: - format: int32 - type: integer - tls: + peerUrlTls: description: TLSConfig hold the TLS configuration details. properties: clientTLSSecretRef: @@ -404,6 +414,43 @@ spec: - serverTLSSecretRef - tlsCASecretRef type: object + quota: + anyOf: + - type: integer + - type: string + description: Quota defines the etcd DB quota. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resources: + description: 'Resources defines the compute Resources required + by etcd container. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + serverPort: + format: int32 + type: integer type: object labels: additionalProperties: diff --git a/config/default/manager_image_patch.yaml b/config/default/manager_image_patch.yaml index 89b0a861e..02396b360 100644 --- a/config/default/manager_image_patch.yaml +++ b/config/default/manager_image_patch.yaml @@ -8,5 +8,5 @@ spec: spec: containers: # Change the value of image field below to your controller image URL - - image: eu.gcr.io/gardener-project/gardener/etcd-druid:v0.7.0-dev + - image: abdasgupta/etcd-druid:0.8.0-dev name: druid diff --git a/config/rbac/leader_election_role.yaml b/config/rbac/leader_election_role.yaml index c9f7a0c3a..947dabd56 100644 --- a/config/rbac/leader_election_role.yaml +++ b/config/rbac/leader_election_role.yaml @@ -5,7 +5,7 @@ metadata: name: leader-election-role rules: - apiGroups: - - "" + - "coordination.k8s.io resources: - leases verbs: diff --git a/controllers/compaction_lease_controller.go b/controllers/compaction_lease_controller.go index 814b7c84b..563912947 100644 --- a/controllers/compaction_lease_controller.go +++ b/controllers/compaction_lease_controller.go @@ -173,7 +173,7 @@ func (lc *CompactionLeaseController) reconcileJob(ctx context.Context, logger lo // First check if a job is already running job := &batchv1.Job{} - err := lc.Get(ctx, types.NamespacedName{Name: getJobName(etcd), Namespace: etcd.Namespace}, job) + err := lc.Get(ctx, types.NamespacedName{Name: utils.GetJobName(etcd), Namespace: etcd.Namespace}, job) if err != nil { if !errors.IsNotFound(err) { @@ -228,7 +228,7 @@ func (lc *CompactionLeaseController) reconcileJob(ctx context.Context, logger lo func (lc *CompactionLeaseController) delete(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) (ctrl.Result, error) { job := &batchv1.Job{} - err := lc.Get(ctx, types.NamespacedName{Name: getJobName(etcd), Namespace: etcd.Namespace}, job) + err := lc.Get(ctx, types.NamespacedName{Name: utils.GetJobName(etcd), Namespace: etcd.Namespace}, job) if err != nil { if !errors.IsNotFound(err) { return ctrl.Result{RequeueAfter: 10 * time.Second}, fmt.Errorf("error while fetching compaction job: %v", err) @@ -246,7 +246,9 @@ func (lc *CompactionLeaseController) delete(ctx context.Context, logger logr.Log } logger.Info("No compaction job is running") - return ctrl.Result{Requeue: false}, nil + return ctrl.Result{ + Requeue: false, + }, nil } func (lc *CompactionLeaseController) createCompactJob(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) (*batchv1.Job, error) { @@ -267,7 +269,7 @@ func (lc *CompactionLeaseController) createCompactJob(ctx context.Context, logge job := &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ - Name: getJobName(etcd), + Name: utils.GetJobName(etcd), Namespace: etcd.Namespace, Labels: getLabels(etcd), OwnerReferences: []metav1.OwnerReference{ @@ -293,7 +295,7 @@ func (lc *CompactionLeaseController) createCompactJob(ctx context.Context, logge }, Spec: v1.PodSpec{ ActiveDeadlineSeconds: pointer.Int64Ptr(int64(activeDeadlineSeconds)), - ServiceAccountName: getServiceAccountName(etcd), + ServiceAccountName: utils.GetServiceAccountName(etcd), RestartPolicy: v1.RestartPolicyNever, Containers: []v1.Container{{ Name: "compact-backup", @@ -323,14 +325,6 @@ func (lc *CompactionLeaseController) createCompactJob(ctx context.Context, logge return job, nil } -func getCronJobName(etcd *druidv1alpha1.Etcd) string { - return fmt.Sprintf("%s-compact-backup", etcd.Name) -} - -func getJobName(etcd *druidv1alpha1.Etcd) string { - return fmt.Sprintf("%s-compact-job", string(etcd.UID[:6])) -} - func getLabels(etcd *druidv1alpha1.Etcd) map[string]string { return map[string]string{ "name": "etcd-backup-compaction", diff --git a/controllers/compaction_lease_controller_test.go b/controllers/compaction_lease_controller_test.go index 29fce7151..d8b1c36ce 100644 --- a/controllers/compaction_lease_controller_test.go +++ b/controllers/compaction_lease_controller_test.go @@ -68,7 +68,7 @@ var _ = Describe("Lease Controller", func() { cm = &corev1.ConfigMap{} Eventually(func() error { return configMapIsCorrectlyReconciled(c, instance, cm) }, timeout, pollingInterval).Should(BeNil()) svc = &corev1.Service{} - Eventually(func() error { return serviceIsCorrectlyReconciled(c, instance, svc) }, timeout, pollingInterval).Should(BeNil()) + Eventually(func() error { return clientServiceIsCorrectlyReconciled(c, instance, svc) }, timeout, pollingInterval).Should(BeNil()) }) AfterEach(func() { @@ -382,7 +382,7 @@ func validateEtcdForCmpctJob(instance *druidv1alpha1.Etcd, j *batchv1.Job) { Expect(*j).To(MatchFields(IgnoreExtras, Fields{ "ObjectMeta": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(getJobName(instance)), + "Name": Equal(utils.GetJobName(instance)), "Namespace": Equal(instance.Namespace), "OwnerReferences": MatchElements(ownerRefIterator, IgnoreExtras, Elements{ instance.Name: MatchFields(IgnoreExtras, Fields{ @@ -708,7 +708,7 @@ func jobIsCorrectlyReconciled(c client.Client, instance *druidv1alpha1.Etcd, job defer cancel() req := types.NamespacedName{ - Name: getJobName(instance), + Name: utils.GetJobName(instance), Namespace: instance.Namespace, } @@ -726,7 +726,7 @@ func jobIsCorrectlyReconciled(c client.Client, instance *druidv1alpha1.Etcd, job func createJob(instance *druidv1alpha1.Etcd) *batchv1.Job { j := batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ - Name: getJobName(instance), + Name: utils.GetJobName(instance), Namespace: instance.Namespace, Labels: instance.Labels, }, diff --git a/controllers/controller_ref_manager.go b/controllers/controller_ref_manager.go index 59dc98344..d00162d21 100644 --- a/controllers/controller_ref_manager.go +++ b/controllers/controller_ref_manager.go @@ -290,54 +290,6 @@ func (m *EtcdDruidRefManager) ClaimPodDisruptionBudget(ctx context.Context, pdb return claimed, utilerrors.NewAggregate(errlist) } -// ClaimConfigMaps tries to take ownership of a list of ConfigMaps. -// -// It will reconcile the following: -// * Adopt orphans if the selector matches. -// * Release owned objects if the selector no longer matches. -// -// Optional: If one or more filters are specified, a Service will only be claimed if -// all filters return true. -// -// A non-nil error is returned if some form of reconciliation was attempted and -// failed. Usually, controllers should try again later in case reconciliation -// is still needed. -// -// If the error is nil, either the reconciliation succeeded, or no -// reconciliation was necessary. The list of Services that you now own is returned. -func (m *EtcdDruidRefManager) ClaimConfigMaps(ctx context.Context, cms *corev1.ConfigMapList, filters ...func(*corev1.ConfigMap) bool) ([]*corev1.ConfigMap, error) { - var claimed []*corev1.ConfigMap - var errlist []error - - match := func(obj metav1.Object) bool { - cm := obj.(*corev1.ConfigMap) - // Check selector first so filters only run on potentially matching configmaps. - if !m.Selector.Matches(labels.Set(cm.Labels)) { - return false - } - for _, filter := range filters { - if !filter(cm) { - return false - } - } - return true - } - - for k := range cms.Items { - cm := &cms.Items[k] - ok, err := m.claimObject(ctx, cm, match, m.AdoptResource, m.ReleaseResource) - - if err != nil { - errlist = append(errlist, err) - continue - } - if ok { - claimed = append(claimed, cm) - } - } - return claimed, utilerrors.NewAggregate(errlist) -} - // AdoptResource sends a patch to take control of the Etcd. It returns the error if // the patching fails. func (m *EtcdDruidRefManager) AdoptResource(ctx context.Context, obj client.Object) error { @@ -362,13 +314,6 @@ func (m *EtcdDruidRefManager) AdoptResource(ctx context.Context, obj client.Obje annotations[common.GardenerOwnedBy] = objectKey annotations[common.GardenerOwnerType] = strings.ToLower(etcdGVK.Kind) clone.SetAnnotations(annotations) - case *corev1.ConfigMap: - clone = obj.(*corev1.ConfigMap).DeepCopy() - // Note that ValidateOwnerReferences() will reject this patch if another - // OwnerReference exists with controller=true. - if err := controllerutil.SetControllerReference(m.Controller, clone, m.scheme); err != nil { - return err - } case *batchv1.Job: clone = obj.(*batchv1.Job).DeepCopy() // Note that ValidateOwnerReferences() will reject this patch if another @@ -423,8 +368,6 @@ func (m *EtcdDruidRefManager) ReleaseResource(ctx context.Context, obj client.Ob // recommend if statefulset has an owner reference set. delete(clone.GetAnnotations(), common.GardenerOwnedBy) delete(clone.GetAnnotations(), common.GardenerOwnerType) - case *corev1.ConfigMap: - clone = obj.(*corev1.ConfigMap).DeepCopy() case *policyv1beta1.PodDisruptionBudget: clone = obj.(*policyv1beta1.PodDisruptionBudget).DeepCopy() default: diff --git a/controllers/etcd_controller.go b/controllers/etcd_controller.go index 025ed8d41..49701514c 100644 --- a/controllers/etcd_controller.go +++ b/controllers/etcd_controller.go @@ -26,6 +26,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/pkg/common" componentetcd "github.com/gardener/etcd-druid/pkg/component/etcd" + componentconfigmap "github.com/gardener/etcd-druid/pkg/component/etcd/configmap" componentlease "github.com/gardener/etcd-druid/pkg/component/etcd/lease" componentservice "github.com/gardener/etcd-druid/pkg/component/etcd/service" druidpredicates "github.com/gardener/etcd-druid/pkg/predicate" @@ -61,7 +62,6 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/predicate" @@ -144,10 +144,6 @@ func getChartPathForStatefulSet() string { return filepath.Join("etcd", "templates", "etcd-statefulset.yaml") } -func getChartPathForConfigMap() string { - return filepath.Join("etcd", "templates", "etcd-configmap.yaml") -} - func getChartPathForService() string { return filepath.Join("etcd", "templates", "etcd-service.yaml") } @@ -202,12 +198,39 @@ func (r *EtcdReconciler) InitializeControllerWithImageVector() (*EtcdReconciler, return r, nil } +// SetupWithManager sets up manager with a new controller and r as the reconcile.Reconciler +func (r *EtcdReconciler) SetupWithManager(mgr ctrl.Manager, workers int, ignoreOperationAnnotation bool) error { + builder := ctrl.NewControllerManagedBy(mgr).WithOptions(controller.Options{ + MaxConcurrentReconciles: workers, + }) + builder = builder.WithEventFilter(buildPredicate(ignoreOperationAnnotation)).For(&druidv1alpha1.Etcd{}) + if ignoreOperationAnnotation { + builder = builder.Owns(&corev1.Service{}). + Owns(&corev1.ConfigMap{}). + Owns(&appsv1.StatefulSet{}) + } + return builder.Complete(r) +} + +func buildPredicate(ignoreOperationAnnotation bool) predicate.Predicate { + if ignoreOperationAnnotation { + return predicate.GenerationChangedPredicate{} + } + + return predicate.Or( + druidpredicates.HasOperationAnnotation(), + druidpredicates.LastOperationNotSuccessful(), + extensionspredicate.IsDeleting(), + ) +} + // +kubebuilder:rbac:groups=druid.gardener.cloud,resources=etcds,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=druid.gardener.cloud,resources=etcds/status,verbs=get;update;patch // +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;watch;create;update;patch;delete // Reconcile reconciles the etcd. func (r *EtcdReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + r.logger.Info("ETCD controller reconciliation started") etcd := &druidv1alpha1.Etcd{} if err := r.Get(ctx, req.NamespacedName, etcd); err != nil { if apierrors.IsNotFound(err) { @@ -227,7 +250,6 @@ func (r *EtcdReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. func (r *EtcdReconciler) reconcile(ctx context.Context, etcd *druidv1alpha1.Etcd) (ctrl.Result, error) { logger := r.logger.WithValues("etcd", kutil.Key(etcd.Namespace, etcd.Name).String(), "operation", "reconcile") - logger.Info("ETCD controller reconciliation started") // Add Finalizers to Etcd if finalizers := sets.NewString(etcd.Finalizers...); !finalizers.Has(FinalizerName) { @@ -309,7 +331,7 @@ func (r *EtcdReconciler) reconcile(ctx context.Context, etcd *druidv1alpha1.Etcd func (r *EtcdReconciler) cleanCronJobs(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) (*batchv1beta1.CronJob, error) { cronJob := &batchv1beta1.CronJob{} - err := r.Get(ctx, types.NamespacedName{Name: getCronJobName(etcd), Namespace: etcd.Namespace}, cronJob) + err := r.Get(ctx, types.NamespacedName{Name: utils.GetCronJobName(etcd), Namespace: etcd.Namespace}, cronJob) if err != nil { if !apierrors.IsNotFound(err) { return nil, err @@ -346,11 +368,11 @@ func (r *EtcdReconciler) delete(ctx context.Context, etcd *druidv1alpha1.Etcd) ( // TODO(abdasgupta) : This is for backward compatibility towards ETCD-Druid 0.6.0. Remove it. cronJob := &batchv1beta1.CronJob{} - if err := client.IgnoreNotFound(r.Get(ctx, types.NamespacedName{Name: getCronJobName(etcd), Namespace: etcd.Namespace}, cronJob)); err != nil { + if err := client.IgnoreNotFound(r.Get(ctx, types.NamespacedName{Name: utils.GetCronJobName(etcd), Namespace: etcd.Namespace}, cronJob)); err != nil { return ctrl.Result{RequeueAfter: 10 * time.Second}, fmt.Errorf("error while fetching compaction cron job: %v", err) } - if cronJob.Name == getCronJobName(etcd) && cronJob.DeletionTimestamp == nil { + if cronJob.Name == utils.GetCronJobName(etcd) && cronJob.DeletionTimestamp == nil { logger.Info("Deleting cron job", "cronjob", kutil.ObjectName(cronJob)) if err := client.IgnoreNotFound(r.Delete(ctx, cronJob, client.PropagationPolicy(metav1.DeletePropagationForeground))); err != nil { return ctrl.Result{ @@ -392,6 +414,13 @@ func (r *EtcdReconciler) delete(ctx context.Context, etcd *druidv1alpha1.Etcd) ( }, err } + cmDeployer := componentconfigmap.New(r.Client, etcd.Namespace, componentconfigmap.GenerateValues(etcd)) + if err := cmDeployer.Destroy(ctx); err != nil { + return ctrl.Result{ + Requeue: true, + }, err + } + if sets.NewString(etcd.Finalizers...).Has(FinalizerName) { logger.Info("Removing finalizer") if err := controllerutils.PatchRemoveFinalizers(ctx, r.Client, etcd, FinalizerName); client.IgnoreNotFound(err) != nil { @@ -420,132 +449,6 @@ func (r *EtcdReconciler) getServiceFromEtcd(etcd *druidv1alpha1.Etcd, renderedCh return decoded, nil } -func (r *EtcdReconciler) reconcileConfigMaps(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd, renderedChart *chartrenderer.RenderedChart) (*corev1.ConfigMap, error) { - logger.Info("Reconciling etcd configmap") - - selector, err := metav1.LabelSelectorAsSelector(etcd.Spec.Selector) - if err != nil { - logger.Error(err, "Error converting etcd selector to selector") - return nil, err - } - - // list all configmaps to include the configmaps that don't match the etcd`s selector - // anymore but has the stale controller ref. - cms := &corev1.ConfigMapList{} - err = r.List(ctx, cms, client.InNamespace(etcd.Namespace), client.MatchingLabelsSelector{Selector: selector}) - if err != nil { - logger.Error(err, "Error listing configmaps") - return nil, err - } - - // NOTE: filteredCMs are pointing to deepcopies of the cache, but this could change in the future. - // Ref: https://github.com/kubernetes-sigs/controller-runtime/blob/release-0.2/pkg/cache/internal/cache_reader.go#L74 - // if you need to modify them, you need to copy it first. - filteredCMs, err := r.claimConfigMaps(ctx, etcd, selector, cms) - if err != nil { - return nil, err - } - - if len(filteredCMs) > 0 { - logger.Info("Claiming existing etcd configmaps") - - // Keep only 1 Configmap. Delete the rest - for i := 1; i < len(filteredCMs); i++ { - ss := filteredCMs[i] - if err := r.Delete(ctx, ss); err != nil { - logger.Error(err, "Error in deleting duplicate StatefulSet") - continue - } - } - - // Return the updated Configmap - cm := &corev1.ConfigMap{} - err = r.Get(ctx, types.NamespacedName{Name: filteredCMs[0].Name, Namespace: filteredCMs[0].Namespace}, cm) - if err != nil { - return nil, err - } - - // ConfigMap is claimed by for this etcd. Just sync the data - if cm, err = r.syncConfigMapData(ctx, logger, cm, etcd, renderedChart); err != nil { - return nil, err - } - - return cm, err - } - - // Required Configmap doesn't exist. Create new - - cm, err := r.getConfigMapFromEtcd(etcd, renderedChart) - if err != nil { - return nil, err - } - - err = r.Create(ctx, cm) - - // Ignore the precondition violated error, this machine is already updated - // with the desired label. - if err == errorsutil.ErrPreconditionViolated { - logger.Info("ConfigMap precondition doesn't hold, skip updating it.", "configmap", kutil.Key(cm.Namespace, cm.Name).String()) - err = nil - } - - if err != nil { - return nil, err - } - - if err := controllerutil.SetControllerReference(etcd, cm, r.Scheme); err != nil { - return nil, err - } - - return cm.DeepCopy(), err -} - -func (r *EtcdReconciler) syncConfigMapData(ctx context.Context, logger logr.Logger, cm *corev1.ConfigMap, etcd *druidv1alpha1.Etcd, renderedChart *chartrenderer.RenderedChart) (*corev1.ConfigMap, error) { - decoded, err := r.getConfigMapFromEtcd(etcd, renderedChart) - if err != nil { - return nil, err - } - - if reflect.DeepEqual(cm.Data, decoded.Data) { - return cm, nil - } - cmCopy := cm.DeepCopy() - cmCopy.Data = decoded.Data - - err = retry.RetryOnConflict(retry.DefaultBackoff, func() error { - return r.Patch(ctx, cmCopy, client.MergeFrom(cm)) - }) - - // Ignore the precondition violated error, this machine is already updated - // with the desired label. - if err == errorsutil.ErrPreconditionViolated { - logger.Info("ConfigMap precondition doesn't hold, skip updating it.", "configmap", kutil.Key(cm.Namespace, cm.Name).String()) - err = nil - } - if err != nil { - return nil, err - } - return cmCopy, err -} - -func (r *EtcdReconciler) getConfigMapFromEtcd(etcd *druidv1alpha1.Etcd, renderedChart *chartrenderer.RenderedChart) (*corev1.ConfigMap, error) { - var err error - decoded := &corev1.ConfigMap{} - configMapPath := getChartPathForConfigMap() - - if _, ok := renderedChart.Files()[configMapPath]; !ok { - return nil, fmt.Errorf("missing configmap template file in the charts: %v", configMapPath) - } - - //logger.Infof("%v: %v", statefulsetPath, renderer.Files()[statefulsetPath]) - decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader([]byte(renderedChart.Files()[configMapPath])), 1024) - - if err = decoder.Decode(&decoded); err != nil { - return nil, err - } - return decoded, nil -} - func (r *EtcdReconciler) reconcilePodDisruptionBudget(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd, values map[string]interface{}) error { logger.Info("Reconcile PodDisruptionBudget") pdb := &policyv1beta1.PodDisruptionBudget{} @@ -958,21 +861,15 @@ func (r *EtcdReconciler) reconcileRoleBinding(ctx context.Context, logger logr.L } func (r *EtcdReconciler) reconcileEtcd(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) (*string, *appsv1.StatefulSet, error) { - val := componentetcd.Values{ - Lease: componentlease.GenerateValues(etcd), - Service: componentservice.GenerateValues(etcd), - } - - values, err := getMapFromEtcd(ctx, r.Client, r.ImageVector, etcd, val, r.disableEtcdServiceAccountAutomount) - - if err != nil { - return nil, nil, err + // Check if Spec.Replicas is odd or even. + if etcd.Status.Replicas > 1 && etcd.Spec.Replicas&1 == 0 { + return nil, nil, fmt.Errorf("Spec.Replicas should not be even number: %d", etcd.Spec.Replicas) } - chartPath := getChartPath() - renderedChart, err := r.chartApplier.Render(chartPath, etcd.Name, etcd.Namespace, values) - if err != nil { - return nil, nil, err + val := componentetcd.Values{ + ConfigMap: componentconfigmap.GenerateValues(etcd), + Lease: componentlease.GenerateValues(etcd), + Service: componentservice.GenerateValues(etcd), } leaseDeployer := componentlease.New(r.Client, etcd.Namespace, val.Lease) @@ -987,12 +884,15 @@ func (r *EtcdReconciler) reconcileEtcd(ctx context.Context, logger logr.Logger, return nil, nil, err } - cm, err := r.reconcileConfigMaps(ctx, logger, etcd, renderedChart) - if err != nil { + cmDeployer := componentconfigmap.New(r.Client, etcd.Namespace, val.ConfigMap) + if err := cmDeployer.Deploy(ctx); err != nil { return nil, nil, err } - if cm != nil { - values["configMapName"] = cm.Name + + values, err := r.getMapFromEtcd(r.ImageVector, etcd, val, r.disableEtcdServiceAccountAutomount) + + if err != nil { + return nil, nil, err } err = r.reconcileServiceAccount(ctx, logger, etcd, values) @@ -1051,16 +951,14 @@ func checkEtcdAnnotations(annotations map[string]string, etcd metav1.Object) boo } -func getMapFromEtcd(ctx context.Context, client client.Client, im imagevector.ImageVector, etcd *druidv1alpha1.Etcd, val componentetcd.Values, disableEtcdServiceAccountAutomount bool) (map[string]interface{}, error) { - var statefulsetReplicas int - if etcd.Spec.Replicas != 0 { - statefulsetReplicas = 1 - } +func (r *EtcdReconciler) getMapFromEtcd(im imagevector.ImageVector, etcd *druidv1alpha1.Etcd, val componentetcd.Values, disableEtcdServiceAccountAutomount bool) (map[string]interface{}, error) { + statefulsetReplicas := int(etcd.Spec.Replicas) etcdValues := map[string]interface{}{ "clientPort": val.Service.ClientPort, "defragmentationSchedule": etcd.Spec.Etcd.DefragmentationSchedule, - "enableTLS": (etcd.Spec.Etcd.TLS != nil), + "enableClientTLS": (etcd.Spec.Etcd.ClientUrlTLS != nil), + "enablePeerTLS": (etcd.Spec.Etcd.PeerUrlTLS != nil), "pullPolicy": corev1.PullIfNotPresent, "serverPort": val.Service.ServerPort, // "username": etcd.Spec.Etcd.Username, @@ -1108,6 +1006,7 @@ func getMapFromEtcd(ctx context.Context, client client.Client, im imagevector.Im } backupValues := map[string]interface{}{ + "enableTLS": (etcd.Spec.Backup.TLS != nil), "pullPolicy": corev1.PullIfNotPresent, "port": val.Service.BackupPort, "etcdQuotaBytes": quota, @@ -1207,6 +1106,15 @@ func getMapFromEtcd(ctx context.Context, client client.Client, im imagevector.Im sharedConfigValues["autoCompactionRetention"] = etcd.Spec.Common.AutoCompactionRetention } + annotations := make(map[string]string) + if etcd.Spec.Annotations != nil { + for key, value := range etcd.Spec.Annotations { + annotations[key] = value + } + } + + annotations["checksum/etcd-configmap"] = val.ConfigMap.ConfigMapChecksum + pdbMinAvailable := 0 if etcd.Spec.Replicas > 1 { pdbMinAvailable = int(etcd.Spec.Replicas) @@ -1217,18 +1125,18 @@ func getMapFromEtcd(ctx context.Context, client client.Client, im imagevector.Im "uid": etcd.UID, "selector": etcd.Spec.Selector, "labels": etcd.Spec.Labels, - "annotations": etcd.Spec.Annotations, + "annotations": annotations, "etcd": etcdValues, "backup": backupValues, "sharedConfig": sharedConfigValues, "replicas": etcd.Spec.Replicas, "statefulsetReplicas": statefulsetReplicas, "serviceName": val.Service.PeerServiceName, - "configMapName": fmt.Sprintf("etcd-bootstrap-%s", string(etcd.UID[:6])), - "jobName": getJobName(etcd), + "configMapName": val.ConfigMap.ConfigMapName, + "jobName": utils.GetJobName(etcd), "pdbMinAvailable": pdbMinAvailable, "volumeClaimTemplateName": volumeClaimTemplateName, - "serviceAccountName": getServiceAccountName(etcd), + "serviceAccountName": utils.GetServiceAccountName(etcd), "disableEtcdServiceAccountAutomount": disableEtcdServiceAccountAutomount, "roleName": fmt.Sprintf("druid.gardener.cloud:etcd:%s", etcd.Name), "roleBindingName": fmt.Sprintf("druid.gardener.cloud:etcd:%s", etcd.Name), @@ -1246,10 +1154,15 @@ func getMapFromEtcd(ctx context.Context, client client.Client, im imagevector.Im values["priorityClassName"] = *etcd.Spec.PriorityClassName } - if etcd.Spec.Etcd.TLS != nil { - values["tlsServerSecret"] = etcd.Spec.Etcd.TLS.ServerTLSSecretRef.Name - values["tlsClientSecret"] = etcd.Spec.Etcd.TLS.ClientTLSSecretRef.Name - values["tlsCASecret"] = etcd.Spec.Etcd.TLS.TLSCASecretRef.Name + if etcd.Spec.Etcd.ClientUrlTLS != nil { + values["clientUrlTlsCASecret"] = etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.Name + values["clientUrlTlsServerSecret"] = etcd.Spec.Etcd.ClientUrlTLS.ServerTLSSecretRef.Name + values["clientUrlTlsClientSecret"] = etcd.Spec.Etcd.ClientUrlTLS.ClientTLSSecretRef.Name + } + + if etcd.Spec.Etcd.PeerUrlTLS != nil { + values["peerUrlTlsCASecret"] = etcd.Spec.Etcd.PeerUrlTLS.TLSCASecretRef.Name + values["peerUrlTlsServerSecret"] = etcd.Spec.Etcd.PeerUrlTLS.ServerTLSSecretRef.Name } if heartBeatDuration := etcd.Spec.Etcd.HeartbeatDuration; heartBeatDuration != nil { @@ -1257,7 +1170,7 @@ func getMapFromEtcd(ctx context.Context, client client.Client, im imagevector.Im } if etcd.Spec.Backup.Store != nil { - if values["store"], err = utils.GetStoreValues(ctx, client, etcd.Spec.Backup.Store, etcd.Namespace); err != nil { + if values["store"], err = utils.GetStoreValues(context.Background(), r.Client, etcd.Spec.Backup.Store, etcd.Namespace); err != nil { return nil, err } @@ -1268,10 +1181,6 @@ func getMapFromEtcd(ctx context.Context, client client.Client, im imagevector.Im return values, nil } -func getServiceAccountName(etcd *druidv1alpha1.Etcd) string { - return etcd.Name -} - func getEtcdImages(im imagevector.ImageVector, etcd *druidv1alpha1.Etcd) (string, string, error) { var ( err error @@ -1310,14 +1219,25 @@ func getEtcdImages(im imagevector.ImageVector, etcd *druidv1alpha1.Etcd) (string func (r *EtcdReconciler) addFinalizersToDependantSecrets(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) error { secrets := []*corev1.SecretReference{} - if etcd.Spec.Etcd.TLS != nil { + + if etcd.Spec.Etcd.ClientUrlTLS != nil { // As the secrets inside TLS field are required, we error in case they are not found. secrets = append(secrets, - &etcd.Spec.Etcd.TLS.ClientTLSSecretRef, - &etcd.Spec.Etcd.TLS.ServerTLSSecretRef, - &etcd.Spec.Etcd.TLS.TLSCASecretRef, + &etcd.Spec.Etcd.ClientUrlTLS.ClientTLSSecretRef, + &etcd.Spec.Etcd.ClientUrlTLS.ServerTLSSecretRef, + &etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef, ) } + + if etcd.Spec.Etcd.PeerUrlTLS != nil { + // As the secrets inside TLS field are required, we error in case they are not found. + secrets = append(secrets, + &etcd.Spec.Etcd.PeerUrlTLS.ClientTLSSecretRef, + &etcd.Spec.Etcd.PeerUrlTLS.ServerTLSSecretRef, + &etcd.Spec.Etcd.PeerUrlTLS.TLSCASecretRef, + ) + } + if etcd.Spec.Backup.Store != nil && etcd.Spec.Backup.Store.SecretRef != nil { // As the store secret is required, we error in case it is not found as well. secrets = append(secrets, etcd.Spec.Backup.Store.SecretRef) @@ -1343,13 +1263,23 @@ func (r *EtcdReconciler) addFinalizersToDependantSecrets(ctx context.Context, lo func (r *EtcdReconciler) removeFinalizersToDependantSecrets(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) error { secrets := []*corev1.SecretReference{} - if etcd.Spec.Etcd.TLS != nil { + + if etcd.Spec.Etcd.ClientUrlTLS != nil { + secrets = append(secrets, + &etcd.Spec.Etcd.ClientUrlTLS.ClientTLSSecretRef, + &etcd.Spec.Etcd.ClientUrlTLS.ServerTLSSecretRef, + &etcd.Spec.Etcd.ClientUrlTLS.TLSCASecretRef, + ) + } + + if etcd.Spec.Etcd.PeerUrlTLS != nil { secrets = append(secrets, - &etcd.Spec.Etcd.TLS.ClientTLSSecretRef, - &etcd.Spec.Etcd.TLS.ServerTLSSecretRef, - &etcd.Spec.Etcd.TLS.TLSCASecretRef, + &etcd.Spec.Etcd.PeerUrlTLS.ClientTLSSecretRef, + &etcd.Spec.Etcd.PeerUrlTLS.ServerTLSSecretRef, + &etcd.Spec.Etcd.PeerUrlTLS.TLSCASecretRef, ) } + if etcd.Spec.Backup.Store != nil && etcd.Spec.Backup.Store.SecretRef != nil { secrets = append(secrets, etcd.Spec.Backup.Store.SecretRef) } @@ -1554,47 +1484,3 @@ func (r *EtcdReconciler) claimPodDisruptionBudget(ctx context.Context, etcd *dru cm := NewEtcdDruidRefManager(r.Client, r.Scheme, etcd, selector, etcdGVK, canAdoptFunc) return cm.ClaimPodDisruptionBudget(ctx, pdb) } - -func (r *EtcdReconciler) claimConfigMaps(ctx context.Context, etcd *druidv1alpha1.Etcd, selector labels.Selector, configMaps *corev1.ConfigMapList) ([]*corev1.ConfigMap, error) { - // If any adoptions are attempted, we should first recheck for deletion with - // an uncached quorum read sometime after listing Machines (see #42639). - canAdoptFunc := RecheckDeletionTimestamp(func() (metav1.Object, error) { - foundEtcd := &druidv1alpha1.Etcd{} - err := r.Get(ctx, types.NamespacedName{Name: etcd.Name, Namespace: etcd.Namespace}, foundEtcd) - if err != nil { - return nil, err - } - if foundEtcd.UID != etcd.UID { - return nil, fmt.Errorf("original %v/%v hvpa gone: got uid %v, wanted %v", etcd.Namespace, etcd.Name, foundEtcd.UID, etcd.UID) - } - return foundEtcd, nil - }) - cm := NewEtcdDruidRefManager(r.Client, r.Scheme, etcd, selector, etcdGVK, canAdoptFunc) - return cm.ClaimConfigMaps(ctx, configMaps) -} - -// SetupWithManager sets up manager with a new controller and r as the reconcile.Reconciler -func (r *EtcdReconciler) SetupWithManager(mgr ctrl.Manager, workers int, ignoreOperationAnnotation bool) error { - builder := ctrl.NewControllerManagedBy(mgr).WithOptions(controller.Options{ - MaxConcurrentReconciles: workers, - }) - builder = builder.WithEventFilter(buildPredicate(ignoreOperationAnnotation)).For(&druidv1alpha1.Etcd{}) - if ignoreOperationAnnotation { - builder = builder.Owns(&corev1.Service{}). - Owns(&corev1.ConfigMap{}). - Owns(&appsv1.StatefulSet{}) - } - return builder.Complete(r) -} - -func buildPredicate(ignoreOperationAnnotation bool) predicate.Predicate { - if ignoreOperationAnnotation { - return predicate.GenerationChangedPredicate{} - } - - return predicate.Or( - druidpredicates.HasOperationAnnotation(), - druidpredicates.LastOperationNotSuccessful(), - extensionspredicate.IsDeleting(), - ) -} diff --git a/controllers/etcd_controller_test.go b/controllers/etcd_controller_test.go index 694925563..b56260bb2 100644 --- a/controllers/etcd_controller_test.go +++ b/controllers/etcd_controller_test.go @@ -16,8 +16,10 @@ package controllers import ( "context" + "encoding/json" "fmt" "os" + "strings" "time" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" @@ -27,6 +29,7 @@ import ( v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/gardener/gardener/pkg/controllerutils" + gardenerUtils "github.com/gardener/gardener/pkg/utils" "github.com/gardener/gardener/pkg/utils/imagevector" "github.com/gardener/gardener/pkg/utils/kubernetes/health" "github.com/gardener/gardener/pkg/utils/test/matchers" @@ -66,7 +69,6 @@ const ( timeout = time.Minute * 2 pollingInterval = time.Second * 2 etcdConfig = "etcd.conf.yaml" - quotaKey = "quota-backend-bytes" backupRestore = "backup-restore" metricsKey = "metrics" ) @@ -215,7 +217,7 @@ var _ = Describe("Druid", func() { // Wait until Service has been created by controller Eventually(func() error { return c.Get(context.TODO(), types.NamespacedName{ - Name: fmt.Sprintf("%s-client", instance.Name), + Name: utils.GetClientServiceName(instance), Namespace: instance.Namespace, }, svc) }, timeout, pollingInterval).Should(BeNil()) @@ -590,13 +592,13 @@ var _ = Describe("Druid", func() { ) DescribeTable("when etcd resource is created", - func(name string, generateEtcd func(string, string) *druidv1alpha1.Etcd, validate func(*druidv1alpha1.Etcd, *appsv1.StatefulSet, *corev1.ConfigMap, *corev1.Service)) { + func(name string, generateEtcd func(string, string) *druidv1alpha1.Etcd, validate func(*druidv1alpha1.Etcd, *appsv1.StatefulSet, *corev1.ConfigMap, *corev1.Service, *corev1.Service)) { var err error var instance *druidv1alpha1.Etcd var c client.Client var s *appsv1.StatefulSet var cm *corev1.ConfigMap - var svc *corev1.Service + var clSvc, prSvc *corev1.Service var sa *corev1.ServiceAccount var role *rbac.Role var rb *rbac.RoleBinding @@ -623,8 +625,10 @@ var _ = Describe("Druid", func() { Eventually(func() error { return statefulsetIsCorrectlyReconciled(c, instance, s) }, timeout, pollingInterval).Should(BeNil()) cm = &corev1.ConfigMap{} Eventually(func() error { return configMapIsCorrectlyReconciled(c, instance, cm) }, timeout, pollingInterval).Should(BeNil()) - svc = &corev1.Service{} - Eventually(func() error { return serviceIsCorrectlyReconciled(c, instance, svc) }, timeout, pollingInterval).Should(BeNil()) + clSvc = &corev1.Service{} + Eventually(func() error { return clientServiceIsCorrectlyReconciled(c, instance, clSvc) }, timeout, pollingInterval).Should(BeNil()) + prSvc = &corev1.Service{} + Eventually(func() error { return peerServiceIsCorrectlyReconciled(c, instance, prSvc) }, timeout, pollingInterval).Should(BeNil()) sa = &corev1.ServiceAccount{} Eventually(func() error { return serviceAccountIsCorrectlyReconciled(c, instance, sa) }, timeout, pollingInterval).Should(BeNil()) role = &rbac.Role{} @@ -632,7 +636,7 @@ var _ = Describe("Druid", func() { rb = &rbac.RoleBinding{} Eventually(func() error { return roleBindingIsCorrectlyReconciled(c, instance, rb) }, timeout, pollingInterval).Should(BeNil()) - validate(instance, s, cm, svc) + validate(instance, s, cm, clSvc, prSvc) validateRole(instance, role) setStatefulSetReady(s) @@ -764,6 +768,144 @@ var _ = Describe("Cron Job", func() { }) }) +var _ = Describe("Multinode ETCD", func() { + //Reconciliation of new etcd resource deployment without any existing statefulsets. + Context("when adding etcd resources", func() { + var ( + err error + instance *druidv1alpha1.Etcd + sts *appsv1.StatefulSet + svc *corev1.Service + c client.Client + ) + + BeforeEach(func() { + instance = getEtcd("foo82", "default", false) + c = mgr.GetClient() + ns := corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.Namespace, + }, + } + _, err = controllerutil.CreateOrUpdate(context.TODO(), c, &ns, func() error { return nil }) + Expect(err).To(Not(HaveOccurred())) + + storeSecret := instance.Spec.Backup.Store.SecretRef.Name + errors := createSecrets(c, instance.Namespace, storeSecret) + Expect(len(errors)).Should(BeZero()) + }) + It("no statefulsets are created when ETCD replicas are even number", func() { + // Update replicas in ETCD resource with 0 + instance.Spec.Replicas = 4 + Expect(c.Create(context.TODO(), instance)).To(Succeed()) + + Eventually(func() error { + return c.Get(context.TODO(), types.NamespacedName{ + Name: instance.Name, + Namespace: instance.Namespace, + }, instance) + }, timeout, pollingInterval).Should(BeNil()) + + // No StatefulSet has been created by controller as even number of replicas are not allowed + sts = &appsv1.StatefulSet{} + Eventually(func() error { + return c.Get(context.TODO(), types.NamespacedName{ + Name: instance.Name, + Namespace: instance.Namespace, + }, sts) + }, timeout, pollingInterval).Should(matchers.BeNotFoundError()) + + // No Service has been created by controller as even number of replicas are not allowed + svc = &corev1.Service{} + Expect(c.Get(context.TODO(), types.NamespacedName{ + Name: utils.GetClientServiceName(instance), + Namespace: instance.Namespace, + }, svc)).Should(matchers.BeNotFoundError()) + svc = nil + }) + It("statefulsets are created when ETCD replicas are odd number", func() { + // Update replicas in ETCD resource with 0 + instance.Spec.Replicas = 3 + Expect(c.Create(context.TODO(), instance)).To(Succeed()) + + Eventually(func() error { + return c.Get(context.TODO(), types.NamespacedName{ + Name: instance.Name, + Namespace: instance.Namespace, + }, instance) + }, timeout, pollingInterval).Should(BeNil()) + + sts = &appsv1.StatefulSet{} + // StatefulSet has been created by controller + Eventually(func() error { return statefulsetIsCorrectlyReconciled(c, instance, sts) }, timeout, pollingInterval).Should(BeNil()) + Expect(int(*sts.Spec.Replicas)).To(Equal(3)) + }) + AfterEach(func() { + // Delete `etcd` instance + Expect(c.Delete(context.TODO(), instance)).To(Succeed()) + Eventually(func() error { + return c.Get(context.TODO(), client.ObjectKeyFromObject(instance), &druidv1alpha1.Etcd{}) + }, timeout, pollingInterval).Should(matchers.BeNotFoundError()) + // Delete service manually because garbage collection is not available in `envtest` + if svc != nil { + Expect(c.Delete(context.TODO(), svc)).To(Succeed()) + Eventually(func() error { + return c.Get(context.TODO(), client.ObjectKeyFromObject(svc), &corev1.Service{}) + }, timeout, pollingInterval).Should(matchers.BeNotFoundError()) + } + }) + }) + DescribeTable("configmaps are mounted properly when ETCD replicas are odd number", func(name string, replicas int, getEtcdWithReplicas func(string, string, int) *druidv1alpha1.Etcd) { + var err error + var instance *druidv1alpha1.Etcd + var c client.Client + var sts *appsv1.StatefulSet + var cm *corev1.ConfigMap + var svc *corev1.Service + + instance = getEtcdWithReplicas(name, "default", replicas) + c = mgr.GetClient() + ns := corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.Namespace, + }, + } + + _, err = controllerutil.CreateOrUpdate(context.TODO(), c, &ns, func() error { return nil }) + Expect(err).To(Not(HaveOccurred())) + + if instance.Spec.Backup.Store != nil && instance.Spec.Backup.Store.SecretRef != nil { + storeSecret := instance.Spec.Backup.Store.SecretRef.Name + errors := createSecrets(c, instance.Namespace, storeSecret) + Expect(len(errors)).Should(BeZero()) + } + err = c.Create(context.TODO(), instance) + Expect(err).NotTo(HaveOccurred()) + sts = &appsv1.StatefulSet{} + Eventually(func() error { return statefulsetIsCorrectlyReconciled(c, instance, sts) }, timeout, pollingInterval).Should(BeNil()) + cm = &corev1.ConfigMap{} + Eventually(func() error { return configMapIsCorrectlyReconciled(c, instance, cm) }, timeout, pollingInterval).Should(BeNil()) + svc = &corev1.Service{} + Eventually(func() error { return clientServiceIsCorrectlyReconciled(c, instance, svc) }, timeout, pollingInterval).Should(BeNil()) + + // Validate statefulset + Expect(*sts.Spec.Replicas).To(Equal(int32(instance.Spec.Replicas))) + + if instance.Spec.Replicas == 1 { + matcher := "initial-cluster: foo83-0=http://foo83-0.foo83-peer.default.svc:2380" + Expect(strings.Contains(cm.Data["etcd.conf.yaml"], matcher)).To(BeTrue()) + } + + if instance.Spec.Replicas > 1 { + matcher := "initial-cluster: foo84-0=http://foo84-0.foo84-peer.default.svc:2380,foo84-1=http://foo84-1.foo84-peer.default.svc:2380,foo84-2=http://foo84-2.foo84-peer.default.svc:2380" + Expect(strings.Contains(cm.Data["etcd.conf.yaml"], matcher)).To(BeTrue()) + } + }, + Entry("verify configmap mount path and etcd.conf.yaml when replica is 1 ", "foo83", 1, getEtcdWithReplicas), + Entry("verify configmap mount path and etcd.conf.yaml when replica is 3 ", "foo84", 3, getEtcdWithReplicas), + ) +}) + func validateRole(instance *druidv1alpha1.Etcd, role *rbac.Role) { Expect(*role).To(MatchFields(IgnoreExtras, Fields{ "ObjectMeta": MatchFields(IgnoreExtras, Fields{ @@ -824,14 +966,16 @@ func podDeleted(c client.Client, etcd *druidv1alpha1.Etcd) error { } -func validateEtcdWithDefaults(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev1.ConfigMap, svc *corev1.Service) { - // Validate Quota +func validateEtcdWithDefaults(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev1.ConfigMap, clSvc *corev1.Service, prSvc *corev1.Service) { configYML := cm.Data[etcdConfig] - config := map[string]string{} + config := map[string]interface{}{} err := yaml.Unmarshal([]byte(configYML), &config) Expect(err).NotTo(HaveOccurred()) - Expect(instance.Spec.Etcd.Quota).To(BeNil()) - Expect(config).To(HaveKeyWithValue(quotaKey, fmt.Sprintf("%d", int64(quota.Value())))) + + // Validate ETCD annotation for configmap checksum + jsonString, err := json.Marshal(cm.Data) + Expect(err).NotTo(HaveOccurred()) + configMapChecksum := gardenerUtils.ComputeSHA256Hex(jsonString) // Validate Metrics MetricsLevel Expect(instance.Spec.Etcd.Metrics).To(BeNil()) @@ -861,26 +1005,33 @@ func validateEtcdWithDefaults(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSe Expect(instance.Spec.Etcd.Resources).To(BeNil()) // Validate TLS. Ensure that enableTLS flag is not triggered in the go-template - Expect(instance.Spec.Etcd.TLS).To(BeNil()) + Expect(instance.Spec.Etcd.PeerUrlTLS).To(BeNil()) + + readinessProbeUrl := fmt.Sprintf("http://%s-local:%d/health", instance.Name, clientPort) + if int(instance.Spec.Replicas) == 1 { + readinessProbeUrl = fmt.Sprintf("http://%s-local:%d/healthz", instance.Name, backupPort) + } Expect(config).To(MatchKeys(IgnoreExtras, Keys{ - "name": Equal(fmt.Sprintf("etcd-%s", instance.UID[:6])), - "data-dir": Equal("/var/etcd/data/new.etcd"), - "metrics": Equal(string(druidv1alpha1.Basic)), - "snapshot-count": Equal("75000"), - "enable-v2": Equal("false"), - "quota-backend-bytes": Equal("8589934592"), - "listen-client-urls": Equal(fmt.Sprintf("http://0.0.0.0:%d", clientPort)), - "advertise-client-urls": Equal(fmt.Sprintf("http://0.0.0.0:%d", clientPort)), - "initial-cluster-token": Equal("initial"), - "initial-cluster-state": Equal("new"), - "auto-compaction-mode": Equal(string(druidv1alpha1.Periodic)), - "auto-compaction-retention": Equal(DefaultAutoCompactionRetention), + "name": Equal(fmt.Sprintf("etcd-%s", instance.UID[:6])), + "data-dir": Equal("/var/etcd/data/new.etcd"), + "metrics": Equal(string(druidv1alpha1.Basic)), + "snapshot-count": Equal(float64(75000)), + "enable-v2": Equal(false), + "quota-backend-bytes": Equal(float64(8589934592)), + "listen-client-urls": Equal(fmt.Sprintf("http://0.0.0.0:%d", clientPort)), + "advertise-client-urls": Equal(fmt.Sprintf("%s@%s@%s@%d", "http", prSvc.Name, instance.Namespace, clientPort)), + "listen-peer-urls": Equal(fmt.Sprintf("http://0.0.0.0:%d", serverPort)), + "initial-advertise-peer-urls": Equal(fmt.Sprintf("%s@%s@%s@%d", "http", prSvc.Name, instance.Namespace, serverPort)), + "initial-cluster-token": Equal("etcd-cluster"), + "initial-cluster-state": Equal("new"), + "auto-compaction-mode": Equal(string(druidv1alpha1.Periodic)), + "auto-compaction-retention": Equal(DefaultAutoCompactionRetention), })) - Expect(*svc).To(MatchFields(IgnoreExtras, Fields{ + Expect(*clSvc).To(MatchFields(IgnoreExtras, Fields{ "ObjectMeta": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(fmt.Sprintf("%s-client", instance.Name)), + "Name": Equal(utils.GetClientServiceName(instance)), "Namespace": Equal(instance.Namespace), "Labels": MatchAllKeys(Keys{ "name": Equal("etcd"), @@ -943,6 +1094,7 @@ func validateEtcdWithDefaults(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSe "app": Equal("etcd-statefulset"), "role": Equal("test"), "instance": Equal(instance.Name), + "checksum/etcd-configmap": Equal(configMapChecksum), }), "Labels": MatchAllKeys(Keys{ "name": Equal("etcd"), @@ -953,7 +1105,7 @@ func validateEtcdWithDefaults(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSe "UpdateStrategy": MatchFields(IgnoreExtras, Fields{ "Type": Equal(appsv1.RollingUpdateStatefulSetStrategyType), }), - "ServiceName": Equal(fmt.Sprintf("%s-peer", instance.Name)), + "ServiceName": Equal(utils.GetPeerServiceName(instance)), "Replicas": PointTo(Equal(int32(instance.Spec.Replicas))), "Selector": PointTo(MatchFields(IgnoreExtras, Fields{ "MatchLabels": MatchAllKeys(Keys{ @@ -1016,12 +1168,11 @@ func validateEtcdWithDefaults(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSe }), "ReadinessProbe": PointTo(MatchFields(IgnoreExtras, Fields{ "Handler": MatchFields(IgnoreExtras, Fields{ - "HTTPGet": PointTo(MatchFields(IgnoreExtras, Fields{ - "Path": Equal("/healthz"), - "Port": MatchFields(IgnoreExtras, Fields{ - "IntVal": Equal(int32(8080)), + "Exec": PointTo(MatchFields(IgnoreExtras, Fields{ + "Command": MatchAllElements(cmdIterator, Elements{ + "/usr/bin/curl": Equal("/usr/bin/curl"), + readinessProbeUrl: Equal(readinessProbeUrl), }), - "Scheme": Equal(corev1.URISchemeHTTP), })), }), "InitialDelaySeconds": Equal(int32(15)), @@ -1049,10 +1200,6 @@ func validateEtcdWithDefaults(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSe "Name": Equal(instance.Name), "MountPath": Equal("/var/etcd/data/"), }), - "etcd-config-file": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("etcd-config-file"), - "MountPath": Equal("/var/etcd/config/"), - }), }), }), @@ -1173,14 +1320,16 @@ func validateEtcdWithDefaults(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSe })) } -func validateEtcd(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev1.ConfigMap, svc *corev1.Service) { - // Validate Quota +func validateEtcd(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev1.ConfigMap, clSvc *corev1.Service, prSvc *corev1.Service) { configYML := cm.Data[etcdConfig] config := map[string]interface{}{} err := yaml.Unmarshal([]byte(configYML), &config) Expect(err).NotTo(HaveOccurred()) - Expect(instance.Spec.Etcd.Quota).NotTo(BeNil()) - Expect(config).To(HaveKeyWithValue(quotaKey, float64(instance.Spec.Etcd.Quota.Value()))) + + // Validate ETCD annotation for configmap checksum + jsonString, err := json.Marshal(cm.Data) + Expect(err).NotTo(HaveOccurred()) + configMapChecksum := gardenerUtils.ComputeSHA256Hex(jsonString) // Validate Metrics MetricsLevel Expect(instance.Spec.Etcd.Metrics).NotTo(BeNil()) @@ -1198,6 +1347,11 @@ func validateEtcd(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev store, err := utils.StorageProviderFromInfraProvider(instance.Spec.Backup.Store.Provider) Expect(err).NotTo(HaveOccurred()) + readinessProbeUrl := fmt.Sprintf("https://%s-local:%d/health", instance.Name, clientPort) + if int(instance.Spec.Replicas) == 1 { + readinessProbeUrl = fmt.Sprintf("https://%s-local:%d/healthz", instance.Name, backupPort) + } + Expect(*cm).To(MatchFields(IgnoreExtras, Fields{ "ObjectMeta": MatchFields(IgnoreExtras, Fields{ "Name": Equal(fmt.Sprintf("etcd-bootstrap-%s", string(instance.UID[:6]))), @@ -1220,31 +1374,42 @@ func validateEtcd(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev })) Expect(config).To(MatchKeys(IgnoreExtras, Keys{ - "name": Equal(fmt.Sprintf("etcd-%s", instance.UID[:6])), - "data-dir": Equal("/var/etcd/data/new.etcd"), - "metrics": Equal(string(*instance.Spec.Etcd.Metrics)), - "snapshot-count": Equal(float64(75000)), - "enable-v2": Equal(false), - "quota-backend-bytes": Equal(float64(instance.Spec.Etcd.Quota.Value())), - "listen-client-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", *instance.Spec.Etcd.ClientPort)), - "advertise-client-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", *instance.Spec.Etcd.ClientPort)), - "initial-cluster-token": Equal("initial"), - "initial-cluster-state": Equal("new"), - "auto-compaction-mode": Equal(string(*instance.Spec.Common.AutoCompactionMode)), - "auto-compaction-retention": Equal(*instance.Spec.Common.AutoCompactionRetention), + "name": Equal(fmt.Sprintf("etcd-%s", instance.UID[:6])), + "data-dir": Equal("/var/etcd/data/new.etcd"), + "metrics": Equal(string(*instance.Spec.Etcd.Metrics)), + "snapshot-count": Equal(float64(75000)), + "enable-v2": Equal(false), + "quota-backend-bytes": Equal(float64(instance.Spec.Etcd.Quota.Value())), "client-transport-security": MatchKeys(IgnoreExtras, Keys{ - "cert-file": Equal("/var/etcd/ssl/server/tls.crt"), - "key-file": Equal("/var/etcd/ssl/server/tls.key"), + "cert-file": Equal("/var/etcd/ssl/client/server/tls.crt"), + "key-file": Equal("/var/etcd/ssl/client/server/tls.key"), + "client-cert-auth": Equal(true), + "trusted-ca-file": Equal("/var/etcd/ssl/client/ca/ca.crt"), + "auto-tls": Equal(false), + }), + "listen-client-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", *instance.Spec.Etcd.ClientPort)), + "advertise-client-urls": Equal(fmt.Sprintf("%s@%s@%s@%d", "https", prSvc.Name, instance.Namespace, *instance.Spec.Etcd.ClientPort)), + + "peer-transport-security": MatchKeys(IgnoreExtras, Keys{ + "cert-file": Equal("/var/etcd/ssl/peer/server/tls.crt"), + "key-file": Equal("/var/etcd/ssl/peer/server/tls.key"), "client-cert-auth": Equal(true), - "trusted-ca-file": Equal("/var/etcd/ssl/ca/ca.crt"), + "trusted-ca-file": Equal("/var/etcd/ssl/peer/ca/ca.crt"), "auto-tls": Equal(false), }), + "listen-peer-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", *instance.Spec.Etcd.ServerPort)), + "initial-advertise-peer-urls": Equal(fmt.Sprintf("%s@%s@%s@%d", "https", prSvc.Name, instance.Namespace, *instance.Spec.Etcd.ServerPort)), + + "initial-cluster-token": Equal("etcd-cluster"), + "initial-cluster-state": Equal("new"), + "auto-compaction-mode": Equal(string(*instance.Spec.Common.AutoCompactionMode)), + "auto-compaction-retention": Equal(*instance.Spec.Common.AutoCompactionRetention), })) - Expect(*svc).To(MatchFields(IgnoreExtras, Fields{ + Expect(*clSvc).To(MatchFields(IgnoreExtras, Fields{ "ObjectMeta": MatchFields(IgnoreExtras, Fields{ - "Name": Equal(fmt.Sprintf("%s-client", instance.Name)), + "Name": Equal(utils.GetClientServiceName(instance)), "Namespace": Equal(instance.Namespace), "Labels": MatchAllKeys(Keys{ "name": Equal("etcd"), @@ -1307,6 +1472,7 @@ func validateEtcd(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev "app": Equal("etcd-statefulset"), "role": Equal("test"), "instance": Equal(instance.Name), + "checksum/etcd-configmap": Equal(configMapChecksum), }), "Labels": MatchAllKeys(Keys{ "name": Equal("etcd"), @@ -1318,7 +1484,7 @@ func validateEtcd(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev "UpdateStrategy": MatchFields(IgnoreExtras, Fields{ "Type": Equal(appsv1.RollingUpdateStatefulSetStrategyType), }), - "ServiceName": Equal(fmt.Sprintf("%s-peer", instance.Name)), + "ServiceName": Equal(utils.GetPeerServiceName(instance)), "Replicas": PointTo(Equal(int32(instance.Spec.Replicas))), "Selector": PointTo(MatchFields(IgnoreExtras, Fields{ "MatchLabels": MatchAllKeys(Keys{ @@ -1352,13 +1518,13 @@ func validateEtcd(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev "Containers": MatchAllElements(containerIterator, Elements{ common.Etcd: MatchFields(IgnoreExtras, Fields{ "Ports": ConsistOf([]corev1.ContainerPort{ - corev1.ContainerPort{ + { Name: "server", Protocol: corev1.ProtocolTCP, HostPort: 0, ContainerPort: *instance.Spec.Etcd.ServerPort, }, - corev1.ContainerPort{ + { Name: "client", Protocol: corev1.ProtocolTCP, HostPort: 0, @@ -1382,12 +1548,17 @@ func validateEtcd(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev }), "ReadinessProbe": PointTo(MatchFields(IgnoreExtras, Fields{ "Handler": MatchFields(IgnoreExtras, Fields{ - "HTTPGet": PointTo(MatchFields(IgnoreExtras, Fields{ - "Path": Equal("/healthz"), - "Port": MatchFields(IgnoreExtras, Fields{ - "IntVal": Equal(int32(8080)), + "Exec": PointTo(MatchFields(IgnoreExtras, Fields{ + "Command": MatchAllElements(cmdIterator, Elements{ + "/usr/bin/curl": Equal("/usr/bin/curl"), + "--cert": Equal("--cert"), + "/var/etcd/ssl/client/client/tls.crt": Equal("/var/etcd/ssl/client/client/tls.crt"), + "--key": Equal("--key"), + "/var/etcd/ssl/client/client/tls.key": Equal("/var/etcd/ssl/client/client/tls.key"), + "--cacert": Equal("--cacert"), + "/var/etcd/ssl/client/ca/ca.crt": Equal("/var/etcd/ssl/client/ca/ca.crt"), + readinessProbeUrl: Equal(readinessProbeUrl), }), - "Scheme": Equal(corev1.URISchemeHTTPS), })), }), "InitialDelaySeconds": Equal(int32(15)), @@ -1397,13 +1568,13 @@ func validateEtcd(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev "Handler": MatchFields(IgnoreExtras, Fields{ "Exec": PointTo(MatchFields(IgnoreExtras, Fields{ "Command": MatchAllElements(cmdIterator, Elements{ - "/bin/sh": Equal("/bin/sh"), - "-ec": Equal("-ec"), - "ETCDCTL_API=3": Equal("ETCDCTL_API=3"), - "etcdctl": Equal("etcdctl"), - "--cert=/var/etcd/ssl/client/tls.crt": Equal("--cert=/var/etcd/ssl/client/tls.crt"), - "--key=/var/etcd/ssl/client/tls.key": Equal("--key=/var/etcd/ssl/client/tls.key"), - "--cacert=/var/etcd/ssl/ca/ca.crt": Equal("--cacert=/var/etcd/ssl/ca/ca.crt"), + "/bin/sh": Equal("/bin/sh"), + "-ec": Equal("-ec"), + "ETCDCTL_API=3": Equal("ETCDCTL_API=3"), + "etcdctl": Equal("etcdctl"), + "--cert=/var/etcd/ssl/client/client/tls.crt": Equal("--cert=/var/etcd/ssl/client/client/tls.crt"), + "--key=/var/etcd/ssl/client/client/tls.key": Equal("--key=/var/etcd/ssl/client/client/tls.key"), + "--cacert=/var/etcd/ssl/client/ca/ca.crt": Equal("--cacert=/var/etcd/ssl/client/ca/ca.crt"), fmt.Sprintf("--endpoints=https://%s-local:%d", instance.Name, clientPort): Equal(fmt.Sprintf("--endpoints=https://%s-local:%d", instance.Name, clientPort)), "get": Equal("get"), "foo": Equal("foo"), @@ -1418,43 +1589,47 @@ func validateEtcd(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev "Name": Equal(*instance.Spec.VolumeClaimTemplate), "MountPath": Equal("/var/etcd/data/"), }), - "etcd-config-file": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("etcd-config-file"), - "MountPath": Equal("/var/etcd/config/"), + "client-url-ca-etcd": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("client-url-ca-etcd"), + "MountPath": Equal("/var/etcd/ssl/client/ca"), }), - "ca-etcd": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("ca-etcd"), - "MountPath": Equal("/var/etcd/ssl/ca"), + "client-url-etcd-server-tls": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("client-url-etcd-server-tls"), + "MountPath": Equal("/var/etcd/ssl/client/server"), }), - "etcd-server-tls": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("etcd-server-tls"), - "MountPath": Equal("/var/etcd/ssl/server"), + "client-url-etcd-client-tls": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("client-url-etcd-client-tls"), + "MountPath": Equal("/var/etcd/ssl/client/client"), }), - "etcd-client-tls": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("etcd-client-tls"), - "MountPath": Equal("/var/etcd/ssl/client"), + "peer-url-ca-etcd": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("peer-url-ca-etcd"), + "MountPath": Equal("/var/etcd/ssl/peer/ca"), + }), + "peer-url-etcd-server-tls": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("peer-url-etcd-server-tls"), + "MountPath": Equal("/var/etcd/ssl/peer/server"), }), }), }), backupRestore: MatchFields(IgnoreExtras, Fields{ "Command": MatchAllElements(cmdIterator, Elements{ - "etcdbrctl": Equal("etcdbrctl"), - "server": Equal("server"), - "--cert=/var/etcd/ssl/client/tls.crt": Equal("--cert=/var/etcd/ssl/client/tls.crt"), - "--key=/var/etcd/ssl/client/tls.key": Equal("--key=/var/etcd/ssl/client/tls.key"), - "--cacert=/var/etcd/ssl/ca/ca.crt": Equal("--cacert=/var/etcd/ssl/ca/ca.crt"), - "--server-cert=/var/etcd/ssl/server/tls.crt": Equal("--server-cert=/var/etcd/ssl/server/tls.crt"), - "--server-key=/var/etcd/ssl/server/tls.key": Equal("--server-key=/var/etcd/ssl/server/tls.key"), - "--data-dir=/var/etcd/data/new.etcd": Equal("--data-dir=/var/etcd/data/new.etcd"), - "--insecure-transport=false": Equal("--insecure-transport=false"), - "--insecure-skip-tls-verify=false": Equal("--insecure-skip-tls-verify=false"), - "--snapstore-temp-directory=/var/etcd/data/temp": Equal("--snapstore-temp-directory=/var/etcd/data/temp"), - "--etcd-process-name=etcd": Equal("--etcd-process-name=etcd"), - "--etcd-connection-timeout=5m": Equal("--etcd-connection-timeout=5m"), - "--enable-snapshot-lease-renewal=true": Equal("--enable-snapshot-lease-renewal=true"), - "--enable-member-lease-renewal=true": Equal("--enable-member-lease-renewal=true"), - "--k8s-heartbeat-duration=10s": Equal("--k8s-heartbeat-duration=10s"), + "etcdbrctl": Equal("etcdbrctl"), + "server": Equal("server"), + "--cert=/var/etcd/ssl/client/client/tls.crt": Equal("--cert=/var/etcd/ssl/client/client/tls.crt"), + "--key=/var/etcd/ssl/client/client/tls.key": Equal("--key=/var/etcd/ssl/client/client/tls.key"), + "--cacert=/var/etcd/ssl/client/ca/ca.crt": Equal("--cacert=/var/etcd/ssl/client/ca/ca.crt"), + "--server-cert=/var/etcd/ssl/client/server/tls.crt": Equal("--server-cert=/var/etcd/ssl/client/server/tls.crt"), + "--server-key=/var/etcd/ssl/client/server/tls.key": Equal("--server-key=/var/etcd/ssl/client/server/tls.key"), + "--data-dir=/var/etcd/data/new.etcd": Equal("--data-dir=/var/etcd/data/new.etcd"), + "--insecure-transport=false": Equal("--insecure-transport=false"), + "--insecure-skip-tls-verify=false": Equal("--insecure-skip-tls-verify=false"), + "--snapstore-temp-directory=/var/etcd/data/temp": Equal("--snapstore-temp-directory=/var/etcd/data/temp"), + "--etcd-process-name=etcd": Equal("--etcd-process-name=etcd"), + "--etcd-connection-timeout=5m": Equal("--etcd-connection-timeout=5m"), + "--enable-snapshot-lease-renewal=true": Equal("--enable-snapshot-lease-renewal=true"), + "--enable-member-lease-renewal=true": Equal("--enable-member-lease-renewal=true"), + "--k8s-heartbeat-duration=10s": Equal("--k8s-heartbeat-duration=10s"), fmt.Sprintf("--defragmentation-schedule=%s", *instance.Spec.Etcd.DefragmentationSchedule): Equal(fmt.Sprintf("--defragmentation-schedule=%s", *instance.Spec.Etcd.DefragmentationSchedule)), fmt.Sprintf("--schedule=%s", *instance.Spec.Backup.FullSnapshotSchedule): Equal(fmt.Sprintf("--schedule=%s", *instance.Spec.Backup.FullSnapshotSchedule)), fmt.Sprintf("%s=%s", "--garbage-collection-policy", *instance.Spec.Backup.GarbageCollectionPolicy): Equal(fmt.Sprintf("%s=%s", "--garbage-collection-policy", *instance.Spec.Backup.GarbageCollectionPolicy)), @@ -1479,7 +1654,7 @@ func validateEtcd(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", componentlease.GetFullSnapshotLeaseName(instance)): Equal(fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", componentlease.GetFullSnapshotLeaseName(instance))), }), "Ports": ConsistOf([]corev1.ContainerPort{ - corev1.ContainerPort{ + { Name: "server", Protocol: corev1.ProtocolTCP, HostPort: 0, @@ -1497,18 +1672,6 @@ func validateEtcd(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev "Name": Equal("etcd-config-file"), "MountPath": Equal("/var/etcd/config/"), }), - "ca-etcd": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("ca-etcd"), - "MountPath": Equal("/var/etcd/ssl/ca"), - }), - "etcd-server-tls": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("etcd-server-tls"), - "MountPath": Equal("/var/etcd/ssl/server"), - }), - "etcd-client-tls": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("etcd-client-tls"), - "MountPath": Equal("/var/etcd/ssl/client"), - }), "host-storage": MatchFields(IgnoreExtras, Fields{ "Name": Equal("host-storage"), "MountPath": Equal(*instance.Spec.Backup.Store.Container), @@ -1573,27 +1736,43 @@ func validateEtcd(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev })), }), }), - "etcd-server-tls": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("etcd-server-tls"), + "client-url-etcd-server-tls": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("client-url-etcd-server-tls"), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(instance.Spec.Etcd.TLS.ServerTLSSecretRef.Name), + "SecretName": Equal(instance.Spec.Etcd.ClientUrlTLS.ServerTLSSecretRef.Name), })), }), }), - "etcd-client-tls": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("etcd-client-tls"), + "client-url-etcd-client-tls": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("client-url-etcd-client-tls"), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(instance.Spec.Etcd.TLS.ClientTLSSecretRef.Name), + "SecretName": Equal(instance.Spec.Etcd.ClientUrlTLS.ClientTLSSecretRef.Name), })), }), }), - "ca-etcd": MatchFields(IgnoreExtras, Fields{ - "Name": Equal("ca-etcd"), + "client-url-ca-etcd": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("client-url-ca-etcd"), "VolumeSource": MatchFields(IgnoreExtras, Fields{ "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ - "SecretName": Equal(instance.Spec.Etcd.TLS.TLSCASecretRef.Name), + "SecretName": Equal(instance.Spec.Etcd.ClientUrlTLS.TLSCASecretRef.Name), + })), + }), + }), + "peer-url-etcd-server-tls": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("peer-url-etcd-server-tls"), + "VolumeSource": MatchFields(IgnoreExtras, Fields{ + "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ + "SecretName": Equal(instance.Spec.Etcd.PeerUrlTLS.ServerTLSSecretRef.Name), + })), + }), + }), + "peer-url-ca-etcd": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("peer-url-ca-etcd"), + "VolumeSource": MatchFields(IgnoreExtras, Fields{ + "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ + "SecretName": Equal(instance.Spec.Etcd.PeerUrlTLS.TLSCASecretRef.Name), })), }), }), @@ -1622,7 +1801,7 @@ func validateEtcd(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev })) } -func validateStoreGCP(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev1.ConfigMap, svc *corev1.Service) { +func validateStoreGCP(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev1.ConfigMap, clSvc *corev1.Service, prSvc *corev1.Service) { Expect(*s).To(MatchFields(IgnoreExtras, Fields{ "Spec": MatchFields(IgnoreExtras, Fields{ @@ -1686,7 +1865,7 @@ func validateStoreGCP(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *c } -func validateStoreAzure(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev1.ConfigMap, svc *corev1.Service) { +func validateStoreAzure(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev1.ConfigMap, clSvc *corev1.Service, prSvc *corev1.Service) { Expect(*s).To(MatchFields(IgnoreExtras, Fields{ "Spec": MatchFields(IgnoreExtras, Fields{ "Template": MatchFields(IgnoreExtras, Fields{ @@ -1742,7 +1921,7 @@ func validateStoreAzure(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm })) } -func validateStoreOpenstack(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev1.ConfigMap, svc *corev1.Service) { +func validateStoreOpenstack(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev1.ConfigMap, clSvc *corev1.Service, prSvc *corev1.Service) { Expect(*s).To(MatchFields(IgnoreExtras, Fields{ "Spec": MatchFields(IgnoreExtras, Fields{ "Template": MatchFields(IgnoreExtras, Fields{ @@ -1798,7 +1977,7 @@ func validateStoreOpenstack(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, })) } -func validateStoreAlicloud(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev1.ConfigMap, svc *corev1.Service) { +func validateStoreAlicloud(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev1.ConfigMap, clSvc *corev1.Service, prSvc *corev1.Service) { Expect(*s).To(MatchFields(IgnoreExtras, Fields{ "Spec": MatchFields(IgnoreExtras, Fields{ "Template": MatchFields(IgnoreExtras, Fields{ @@ -1856,7 +2035,7 @@ func validateStoreAlicloud(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, })) } -func validateStoreAWS(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev1.ConfigMap, svc *corev1.Service) { +func validateStoreAWS(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev1.ConfigMap, clSvc *corev1.Service, prSvc *corev1.Service) { Expect(*s).To(MatchFields(IgnoreExtras, Fields{ "Spec": MatchFields(IgnoreExtras, Fields{ "Template": MatchFields(IgnoreExtras, Fields{ @@ -2028,11 +2207,29 @@ func configMapIsCorrectlyReconciled(c client.Client, instance *druidv1alpha1.Etc return nil } -func serviceIsCorrectlyReconciled(c client.Client, instance *druidv1alpha1.Etcd, svc *corev1.Service) error { +func clientServiceIsCorrectlyReconciled(c client.Client, instance *druidv1alpha1.Etcd, svc *corev1.Service) error { + ctx, cancel := context.WithTimeout(context.TODO(), timeout) + defer cancel() + req := types.NamespacedName{ + Name: utils.GetClientServiceName(instance), + Namespace: instance.Namespace, + } + + if err := c.Get(ctx, req, svc); err != nil { + return err + } + + if !checkEtcdOwnerReference(svc.GetOwnerReferences(), instance) { + return fmt.Errorf("ownerReference does not exists") + } + return nil +} + +func peerServiceIsCorrectlyReconciled(c client.Client, instance *druidv1alpha1.Etcd, svc *corev1.Service) error { ctx, cancel := context.WithTimeout(context.TODO(), timeout) defer cancel() req := types.NamespacedName{ - Name: fmt.Sprintf("%s-client", instance.Name), + Name: utils.GetPeerServiceName(instance), Namespace: instance.Namespace, } @@ -2372,22 +2569,52 @@ func getEtcd(name, namespace string, tlsEnabled bool) *druidv1alpha1.Etcd { } if tlsEnabled { - tlsConfig := &druidv1alpha1.TLSConfig{ + clientTlsConfig := &druidv1alpha1.TLSConfig{ + TLSCASecretRef: corev1.SecretReference{ + Name: "client-url-ca-etcd", + }, + ClientTLSSecretRef: corev1.SecretReference{ + Name: "client-url-etcd-client-tls", + }, + ServerTLSSecretRef: corev1.SecretReference{ + Name: "client-url-etcd-server-tls", + }, + } + + peerTlsConfig := &druidv1alpha1.TLSConfig{ + TLSCASecretRef: corev1.SecretReference{ + Name: "peer-url-ca-etcd", + }, + ServerTLSSecretRef: corev1.SecretReference{ + Name: "peer-url-etcd-server-tls", + }, + } + + backupTlsConfig := &druidv1alpha1.TLSConfig{ ClientTLSSecretRef: corev1.SecretReference{ - Name: "etcd-client-tls", + Name: "etcdbr-client-tls", }, ServerTLSSecretRef: corev1.SecretReference{ - Name: "etcd-server-tls", + Name: "etcdbr-server-tls", }, TLSCASecretRef: corev1.SecretReference{ - Name: "ca-etcd", + Name: "ca-etcdbr", }, } - instance.Spec.Etcd.TLS = tlsConfig + + instance.Spec.Etcd.ClientUrlTLS = clientTlsConfig + instance.Spec.Etcd.PeerUrlTLS = peerTlsConfig + instance.Spec.Backup.TLS = backupTlsConfig } return instance } +func getEtcdWithReplicas(name, namespace string, replicas int) *druidv1alpha1.Etcd { + instance := getEtcdWithDefault(name, namespace) + instance.Spec.Replicas = int32(replicas) + return instance +} + func parseQuantity(q string) resource.Quantity { val, _ := resource.ParseQuantity(q) return val @@ -2449,7 +2676,7 @@ func cronJobIsCorrectlyReconciled(c client.Client, instance *druidv1alpha1.Etcd, ctx, cancel := context.WithTimeout(context.TODO(), timeout) defer cancel() req := types.NamespacedName{ - Name: getCronJobName(instance), + Name: utils.GetCronJobName(instance), Namespace: instance.Namespace, } @@ -2462,7 +2689,7 @@ func cronJobIsCorrectlyReconciled(c client.Client, instance *druidv1alpha1.Etcd, func createCronJob(instance *druidv1alpha1.Etcd) *batchv1beta1.CronJob { cj := batchv1beta1.CronJob{ ObjectMeta: metav1.ObjectMeta{ - Name: getCronJobName(instance), + Name: utils.GetCronJobName(instance), Namespace: instance.Namespace, Labels: instance.Labels, }, diff --git a/pkg/component/etcd/configmap/configmap.go b/pkg/component/etcd/configmap/configmap.go new file mode 100644 index 000000000..6a9251f58 --- /dev/null +++ b/pkg/component/etcd/configmap/configmap.go @@ -0,0 +1,242 @@ +// Copyright (c) 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package configmap + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + + "github.com/gardener/gardener/pkg/controllerutils" + gardenercomponent "github.com/gardener/gardener/pkg/operation/botanist/component" + "github.com/gardener/gardener/pkg/utils" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type component struct { + client client.Client + namespace string + + values *Values +} + +// New creates a new configmap deployer instance. +func New(c client.Client, namespace string, values *Values) gardenercomponent.Deployer { + return &component{ + client: c, + namespace: namespace, + values: values, + } +} + +func (c *component) Deploy(ctx context.Context) error { + cm := c.emptyConfigmap(c.values.ConfigMapName) + + if err := c.syncConfigmap(ctx, cm); err != nil { + return err + } + + return nil +} + +func (c *component) syncConfigmap(ctx context.Context, cm *corev1.ConfigMap) error { + etcdConfigYaml := `# Human-readable name for this member. + name: ` + fmt.Sprintf("etcd-%s", c.values.EtcdUID[:6]) + ` + # Path to the data directory. + data-dir: /var/etcd/data/new.etcd` + metricsLevel := druidv1alpha1.Basic + if c.values.Metrics != nil { + metricsLevel = *c.values.Metrics + } + + etcdConfigYaml = etcdConfigYaml + ` + # metrics configuration + metrics: ` + string(metricsLevel) + ` + # Number of committed transactions to trigger a snapshot to disk. + snapshot-count: 75000 + + # Accept etcd V2 client requests + enable-v2: false` + var quota int64 = 8 * 1024 * 1024 * 1024 // 8Gi + if c.values.Quota != nil { + quota = c.values.Quota.Value() + } + + etcdConfigYaml = etcdConfigYaml + ` + # Raise alarms when backend size exceeds the given quota. 0 means use the + # default quota. + quota-backend-bytes: ` + fmt.Sprint(quota) + + enableTLS := (c.values.ClientUrlTLS != nil) + + protocol := "http" + if enableTLS { + protocol = "https" + etcdConfigYaml = etcdConfigYaml + ` + client-transport-security: + # Path to the client server TLS cert file. + cert-file: /var/etcd/ssl/client/server/tls.crt + + # Path to the client server TLS key file. + key-file: /var/etcd/ssl/client/server/tls.key + + # Enable client cert authentication. + client-cert-auth: true + + # Path to the client server TLS trusted CA cert file. + trusted-ca-file: /var/etcd/ssl/client/ca/ca.crt + + # Client TLS using generated certificates + auto-tls: false` + } + + clientPort := strconv.Itoa(int(pointer.Int32Deref(c.values.ClientPort, defaultClientPort))) + + serverPort := strconv.Itoa(int(pointer.Int32Deref(c.values.ServerPort, defaultServerPort))) + + etcdConfigYaml = etcdConfigYaml + ` + # List of comma separated URLs to listen on for client traffic. + listen-client-urls: ` + protocol + `://0.0.0.0:` + clientPort + ` + + # List of this member's client URLs to advertise to the public. + # The URLs needed to be a comma-separated list. + advertise-client-urls: ` + protocol + `@` + c.values.PeerServiceName + `@` + c.values.EtcdNameSpace + `@` + clientPort + + enableTLS = (c.values.PeerUrlTLS != nil) + protocol = "http" + if enableTLS { + protocol = "https" + etcdConfigYaml = etcdConfigYaml + ` + peer-transport-security: + # Path to the peer server TLS cert file. + cert-file: /var/etcd/ssl/peer/server/tls.crt + + # Path to the peer server TLS key file. + key-file: /var/etcd/ssl/peer/server/tls.key + + # Enable peer cert authentication. + client-cert-auth: true + + # Path to the peer server TLS trusted CA cert file. + trusted-ca-file: /var/etcd/ssl/peer/ca/ca.crt + + # Peer TLS using generated certificates + auto-tls: false` + } + + etcdConfigYaml = etcdConfigYaml + ` + # List of comma separated URLs to listen on for peer traffic. + listen-peer-urls: ` + protocol + `://0.0.0.0:` + serverPort + ` + + # List of this member's peer URLs to advertise to the public. + # The URLs needed to be a comma-separated list. + initial-advertise-peer-urls: ` + protocol + `@` + c.values.PeerServiceName + `@` + c.values.EtcdNameSpace + `@` + serverPort + ` + + # Initial cluster token for the etcd cluster during bootstrap. + initial-cluster-token: etcd-cluster + + # Initial cluster state ('new' or 'existing'). + initial-cluster-state: new + + # Initial cluster + initial-cluster: ` + c.values.InitialCluster + ` + + # auto-compaction-mode ("periodic" or "revision").` + + autoCompactionMode := "periodic" + if c.values.AutoCompactionMode != nil { + autoCompactionMode = string(*c.values.AutoCompactionMode) + } + etcdConfigYaml = etcdConfigYaml + ` + auto-compaction-mode: ` + autoCompactionMode + + autoCompactionRetention := "30m" + if c.values.AutoCompactionRetention != nil { + autoCompactionRetention = string(*c.values.AutoCompactionRetention) + } + etcdConfigYaml = etcdConfigYaml + ` + # auto-compaction-retention defines Auto compaction retention length for etcd. + auto-compaction-retention: ` + autoCompactionRetention + + _, err := controllerutils.GetAndCreateOrMergePatch(ctx, c.client, cm, func() error { + cm.ObjectMeta = getObjectMeta(c.values) + cm.OwnerReferences = getOwnerReferences(c.values) + cm.Data = map[string]string{"etcd.conf.yaml": etcdConfigYaml} + return nil + }) + if err != nil { + return err + } + + // save the checksum value of generated etcd config + jsonString, err := json.Marshal(cm.Data) + if err != nil { + return err + } + c.values.ConfigMapChecksum = utils.ComputeSHA256Hex(jsonString) + + return nil + +} + +func (c *component) Destroy(ctx context.Context) error { + configMap := c.emptyConfigmap(c.values.ConfigMapName) + + if err := c.deleteConfigmap(ctx, configMap); err != nil { + return err + } + return nil +} + +func (c *component) deleteConfigmap(ctx context.Context, cm *corev1.ConfigMap) error { + return client.IgnoreNotFound(c.client.Delete(ctx, cm)) +} + +func (c *component) emptyConfigmap(name string) *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: c.namespace, + }, + } +} + +func getObjectMeta(val *Values) metav1.ObjectMeta { + labels := map[string]string{"name": "etcd", "instance": val.EtcdName} + return metav1.ObjectMeta{ + Name: val.ConfigMapName, + Namespace: val.EtcdNameSpace, + Labels: labels, + } +} + +func getOwnerReferences(val *Values) []metav1.OwnerReference { + return []metav1.OwnerReference{ + { + APIVersion: druidv1alpha1.GroupVersion.String(), + Kind: "Etcd", + Name: val.EtcdName, + UID: val.EtcdUID, + Controller: pointer.BoolPtr(true), + BlockOwnerDeletion: pointer.BoolPtr(true), + }, + } +} diff --git a/pkg/component/etcd/configmap/configmap_suite_test.go b/pkg/component/etcd/configmap/configmap_suite_test.go new file mode 100644 index 000000000..471a3c93c --- /dev/null +++ b/pkg/component/etcd/configmap/configmap_suite_test.go @@ -0,0 +1,27 @@ +// Copyright (c) 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package configmap + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestService(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Configmap Component Suite") +} diff --git a/pkg/component/etcd/configmap/configmap_test.go b/pkg/component/etcd/configmap/configmap_test.go new file mode 100644 index 000000000..6c3fb8c33 --- /dev/null +++ b/pkg/component/etcd/configmap/configmap_test.go @@ -0,0 +1,255 @@ +// Copyright (c) 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package configmap_test + +import ( + "context" + "encoding/json" + "fmt" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/pkg/client/kubernetes" + . "github.com/gardener/etcd-druid/pkg/component/etcd/configmap" + "github.com/ghodss/yaml" + + "github.com/gardener/gardener/pkg/operation/botanist/component" + "github.com/gardener/gardener/pkg/utils" + kutil "github.com/gardener/gardener/pkg/utils/kubernetes" + . "github.com/gardener/gardener/pkg/utils/test/matchers" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +const etcdConfig = "etcd.conf.yaml" + +var _ = Describe("Configmap", func() { + var ( + ctx context.Context + cl client.Client + + etcd *druidv1alpha1.Etcd + namespace string + name string + uid types.UID + + metricsLevel druidv1alpha1.MetricsLevel + quota resource.Quantity + clientPort, serverPort int32 + autoCompactionMode druidv1alpha1.CompactionMode + autoCompactionRetention string + labels map[string]string + + cm *corev1.ConfigMap + + values *Values + configmapDeployer component.Deployer + ) + + BeforeEach(func() { + ctx = context.Background() + cl = fakeclient.NewClientBuilder().WithScheme(kubernetes.Scheme).Build() + + name = "configmap" + namespace = "default" + uid = "a9b8c7d6e5f4" + + metricsLevel = druidv1alpha1.Basic + quota = resource.MustParse("8Gi") + clientPort = 2222 + serverPort = 3333 + autoCompactionMode = "periodic" + autoCompactionRetention = "30m" + + labels = map[string]string{ + "foo": "bar", + } + + etcd = &druidv1alpha1.Etcd{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + UID: uid, + }, + Spec: druidv1alpha1.EtcdSpec{ + Selector: metav1.SetAsLabelSelector(labels), + Replicas: 3, + Etcd: druidv1alpha1.EtcdConfig{ + Quota: "a, + Metrics: &metricsLevel, + ClientUrlTLS: &druidv1alpha1.TLSConfig{ + ClientTLSSecretRef: corev1.SecretReference{ + Name: "client-url-etcd-client-tls", + }, + ServerTLSSecretRef: corev1.SecretReference{ + Name: "client-url-etcd-server-cert", + }, + TLSCASecretRef: corev1.SecretReference{ + Name: "client-url-ca-etcd", + }, + }, + PeerUrlTLS: &druidv1alpha1.TLSConfig{ + TLSCASecretRef: corev1.SecretReference{ + Name: "peer-url-ca-etcd", + }, + ServerTLSSecretRef: corev1.SecretReference{ + Name: "peer-url-etcd-server-tls", + }, + }, + ClientPort: &clientPort, + ServerPort: &serverPort, + }, + Common: druidv1alpha1.SharedConfig{ + AutoCompactionMode: &autoCompactionMode, + AutoCompactionRetention: &autoCompactionRetention, + }, + }, + } + + values = GenerateValues(etcd) + + cm = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("etcd-bootstrap-%s", string(values.EtcdUID[:6])), + Namespace: values.EtcdName, + }, + } + + configmapDeployer = New(cl, namespace, values) + }) + + Describe("#Deploy", func() { + Context("when configmap does not exist", func() { + It("should create the configmap successfully", func() { + Expect(configmapDeployer.Deploy(ctx)).To(Succeed()) + + cm := &corev1.ConfigMap{} + + Expect(cl.Get(ctx, kutil.Key(namespace, values.ConfigMapName), cm)).To(Succeed()) + checkConfigmap(cm, values) + + }) + }) + + Context("when configmap exists", func() { + It("should update the configmap successfully", func() { + Expect(cl.Create(ctx, cm)).To(Succeed()) + + Expect(configmapDeployer.Deploy(ctx)).To(Succeed()) + + cm := &corev1.ConfigMap{} + + Expect(cl.Get(ctx, kutil.Key(namespace, values.ConfigMapName), cm)).To(Succeed()) + checkConfigmap(cm, values) + }) + }) + }) + + Describe("#Destroy", func() { + Context("when services do not exist", func() { + It("should destroy successfully", func() { + Expect(configmapDeployer.Destroy(ctx)).To(Succeed()) + Expect(cl.Get(ctx, client.ObjectKeyFromObject(cm), &corev1.ConfigMap{})).To(BeNotFoundError()) + }) + }) + + Context("when services exist", func() { + It("should destroy successfully", func() { + Expect(cl.Create(ctx, cm)).To(Succeed()) + + Expect(configmapDeployer.Destroy(ctx)).To(Succeed()) + + Expect(cl.Get(ctx, kutil.Key(namespace, cm.Name), &corev1.ConfigMap{})).To(BeNotFoundError()) + }) + }) + }) +}) + +func checkConfigmap(cm *corev1.ConfigMap, values *Values) { + checkConfigmapMetadata(&cm.ObjectMeta, values) + + configYML := cm.Data[etcdConfig] + config := map[string]interface{}{} + err := yaml.Unmarshal([]byte(configYML), &config) + Expect(err).NotTo(HaveOccurred()) + + Expect(config).To(MatchKeys(IgnoreExtras, Keys{ + "name": Equal(fmt.Sprintf("etcd-%s", values.EtcdUID[:6])), + "data-dir": Equal("/var/etcd/data/new.etcd"), + "metrics": Equal(string(druidv1alpha1.Basic)), + "snapshot-count": Equal(float64(75000)), + "enable-v2": Equal(false), + "quota-backend-bytes": Equal(float64(values.Quota.Value())), + + "client-transport-security": MatchKeys(IgnoreExtras, Keys{ + "cert-file": Equal("/var/etcd/ssl/client/server/tls.crt"), + "key-file": Equal("/var/etcd/ssl/client/server/tls.key"), + "client-cert-auth": Equal(true), + "trusted-ca-file": Equal("/var/etcd/ssl/client/ca/ca.crt"), + "auto-tls": Equal(false), + }), + "listen-client-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", *values.ClientPort)), + "advertise-client-urls": Equal(fmt.Sprintf("%s@%s@%s@%d", "https", values.PeerServiceName, values.EtcdNameSpace, *values.ClientPort)), + + "peer-transport-security": MatchKeys(IgnoreExtras, Keys{ + "cert-file": Equal("/var/etcd/ssl/peer/server/tls.crt"), + "key-file": Equal("/var/etcd/ssl/peer/server/tls.key"), + "client-cert-auth": Equal(true), + "trusted-ca-file": Equal("/var/etcd/ssl/peer/ca/ca.crt"), + "auto-tls": Equal(false), + }), + "listen-peer-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", *values.ServerPort)), + "initial-advertise-peer-urls": Equal(fmt.Sprintf("%s@%s@%s@%d", "https", values.PeerServiceName, values.EtcdNameSpace, *values.ServerPort)), + + "initial-cluster-token": Equal("etcd-cluster"), + "initial-cluster-state": Equal("new"), + "auto-compaction-mode": Equal(string(*values.AutoCompactionMode)), + "auto-compaction-retention": Equal(*values.AutoCompactionRetention), + })) + + jsonString, err := json.Marshal(cm.Data) + Expect(err).NotTo(HaveOccurred()) + configMapChecksum := utils.ComputeSHA256Hex(jsonString) + + Expect(configMapChecksum).To(Equal(values.ConfigMapChecksum)) +} + +func checkConfigmapMetadata(meta *metav1.ObjectMeta, values *Values) { + Expect(meta.OwnerReferences).To(ConsistOf(Equal(metav1.OwnerReference{ + APIVersion: druidv1alpha1.GroupVersion.String(), + Kind: "Etcd", + Name: values.EtcdName, + UID: values.EtcdUID, + Controller: pointer.BoolPtr(true), + BlockOwnerDeletion: pointer.BoolPtr(true), + }))) + Expect(meta.Labels).To(Equal(configmapLabels(values))) +} + +func configmapLabels(val *Values) map[string]string { + labels := map[string]string{ + "name": "etcd", + "instance": val.EtcdName, + } + + return labels +} diff --git a/pkg/component/etcd/configmap/values.go b/pkg/component/etcd/configmap/values.go new file mode 100644 index 000000000..ce266808c --- /dev/null +++ b/pkg/component/etcd/configmap/values.go @@ -0,0 +1,62 @@ +// Copyright (c) 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package configmap + +import ( + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/types" +) + +const ( + defaultClientPort = 2379 + defaultServerPort = 2380 +) + +type Values struct { + // EtcdName is the name of the etcd resource. + EtcdName string + // EtcdNameSpace is the namespace of etcd resource + EtcdNameSpace string + // EtcdName is the UID of the etcd resource. + EtcdUID types.UID + // Metrics defines the level of detail for exported metrics of etcd, specify 'extensive' to include histogram metrics. + Metrics *druidv1alpha1.MetricsLevel + // Quota defines the etcd DB quota. + Quota *resource.Quantity + // InitialCluster is the initial cluster value to bootstrap ETCD. + InitialCluster string + // ClientUrlTLS hold the TLS configuration details for Client Communication + ClientUrlTLS *druidv1alpha1.TLSConfig + // PeerUrlTLS hold the TLS configuration details for Peer Communication + PeerUrlTLS *druidv1alpha1.TLSConfig + //ClientServiceName is name of the etcd client service + ClientServiceName string + // ClientPort holds the client port + ClientPort *int32 + //PeerServiceName is name of the etcd peer service + PeerServiceName string + // ServerPort holds the peer port + ServerPort *int32 + // AutoCompactionMode defines the auto-compaction-mode: 'periodic' or 'revision'. + AutoCompactionMode *druidv1alpha1.CompactionMode + //AutoCompactionRetention defines the auto-compaction-retention length for etcd as well as for embedded-Etcd of backup-restore sidecar. + AutoCompactionRetention *string + // ConfigMapName is the name of the configmap that holds the ETCD config + ConfigMapName string + // ConfigMapChecksum is the checksum of deployed configmap + ConfigMapChecksum string +} diff --git a/pkg/component/etcd/configmap/values_helper.go b/pkg/component/etcd/configmap/values_helper.go new file mode 100644 index 000000000..f03a500f7 --- /dev/null +++ b/pkg/component/etcd/configmap/values_helper.go @@ -0,0 +1,75 @@ +// Copyright (c) 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package configmap + +import ( + "fmt" + "strconv" + "strings" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/pkg/utils" + "k8s.io/utils/pointer" +) + +// GenerateValues generates `configmap.Values` for the configmap component with the given parameters. +func GenerateValues(etcd *druidv1alpha1.Etcd) *Values { + initialCluster := prepareMultiNodeCluster(etcd) + values := &Values{ + EtcdName: etcd.Name, + EtcdNameSpace: etcd.Namespace, + EtcdUID: etcd.UID, + Metrics: etcd.Spec.Etcd.Metrics, + Quota: etcd.Spec.Etcd.Quota, + InitialCluster: initialCluster, + ClientUrlTLS: etcd.Spec.Etcd.ClientUrlTLS, + PeerUrlTLS: etcd.Spec.Etcd.PeerUrlTLS, + ClientServiceName: utils.GetClientServiceName(etcd), + ClientPort: etcd.Spec.Etcd.ClientPort, + PeerServiceName: utils.GetPeerServiceName(etcd), + ServerPort: etcd.Spec.Etcd.ServerPort, + AutoCompactionMode: etcd.Spec.Common.AutoCompactionMode, + AutoCompactionRetention: etcd.Spec.Common.AutoCompactionRetention, + ConfigMapName: utils.GetConfigmapName(etcd), + } + return values +} + +func prepareMultiNodeCluster(etcd *druidv1alpha1.Etcd) string { + protocol := "http" + if etcd.Spec.Etcd.PeerUrlTLS != nil { + protocol = "https" + } + + statefulsetReplicas := int(etcd.Spec.Replicas) + + // Form the service name and pod name for mutinode cluster with the help of ETCD name + podName := utils.GetOrdinalPodName(etcd, 0) + domaiName := fmt.Sprintf("%s.%s.%s", utils.GetPeerServiceName(etcd), etcd.Namespace, "svc") + serverPort := strconv.Itoa(int(pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, defaultServerPort))) + + initialCluster := fmt.Sprintf("%s=%s://%s.%s:%s", podName, protocol, podName, domaiName, serverPort) + if statefulsetReplicas > 1 { + // form initial cluster + initialCluster = "" + for i := 0; i < statefulsetReplicas; i++ { + podName = utils.GetOrdinalPodName(etcd, i) + initialCluster = initialCluster + fmt.Sprintf("%s=%s://%s.%s:%s,", podName, protocol, podName, domaiName, serverPort) + } + } + + initialCluster = strings.Trim(initialCluster, ",") + return initialCluster +} diff --git a/pkg/component/etcd/lease/lease.go b/pkg/component/etcd/lease/lease.go index 1ab50b278..2fa2dcadc 100644 --- a/pkg/component/etcd/lease/lease.go +++ b/pkg/component/etcd/lease/lease.go @@ -85,10 +85,12 @@ func New(c client.Client, namespace string, values Values) gardenercomponent.Dep } func (c *component) emptyLease(name string) *coordinationv1.Lease { + labels := map[string]string{"name": "etcd", "instance": c.values.EtcdName} return &coordinationv1.Lease{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: c.namespace, + Labels: labels, }, } } diff --git a/pkg/component/etcd/service/values_helper.go b/pkg/component/etcd/service/values_helper.go index 2b31953e8..4fbfa8a91 100644 --- a/pkg/component/etcd/service/values_helper.go +++ b/pkg/component/etcd/service/values_helper.go @@ -15,11 +15,10 @@ package service import ( - "fmt" - "k8s.io/utils/pointer" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/pkg/utils" ) const ( @@ -33,11 +32,11 @@ func GenerateValues(etcd *druidv1alpha1.Etcd) Values { return Values{ BackupPort: pointer.Int32Deref(etcd.Spec.Backup.Port, defaultBackupPort), ClientPort: pointer.Int32Deref(etcd.Spec.Etcd.ClientPort, defaultClientPort), - ClientServiceName: fmt.Sprintf("%s-client", etcd.Name), + ClientServiceName: utils.GetClientServiceName(etcd), EtcdName: etcd.Name, EtcdUID: etcd.UID, Labels: etcd.Spec.Labels, - PeerServiceName: fmt.Sprintf("%s-peer", etcd.Name), + PeerServiceName: utils.GetPeerServiceName(etcd), ServerPort: pointer.Int32Deref(etcd.Spec.Etcd.ServerPort, defaultServerPort), } } diff --git a/pkg/component/etcd/values.go b/pkg/component/etcd/values.go index 2664a40d0..cc95326ba 100644 --- a/pkg/component/etcd/values.go +++ b/pkg/component/etcd/values.go @@ -15,12 +15,14 @@ package etcd import ( + "github.com/gardener/etcd-druid/pkg/component/etcd/configmap" "github.com/gardener/etcd-druid/pkg/component/etcd/lease" "github.com/gardener/etcd-druid/pkg/component/etcd/service" ) // Values contains all values relevant for deploying etcd components. type Values struct { - Lease lease.Values - Service service.Values + ConfigMap *configmap.Values + Service service.Values + Lease lease.Values } diff --git a/pkg/utils/names.go b/pkg/utils/names.go new file mode 100644 index 000000000..8ad48319a --- /dev/null +++ b/pkg/utils/names.go @@ -0,0 +1,56 @@ +// Copyright (c) 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "fmt" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" +) + +// GetPeerServiceName returns the peer service name of ETCD cluster reachable by members within the ETCD cluster +func GetPeerServiceName(etcd *druidv1alpha1.Etcd) string { + return fmt.Sprintf("%s-peer", etcd.Name) +} + +// GetClientServiceName returns the client service name of ETCD cluster reachable by external client +func GetClientServiceName(etcd *druidv1alpha1.Etcd) string { + return fmt.Sprintf("%s-client", etcd.Name) +} + +// GetServiceAccountName returns the service account name +func GetServiceAccountName(etcd *druidv1alpha1.Etcd) string { + return etcd.Name +} + +// GetConfigmapName returns the name of the configmap based on the given `etcd` object. +func GetConfigmapName(etcd *druidv1alpha1.Etcd) string { + return fmt.Sprintf("etcd-bootstrap-%s", string(etcd.UID[:6])) +} + +// GetCronJobName returns the legacy compaction cron job name +func GetCronJobName(etcd *druidv1alpha1.Etcd) string { + return fmt.Sprintf("%s-compact-backup", etcd.Name) +} + +// GetJobName returns the compaction job name +func GetJobName(etcd *druidv1alpha1.Etcd) string { + return fmt.Sprintf("%s-compact-job", string(etcd.UID[:6])) +} + +// GetOrdinalPodName returns the ETCD pod name based on the order +func GetOrdinalPodName(etcd *druidv1alpha1.Etcd, order int) string { + return fmt.Sprintf("%s-%d", etcd.Name, order) +}