From 842e731f231c399289373c3f7cf66b9cc0c2f4d8 Mon Sep 17 00:00:00 2001 From: Abhishek Dasgupta Date: Wed, 16 Jun 2021 23:11:45 +0530 Subject: [PATCH] Updated configmap mountpoints based on ETCD replicas. --- charts/etcd/templates/etcd-configmap.yaml | 3 + charts/etcd/templates/etcd-statefulset.yaml | 4 +- charts/etcd/values.yaml | 2 + .../crd/bases/druid.gardener.cloud_etcds.yaml | 267 ++++++------------ controllers/etcd_controller.go | 61 +++- controllers/etcd_controller_test.go | 190 ++++++++++++- 6 files changed, 323 insertions(+), 204 deletions(-) diff --git a/charts/etcd/templates/etcd-configmap.yaml b/charts/etcd/templates/etcd-configmap.yaml index ce8c0b0c8..1ca976957 100644 --- a/charts/etcd/templates/etcd-configmap.yaml +++ b/charts/etcd/templates/etcd-configmap.yaml @@ -53,6 +53,9 @@ data: # Initial cluster state ('new' or 'existing'). initial-cluster-state: {{ .Values.etcd.initialClusterState }} + # Initial cluster + initial-cluster: {{ .Values.etcd.initialCluster }} + {{- if .Values.sharedConfig }} # auto-compaction-mode ("periodic" or "revision"). {{- if .Values.sharedConfig.autoCompactionMode }} diff --git a/charts/etcd/templates/etcd-statefulset.yaml b/charts/etcd/templates/etcd-statefulset.yaml index 94219df5c..df6dfa274 100644 --- a/charts/etcd/templates/etcd-statefulset.yaml +++ b/charts/etcd/templates/etcd-statefulset.yaml @@ -102,7 +102,7 @@ spec: - name: {{ .Values.volumeClaimTemplateName }} mountPath: /var/etcd/data/ - name: etcd-config-file - mountPath: /var/etcd/config/ + mountPath: {{ .Values.etcdConfigMountPath }} {{- if .Values.etcd.enableTLS }} - name: ca-etcd mountPath: /var/etcd/ssl/ca @@ -321,7 +321,7 @@ spec: - name: {{ .Values.volumeClaimTemplateName }} mountPath: /var/etcd/data - name: etcd-config-file - mountPath: /var/etcd/config/ + mountPath: {{ .Values.etcdConfigMountPath }} {{- if .Values.etcd.enableTLS }} - name: ca-etcd mountPath: /var/etcd/ssl/ca diff --git a/charts/etcd/values.yaml b/charts/etcd/values.yaml index 8cd4b946c..75fdf2e37 100644 --- a/charts/etcd/values.yaml +++ b/charts/etcd/values.yaml @@ -15,6 +15,7 @@ labels: {} etcd: initialClusterToken: initial initialClusterState: new + initialCluster: test enableTLS: false pullPolicy: IfNotPresent metrics: basic @@ -30,6 +31,7 @@ etcd: #username: username #password: password +etcdConfigMountPath: "/var/etcd/config/" backup: port: 8080 pullPolicy: IfNotPresent diff --git a/config/crd/bases/druid.gardener.cloud_etcds.yaml b/config/crd/bases/druid.gardener.cloud_etcds.yaml index ae2a41c9f..6d7a4ccfa 100644 --- a/config/crd/bases/druid.gardener.cloud_etcds.yaml +++ b/config/crd/bases/druid.gardener.cloud_etcds.yaml @@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.5.0 + controller-gen.kubebuilder.io/version: v0.4.1 creationTimestamp: null name: etcds.druid.gardener.cloud spec: @@ -29,14 +29,10 @@ spec: description: Etcd is the Schema for the etcds API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object @@ -48,21 +44,18 @@ spec: type: string type: object backup: - description: BackupSpec defines parametes associated with the full - and delta snapshots of etcd + description: BackupSpec defines parametes associated with the full and delta snapshots of etcd properties: backupCompactionSchedule: description: BackupCompactionSchedule defines the cron standard for compacting the snapstore type: string compression: - description: SnapshotCompression defines the specification for - compression of Snapshots. + description: SnapshotCompression defines the specification for compression of Snapshots. properties: enabled: type: boolean policy: - description: CompressionPolicy defines the type of policy - for compression of snapshots. + description: CompressionPolicy defines the type of policy for compression of snapshots. enum: - gzip - lzw @@ -73,29 +66,23 @@ spec: anyOf: - type: integer - type: string - description: DeltaSnapshotMemoryLimit defines the memory limit - after which delta snapshots will be taken + description: DeltaSnapshotMemoryLimit defines the memory limit after which delta snapshots will be taken pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true deltaSnapshotPeriod: - description: DeltaSnapshotPeriod defines the period after which - delta snapshots will be taken + description: DeltaSnapshotPeriod defines the period after which delta snapshots will be taken type: string enableProfiling: - description: EnableProfiling defines if profiling should be enabled - for the etcd-backup-restore-sidecar + description: EnableProfiling defines if profiling should be enabled for the etcd-backup-restore-sidecar type: boolean fullSnapshotSchedule: - description: FullSnapshotSchedule defines the cron standard schedule - for full snapshots. + description: FullSnapshotSchedule defines the cron standard schedule for full snapshots. type: string garbageCollectionPeriod: - description: GarbageCollectionPeriod defines the period for garbage - collecting old backups + description: GarbageCollectionPeriod defines the period for garbage collecting old backups type: string garbageCollectionPolicy: - description: GarbageCollectionPolicy defines the policy for garbage - collecting old backups + description: GarbageCollectionPolicy defines the policy for garbage collecting old backups enum: - Exponential - LimitBased @@ -104,13 +91,11 @@ spec: description: Image defines the etcd container image and tag type: string port: - description: Port define the port on which etcd-backup-restore - server will exposed. + description: Port define the port on which etcd-backup-restore server will exposed. format: int32 type: integer resources: - description: 'Resources defines the compute Resources required - by backup-restore container. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Resources defines the compute Resources required by backup-restore container. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' properties: limits: additionalProperties: @@ -119,8 +104,7 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute - resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object requests: additionalProperties: @@ -129,19 +113,14 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount of compute - resources required. If Requests is omitted for a container, - it defaults to Limits if that is explicitly specified, otherwise - to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object store: - description: Store defines the specification of object store provider - for storing backups. + description: Store defines the specification of object store provider for storing backups. properties: container: - description: Container is the name of the container the backup - is stored at. + description: Container is the name of the container the backup is stored at. type: string prefix: description: Prefix is the prefix used for the store. @@ -150,16 +129,13 @@ spec: description: Provider is the name of the backup provider. type: string secretRef: - description: SecretRef is the reference to the secret which - used to connect to the backup store. + description: SecretRef is the reference to the secret which used to connect to the backup store. properties: name: - description: Name is unique within a namespace to reference - a secret resource. + description: Name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which - the secret name must be unique. + description: Namespace defines the space within which the secret name must be unique. type: string type: object required: @@ -169,42 +145,33 @@ spec: description: TLSConfig hold the TLS configuration details. properties: clientTLSSecretRef: - description: SecretReference represents a Secret Reference. - It has enough information to retrieve secret in any namespace + description: SecretReference represents a Secret Reference. It has enough information to retrieve secret in any namespace properties: name: - description: Name is unique within a namespace to reference - a secret resource. + description: Name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which - the secret name must be unique. + description: Namespace defines the space within which the secret name must be unique. type: string type: object serverTLSSecretRef: - description: SecretReference represents a Secret Reference. - It has enough information to retrieve secret in any namespace + description: SecretReference represents a Secret Reference. It has enough information to retrieve secret in any namespace properties: name: - description: Name is unique within a namespace to reference - a secret resource. + description: Name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which - the secret name must be unique. + description: Namespace defines the space within which the secret name must be unique. type: string type: object tlsCASecretRef: - description: SecretReference represents a Secret Reference. - It has enough information to retrieve secret in any namespace + description: SecretReference represents a Secret Reference. It has enough information to retrieve secret in any namespace properties: name: - description: Name is unique within a namespace to reference - a secret resource. + description: Name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which - the secret name must be unique. + description: Namespace defines the space within which the secret name must be unique. type: string type: object required: @@ -217,31 +184,26 @@ spec: description: EtcdConfig defines parameters associated etcd deployed properties: authSecretRef: - description: SecretReference represents a Secret Reference. It - has enough information to retrieve secret in any namespace + description: SecretReference represents a Secret Reference. It has enough information to retrieve secret in any namespace properties: name: - description: Name is unique within a namespace to reference - a secret resource. + description: Name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which the - secret name must be unique. + description: Namespace defines the space within which the secret name must be unique. type: string type: object clientPort: format: int32 type: integer defragmentationSchedule: - description: DefragmentationSchedule defines the cron standard - schedule for defragmentation of etcd. + description: DefragmentationSchedule defines the cron standard schedule for defragmentation of etcd. type: string image: description: Image defines the etcd container image and tag type: string metrics: - description: Metrics defines the level of detail for exported - metrics of etcd, specify 'extensive' to include histogram metrics. + description: Metrics defines the level of detail for exported metrics of etcd, specify 'extensive' to include histogram metrics. enum: - basic - extensive @@ -254,8 +216,7 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resources: - description: 'Resources defines the compute Resources required - by etcd container. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Resources defines the compute Resources required by etcd container. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' properties: limits: additionalProperties: @@ -264,8 +225,7 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute - resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object requests: additionalProperties: @@ -274,10 +234,7 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount of compute - resources required. If Requests is omitted for a container, - it defaults to Limits if that is explicitly specified, otherwise - to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object serverPort: @@ -287,42 +244,33 @@ spec: description: TLSConfig hold the TLS configuration details. properties: clientTLSSecretRef: - description: SecretReference represents a Secret Reference. - It has enough information to retrieve secret in any namespace + description: SecretReference represents a Secret Reference. It has enough information to retrieve secret in any namespace properties: name: - description: Name is unique within a namespace to reference - a secret resource. + description: Name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which - the secret name must be unique. + description: Namespace defines the space within which the secret name must be unique. type: string type: object serverTLSSecretRef: - description: SecretReference represents a Secret Reference. - It has enough information to retrieve secret in any namespace + description: SecretReference represents a Secret Reference. It has enough information to retrieve secret in any namespace properties: name: - description: Name is unique within a namespace to reference - a secret resource. + description: Name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which - the secret name must be unique. + description: Namespace defines the space within which the secret name must be unique. type: string type: object tlsCASecretRef: - description: SecretReference represents a Secret Reference. - It has enough information to retrieve secret in any namespace + description: SecretReference represents a Secret Reference. It has enough information to retrieve secret in any namespace properties: name: - description: Name is unique within a namespace to reference - a secret resource. + description: Name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which - the secret name must be unique. + description: Namespace defines the space within which the secret name must be unique. type: string type: object required: @@ -336,39 +284,26 @@ spec: type: string type: object priorityClassName: - description: PriorityClassName is the name of a priority class that - shall be used for the etcd pods. + description: PriorityClassName is the name of a priority class that shall be used for the etcd pods. type: string replicas: type: integer selector: - description: 'selector is a label query over pods that should match - the replica count. It must match the pod template''s labels. More - info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors' + description: 'selector is a label query over pods that should match the replica count. It must match the pod template''s labels. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors' properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. - The requirements are ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: A label selector requirement is a selector that - contains values, a key, and an operator that relates the key - and values. + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies - to. + description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to - a set of values. Valid operators are In, NotIn, Exists - and DoesNotExist. + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the - operator is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. This array is replaced during a strategic - merge patch. + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array @@ -380,29 +315,20 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A single - {key,value} in the matchLabels map is equivalent to an element - of matchExpressions, whose key field is "key", the operator - is "In", and the values array contains only "value". The requirements - are ANDed. + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object sharedConfig: - description: SharedConfig defines parameters shared and used by Etcd - as well as backup-restore sidecar. + description: SharedConfig defines parameters shared and used by Etcd as well as backup-restore sidecar. properties: autoCompactionMode: - description: AutoCompactionMode defines the auto-compaction-mode:'periodic' - mode or 'revision' mode for etcd and embedded-Etcd of backup-restore - sidecar. + description: AutoCompactionMode defines the auto-compaction-mode:'periodic' mode or 'revision' mode for etcd and embedded-Etcd of backup-restore sidecar. enum: - periodic - revision type: string autoCompactionRetention: - description: AutoCompactionRetention defines the auto-compaction-retention - length for etcd as well as for embedded-Etcd of backup-restore - sidecar. + description: AutoCompactionRetention defines the auto-compaction-retention length for etcd as well as for embedded-Etcd of backup-restore sidecar. type: string type: object storageCapacity: @@ -413,12 +339,10 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true storageClass: - description: 'StorageClass defines the name of the StorageClass required - by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + description: 'StorageClass defines the name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' type: string volumeClaimTemplate: - description: VolumeClaimTemplate defines the volume claim template - to be created + description: VolumeClaimTemplate defines the volume claim template to be created type: string required: - backup @@ -435,15 +359,12 @@ spec: format: int32 type: integer conditions: - description: Conditions represents the latest available observations - of an etcd's current state. + description: Conditions represents the latest available observations of an etcd's current state. items: - description: Condition holds the information about the state of - a resource. + description: Condition holds the information about the state of a resource. properties: lastTransitionTime: - description: Last time the condition transitioned from one status - to another. + description: Last time the condition transitioned from one status to another. format: date-time type: string lastUpdateTime: @@ -451,8 +372,7 @@ spec: format: date-time type: string message: - description: A human readable message indicating details about - the transition. + description: A human readable message indicating details about the transition. type: string reason: description: The reason for the condition's last transition. @@ -473,13 +393,11 @@ spec: type: object type: array currentReplicas: - description: CurrentReplicas is the current replica count for the - etcd cluster. + description: CurrentReplicas is the current replica count for the etcd cluster. format: int32 type: integer etcd: - description: CrossVersionObjectReference contains enough information - to let you identify the referred resource. + description: CrossVersionObjectReference contains enough information to let you identify the referred resource. properties: apiVersion: description: API version of the referent @@ -492,32 +410,21 @@ spec: type: string type: object labelSelector: - description: LabelSelector is a label query over pods that should - match the replica count. It must match the pod template's labels. + description: LabelSelector is a label query over pods that should match the replica count. It must match the pod template's labels. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. - The requirements are ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: A label selector requirement is a selector that - contains values, a key, and an operator that relates the key - and values. + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies - to. + description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to - a set of values. Valid operators are In, NotIn, Exists - and DoesNotExist. + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the - operator is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. This array is replaced during a strategic - merge patch. + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array @@ -529,11 +436,7 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A single - {key,value} in the matchLabels map is equivalent to an element - of matchExpressions, whose key field is "key", the operator - is "In", and the values array contains only "value". The requirements - are ANDed. + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object lastError: @@ -542,32 +445,27 @@ spec: members: description: Members represents the members of the etcd cluster items: - description: EtcdMemberStatus holds information about a etcd cluster - membership. + description: EtcdMemberStatus holds information about a etcd cluster membership. properties: id: description: ID is the ID of the etcd member. type: string lastTransitionTime: - description: LastTransitionTime is the last time the condition's - status changed. + description: LastTransitionTime is the last time the condition's status changed. format: date-time type: string lastUpdateTime: - description: LastUpdateTime is the last time this condition - was updated. + description: LastUpdateTime is the last time this condition was updated. format: date-time type: string name: - description: Name is the name of the etcd member. It is the - name of the backing `Pod`. + description: Name is the name of the etcd member. It is the name of the backing `Pod`. type: string reason: description: The reason for the condition's last transition. type: string role: - description: Role is the role in the etcd cluster, either `Member` - or `Learner`. + description: Role is the role in the etcd cluster, either `Member` or `Learner`. type: string status: description: Status of the condition, one of True, False, Unknown. @@ -583,16 +481,14 @@ spec: type: object type: array observedGeneration: - description: ObservedGeneration is the most recent generation observed - for this resource. + description: ObservedGeneration is the most recent generation observed for this resource. format: int64 type: integer ready: description: Ready represents the readiness of the etcd resource. type: boolean readyReplicas: - description: ReadyReplicas is the count of replicas being ready in - the etcd cluster. + description: ReadyReplicas is the count of replicas being ready in the etcd cluster. format: int32 type: integer replicas: @@ -603,8 +499,7 @@ spec: description: ServiceName is the name of the etcd service. type: string updatedReplicas: - description: UpdatedReplicas is the count of updated replicas in the - etcd cluster. + description: UpdatedReplicas is the count of updated replicas in the etcd cluster. format: int32 type: integer type: object diff --git a/controllers/etcd_controller.go b/controllers/etcd_controller.go index e4077faf8..98179c9a4 100644 --- a/controllers/etcd_controller.go +++ b/controllers/etcd_controller.go @@ -428,6 +428,7 @@ func (r *EtcdReconciler) syncServiceSpec(ctx context.Context, logger logr.Logger if err != nil { return nil, err } + return svcCopy, err } @@ -501,7 +502,6 @@ func (r *EtcdReconciler) reconcileConfigMaps(ctx context.Context, logger logr.Lo } // Required Configmap doesn't exist. Create new - cm, err := r.getConfigMapFromEtcd(etcd, renderedChart) if err != nil { return nil, err @@ -570,6 +570,7 @@ func (r *EtcdReconciler) getConfigMapFromEtcd(etcd *druidv1alpha1.Etcd, rendered if err = decoder.Decode(&decoded); err != nil { return nil, err } + return decoded, nil } @@ -582,7 +583,7 @@ const ( noOp operationResult = "none" ) -func (r *EtcdReconciler) reconcileStatefulSet(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd, values map[string]interface{}) (operationResult, *appsv1.StatefulSet, error) { +func (r *EtcdReconciler) reconcileStatefulSet(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd, renderedChart *chartrenderer.RenderedChart) (operationResult, *appsv1.StatefulSet, error) { logger.Info("Reconciling etcd statefulset") // If any adoptions are attempted, we should first recheck for deletion with @@ -641,7 +642,7 @@ func (r *EtcdReconciler) reconcileStatefulSet(ctx context.Context, logger logr.L } // Statefulset is claimed by for this etcd. Just sync the specs - if sts, err = r.syncStatefulSetSpec(ctx, logger, sts, etcd, values); err != nil { + if sts, err = r.syncStatefulSetSpec(ctx, logger, sts, etcd, renderedChart); err != nil { return noOp, nil, err } @@ -670,7 +671,7 @@ func (r *EtcdReconciler) reconcileStatefulSet(ctx context.Context, logger logr.L } // Required statefulset doesn't exist. Create new - sts, err := r.getStatefulSetFromEtcd(etcd, values) + sts, err := r.getStatefulSetFromEtcd(etcd, renderedChart) if err != nil { return noOp, nil, err } @@ -699,8 +700,8 @@ func getContainerMapFromPodTemplateSpec(spec v1.PodSpec) map[string]v1.Container return containers } -func (r *EtcdReconciler) syncStatefulSetSpec(ctx context.Context, logger logr.Logger, ss *appsv1.StatefulSet, etcd *druidv1alpha1.Etcd, values map[string]interface{}) (*appsv1.StatefulSet, error) { - decoded, err := r.getStatefulSetFromEtcd(etcd, values) +func (r *EtcdReconciler) syncStatefulSetSpec(ctx context.Context, logger logr.Logger, ss *appsv1.StatefulSet, etcd *druidv1alpha1.Etcd, renderedChart *chartrenderer.RenderedChart) (*appsv1.StatefulSet, error) { + decoded, err := r.getStatefulSetFromEtcd(etcd, renderedChart) if err != nil { return nil, err } @@ -765,15 +766,11 @@ func (r *EtcdReconciler) recreateStatefulset(ctx context.Context, ss *appsv1.Sta return err } -func (r *EtcdReconciler) getStatefulSetFromEtcd(etcd *druidv1alpha1.Etcd, values map[string]interface{}) (*appsv1.StatefulSet, error) { +func (r *EtcdReconciler) getStatefulSetFromEtcd(etcd *druidv1alpha1.Etcd, renderedChart *chartrenderer.RenderedChart) (*appsv1.StatefulSet, error) { var err error decoded := &appsv1.StatefulSet{} statefulSetPath := getChartPathForStatefulSet() - chartPath := getChartPath() - renderedChart, err := r.chartApplier.Render(chartPath, etcd.Name, etcd.Namespace, values) - if err != nil { - return nil, err - } + if _, ok := renderedChart.Files()[statefulSetPath]; !ok { return nil, fmt.Errorf("missing configmap template file in the charts: %v", statefulSetPath) } @@ -959,7 +956,7 @@ func (r *EtcdReconciler) reconcileEtcd(ctx context.Context, logger logr.Logger, values["cronJobName"] = cj.Name } - op, sts, err := r.reconcileStatefulSet(ctx, logger, etcd, values) + op, sts, err := r.reconcileStatefulSet(ctx, logger, etcd, renderedChart) if err != nil { return noOp, nil, nil, err } @@ -1015,12 +1012,18 @@ func (r *EtcdReconciler) getMapFromEtcd(etcd *druidv1alpha1.Etcd) (map[string]in } var statefulsetReplicas int - if etcd.Spec.Replicas != 0 { - statefulsetReplicas = 1 + // Check if Spec.Replicas is odd or even + if etcd.Spec.Replicas&1 == 1 { + statefulsetReplicas = etcd.Spec.Replicas + } else { + return map[string]interface{}{}, fmt.Errorf("Spec.Replicas should not be even number: %d", etcd.Spec.Replicas) } + etcdConfigMountPath, initialCluster := prepareMultiNodeCluster(etcd) + etcdValues := map[string]interface{}{ "defragmentationSchedule": etcd.Spec.Etcd.DefragmentationSchedule, + "initialCluster": initialCluster, "enableTLS": (etcd.Spec.Etcd.TLS != nil), "pullPolicy": corev1.PullIfNotPresent, // "username": etcd.Spec.Etcd.Username, @@ -1156,6 +1159,7 @@ func (r *EtcdReconciler) getMapFromEtcd(etcd *druidv1alpha1.Etcd) (map[string]in "sharedConfig": sharedConfigValues, "replicas": etcd.Spec.Replicas, "statefulsetReplicas": statefulsetReplicas, + "etcdConfigMountPath": etcdConfigMountPath, "serviceName": fmt.Sprintf("%s-client", etcd.Name), "configMapName": fmt.Sprintf("etcd-bootstrap-%s", string(etcd.UID[:6])), "cronJobName": getCronJobName(etcd), @@ -1202,6 +1206,33 @@ func (r *EtcdReconciler) getMapFromEtcd(etcd *druidv1alpha1.Etcd) (map[string]in return values, nil } +func prepareMultiNodeCluster(etcd *druidv1alpha1.Etcd) (string, string) { + protocol := "http" + if etcd.Spec.Etcd.TLS != nil { + protocol = "https" + } + + statefulsetReplicas := etcd.Spec.Replicas + + etcdConfigMountPath := "/var/etcd/config/" + // Form the service name and pod name for mutinode cluster with the help of ETCD name + svcName := etcd.Name + podName := fmt.Sprintf("%s-%d", etcd.Name, 0) + initialCluster := fmt.Sprintf("%s=%s://%s.%s:2380", podName, protocol, podName, svcName) + if statefulsetReplicas > 1 { + etcdConfigMountPath = "/var/etcd/template/config/" + // form initial cluster + initialCluster = "" + for i := 0; i < statefulsetReplicas; i++ { + podName = fmt.Sprintf("%s-%d", etcd.Name, i) + initialCluster = initialCluster + fmt.Sprintf("%s=%s://%s.%s:2380,", podName, protocol, podName, svcName) + } + } + + initialCluster = strings.Trim(initialCluster, ",") + return etcdConfigMountPath, initialCluster +} + func (r *EtcdReconciler) addFinalizersToDependantSecrets(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) error { secrets := []*corev1.SecretReference{} if etcd.Spec.Etcd.TLS != nil { diff --git a/controllers/etcd_controller_test.go b/controllers/etcd_controller_test.go index 073ad8d32..e60a35e9e 100644 --- a/controllers/etcd_controller_test.go +++ b/controllers/etcd_controller_test.go @@ -18,8 +18,10 @@ import ( "context" "fmt" "os" + "strings" "time" + kutil "github.com/gardener/gardener/pkg/utils/kubernetes" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "github.com/gardener/gardener/pkg/utils/imagevector" @@ -44,6 +46,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" @@ -773,6 +776,160 @@ var _ = Describe("Druid", func() { }) +var _ = Describe("Multinode ETCD", func() { + //Reconciliation of new etcd resource deployment without any existing statefulsets. + Context("when adding etcd resources", func() { + var ( + err error + instance *druidv1alpha1.Etcd + sts *appsv1.StatefulSet + svc *corev1.Service + c client.Client + ) + + BeforeEach(func() { + instance = getEtcd("foo34", "default", false) + c = mgr.GetClient() + ns := corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.Namespace, + }, + } + _, err = controllerutil.CreateOrUpdate(context.TODO(), c, &ns, func() error { return nil }) + Expect(err).To(Not(HaveOccurred())) + + storeSecret := instance.Spec.Backup.Store.SecretRef.Name + errors := createSecrets(c, instance.Namespace, storeSecret) + Expect(len(errors)).Should(BeZero()) + }) + It("no statefulsets are created when ETCD replicas are even number", func() { + // Update replicas in ETCD resource with 0 + instance.Spec.Replicas = 0 + Expect(c.Create(context.TODO(), instance)).To(Succeed()) + + Eventually(func() error { + return c.Get(context.TODO(), types.NamespacedName{ + Name: instance.Name, + Namespace: instance.Namespace, + }, instance) + }, timeout, pollingInterval).Should(BeNil()) + + sts = &appsv1.StatefulSet{} + // No StatefulSet has been created by controller as even number of replicas are not allowed + Eventually(func() error { + return c.Get(context.TODO(), types.NamespacedName{ + Name: instance.Name, + Namespace: instance.Namespace, + }, sts) + }, timeout, pollingInterval).Should(matchers.BeNotFoundError()) + + // Update replicas in ETCD resource with even number + Expect(kutil.TryUpdate(context.TODO(), retry.DefaultBackoff, c, instance, func() error { + instance.Spec.Replicas = 4 + return nil + })).To(Succeed()) + + Eventually(func() error { + return c.Get(context.TODO(), types.NamespacedName{ + Name: instance.Name, + Namespace: instance.Namespace, + }, instance) + }, timeout, pollingInterval).Should(BeNil()) + + // No StatefulSet has been created by controller as even number of replicas are not allowed + sts = &appsv1.StatefulSet{} + Eventually(func() error { + return c.Get(context.TODO(), types.NamespacedName{ + Name: instance.Name, + Namespace: instance.Namespace, + }, sts) + }, timeout, pollingInterval).Should(matchers.BeNotFoundError()) + + // No Service has been created by controller as even number of replicas are not allowed + svc = &corev1.Service{} + Expect(c.Get(context.TODO(), types.NamespacedName{ + Name: fmt.Sprintf("%s-client", instance.Name), + Namespace: instance.Namespace, + }, svc)).Should(matchers.BeNotFoundError()) + svc = nil + }) + AfterEach(func() { + // Delete `etcd` instance + Expect(c.Delete(context.TODO(), instance)).To(Succeed()) + Eventually(func() error { + return c.Get(context.TODO(), client.ObjectKeyFromObject(instance), &druidv1alpha1.Etcd{}) + }, timeout, pollingInterval).Should(matchers.BeNotFoundError()) + // Delete service manually because garbage collection is not available in `envtest` + if svc != nil { + Expect(c.Delete(context.TODO(), svc)).To(Succeed()) + Eventually(func() error { + return c.Get(context.TODO(), client.ObjectKeyFromObject(svc), &corev1.Service{}) + }, timeout, pollingInterval).Should(matchers.BeNotFoundError()) + } + }) + }) + DescribeTable("configmaps are mounted properly when ETCD replicas are odd number", func(name string, replicas int, getEtcdWithReplicas func(string, string, int) *druidv1alpha1.Etcd) { + var err error + var instance *druidv1alpha1.Etcd + var c client.Client + var sts *appsv1.StatefulSet + var cm *corev1.ConfigMap + var svc *corev1.Service + + instance = getEtcdWithReplicas(name, "default", replicas) + c = mgr.GetClient() + ns := corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.Namespace, + }, + } + + _, err = controllerutil.CreateOrUpdate(context.TODO(), c, &ns, func() error { return nil }) + Expect(err).To(Not(HaveOccurred())) + + if instance.Spec.Backup.Store != nil && instance.Spec.Backup.Store.SecretRef != nil { + storeSecret := instance.Spec.Backup.Store.SecretRef.Name + errors := createSecrets(c, instance.Namespace, storeSecret) + Expect(len(errors)).Should(BeZero()) + } + err = c.Create(context.TODO(), instance) + Expect(err).NotTo(HaveOccurred()) + sts = &appsv1.StatefulSet{} + Eventually(func() error { return statefulsetIsCorrectlyReconciled(c, instance, sts) }, timeout, pollingInterval).Should(BeNil()) + cm = &corev1.ConfigMap{} + Eventually(func() error { return configMapIsCorrectlyReconciled(c, instance, cm) }, timeout, pollingInterval).Should(BeNil()) + svc = &corev1.Service{} + Eventually(func() error { return serviceIsCorrectlyReconciled(c, instance, svc) }, timeout, pollingInterval).Should(BeNil()) + + // Validate statefulset + Expect(*sts.Spec.Replicas).To(Equal(int32(instance.Spec.Replicas))) + + if instance.Spec.Replicas == 1 { + // configmap volume in statefulset container should be mounted in /var/etcd/config/ + Expect(sts.Spec.Template.Spec.Containers[0].VolumeMounts[1].MountPath).To(Equal("/var/etcd/config/")) + Expect(sts.Spec.Template.Spec.Containers[1].VolumeMounts[1].MountPath).To(Equal("/var/etcd/config/")) + } + + if instance.Spec.Replicas > 1 { + Expect(sts.Spec.Template.Spec.Containers[0].VolumeMounts[1].MountPath).To(Equal("/var/etcd/template/config/")) + Expect(sts.Spec.Template.Spec.Containers[1].VolumeMounts[1].MountPath).To(Equal("/var/etcd/template/config/")) + } + + if instance.Spec.Replicas == 1 { + matcher := "initial-cluster: foo07-0=http://foo07-0.foo07:2380" + Expect(strings.Contains(cm.Data["etcd.conf.yaml"], matcher)).To(BeTrue()) + } + + if instance.Spec.Replicas > 1 { + matcher := "initial-cluster: foo08-0=http://foo08-0.foo08:2380,foo08-1=http://foo08-1.foo08:2380,foo08-2=http://foo08-2.foo08:2380" + Expect(strings.Contains(cm.Data["etcd.conf.yaml"], matcher)).To(BeTrue()) + } + }, + Entry("verify configmap mount path and etcd.conf.yaml when replica is 1 ", "foo07", 1, getEtcdWithReplicas), + Entry("verify configmap mount path and etcd.conf.yaml when replica is 3 ", "foo08", 3, getEtcdWithReplicas), + ) +}) + func podDeleted(c client.Client, etcd *druidv1alpha1.Etcd) error { ctx, cancel := context.WithTimeout(context.TODO(), timeout) defer cancel() @@ -1845,7 +2002,7 @@ func validateEtcd(s *appsv1.StatefulSet, cm *corev1.ConfigMap, svc *corev1.Servi fmt.Sprintf("%s=%s", "--auto-compaction-retention", *instance.Spec.Common.AutoCompactionRetention): Equal(fmt.Sprintf("%s=%s", "--auto-compaction-retention", autoCompactionRetention)), }), "Ports": ConsistOf([]corev1.ContainerPort{ - corev1.ContainerPort{ + { Name: "server", Protocol: corev1.ProtocolTCP, HostPort: 0, @@ -2677,6 +2834,37 @@ func getEtcdWithDefault(name, namespace string) *druidv1alpha1.Etcd { return instance } +func getEtcdWithReplicas(name, namespace string, replicas int) *druidv1alpha1.Etcd { + instance := &druidv1alpha1.Etcd{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: druidv1alpha1.EtcdSpec{ + Annotations: map[string]string{ + "app": "etcd-statefulset", + "role": "test", + "instance": name, + }, + Labels: map[string]string{ + "name": "etcd", + "instance": name, + }, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "name": "etcd", + "instance": name, + }, + }, + Replicas: replicas, + Backup: druidv1alpha1.BackupSpec{}, + Etcd: druidv1alpha1.EtcdConfig{}, + Common: druidv1alpha1.SharedConfig{}, + }, + } + return instance +} + func getEtcd(name, namespace string, tlsEnabled bool) *druidv1alpha1.Etcd { instance := &druidv1alpha1.Etcd{