diff --git a/charts/etcd/templates/etcd-statefulset.yaml b/charts/etcd/templates/etcd-statefulset.yaml index 94219df5c..2412192b6 100644 --- a/charts/etcd/templates/etcd-statefulset.yaml +++ b/charts/etcd/templates/etcd-statefulset.yaml @@ -102,7 +102,7 @@ spec: - name: {{ .Values.volumeClaimTemplateName }} mountPath: /var/etcd/data/ - name: etcd-config-file - mountPath: /var/etcd/config/ + mountPath: {{ .Values.etcdConfigMountPath }} {{- if .Values.etcd.enableTLS }} - name: ca-etcd mountPath: /var/etcd/ssl/ca @@ -193,6 +193,8 @@ spec: resources: {{ toYaml .Values.backup.resources | indent 10 }} env: + - name: ETCD_INITIAL_CLUSTER + value: {{ .Values.initialCluster }} - name: STORAGE_CONTAINER value: {{ .Values.store.storageContainer }} {{- if eq .Values.store.storageProvider "S3" }} @@ -321,7 +323,7 @@ spec: - name: {{ .Values.volumeClaimTemplateName }} mountPath: /var/etcd/data - name: etcd-config-file - mountPath: /var/etcd/config/ + mountPath: {{ .Values.etcdConfigMountPath }} {{- if .Values.etcd.enableTLS }} - name: ca-etcd mountPath: /var/etcd/ssl/ca diff --git a/charts/etcd/values.yaml b/charts/etcd/values.yaml index 8cd4b946c..6c490f698 100644 --- a/charts/etcd/values.yaml +++ b/charts/etcd/values.yaml @@ -3,6 +3,8 @@ uid: uuid-of-etcd-resource serviceName: test configMapName: test +initialCluster: test + replicas: 1 #priorityClassName: foo @@ -30,6 +32,7 @@ etcd: #username: username #password: password +etcdConfigMountPath: "/var/etcd/config/" backup: port: 8080 pullPolicy: IfNotPresent diff --git a/config/crd/bases/druid.gardener.cloud_etcds.yaml b/config/crd/bases/druid.gardener.cloud_etcds.yaml index d23880ef8..13c118940 100644 --- a/config/crd/bases/druid.gardener.cloud_etcds.yaml +++ b/config/crd/bases/druid.gardener.cloud_etcds.yaml @@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.5.0 + controller-gen.kubebuilder.io/version: v0.4.1 creationTimestamp: null name: etcds.druid.gardener.cloud spec: @@ -29,14 +29,10 @@ spec: description: Etcd is the Schema for the etcds API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object @@ -48,18 +44,15 @@ spec: type: string type: object backup: - description: BackupSpec defines parametes associated with the full - and delta snapshots of etcd + description: BackupSpec defines parametes associated with the full and delta snapshots of etcd properties: compression: - description: SnapshotCompression defines the specification for - compression of Snapshots. + description: SnapshotCompression defines the specification for compression of Snapshots. properties: enabled: type: boolean policy: - description: CompressionPolicy defines the type of policy - for compression of snapshots. + description: CompressionPolicy defines the type of policy for compression of snapshots. enum: - gzip - lzw @@ -70,29 +63,23 @@ spec: anyOf: - type: integer - type: string - description: DeltaSnapshotMemoryLimit defines the memory limit - after which delta snapshots will be taken + description: DeltaSnapshotMemoryLimit defines the memory limit after which delta snapshots will be taken pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true deltaSnapshotPeriod: - description: DeltaSnapshotPeriod defines the period after which - delta snapshots will be taken + description: DeltaSnapshotPeriod defines the period after which delta snapshots will be taken type: string enableProfiling: - description: EnableProfiling defines if profiling should be enabled - for the etcd-backup-restore-sidecar + description: EnableProfiling defines if profiling should be enabled for the etcd-backup-restore-sidecar type: boolean fullSnapshotSchedule: - description: FullSnapshotSchedule defines the cron standard schedule - for full snapshots. + description: FullSnapshotSchedule defines the cron standard schedule for full snapshots. type: string garbageCollectionPeriod: - description: GarbageCollectionPeriod defines the period for garbage - collecting old backups + description: GarbageCollectionPeriod defines the period for garbage collecting old backups type: string garbageCollectionPolicy: - description: GarbageCollectionPolicy defines the policy for garbage - collecting old backups + description: GarbageCollectionPolicy defines the policy for garbage collecting old backups enum: - Exponential - LimitBased @@ -101,13 +88,11 @@ spec: description: Image defines the etcd container image and tag type: string port: - description: Port define the port on which etcd-backup-restore - server will exposed. + description: Port define the port on which etcd-backup-restore server will exposed. format: int32 type: integer resources: - description: 'Resources defines the compute Resources required - by backup-restore container. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Resources defines the compute Resources required by backup-restore container. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' properties: limits: additionalProperties: @@ -116,8 +101,7 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute - resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object requests: additionalProperties: @@ -126,19 +110,14 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount of compute - resources required. If Requests is omitted for a container, - it defaults to Limits if that is explicitly specified, otherwise - to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object store: - description: Store defines the specification of object store provider - for storing backups. + description: Store defines the specification of object store provider for storing backups. properties: container: - description: Container is the name of the container the backup - is stored at. + description: Container is the name of the container the backup is stored at. type: string prefix: description: Prefix is the prefix used for the store. @@ -147,16 +126,13 @@ spec: description: Provider is the name of the backup provider. type: string secretRef: - description: SecretRef is the reference to the secret which - used to connect to the backup store. + description: SecretRef is the reference to the secret which used to connect to the backup store. properties: name: - description: Name is unique within a namespace to reference - a secret resource. + description: Name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which - the secret name must be unique. + description: Namespace defines the space within which the secret name must be unique. type: string type: object required: @@ -166,42 +142,33 @@ spec: description: TLSConfig hold the TLS configuration details. properties: clientTLSSecretRef: - description: SecretReference represents a Secret Reference. - It has enough information to retrieve secret in any namespace + description: SecretReference represents a Secret Reference. It has enough information to retrieve secret in any namespace properties: name: - description: Name is unique within a namespace to reference - a secret resource. + description: Name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which - the secret name must be unique. + description: Namespace defines the space within which the secret name must be unique. type: string type: object serverTLSSecretRef: - description: SecretReference represents a Secret Reference. - It has enough information to retrieve secret in any namespace + description: SecretReference represents a Secret Reference. It has enough information to retrieve secret in any namespace properties: name: - description: Name is unique within a namespace to reference - a secret resource. + description: Name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which - the secret name must be unique. + description: Namespace defines the space within which the secret name must be unique. type: string type: object tlsCASecretRef: - description: SecretReference represents a Secret Reference. - It has enough information to retrieve secret in any namespace + description: SecretReference represents a Secret Reference. It has enough information to retrieve secret in any namespace properties: name: - description: Name is unique within a namespace to reference - a secret resource. + description: Name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which - the secret name must be unique. + description: Namespace defines the space within which the secret name must be unique. type: string type: object required: @@ -214,31 +181,26 @@ spec: description: EtcdConfig defines parameters associated etcd deployed properties: authSecretRef: - description: SecretReference represents a Secret Reference. It - has enough information to retrieve secret in any namespace + description: SecretReference represents a Secret Reference. It has enough information to retrieve secret in any namespace properties: name: - description: Name is unique within a namespace to reference - a secret resource. + description: Name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which the - secret name must be unique. + description: Namespace defines the space within which the secret name must be unique. type: string type: object clientPort: format: int32 type: integer defragmentationSchedule: - description: DefragmentationSchedule defines the cron standard - schedule for defragmentation of etcd. + description: DefragmentationSchedule defines the cron standard schedule for defragmentation of etcd. type: string image: description: Image defines the etcd container image and tag type: string metrics: - description: Metrics defines the level of detail for exported - metrics of etcd, specify 'extensive' to include histogram metrics. + description: Metrics defines the level of detail for exported metrics of etcd, specify 'extensive' to include histogram metrics. enum: - basic - extensive @@ -251,8 +213,7 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resources: - description: 'Resources defines the compute Resources required - by etcd container. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Resources defines the compute Resources required by etcd container. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' properties: limits: additionalProperties: @@ -261,8 +222,7 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute - resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object requests: additionalProperties: @@ -271,10 +231,7 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount of compute - resources required. If Requests is omitted for a container, - it defaults to Limits if that is explicitly specified, otherwise - to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object serverPort: @@ -284,42 +241,33 @@ spec: description: TLSConfig hold the TLS configuration details. properties: clientTLSSecretRef: - description: SecretReference represents a Secret Reference. - It has enough information to retrieve secret in any namespace + description: SecretReference represents a Secret Reference. It has enough information to retrieve secret in any namespace properties: name: - description: Name is unique within a namespace to reference - a secret resource. + description: Name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which - the secret name must be unique. + description: Namespace defines the space within which the secret name must be unique. type: string type: object serverTLSSecretRef: - description: SecretReference represents a Secret Reference. - It has enough information to retrieve secret in any namespace + description: SecretReference represents a Secret Reference. It has enough information to retrieve secret in any namespace properties: name: - description: Name is unique within a namespace to reference - a secret resource. + description: Name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which - the secret name must be unique. + description: Namespace defines the space within which the secret name must be unique. type: string type: object tlsCASecretRef: - description: SecretReference represents a Secret Reference. - It has enough information to retrieve secret in any namespace + description: SecretReference represents a Secret Reference. It has enough information to retrieve secret in any namespace properties: name: - description: Name is unique within a namespace to reference - a secret resource. + description: Name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which - the secret name must be unique. + description: Namespace defines the space within which the secret name must be unique. type: string type: object required: @@ -333,39 +281,26 @@ spec: type: string type: object priorityClassName: - description: PriorityClassName is the name of a priority class that - shall be used for the etcd pods. + description: PriorityClassName is the name of a priority class that shall be used for the etcd pods. type: string replicas: type: integer selector: - description: 'selector is a label query over pods that should match - the replica count. It must match the pod template''s labels. More - info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors' + description: 'selector is a label query over pods that should match the replica count. It must match the pod template''s labels. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors' properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. - The requirements are ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: A label selector requirement is a selector that - contains values, a key, and an operator that relates the key - and values. + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies - to. + description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to - a set of values. Valid operators are In, NotIn, Exists - and DoesNotExist. + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the - operator is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. This array is replaced during a strategic - merge patch. + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array @@ -377,29 +312,20 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A single - {key,value} in the matchLabels map is equivalent to an element - of matchExpressions, whose key field is "key", the operator - is "In", and the values array contains only "value". The requirements - are ANDed. + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object sharedConfig: - description: SharedConfig defines parameters shared and used by Etcd - as well as backup-restore sidecar. + description: SharedConfig defines parameters shared and used by Etcd as well as backup-restore sidecar. properties: autoCompactionMode: - description: AutoCompactionMode defines the auto-compaction-mode:'periodic' - mode or 'revision' mode for etcd and embedded-Etcd of backup-restore - sidecar. + description: AutoCompactionMode defines the auto-compaction-mode:'periodic' mode or 'revision' mode for etcd and embedded-Etcd of backup-restore sidecar. enum: - periodic - revision type: string autoCompactionRetention: - description: AutoCompactionRetention defines the auto-compaction-retention - length for etcd as well as for embedded-Etcd of backup-restore - sidecar. + description: AutoCompactionRetention defines the auto-compaction-retention length for etcd as well as for embedded-Etcd of backup-restore sidecar. type: string type: object storageCapacity: @@ -410,12 +336,10 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true storageClass: - description: 'StorageClass defines the name of the StorageClass required - by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + description: 'StorageClass defines the name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' type: string volumeClaimTemplate: - description: VolumeClaimTemplate defines the volume claim template - to be created + description: VolumeClaimTemplate defines the volume claim template to be created type: string required: - backup @@ -432,15 +356,12 @@ spec: format: int32 type: integer conditions: - description: Conditions represents the latest available observations - of an etcd's current state. + description: Conditions represents the latest available observations of an etcd's current state. items: - description: Condition holds the information about the state of - a resource. + description: Condition holds the information about the state of a resource. properties: lastTransitionTime: - description: Last time the condition transitioned from one status - to another. + description: Last time the condition transitioned from one status to another. format: date-time type: string lastUpdateTime: @@ -448,8 +369,7 @@ spec: format: date-time type: string message: - description: A human readable message indicating details about - the transition. + description: A human readable message indicating details about the transition. type: string reason: description: The reason for the condition's last transition. @@ -470,13 +390,11 @@ spec: type: object type: array currentReplicas: - description: CurrentReplicas is the current replica count for the - etcd cluster. + description: CurrentReplicas is the current replica count for the etcd cluster. format: int32 type: integer etcd: - description: CrossVersionObjectReference contains enough information - to let you identify the referred resource. + description: CrossVersionObjectReference contains enough information to let you identify the referred resource. properties: apiVersion: description: API version of the referent @@ -489,32 +407,21 @@ spec: type: string type: object labelSelector: - description: LabelSelector is a label query over pods that should - match the replica count. It must match the pod template's labels. + description: LabelSelector is a label query over pods that should match the replica count. It must match the pod template's labels. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. - The requirements are ANDed. + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: A label selector requirement is a selector that - contains values, a key, and an operator that relates the key - and values. + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies - to. + description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to - a set of values. Valid operators are In, NotIn, Exists - and DoesNotExist. + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the - operator is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. This array is replaced during a strategic - merge patch. + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array @@ -526,11 +433,7 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A single - {key,value} in the matchLabels map is equivalent to an element - of matchExpressions, whose key field is "key", the operator - is "In", and the values array contains only "value". The requirements - are ANDed. + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object lastError: @@ -539,32 +442,27 @@ spec: members: description: Members represents the members of the etcd cluster items: - description: EtcdMemberStatus holds information about a etcd cluster - membership. + description: EtcdMemberStatus holds information about a etcd cluster membership. properties: id: description: ID is the ID of the etcd member. type: string lastTransitionTime: - description: LastTransitionTime is the last time the condition's - status changed. + description: LastTransitionTime is the last time the condition's status changed. format: date-time type: string lastUpdateTime: - description: LastUpdateTime is the last time this condition - was updated. + description: LastUpdateTime is the last time this condition was updated. format: date-time type: string name: - description: Name is the name of the etcd member. It is the - name of the backing `Pod`. + description: Name is the name of the etcd member. It is the name of the backing `Pod`. type: string reason: description: The reason for the condition's last transition. type: string role: - description: Role is the role in the etcd cluster, either `Member` - or `Learner`. + description: Role is the role in the etcd cluster, either `Member` or `Learner`. type: string status: description: Status of the condition, one of True, False, Unknown. @@ -580,16 +478,14 @@ spec: type: object type: array observedGeneration: - description: ObservedGeneration is the most recent generation observed - for this resource. + description: ObservedGeneration is the most recent generation observed for this resource. format: int64 type: integer ready: description: Ready represents the readiness of the etcd resource. type: boolean readyReplicas: - description: ReadyReplicas is the count of replicas being ready in - the etcd cluster. + description: ReadyReplicas is the count of replicas being ready in the etcd cluster. format: int32 type: integer replicas: @@ -600,8 +496,7 @@ spec: description: ServiceName is the name of the etcd service. type: string updatedReplicas: - description: UpdatedReplicas is the count of updated replicas in the - etcd cluster. + description: UpdatedReplicas is the count of updated replicas in the etcd cluster. format: int32 type: integer type: object diff --git a/controllers/etcd_controller.go b/controllers/etcd_controller.go index 16f6ebeed..42babcbfd 100644 --- a/controllers/etcd_controller.go +++ b/controllers/etcd_controller.go @@ -365,6 +365,10 @@ func (r *EtcdReconciler) reconcileServices(ctx context.Context, logger logr.Logg return nil, err } + if etcd.Spec.Replicas > 1 { + svc.Spec.ClusterIP = "None" + } + err = r.Create(ctx, svc) // Ignore the precondition violated error, this service is already updated @@ -412,6 +416,10 @@ func (r *EtcdReconciler) syncServiceSpec(ctx context.Context, logger logr.Logger if err != nil { return nil, err } + + if etcd.Spec.Replicas > 1 { + svcCopy.Spec.ClusterIP = "None" + } return svcCopy, err } @@ -431,7 +439,7 @@ func (r *EtcdReconciler) getServiceFromEtcd(etcd *druidv1alpha1.Etcd, renderedCh return decoded, nil } -func (r *EtcdReconciler) reconcileConfigMaps(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd, renderedChart *chartrenderer.RenderedChart) (*corev1.ConfigMap, error) { +func (r *EtcdReconciler) reconcileConfigMaps(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd, renderedChart *chartrenderer.RenderedChart, values map[string]interface{}) (*corev1.ConfigMap, error) { logger.Info("Reconciling etcd configmap") selector, err := metav1.LabelSelectorAsSelector(etcd.Spec.Selector) @@ -477,7 +485,7 @@ func (r *EtcdReconciler) reconcileConfigMaps(ctx context.Context, logger logr.Lo } // ConfigMap is claimed by for this etcd. Just sync the data - if cm, err = r.syncConfigMapData(ctx, logger, cm, etcd, renderedChart); err != nil { + if cm, err = r.syncConfigMapData(ctx, logger, cm, etcd, renderedChart, values); err != nil { return nil, err } @@ -485,8 +493,7 @@ func (r *EtcdReconciler) reconcileConfigMaps(ctx context.Context, logger logr.Lo } // Required Configmap doesn't exist. Create new - - cm, err := r.getConfigMapFromEtcd(etcd, renderedChart) + cm, err := r.getConfigMapFromEtcd(etcd, renderedChart, values) if err != nil { return nil, err } @@ -511,8 +518,8 @@ func (r *EtcdReconciler) reconcileConfigMaps(ctx context.Context, logger logr.Lo return cm.DeepCopy(), err } -func (r *EtcdReconciler) syncConfigMapData(ctx context.Context, logger logr.Logger, cm *corev1.ConfigMap, etcd *druidv1alpha1.Etcd, renderedChart *chartrenderer.RenderedChart) (*corev1.ConfigMap, error) { - decoded, err := r.getConfigMapFromEtcd(etcd, renderedChart) +func (r *EtcdReconciler) syncConfigMapData(ctx context.Context, logger logr.Logger, cm *corev1.ConfigMap, etcd *druidv1alpha1.Etcd, renderedChart *chartrenderer.RenderedChart, values map[string]interface{}) (*corev1.ConfigMap, error) { + decoded, err := r.getConfigMapFromEtcd(etcd, renderedChart, values) if err != nil { return nil, err } @@ -539,7 +546,7 @@ func (r *EtcdReconciler) syncConfigMapData(ctx context.Context, logger logr.Logg return cmCopy, err } -func (r *EtcdReconciler) getConfigMapFromEtcd(etcd *druidv1alpha1.Etcd, renderedChart *chartrenderer.RenderedChart) (*corev1.ConfigMap, error) { +func (r *EtcdReconciler) getConfigMapFromEtcd(etcd *druidv1alpha1.Etcd, renderedChart *chartrenderer.RenderedChart, values map[string]interface{}) (*corev1.ConfigMap, error) { var err error decoded := &corev1.ConfigMap{} configMapPath := getChartPathForConfigMap() @@ -554,6 +561,12 @@ func (r *EtcdReconciler) getConfigMapFromEtcd(etcd *druidv1alpha1.Etcd, rendered if err = decoder.Decode(&decoded); err != nil { return nil, err } + + decoded.Name = fmt.Sprintf("etcd-bootstrap-%s", string(etcd.UID[:6])) + config := decoded.Data["etcd.conf.yaml"] + config = fmt.Sprintf("%s\ninitial-cluster: %s", config, values["initialCluster"]) + + decoded.Data["etcd.conf.yaml"] = config return decoded, nil } @@ -788,7 +801,7 @@ func (r *EtcdReconciler) reconcileEtcd(ctx context.Context, logger logr.Logger, values["serviceName"] = svc.Name } - cm, err := r.reconcileConfigMaps(ctx, logger, etcd, renderedChart) + cm, err := r.reconcileConfigMaps(ctx, logger, etcd, renderedChart, values) if err != nil { return noOp, nil, nil, err } @@ -852,8 +865,11 @@ func (r *EtcdReconciler) getMapFromEtcd(etcd *druidv1alpha1.Etcd) (map[string]in } var statefulsetReplicas int - if etcd.Spec.Replicas != 0 { - statefulsetReplicas = 1 + // Check if Spec.Replicas is odd or even + if etcd.Spec.Replicas&1 == 1 { + statefulsetReplicas = etcd.Spec.Replicas + } else { + return map[string]interface{}{}, fmt.Errorf("Spec.Replicas should not be even number: %d", etcd.Spec.Replicas) } etcdValues := map[string]interface{}{ @@ -978,6 +994,18 @@ func (r *EtcdReconciler) getMapFromEtcd(etcd *druidv1alpha1.Etcd) (map[string]in sharedConfigValues["autoCompactionRetention"] = etcd.Spec.Common.AutoCompactionRetention } + etcdConfigMountPath := "/var/etcd/config/" + initialCluster := fmt.Sprintf("%s=http://0.0.0.0:2380", etcd.Name) + if statefulsetReplicas > 1 { + etcdConfigMountPath = "/var/etcd/template/config/" + // form initial cluster + initialCluster = "" + for i := 0; i < statefulsetReplicas; i++ { + initialCluster = initialCluster + fmt.Sprintf("%s=http://%s-%d.%s:2380,", etcd.Name, etcd.Name, i, etcd.Name) + } + } + + initialCluster = strings.Trim(initialCluster, ",") values := map[string]interface{}{ "name": etcd.Name, "uid": etcd.UID, @@ -989,6 +1017,8 @@ func (r *EtcdReconciler) getMapFromEtcd(etcd *druidv1alpha1.Etcd) (map[string]in "sharedConfig": sharedConfigValues, "replicas": etcd.Spec.Replicas, "statefulsetReplicas": statefulsetReplicas, + "etcdConfigMountPath": etcdConfigMountPath, + "initialCluster": initialCluster, "serviceName": fmt.Sprintf("%s-client", etcd.Name), "configMapName": fmt.Sprintf("etcd-bootstrap-%s", string(etcd.UID[:6])), "volumeClaimTemplateName": volumeClaimTemplateName, diff --git a/controllers/etcd_controller_test.go b/controllers/etcd_controller_test.go index 5aefc4cb9..e94a903e9 100644 --- a/controllers/etcd_controller_test.go +++ b/controllers/etcd_controller_test.go @@ -18,8 +18,10 @@ import ( "context" "fmt" "os" + "strings" "time" + kutil "github.com/gardener/gardener/pkg/utils/kubernetes" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "github.com/gardener/gardener/pkg/utils/imagevector" @@ -41,6 +43,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -602,6 +605,214 @@ var _ = Describe("Druid", func() { ) }) +var _ = Describe("Multinode ETCD", func() { + //Reconciliation of new etcd resource deployment without any existing statefulsets. + Context("when adding etcd resources", func() { + var ( + err error + instance *druidv1alpha1.Etcd + sts *appsv1.StatefulSet + svc *corev1.Service + c client.Client + ) + + BeforeEach(func() { + instance = getEtcd("foo07", "default", false) + c = mgr.GetClient() + ns := corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.Namespace, + }, + } + _, err = controllerutil.CreateOrUpdate(context.TODO(), c, &ns, func() error { return nil }) + Expect(err).To(Not(HaveOccurred())) + + storeSecret := instance.Spec.Backup.Store.SecretRef.Name + errors := createSecrets(c, instance.Namespace, storeSecret) + Expect(len(errors)).Should(BeZero()) + }) + It("no statefulsets are created when ETCD replicas are even number", func() { + // Update replicas in ETCD resource with 0 + instance.Spec.Replicas = 0 + Expect(c.Create(context.TODO(), instance)).To(Succeed()) + + Eventually(func() error { + return c.Get(context.TODO(), types.NamespacedName{ + Name: instance.Name, + Namespace: instance.Namespace, + }, instance) + }, timeout, pollingInterval).Should(BeNil()) + + sts = &appsv1.StatefulSet{} + // No StatefulSet has been created by controller as even number of replicas are not allowed + Eventually(func() error { + return c.Get(context.TODO(), types.NamespacedName{ + Name: instance.Name, + Namespace: instance.Namespace, + }, sts) + }, timeout, pollingInterval).Should(matchers.BeNotFoundError()) + + // Update replicas in ETCD resource with even number + Expect(kutil.TryUpdate(context.TODO(), retry.DefaultBackoff, c, instance, func() error { + instance.Spec.Replicas = 4 + return nil + })).To(Succeed()) + + Eventually(func() error { + return c.Get(context.TODO(), types.NamespacedName{ + Name: instance.Name, + Namespace: instance.Namespace, + }, instance) + }, timeout, pollingInterval).Should(BeNil()) + + // No StatefulSet has been created by controller as even number of replicas are not allowed + sts = &appsv1.StatefulSet{} + Eventually(func() error { + return c.Get(context.TODO(), types.NamespacedName{ + Name: instance.Name, + Namespace: instance.Namespace, + }, sts) + }, timeout, pollingInterval).Should(matchers.BeNotFoundError()) + + // No Service has been created by controller as even number of replicas are not allowed + svc = &corev1.Service{} + Expect(c.Get(context.TODO(), types.NamespacedName{ + Name: fmt.Sprintf("%s-client", instance.Name), + Namespace: instance.Namespace, + }, svc)).Should(matchers.BeNotFoundError()) + svc = nil + }) + It("configmaps are mounted properly when ETCD replicas are odd number", func() { + // Update replicas in ETCD resource with 1 + instance.Spec.Replicas = 1 + Expect(c.Create(context.TODO(), instance)).To(Succeed()) + + sts = &appsv1.StatefulSet{} + // StatefulSet has been created by controller + Eventually(func() error { + if err := c.Get(context.TODO(), types.NamespacedName{ + Name: instance.Name, + Namespace: instance.Namespace, + }, sts); err != nil { + return err + } + + if *sts.Spec.Replicas != int32(instance.Spec.Replicas) { + return fmt.Errorf("Statefulset replicas not updated to: %d", instance.Spec.Replicas) + } + + // configmap volume in statefulset container should be mounted in /var/etcd/config/ + if sts.Spec.Template.Spec.Containers[0].VolumeMounts[1].MountPath != "/var/etcd/config/" { + return fmt.Errorf("Etcd config for ETCD container is not properly updated") + } + + if sts.Spec.Template.Spec.Containers[1].VolumeMounts[1].MountPath != "/var/etcd/config/" { + return fmt.Errorf("Etcd config ETCD backup container is not properly updated") + } + + return nil + }, timeout, pollingInterval).Should(BeNil()) + + Eventually(func() error { + return c.Get(context.TODO(), types.NamespacedName{ + Name: instance.Name, + Namespace: instance.Namespace, + }, instance) + }, timeout, pollingInterval).Should(BeNil()) + + cm := &corev1.ConfigMap{} + // Validate configmap yaml + Eventually(func() error { + if err := c.Get(context.TODO(), types.NamespacedName{ + Name: fmt.Sprintf("etcd-bootstrap-%s", string(instance.UID[:6])), + Namespace: instance.Namespace, + }, cm); err != nil { + return err + } + + matcher := fmt.Sprintf("initial-cluster: %s=http://0.0.0.0:2380", instance.Name) + if !strings.Contains(cm.Data["etcd.conf.yaml"], matcher) { + return fmt.Errorf("Configmap doesn't contain initial clusters: %s", cm.Data["etcd.conf.yaml"]) + } + + return nil + }, timeout, pollingInterval).Should(BeNil()) + + // Update replicas in ETCD resource with 3 + Expect(kutil.TryUpdate(context.TODO(), retry.DefaultBackoff, c, instance, func() error { + instance.Spec.Replicas = 3 + return nil + })).To(Succeed()) + + // StatefulSet has been created by controller + Eventually(func() error { + if err := c.Get(context.TODO(), types.NamespacedName{ + Name: instance.Name, + Namespace: instance.Namespace, + }, sts); err != nil { + return err + } + + if *sts.Spec.Replicas != int32(instance.Spec.Replicas) { + return fmt.Errorf("Statefulset replicas not updated to: %d", instance.Spec.Replicas) + } + + // configmap volume in statefulset container should be mounted in /var/etcd/template/config/ + if sts.Spec.Template.Spec.Containers[0].VolumeMounts[1].MountPath != "/var/etcd/template/config/" { + return fmt.Errorf("Etcd config for ETCD container is not properly updated") + } + + if sts.Spec.Template.Spec.Containers[1].VolumeMounts[1].MountPath != "/var/etcd/template/config/" { + return fmt.Errorf("Etcd config ETCD backup container is not properly updated") + } + + return nil + }, timeout, pollingInterval).Should(BeNil()) + + svc = &corev1.Service{} + // Wait until Service has been created by controller + Eventually(func() error { + return c.Get(context.TODO(), types.NamespacedName{ + Name: fmt.Sprintf("%s-client", instance.Name), + Namespace: instance.Namespace, + }, svc) + }, timeout, pollingInterval).Should(BeNil()) + + cm = &corev1.ConfigMap{} + // Validate configmap yaml + Eventually(func() error { + if err := c.Get(context.TODO(), types.NamespacedName{ + Name: fmt.Sprintf("etcd-bootstrap-%s", string(instance.UID[:6])), + Namespace: instance.Namespace, + }, cm); err != nil { + return err + } + + matcher := "initial-cluster: foo07=http://foo07-0.foo07:2380,foo07=http://foo07-1.foo07:2380,foo07=http://foo07-2.foo07:2380" + if !strings.Contains(cm.Data["etcd.conf.yaml"], matcher) { + return fmt.Errorf("Configmap doesn't contain initial clusters: %s", cm.Data["etcd.conf.yaml"]) + } + + return nil + }, timeout, pollingInterval).Should(BeNil()) + }) + AfterEach(func() { + // Delete `etcd` instance + Expect(c.Delete(context.TODO(), instance)).To(Succeed()) + Eventually(func() error { + return c.Get(context.TODO(), client.ObjectKeyFromObject(instance), &druidv1alpha1.Etcd{}) + }, timeout, pollingInterval).Should(matchers.BeNotFoundError()) + // Delete service manually because garbage collection is not available in `envtest` + if svc != nil { + Expect(c.Delete(context.TODO(), svc)).To(Succeed()) + Eventually(func() error { + return c.Get(context.TODO(), client.ObjectKeyFromObject(svc), &corev1.Service{}) + }, timeout, pollingInterval).Should(matchers.BeNotFoundError()) + } + }) + }) +}) + func podDeleted(c client.Client, etcd *druidv1alpha1.Etcd) error { ctx, cancel := context.WithTimeout(context.TODO(), timeout) defer cancel() @@ -892,6 +1103,10 @@ func validateEtcdWithDefaults(s *appsv1.StatefulSet, cm *corev1.ConfigMap, svc * }), }), "Env": MatchAllElements(envIterator, Elements{ + "ETCD_INITIAL_CLUSTER": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("ETCD_INITIAL_CLUSTER"), + "Value": Equal(fmt.Sprintf("%s=http://0.0.0.0:2380", instance.Name)), + }), "STORAGE_CONTAINER": MatchFields(IgnoreExtras, Fields{ "Name": Equal("STORAGE_CONTAINER"), "Value": Equal(""), @@ -1236,7 +1451,7 @@ func validateEtcd(s *appsv1.StatefulSet, cm *corev1.ConfigMap, svc *corev1.Servi fmt.Sprintf("%s=%s", "--auto-compaction-retention", *instance.Spec.Common.AutoCompactionRetention): Equal(fmt.Sprintf("%s=%s", "--auto-compaction-retention", autoCompactionRetention)), }), "Ports": ConsistOf([]corev1.ContainerPort{ - corev1.ContainerPort{ + { Name: "server", Protocol: corev1.ProtocolTCP, HostPort: 0, @@ -1362,6 +1577,10 @@ func validateStoreGCP(s *appsv1.StatefulSet, cm *corev1.ConfigMap, svc *corev1.S }), }), "Env": MatchAllElements(envIterator, Elements{ + "ETCD_INITIAL_CLUSTER": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("ETCD_INITIAL_CLUSTER"), + "Value": Equal(fmt.Sprintf("%s=http://0.0.0.0:2380", instance.Name)), + }), "STORAGE_CONTAINER": MatchFields(IgnoreExtras, Fields{ "Name": Equal("STORAGE_CONTAINER"), "Value": Equal(*instance.Spec.Backup.Store.Container), @@ -1402,6 +1621,10 @@ func validateStoreAzure(s *appsv1.StatefulSet, cm *corev1.ConfigMap, svc *corev1 fmt.Sprintf("%s=%s", "--store-prefix", instance.Spec.Backup.Store.Prefix): Equal(fmt.Sprintf("%s=%s", "--store-prefix", instance.Spec.Backup.Store.Prefix)), }), "Env": MatchAllElements(envIterator, Elements{ + "ETCD_INITIAL_CLUSTER": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("ETCD_INITIAL_CLUSTER"), + "Value": Equal(fmt.Sprintf("%s=http://0.0.0.0:2380", instance.Name)), + }), "STORAGE_CONTAINER": MatchFields(IgnoreExtras, Fields{ "Name": Equal("STORAGE_CONTAINER"), "Value": Equal(*instance.Spec.Backup.Store.Container), @@ -1450,6 +1673,10 @@ func validateStoreOpenstack(s *appsv1.StatefulSet, cm *corev1.ConfigMap, svc *co fmt.Sprintf("%s=%s", "--store-prefix", instance.Spec.Backup.Store.Prefix): Equal(fmt.Sprintf("%s=%s", "--store-prefix", instance.Spec.Backup.Store.Prefix)), }), "Env": MatchAllElements(envIterator, Elements{ + "ETCD_INITIAL_CLUSTER": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("ETCD_INITIAL_CLUSTER"), + "Value": Equal(fmt.Sprintf("%s=http://0.0.0.0:2380", instance.Name)), + }), "STORAGE_CONTAINER": MatchFields(IgnoreExtras, Fields{ "Name": Equal("STORAGE_CONTAINER"), "Value": Equal(*instance.Spec.Backup.Store.Container), @@ -1533,6 +1760,10 @@ func validateStoreAlicloud(s *appsv1.StatefulSet, cm *corev1.ConfigMap, svc *cor }), "ImagePullPolicy": Equal(corev1.PullIfNotPresent), "Env": MatchAllElements(envIterator, Elements{ + "ETCD_INITIAL_CLUSTER": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("ETCD_INITIAL_CLUSTER"), + "Value": Equal(fmt.Sprintf("%s=http://0.0.0.0:2380", instance.Name)), + }), "STORAGE_CONTAINER": MatchFields(IgnoreExtras, Fields{ "Name": Equal("STORAGE_CONTAINER"), "Value": Equal(*instance.Spec.Backup.Store.Container), @@ -1594,6 +1825,10 @@ func validateStoreAWS(s *appsv1.StatefulSet, cm *corev1.ConfigMap, svc *corev1.S }), "ImagePullPolicy": Equal(corev1.PullIfNotPresent), "Env": MatchAllElements(envIterator, Elements{ + "ETCD_INITIAL_CLUSTER": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("ETCD_INITIAL_CLUSTER"), + "Value": Equal(fmt.Sprintf("%s=http://0.0.0.0:2380", instance.Name)), + }), "STORAGE_CONTAINER": MatchFields(IgnoreExtras, Fields{ "Name": Equal("STORAGE_CONTAINER"), "Value": Equal(*instance.Spec.Backup.Store.Container),