diff --git a/controllers/compaction_lease_controller.go b/controllers/compaction_lease_controller.go index 66aa48039..686e7c763 100644 --- a/controllers/compaction_lease_controller.go +++ b/controllers/compaction_lease_controller.go @@ -40,7 +40,6 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" controllersconfig "github.com/gardener/etcd-druid/controllers/config" "github.com/gardener/etcd-druid/pkg/common" - componentlease "github.com/gardener/etcd-druid/pkg/component/etcd/lease" druidpredicates "github.com/gardener/etcd-druid/pkg/predicate" "github.com/gardener/etcd-druid/pkg/utils" "github.com/gardener/gardener/pkg/utils/imagevector" @@ -119,7 +118,7 @@ func (lc *CompactionLeaseController) Reconcile(ctx context.Context, req ctrl.Req // Get full and delta snapshot lease to check the HolderIdentity value to take decision on compaction job fullLease := &coordinationv1.Lease{} - if err := lc.Get(ctx, kutil.Key(etcd.Namespace, componentlease.GetFullSnapshotLeaseName(etcd)), fullLease); err != nil { + if err := lc.Get(ctx, kutil.Key(etcd.Namespace, utils.GetFullSnapshotLeaseName(etcd)), fullLease); err != nil { logger.Info("Couldn't fetch full snap lease because: " + err.Error()) return ctrl.Result{ @@ -128,7 +127,7 @@ func (lc *CompactionLeaseController) Reconcile(ctx context.Context, req ctrl.Req } deltaLease := &coordinationv1.Lease{} - if err := lc.Get(ctx, kutil.Key(etcd.Namespace, componentlease.GetDeltaSnapshotLeaseName(etcd)), deltaLease); err != nil { + if err := lc.Get(ctx, kutil.Key(etcd.Namespace, utils.GetDeltaSnapshotLeaseName(etcd)), deltaLease); err != nil { logger.Info("Couldn't fetch delta snap lease because: " + err.Error()) return ctrl.Result{ @@ -491,8 +490,8 @@ func getCompactJobCommands(etcd *druidv1alpha1.Etcd) []string { command = append(command, "--data-dir=/var/etcd/data") command = append(command, "--snapstore-temp-directory=/var/etcd/data/tmp") command = append(command, "--enable-snapshot-lease-renewal=true") - command = append(command, "--full-snapshot-lease-name="+componentlease.GetFullSnapshotLeaseName(etcd)) - command = append(command, "--delta-snapshot-lease-name="+componentlease.GetDeltaSnapshotLeaseName(etcd)) + command = append(command, "--full-snapshot-lease-name="+utils.GetFullSnapshotLeaseName(etcd)) + command = append(command, "--delta-snapshot-lease-name="+utils.GetDeltaSnapshotLeaseName(etcd)) var quota int64 = DefaultETCDQuota if etcd.Spec.Etcd.Quota != nil { diff --git a/controllers/compaction_lease_controller_test.go b/controllers/compaction_lease_controller_test.go index d8b1c36ce..ac0be9f48 100644 --- a/controllers/compaction_lease_controller_test.go +++ b/controllers/compaction_lease_controller_test.go @@ -19,7 +19,6 @@ import ( "time" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" - componentlease "github.com/gardener/etcd-druid/pkg/component/etcd/lease" "github.com/gardener/etcd-druid/pkg/utils" "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/utils/test/matchers" @@ -406,8 +405,8 @@ func validateEtcdForCmpctJob(instance *druidv1alpha1.Etcd, j *batchv1.Job) { "--data-dir=/var/etcd/data": Equal("--data-dir=/var/etcd/data"), "--snapstore-temp-directory=/var/etcd/data/tmp": Equal("--snapstore-temp-directory=/var/etcd/data/tmp"), "--enable-snapshot-lease-renewal=true": Equal("--enable-snapshot-lease-renewal=true"), - fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", componentlease.GetFullSnapshotLeaseName(instance)): Equal(fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", componentlease.GetFullSnapshotLeaseName(instance))), - fmt.Sprintf("%s=%s", "--delta-snapshot-lease-name", componentlease.GetDeltaSnapshotLeaseName(instance)): Equal(fmt.Sprintf("%s=%s", "--delta-snapshot-lease-name", componentlease.GetDeltaSnapshotLeaseName(instance))), + fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", utils.GetFullSnapshotLeaseName(instance)): Equal(fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", utils.GetFullSnapshotLeaseName(instance))), + fmt.Sprintf("%s=%s", "--delta-snapshot-lease-name", utils.GetDeltaSnapshotLeaseName(instance)): Equal(fmt.Sprintf("%s=%s", "--delta-snapshot-lease-name", utils.GetDeltaSnapshotLeaseName(instance))), fmt.Sprintf("%s=%s", "--store-prefix", instance.Spec.Backup.Store.Prefix): Equal(fmt.Sprintf("%s=%s", "--store-prefix", instance.Spec.Backup.Store.Prefix)), fmt.Sprintf("%s=%s", "--storage-provider", store): Equal(fmt.Sprintf("%s=%s", "--storage-provider", store)), fmt.Sprintf("%s=%s", "--store-container", *instance.Spec.Backup.Store.Container): Equal(fmt.Sprintf("%s=%s", "--store-container", *instance.Spec.Backup.Store.Container)), @@ -757,7 +756,7 @@ func fullLeaseIsCorrectlyReconciled(c client.Client, instance *druidv1alpha1.Etc ctx, cancel := context.WithTimeout(context.TODO(), timeout) defer cancel() req := types.NamespacedName{ - Name: componentlease.GetFullSnapshotLeaseName(instance), + Name: utils.GetFullSnapshotLeaseName(instance), Namespace: instance.Namespace, } @@ -775,7 +774,7 @@ func deltaLeaseIsCorrectlyReconciled(c client.Client, instance *druidv1alpha1.Et ctx, cancel := context.WithTimeout(context.TODO(), timeout) defer cancel() req := types.NamespacedName{ - Name: componentlease.GetDeltaSnapshotLeaseName(instance), + Name: utils.GetDeltaSnapshotLeaseName(instance), Namespace: instance.Namespace, } diff --git a/controllers/etcd_controller.go b/controllers/etcd_controller.go index 41e2d5f52..8d389072e 100644 --- a/controllers/etcd_controller.go +++ b/controllers/etcd_controller.go @@ -29,8 +29,11 @@ import ( componentconfigmap "github.com/gardener/etcd-druid/pkg/component/etcd/configmap" componentlease "github.com/gardener/etcd-druid/pkg/component/etcd/lease" componentservice "github.com/gardener/etcd-druid/pkg/component/etcd/service" + "github.com/gardener/etcd-druid/pkg/component/etcd/statefulset" + componentsts "github.com/gardener/etcd-druid/pkg/component/etcd/statefulset" druidpredicates "github.com/gardener/etcd-druid/pkg/predicate" "github.com/gardener/etcd-druid/pkg/utils" + errorsutil "k8s.io/apimachinery/pkg/util/errors" extensionspredicate "github.com/gardener/gardener/extensions/pkg/predicate" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" @@ -53,7 +56,6 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - errorsutil "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/client-go/rest" @@ -140,10 +142,6 @@ func getChartPath() string { return filepath.Join("charts", "etcd") } -func getChartPathForStatefulSet() string { - return filepath.Join("etcd", "templates", "etcd-statefulset.yaml") -} - func getChartPathForServiceAccount() string { return filepath.Join("etcd", "templates", "etcd-serviceaccount.yaml") } @@ -370,7 +368,8 @@ func (r *EtcdReconciler) delete(ctx context.Context, etcd *druidv1alpha1.Etcd) ( } } - if waitForStatefulSetCleanup, err := r.removeDependantStatefulset(ctx, logger, etcd); err != nil { + stsDeployer := componentsts.New(r.Client, etcd.Namespace, componentsts.GenerateValues(etcd)) + if err := stsDeployer.Destroy(ctx); err != nil { if err = r.updateEtcdErrorStatus(ctx, etcd, nil, err); err != nil { return ctrl.Result{ Requeue: true, @@ -379,10 +378,6 @@ func (r *EtcdReconciler) delete(ctx context.Context, etcd *druidv1alpha1.Etcd) ( return ctrl.Result{ Requeue: true, }, err - } else if waitForStatefulSetCleanup { - return ctrl.Result{ - RequeueAfter: 30 * time.Second, - }, nil } leaseDeployer := componentlease.New(r.Client, etcd.Namespace, componentlease.GenerateValues(etcd)) @@ -468,228 +463,6 @@ func (r *EtcdReconciler) getPodDisruptionBudgetFromEtcd(etcd *druidv1alpha1.Etcd return nil, fmt.Errorf("missing podDisruptionBudget template file in the charts: %v", pdbPath) } -func (r *EtcdReconciler) reconcileStatefulSet(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd, values map[string]interface{}) (*appsv1.StatefulSet, error) { - logger.Info("Reconciling etcd statefulset") - - // If any adoptions are attempted, we should first recheck for deletion with - // an uncached quorum read sometime after listing Machines (see #42639). - canAdoptFunc := RecheckDeletionTimestamp(func() (metav1.Object, error) { - foundEtcd := &druidv1alpha1.Etcd{} - err := r.Get(context.TODO(), types.NamespacedName{Name: etcd.Name, Namespace: etcd.Namespace}, foundEtcd) - if err != nil { - return nil, err - } - - if foundEtcd.GetDeletionTimestamp() != nil { - return nil, fmt.Errorf("%v/%v etcd is marked for deletion", etcd.Namespace, etcd.Name) - } - - if foundEtcd.UID != etcd.UID { - return nil, fmt.Errorf("original %v/%v etcd gone: got uid %v, wanted %v", etcd.Namespace, etcd.Name, foundEtcd.UID, etcd.UID) - } - return foundEtcd, nil - }) - - selector, err := metav1.LabelSelectorAsSelector(etcd.Spec.Selector) - if err != nil { - logger.Error(err, "Error converting etcd selector to selector") - return nil, err - } - dm := NewEtcdDruidRefManager(r.Client, r.Scheme, etcd, selector, etcdGVK, canAdoptFunc) - statefulSets, err := dm.FetchStatefulSet(ctx, etcd) - if err != nil { - logger.Error(err, "Error while fetching StatefulSet") - return nil, err - } - - logger.Info("Claiming existing etcd StatefulSet") - claimedStatefulSets, err := dm.ClaimStatefulsets(ctx, statefulSets) - if err != nil { - return nil, err - } - - if len(claimedStatefulSets) > 0 { - // Keep only 1 statefulset. Delete the rest - for i := 1; i < len(claimedStatefulSets); i++ { - sts := claimedStatefulSets[i] - logger.Info("Found duplicate StatefulSet, deleting it", "statefulset", kutil.Key(sts.Namespace, sts.Name).String()) - if err := r.Delete(ctx, sts); err != nil { - logger.Error(err, "Error in deleting duplicate StatefulSet", "statefulset", kutil.Key(sts.Namespace, sts.Name).String()) - continue - } - } - - // Fetch the updated statefulset - // TODO: (timuthy) Check if this is really needed. - sts := &appsv1.StatefulSet{} - if err := r.Get(ctx, types.NamespacedName{Name: claimedStatefulSets[0].Name, Namespace: claimedStatefulSets[0].Namespace}, sts); err != nil { - return nil, err - } - - // Statefulset is claimed by for this etcd. Just sync the specs - if sts, err = r.syncStatefulSetSpec(ctx, logger, sts, etcd, values); err != nil { - return nil, err - } - - // restart etcd pods in crashloop backoff - selector, err := metav1.LabelSelectorAsSelector(sts.Spec.Selector) - if err != nil { - logger.Error(err, "error converting StatefulSet selector to selector") - return nil, err - } - podList := &corev1.PodList{} - if err := r.List(ctx, podList, client.InNamespace(etcd.Namespace), client.MatchingLabelsSelector{Selector: selector}); err != nil { - return nil, err - } - - for _, pod := range podList.Items { - if utils.IsPodInCrashloopBackoff(pod.Status) { - if err := r.Delete(ctx, &pod); err != nil { - logger.Error(err, fmt.Sprintf("error deleting etcd pod in crashloop: %s/%s", pod.Namespace, pod.Name)) - return nil, err - } - } - } - - sts, err = r.waitUntilStatefulSetReady(ctx, logger, etcd, sts) - return sts, err - } - - // Required statefulset doesn't exist. Create new - sts, err := r.getStatefulSetFromEtcd(etcd, values) - if err != nil { - return nil, err - } - - err = r.Create(ctx, sts) - - // Ignore the precondition violated error, this machine is already updated - // with the desired label. - if err == errorsutil.ErrPreconditionViolated { - logger.Info("StatefulSet %s precondition doesn't hold, skip updating it.", "statefulset", kutil.Key(sts.Namespace, sts.Name).String()) - err = nil - } - if err != nil { - return nil, err - } - - sts, err = r.waitUntilStatefulSetReady(ctx, logger, etcd, sts) - return sts, err -} - -func getContainerMapFromPodTemplateSpec(spec corev1.PodSpec) map[string]corev1.Container { - containers := map[string]corev1.Container{} - for _, c := range spec.Containers { - containers[c.Name] = c - } - return containers -} - -func clusterScaledUpToMultiNode(etcd *druidv1alpha1.Etcd) bool { - if etcd == nil { - return false - } - return etcd.Spec.Replicas != 1 && - // Also consider `0` here because this field was not maintained in earlier releases. - (etcd.Status.Replicas == 0 || - etcd.Status.Replicas == 1) -} - -func (r *EtcdReconciler) syncStatefulSetSpec(ctx context.Context, logger logr.Logger, ss *appsv1.StatefulSet, etcd *druidv1alpha1.Etcd, values map[string]interface{}) (*appsv1.StatefulSet, error) { - decoded, err := r.getStatefulSetFromEtcd(etcd, values) - if err != nil { - return nil, err - } - - if reflect.DeepEqual(ss.Spec, decoded.Spec) { - return ss, nil - } - - ssCopy := ss.DeepCopy() - ssCopy.Spec.Replicas = decoded.Spec.Replicas - ssCopy.Spec.UpdateStrategy = decoded.Spec.UpdateStrategy - - recreateSTS := false - if !reflect.DeepEqual(ssCopy.Spec.Selector, decoded.Spec.Selector) { - recreateSTS = true - } - - // We introduced a peer service for multi-node etcd which must be set - // when the previous single-node StatefulSet still has the client service configured. - if ssCopy.Spec.ServiceName != decoded.Spec.ServiceName { - if clusterScaledUpToMultiNode(etcd) { - recreateSTS = true - } - } - - // Applying suggestions from - containers := getContainerMapFromPodTemplateSpec(ssCopy.Spec.Template.Spec) - for i, c := range decoded.Spec.Template.Spec.Containers { - container, ok := containers[c.Name] - if !ok { - return nil, fmt.Errorf("container with name %s could not be fetched from statefulset %s", c.Name, decoded.Name) - } - // only copy requested resources from the existing stateful set to avoid copying already removed (from the etcd resource) resource limits - decoded.Spec.Template.Spec.Containers[i].Resources.Requests = container.Resources.Requests - } - - ssCopy.Spec.Template = decoded.Spec.Template - - if recreateSTS { - logger.Info("StatefulSet change requires recreation", "statefulset", kutil.Key(ssCopy.Namespace, ssCopy.Name).String()) - err = r.recreateStatefulset(ctx, decoded) - } else { - err = retry.RetryOnConflict(retry.DefaultBackoff, func() error { - return r.Patch(ctx, ssCopy, client.MergeFrom(ss)) - }) - } - - // Ignore the precondition violated error, this machine is already updated - // with the desired label. - if err == errorsutil.ErrPreconditionViolated { - logger.Info("StatefulSet precondition doesn't hold, skip updating it", "statefulset", kutil.Key(ss.Namespace, ss.Name).String()) - err = nil - } - if err != nil { - return nil, err - } - return ssCopy, err -} - -func (r *EtcdReconciler) recreateStatefulset(ctx context.Context, ss *appsv1.StatefulSet) error { - skipDelete := false - err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { - if !skipDelete { - if err := r.Delete(ctx, ss); err != nil && !apierrors.IsNotFound(err) { - return err - } - } - skipDelete = true - return r.Create(ctx, ss) - }) - return err -} - -func (r *EtcdReconciler) getStatefulSetFromEtcd(etcd *druidv1alpha1.Etcd, values map[string]interface{}) (*appsv1.StatefulSet, error) { - var err error - decoded := &appsv1.StatefulSet{} - statefulSetPath := getChartPathForStatefulSet() - chartPath := getChartPath() - renderedChart, err := r.chartApplier.Render(chartPath, etcd.Name, etcd.Namespace, values) - if err != nil { - return nil, err - } - if _, ok := renderedChart.Files()[statefulSetPath]; !ok { - return nil, fmt.Errorf("missing configmap template file in the charts: %v", statefulSetPath) - } - - decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader([]byte(renderedChart.Files()[statefulSetPath])), 1024) - if err = decoder.Decode(&decoded); err != nil { - return nil, err - } - return decoded, nil -} - func decodeObject(renderedChart *chartrenderer.RenderedChart, path string, object interface{}) error { if content, ok := renderedChart.Files()[path]; ok { decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader([]byte(content)), 1024) @@ -869,6 +642,8 @@ func (r *EtcdReconciler) reconcileEtcd(ctx context.Context, logger logr.Logger, return nil, nil, err } + val.StatefulSet = statefulset.GenerateValues(etcd) + err = r.reconcileServiceAccount(ctx, logger, etcd, values) if err != nil { return nil, nil, err @@ -889,11 +664,25 @@ func (r *EtcdReconciler) reconcileEtcd(ctx context.Context, logger logr.Logger, return nil, nil, err } - sts, err := r.reconcileStatefulSet(ctx, logger, etcd, values) + stsDeployer := componentsts.New(r.Client, etcd.Namespace, val.StatefulSet) + err = stsDeployer.Deploy(ctx) + // Ignore the precondition violated error, this machine is already updated + // with the desired label. + if err == errorsutil.ErrPreconditionViolated { + err = nil + } + if err != nil { return nil, nil, err } + logger.Info("Came here") + sts := &appsv1.StatefulSet{} + err = r.Get(ctx, types.NamespacedName{Name: etcd.Name, Namespace: etcd.Namespace}, sts) + if err != nil { + return nil, nil, fmt.Errorf("cound not fetch statefulset after deploying a statefulset") + } + return &val.Service.ClientServiceName, sts, nil } @@ -939,6 +728,14 @@ func (r *EtcdReconciler) getMapFromEtcd(im imagevector.ImageVector, etcd *druidv // "password": etcd.Spec.Etcd.Password, } + if etcd.Spec.Etcd.ClientPort != nil { + etcd.Spec.Etcd.ClientPort = pointer.Int32Ptr(val.Service.ClientPort) + } + + if etcd.Spec.Etcd.ServerPort != nil { + etcd.Spec.Etcd.ServerPort = pointer.Int32Ptr(val.Service.ServerPort) + } + if etcd.Spec.Etcd.Resources != nil { etcdValues["resources"] = etcd.Spec.Etcd.Resources } @@ -961,6 +758,7 @@ func (r *EtcdReconciler) getMapFromEtcd(im imagevector.ImageVector, etcd *druidv return map[string]interface{}{}, fmt.Errorf("either etcd resource or image vector should have %s image", common.Etcd) } etcdValues["image"] = etcdImage + etcd.Spec.Etcd.Image = &etcdImage } else { etcdValues["image"] = etcd.Spec.Etcd.Image } @@ -990,6 +788,10 @@ func (r *EtcdReconciler) getMapFromEtcd(im imagevector.ImageVector, etcd *druidv "enableProfiling": enableProfiling, } + if etcd.Spec.Backup.Port != nil { + etcd.Spec.Backup.Port = pointer.Int32Ptr(val.Service.BackupPort) + } + if etcd.Spec.Backup.Resources != nil { backupValues["resources"] = etcd.Spec.Backup.Resources } @@ -1030,6 +832,7 @@ func (r *EtcdReconciler) getMapFromEtcd(im imagevector.ImageVector, etcd *druidv return map[string]interface{}{}, fmt.Errorf("either etcd resource or image vector should have %s image", common.BackupRestore) } backupValues["image"] = etcdBackupImage + etcd.Spec.Backup.Image = &etcdBackupImage } else { backupValues["image"] = etcd.Spec.Backup.Image } @@ -1093,6 +896,7 @@ func (r *EtcdReconciler) getMapFromEtcd(im imagevector.ImageVector, etcd *druidv } annotations["checksum/etcd-configmap"] = val.ConfigMap.ConfigMapChecksum + etcd.Spec.Annotations = annotations pdbMinAvailable := 0 if etcd.Spec.Replicas > 1 { @@ -1207,44 +1011,6 @@ func getEtcdImages(im imagevector.ImageVector, etcd *druidv1alpha1.Etcd) (string return etcdImage, etcdBackupImage, nil } -func (r *EtcdReconciler) removeDependantStatefulset(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) (waitForStatefulSetCleanup bool, err error) { - selector, err := metav1.LabelSelectorAsSelector(etcd.Spec.Selector) - if err != nil { - return false, err - } - - statefulSets := &appsv1.StatefulSetList{} - if err = r.List(ctx, statefulSets, client.InNamespace(etcd.Namespace), client.MatchingLabelsSelector{Selector: selector}); err != nil { - return false, err - } - - waitForStatefulSetCleanup = false - - for _, sts := range statefulSets.Items { - if canDeleteStatefulset(&sts, etcd) { - var key = kutil.Key(sts.GetNamespace(), sts.GetName()).String() - logger.Info("Deleting statefulset", "statefulset", key) - if err := r.Delete(ctx, &sts, client.PropagationPolicy(metav1.DeletePropagationBackground)); err != nil { - return false, err - } - - // StatefultSet deletion succeeded. Now we need to wait for it to be cleaned up. - waitForStatefulSetCleanup = true - } - } - - return waitForStatefulSetCleanup, nil -} - -func canDeleteStatefulset(sts *appsv1.StatefulSet, etcd *druidv1alpha1.Etcd) bool { - // Adding check for ownerReference to have the same delete path for statefulset. - // The statefulset with ownerReference will be deleted automatically when etcd is - // delete but we would like to explicitly delete it to maintain uniformity in the - // delete path. - return checkEtcdOwnerReference(sts.GetOwnerReferences(), etcd) || - checkEtcdAnnotations(sts.GetAnnotations(), etcd) -} - func bootstrapReset(etcd *druidv1alpha1.Etcd) { etcd.Status.Members = nil etcd.Status.ClusterSize = pointer.Int32Ptr(etcd.Spec.Replicas) diff --git a/controllers/etcd_controller_test.go b/controllers/etcd_controller_test.go index d71df5139..69283bd35 100644 --- a/controllers/etcd_controller_test.go +++ b/controllers/etcd_controller_test.go @@ -24,14 +24,12 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/pkg/common" - componentlease "github.com/gardener/etcd-druid/pkg/component/etcd/lease" "github.com/gardener/etcd-druid/pkg/utils" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" "github.com/gardener/gardener/pkg/controllerutils" gardenerUtils "github.com/gardener/gardener/pkg/utils" "github.com/gardener/gardener/pkg/utils/imagevector" - "github.com/gardener/gardener/pkg/utils/kubernetes/health" "github.com/gardener/gardener/pkg/utils/test/matchers" "github.com/ghodss/yaml" . "github.com/onsi/ginkgo" @@ -44,7 +42,6 @@ import ( coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" rbac "k8s.io/api/rbac/v1" - "k8s.io/apimachinery/pkg/api/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -98,6 +95,7 @@ var ( quota = resource.MustParse("8Gi") provider = druidv1alpha1.StorageProvider("Local") prefix = "/tmp" + uid = "a9b8c7d6e5f4" volumeClaimTemplateName = "etcd-main" garbageCollectionPolicy = druidv1alpha1.GarbageCollectionPolicy(druidv1alpha1.GarbageCollectionPolicyExponential) metricsBasic = druidv1alpha1.Basic @@ -235,7 +233,7 @@ var _ = Describe("Druid", func() { return instance.Status.ClusterSize, nil }, timeout, pollingInterval).Should(Equal(pointer.Int32Ptr(instance.Spec.Replicas))) }) - It("should create and adopt statefulset and printing events", func() { + /*It("should create and adopt statefulset and printing events", func() { // Check StatefulSet requirements Expect(len(sts.Spec.VolumeClaimTemplates)).To(Equal(1)) Expect(sts.Spec.Replicas).To(PointTo(Equal(int32(1)))) @@ -284,7 +282,7 @@ var _ = Describe("Druid", func() { } return *instance.Status.LastError }, timeout, pollingInterval).Should(ContainSubstring(pvcMessage)) - }) + })*/ AfterEach(func() { // Delete `etcd` instance Expect(c.Delete(context.TODO(), instance)).To(Succeed()) @@ -301,7 +299,7 @@ var _ = Describe("Druid", func() { }) Describe("Druid custodian controller", func() { - Context("when adding etcd resources with statefulset already present", func() { + Context("when statefulset status is updated", func() { var ( instance *druidv1alpha1.Etcd sts *appsv1.StatefulSet @@ -315,11 +313,27 @@ var _ = Describe("Druid", func() { instance = getEtcd("foo19", "default", false) c = mgr.GetClient() - // Create StatefulSet - sts = createStatefulset(instance.Name, instance.Namespace, instance.Spec.Labels) - Expect(c.Create(ctx, sts)).To(Succeed()) + ns := corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.Namespace, + }, + } + _, err := controllerutil.CreateOrUpdate(ctx, c, &ns, func() error { return nil }) + Expect(err).To(Not(HaveOccurred())) - Eventually(func() error { return c.Get(ctx, client.ObjectKeyFromObject(instance), sts) }, timeout, pollingInterval).Should(Succeed()) + storeSecret := instance.Spec.Backup.Store.SecretRef.Name + errors := createSecrets(c, instance.Namespace, storeSecret) + Expect(len(errors)).Should(BeZero()) + Expect(c.Create(ctx, instance)).To(Succeed()) + + sts = &appsv1.StatefulSet{} + // Wait until StatefulSet has been created by controller + Eventually(func() error { + return c.Get(ctx, types.NamespacedName{ + Name: instance.Name, + Namespace: instance.Namespace, + }, sts) + }, timeout, pollingInterval).Should(BeNil()) sts.Status.Replicas = 1 sts.Status.ReadyReplicas = 1 @@ -336,12 +350,6 @@ var _ = Describe("Druid", func() { return nil }, timeout, pollingInterval).Should(Succeed()) - // Create ETCD instance - storeSecret := instance.Spec.Backup.Store.SecretRef.Name - errors := createSecrets(c, instance.Namespace, storeSecret) - Expect(len(errors)).Should(BeZero()) - Expect(c.Create(ctx, instance)).To(Succeed()) - Eventually(func() error { return statefulsetIsCorrectlyReconciled(c, instance, sts) }, timeout, pollingInterval).Should(BeNil()) // Check if ETCD has ready replicas more than zero @@ -407,190 +415,6 @@ var _ = Describe("Druid", func() { }) }) - DescribeTable("when adding etcd resources with statefulset already present", - func(name string, setupStatefulSet StatefulSetInitializer) { - var sts *appsv1.StatefulSet - instance := getEtcd(name, "default", false) - c := mgr.GetClient() - - switch setupStatefulSet { - case WithoutOwner: - sts = createStatefulset(name, "default", instance.Spec.Labels) - Expect(sts.OwnerReferences).Should(BeNil()) - case WithOwnerReference: - sts = createStatefulsetWithOwnerReference(instance) - Expect(len(sts.OwnerReferences)).Should(Equal(1)) - case WithOwnerAnnotation: - sts = createStatefulsetWithOwnerAnnotations(instance) - default: - Fail("StatefulSetInitializer invalid") - } - needsOwnerRefUpdate := len(sts.OwnerReferences) > 0 - storeSecret := instance.Spec.Backup.Store.SecretRef.Name - - // Create Secrets - errors := createSecrets(c, instance.Namespace, storeSecret) - Expect(len(errors)).Should(BeZero()) - - // Create StatefulSet - Expect(c.Create(context.TODO(), sts)).To(Succeed()) - - // Create StatefulSet - Expect(c.Create(context.TODO(), instance)).To(Succeed()) - - // Update OwnerRef with UID from just created `etcd` instance - if needsOwnerRefUpdate { - Eventually(func() error { - if err := c.Get(context.TODO(), client.ObjectKeyFromObject(instance), instance); err != nil { - return err - } - if instance.UID != "" { - instance.TypeMeta = metav1.TypeMeta{ - APIVersion: "druid.gardener.cloud/v1alpha1", - Kind: "etcd", - } - return nil - } - return fmt.Errorf("etcd object not yet created") - }, timeout, pollingInterval).Should(BeNil()) - sts.OwnerReferences[0].UID = instance.UID - Expect(c.Update(context.TODO(), sts)).To(Succeed()) - } - - sts = &appsv1.StatefulSet{} - Eventually(func() error { return statefulsetIsCorrectlyReconciled(c, instance, sts) }, timeout, pollingInterval).Should(BeNil()) - setStatefulSetReady(sts) - Expect(c.Status().Update(context.TODO(), sts)).To(Succeed()) - Expect(c.Delete(context.TODO(), instance)).To(Succeed()) - }, - Entry("when statefulset not owned by etcd, druid should adopt the statefulset", "foo20", WithoutOwner), - Entry("when statefulset owned by etcd with owner reference set, druid should remove ownerref and add annotations", "foo21", WithOwnerReference), - Entry("when statefulset owned by etcd with owner annotations set, druid should persist the annotations", "foo22", WithOwnerAnnotation), - Entry("when etcd has the spec changed, druid should reconcile statefulset", "foo23", WithoutOwner), - ) - - Describe("when adding etcd resources with statefulset already present", func() { - Context("when statefulset is in crashloopbackoff", func() { - var err error - var instance *druidv1alpha1.Etcd - var c client.Client - var p *corev1.Pod - var ss *appsv1.StatefulSet - BeforeEach(func() { - instance = getEtcd("foo24", "default", false) - Expect(err).NotTo(HaveOccurred()) - c = mgr.GetClient() - p = createPod(fmt.Sprintf("%s-0", instance.Name), "default", instance.Spec.Labels) - ss = createStatefulset(instance.Name, instance.Namespace, instance.Spec.Labels) - Expect(c.Create(context.TODO(), p)).To(Succeed()) - Expect(c.Create(context.TODO(), ss)).To(Succeed()) - p.Status.ContainerStatuses = []corev1.ContainerStatus{ - { - Name: "Container-0", - State: corev1.ContainerState{ - Waiting: &corev1.ContainerStateWaiting{ - Reason: "CrashLoopBackOff", - Message: "Container is in CrashLoopBackOff.", - }, - }, - }, - } - err = c.Status().Update(context.TODO(), p) - Expect(err).NotTo(HaveOccurred()) - storeSecret := instance.Spec.Backup.Store.SecretRef.Name - errors := createSecrets(c, instance.Namespace, storeSecret) - Expect(len(errors)).Should(BeZero()) - - }) - It("should restart pod", func() { - Expect(c.Create(context.TODO(), instance)).To(Succeed()) - Eventually(func() error { return podDeleted(c, instance) }, timeout, pollingInterval).Should(BeNil()) - }) - AfterEach(func() { - s := &appsv1.StatefulSet{} - Eventually(func() error { return statefulsetIsCorrectlyReconciled(c, instance, s) }, timeout, pollingInterval).Should(BeNil()) - setStatefulSetReady(s) - Expect(c.Status().Update(context.TODO(), s)).To(Succeed()) - Expect(c.Delete(context.TODO(), instance)).To(Succeed()) - }) - }) - }) - - DescribeTable("when deleting etcd resources", - func(name string, setupStatefulSet StatefulSetInitializer) { - var sts *appsv1.StatefulSet - instance := getEtcd(name, "default", false) - c := mgr.GetClient() - - switch setupStatefulSet { - case WithoutOwner: - sts = createStatefulset(name, "default", instance.Spec.Labels) - Expect(sts.OwnerReferences).Should(BeNil()) - case WithOwnerReference: - sts = createStatefulsetWithOwnerReference(instance) - Expect(len(sts.OwnerReferences)).ShouldNot(BeZero()) - case WithOwnerAnnotation: - sts = createStatefulsetWithOwnerAnnotations(instance) - default: - Fail("StatefulSetInitializer invalid") - } - stopCh := make(chan struct{}) - storeSecret := instance.Spec.Backup.Store.SecretRef.Name - needsOwnerRefUpdate := len(sts.OwnerReferences) > 0 - - // Create Secrets - errors := createSecrets(c, instance.Namespace, storeSecret) - Expect(len(errors)).Should(BeZero()) - - // Create StatefulSet - Expect(c.Create(context.TODO(), sts)).To(Succeed()) - - // Create StatefulSet - Expect(c.Create(context.TODO(), instance)).To(Succeed()) - - // Update OwnerRef with UID from just created `etcd` instance - if needsOwnerRefUpdate { - Eventually(func() error { - if err := c.Get(context.TODO(), client.ObjectKeyFromObject(instance), instance); err != nil { - return err - } - if instance.UID != "" { - instance.TypeMeta = metav1.TypeMeta{ - APIVersion: "druid.gardener.cloud/v1alpha1", - Kind: "etcd", - } - return nil - } - return fmt.Errorf("etcd object not yet created") - }, timeout, pollingInterval).Should(BeNil()) - sts.OwnerReferences[0].UID = instance.UID - Expect(c.Update(context.TODO(), sts)).To(Succeed()) - } - - sts = &appsv1.StatefulSet{} - Eventually(func() error { return statefulsetIsCorrectlyReconciled(c, instance, sts) }, timeout, pollingInterval).Should(BeNil()) - // This go-routine is to set that statefulset is ready manually as statefulset controller is absent for tests. - go func() { - for { - select { - case <-time.After(time.Second * 2): - Expect(setAndCheckStatefulSetReady(c, instance)).To(Succeed()) - case <-stopCh: - return - } - } - }() - Eventually(func() error { return isEtcdReady(c, instance) }, timeout, pollingInterval).Should(BeNil()) - close(stopCh) - Expect(c.Delete(context.TODO(), instance)).To(Succeed()) - Eventually(func() error { return statefulSetRemoved(c, sts) }, timeout, pollingInterval).Should(BeNil()) - Eventually(func() error { return etcdRemoved(c, instance) }, timeout, pollingInterval).Should(BeNil()) - }, - Entry("when statefulset with ownerReference and without owner annotations, druid should adopt and delete statefulset", "foo25", WithOwnerReference), - Entry("when statefulset without ownerReference and without owner annotations, druid should adopt and delete statefulset", "foo26", WithoutOwner), - Entry("when statefulset without ownerReference and with owner annotations, druid should adopt and delete statefulset", "foo27", WithOwnerAnnotation), - ) - DescribeTable("when etcd resource is created", func(name string, generateEtcd func(string, string) *druidv1alpha1.Etcd, validate func(*druidv1alpha1.Etcd, *appsv1.StatefulSet, *corev1.ConfigMap, *corev1.Service, *corev1.Service)) { var err error @@ -942,26 +766,6 @@ func validateRole(instance *druidv1alpha1.Etcd, role *rbac.Role) { })) } -func podDeleted(c client.Client, etcd *druidv1alpha1.Etcd) error { - ctx, cancel := context.WithTimeout(context.TODO(), timeout) - defer cancel() - pod := &corev1.Pod{} - req := types.NamespacedName{ - Name: fmt.Sprintf("%s-0", etcd.Name), - Namespace: etcd.Namespace, - } - if err := c.Get(ctx, req, pod); err != nil { - if errors.IsNotFound(err) { - // Object not found, return. Created objects are automatically garbage collected. - // For additional cleanup logic use finalizers - return nil - } - return err - } - return fmt.Errorf("pod not deleted") - -} - func validateEtcdWithDefaults(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev1.ConfigMap, clSvc *corev1.Service, prSvc *corev1.Service) { configYML := cm.Data[etcdConfig] config := map[string]interface{}{} @@ -1641,8 +1445,8 @@ func validateEtcd(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev fmt.Sprintf("%s=%s", "--owner-check-interval", instance.Spec.Backup.OwnerCheck.Interval.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--owner-check-interval", instance.Spec.Backup.OwnerCheck.Interval.Duration.String())), fmt.Sprintf("%s=%s", "--owner-check-timeout", instance.Spec.Backup.OwnerCheck.Timeout.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--owner-check-timeout", instance.Spec.Backup.OwnerCheck.Timeout.Duration.String())), fmt.Sprintf("%s=%s", "--owner-check-dns-cache-ttl", instance.Spec.Backup.OwnerCheck.DNSCacheTTL.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--owner-check-dns-cache-ttl", instance.Spec.Backup.OwnerCheck.DNSCacheTTL.Duration.String())), - fmt.Sprintf("%s=%s", "--delta-snapshot-lease-name", componentlease.GetDeltaSnapshotLeaseName(instance)): Equal(fmt.Sprintf("%s=%s", "--delta-snapshot-lease-name", componentlease.GetDeltaSnapshotLeaseName(instance))), - fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", componentlease.GetFullSnapshotLeaseName(instance)): Equal(fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", componentlease.GetFullSnapshotLeaseName(instance))), + fmt.Sprintf("%s=%s", "--delta-snapshot-lease-name", utils.GetDeltaSnapshotLeaseName(instance)): Equal(fmt.Sprintf("%s=%s", "--delta-snapshot-lease-name", utils.GetDeltaSnapshotLeaseName(instance))), + fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", utils.GetFullSnapshotLeaseName(instance)): Equal(fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", utils.GetFullSnapshotLeaseName(instance))), }), "Ports": ConsistOf([]corev1.ContainerPort{ { @@ -2093,7 +1897,7 @@ func etcdRemoved(c client.Client, etcd *druidv1alpha1.Etcd) error { Namespace: etcd.Namespace, } if err := c.Get(ctx, req, e); err != nil { - if errors.IsNotFound(err) { + if apierrors.IsNotFound(err) { // Object not found, return. Created objects are automatically garbage collected. // For additional cleanup logic use finalizers return nil @@ -2103,44 +1907,6 @@ func etcdRemoved(c client.Client, etcd *druidv1alpha1.Etcd) error { return fmt.Errorf("etcd not deleted") } -func isEtcdReady(c client.Client, etcd *druidv1alpha1.Etcd) error { - ctx, cancel := context.WithTimeout(context.TODO(), timeout) - defer cancel() - e := &druidv1alpha1.Etcd{} - req := types.NamespacedName{ - Name: etcd.Name, - Namespace: etcd.Namespace, - } - if err := c.Get(ctx, req, e); err != nil { - return err - } - if e.Status.Ready == nil || !*e.Status.Ready { - return fmt.Errorf("etcd not ready") - } - return nil -} - -func setAndCheckStatefulSetReady(c client.Client, etcd *druidv1alpha1.Etcd) error { - ctx, cancel := context.WithTimeout(context.TODO(), timeout) - defer cancel() - ss := &appsv1.StatefulSet{} - req := types.NamespacedName{ - Name: etcd.Name, - Namespace: etcd.Namespace, - } - if err := c.Get(ctx, req, ss); err != nil { - return err - } - setStatefulSetReady(ss) - if err := c.Status().Update(context.TODO(), ss); err != nil { - return err - } - if err := health.CheckStatefulSet(ss); err != nil { - return err - } - return nil -} - func statefulSetRemoved(c client.Client, ss *appsv1.StatefulSet) error { ctx, cancel := context.WithTimeout(context.TODO(), timeout) defer cancel() @@ -2150,7 +1916,7 @@ func statefulSetRemoved(c client.Client, ss *appsv1.StatefulSet) error { Namespace: ss.Namespace, } if err := c.Get(ctx, req, sts); err != nil { - if errors.IsNotFound(err) { + if apierrors.IsNotFound(err) { // Object not found, return. Created objects are automatically garbage collected. // For additional cleanup logic use finalizers return nil @@ -2171,11 +1937,8 @@ func statefulsetIsCorrectlyReconciled(c client.Client, instance *druidv1alpha1.E if err := c.Get(ctx, req, ss); err != nil { return err } - if !checkEtcdAnnotations(ss.GetAnnotations(), instance) { - return fmt.Errorf("no annotations") - } - if checkEtcdOwnerReference(ss.GetOwnerReferences(), instance) { - return fmt.Errorf("ownerReference exists") + if !checkEtcdOwnerReference(ss.GetOwnerReferences(), instance) { + return fmt.Errorf("ownerReference does not exist") } return nil } @@ -2276,93 +2039,6 @@ func roleBindingIsCorrectlyReconciled(c client.Client, instance *druidv1alpha1.E return nil } -func createStatefulset(name, namespace string, labels map[string]string) *appsv1.StatefulSet { - var replicas int32 = 0 - ss := appsv1.StatefulSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - Labels: labels, - }, - Spec: appsv1.StatefulSetSpec{ - Replicas: &replicas, - Selector: &metav1.LabelSelector{ - MatchLabels: labels, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-0", name), - Namespace: namespace, - Labels: labels, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "etcd", - Image: "eu.gcr.io/gardener-project/gardener/etcd:v3.4.13-bootstrap", - }, - { - Name: "backup-restore", - Image: "eu.gcr.io/gardener-project/gardener/etcdbrctl:v0.12.0", - }, - }, - }, - }, - VolumeClaimTemplates: []corev1.PersistentVolumeClaim{}, - ServiceName: "etcd-client", - UpdateStrategy: appsv1.StatefulSetUpdateStrategy{}, - }, - } - return &ss -} - -func createStatefulsetWithOwnerReference(etcd *druidv1alpha1.Etcd) *appsv1.StatefulSet { - ss := createStatefulset(etcd.Name, etcd.Namespace, etcd.Spec.Labels) - ss.SetOwnerReferences([]metav1.OwnerReference{ - { - APIVersion: "druid.gardener.cloud/v1alpha1", - Kind: "etcd", - Name: etcd.Name, - UID: "foo", - Controller: pointer.BoolPtr(true), - BlockOwnerDeletion: pointer.BoolPtr(true), - }, - }) - return ss -} - -func createStatefulsetWithOwnerAnnotations(etcd *druidv1alpha1.Etcd) *appsv1.StatefulSet { - ss := createStatefulset(etcd.Name, etcd.Namespace, etcd.Spec.Labels) - ss.SetAnnotations(map[string]string{ - "gardener.cloud/owned-by": fmt.Sprintf("%s/%s", etcd.Namespace, etcd.Name), - "gardener.cloud/owner-type": "etcd", - }) - return ss -} - -func createPod(name, namespace string, labels map[string]string) *corev1.Pod { - pod := corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - Labels: labels, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "etcd", - Image: "eu.gcr.io/gardener-project/gardener/etcd:v3.4.13-bootstrap", - }, - { - Name: "backup-restore", - Image: "eu.gcr.io/gardener-project/gardener/etcdbrctl:v0.12.0", - }, - }, - }, - } - return &pod -} - func getEtcdWithGCS(name, namespace string) *druidv1alpha1.Etcd { provider := druidv1alpha1.StorageProvider("gcp") etcd := getEtcdWithTLS(name, namespace) @@ -2475,6 +2151,7 @@ func getEtcd(name, namespace string, tlsEnabled bool) *druidv1alpha1.Etcd { ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: namespace, + UID: types.UID(uid), }, Spec: druidv1alpha1.EtcdSpec{ Annotations: map[string]string{ diff --git a/pkg/component/etcd/lease/values_helper.go b/pkg/component/etcd/lease/values_helper.go index 914e56540..31735763a 100644 --- a/pkg/component/etcd/lease/values_helper.go +++ b/pkg/component/etcd/lease/values_helper.go @@ -15,9 +15,8 @@ package lease import ( - "fmt" - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/pkg/utils" ) // GenerateValues generates `lease.Values` for the lease component with the given parameters. @@ -26,18 +25,8 @@ func GenerateValues(etcd *druidv1alpha1.Etcd) Values { BackupEnabled: etcd.Spec.Backup.Store != nil, EtcdName: etcd.Name, EtcdUID: etcd.UID, - DeltaSnapshotLeaseName: GetDeltaSnapshotLeaseName(etcd), - FullSnapshotLeaseName: GetFullSnapshotLeaseName(etcd), + DeltaSnapshotLeaseName: utils.GetDeltaSnapshotLeaseName(etcd), + FullSnapshotLeaseName: utils.GetFullSnapshotLeaseName(etcd), Replicas: etcd.Spec.Replicas, } } - -// GetDeltaSnapshotLeaseName returns the name of the delta snapshot lease based on the given `etcd` object. -func GetDeltaSnapshotLeaseName(etcd *druidv1alpha1.Etcd) string { - return fmt.Sprintf("%s-delta-snap", etcd.Name) -} - -// GetFullSnapshotLeaseName returns the name of the full snapshot lease based on the given `etcd` object. -func GetFullSnapshotLeaseName(etcd *druidv1alpha1.Etcd) string { - return fmt.Sprintf("%s-full-snap", etcd.Name) -} diff --git a/pkg/component/etcd/statefulset/statefulset.go b/pkg/component/etcd/statefulset/statefulset.go new file mode 100644 index 000000000..82b329b62 --- /dev/null +++ b/pkg/component/etcd/statefulset/statefulset.go @@ -0,0 +1,239 @@ +// Copyright (c) 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package statefulset + +import ( + "context" + "fmt" + "strconv" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + + "github.com/gardener/gardener/pkg/controllerutils" + gardenercomponent "github.com/gardener/gardener/pkg/operation/botanist/component" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type component struct { + client client.Client + namespace string + + values Values +} + +func (c *component) Deploy(ctx context.Context) error { + var ( + etcdMainSts = c.emptyStatefulset(c.values.EtcdMainStsName) + //etcdEventSts = c.emptyStatefulset(c.values.EtcdEventStsName) + ) + + if err := c.syncEtcdMainSts(ctx, etcdMainSts); err != nil { + return err + } + + /*if err := c.syncEtcdEventSts(ctx, etcdEventSts); err != nil { + return err + }*/ + + return nil +} + +func (c *component) Destroy(ctx context.Context) error { + etcdMainSts := c.emptyStatefulset(c.values.EtcdMainStsName) + + if err := c.deleteSts(ctx, etcdMainSts); err != nil { + return err + } + return nil +} + +func (c *component) syncEtcdMainSts(ctx context.Context, sts *appsv1.StatefulSet) error { + _, err := controllerutils.GetAndCreateOrStrategicMergePatch(ctx, c.client, sts, func() error { + sts.ObjectMeta = getObjectMeta(&c.values) + sts.Spec = appsv1.StatefulSetSpec{ + VolumeClaimTemplates: []v1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: c.values.VolumeClaimTemplateName, + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{ + v1.ReadWriteOnce, + }, + StorageClassName: c.values.StorageClass, + Resources: getStorageReq(c.values), + }, + }, + }, + PodManagementPolicy: appsv1.ParallelPodManagement, + UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ + Type: appsv1.RollingUpdateStatefulSetStrategyType, + }, + Replicas: pointer.Int32(c.values.Replicas), + ServiceName: c.values.ServiceName, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "name": "etcd", + "instance": c.values.EtcdName, + }, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: c.values.Annotations, + Labels: c.values.Labels, + }, + Spec: v1.PodSpec{ + HostAliases: []v1.HostAlias{ + { + IP: "127.0.0.1", + Hostnames: []string{c.values.EtcdName + "-local"}, + }, + }, + ServiceAccountName: c.values.ServiceAccountName, + Affinity: c.values.Affinity, + TopologySpreadConstraints: c.values.TopologySpreadConstraints, + Containers: []v1.Container{ + { + Name: "etcd", + Image: c.values.EtcdImage, + ImagePullPolicy: v1.PullIfNotPresent, + Command: c.values.EtcdCommand, + ReadinessProbe: &v1.Probe{ + Handler: v1.Handler{ + Exec: &v1.ExecAction{ + Command: c.values.ReadinessProbeCommand, + }, + }, + InitialDelaySeconds: 15, + PeriodSeconds: 5, + FailureThreshold: 5, + }, + LivenessProbe: &v1.Probe{ + Handler: v1.Handler{ + Exec: &v1.ExecAction{ + Command: c.values.LivenessProbCommand, + }, + }, + InitialDelaySeconds: 15, + PeriodSeconds: 5, + FailureThreshold: 5, + }, + Ports: getEtcdPorts(c.values), + Resources: getEtcdResources(c.values), + Env: []v1.EnvVar{ + { + Name: "ENABLE_TLS", + Value: c.values.EnableClientTLS, + }, + { + Name: "BACKUP_ENDPOINT", + Value: strconv.FormatBool(c.values.BackupTLS != nil), + }, + }, + VolumeMounts: getEtcdVolumeMounts(c.values), + }, + { + Name: "backup-restore", + Image: c.values.BackupImage, + ImagePullPolicy: v1.PullIfNotPresent, + Command: c.values.EtcdBackupCommand, + Ports: getBackupPorts(c.values), + Resources: getBackupResources(c.values), + Env: getStsEnvVar(c.values), + VolumeMounts: getBackupRestoreVolumeMounts(c.values), + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{ + Add: []v1.Capability{ + v1.Capability("SYS_PTRACE"), + }, + }, + }, + }, + }, + ShareProcessNamespace: pointer.Bool(true), + Volumes: getBackupRestoreVolumes(c.values), + }, + }, + } + if c.values.PriorityClassName != nil { + sts.Spec.Template.Spec.PriorityClassName = *c.values.PriorityClassName + } + return nil + }) + return err +} + +func (c *component) deleteSts(ctx context.Context, sts *appsv1.StatefulSet) error { + return client.IgnoreNotFound(c.client.Delete(ctx, sts)) +} + +// New creates a new statefulset deployer instance. +func New(c client.Client, namespace string, values Values) gardenercomponent.Deployer { + return &component{ + client: c, + namespace: namespace, + values: values, + } +} + +func (c *component) emptyStatefulset(name string) *appsv1.StatefulSet { + return &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: c.namespace, + }, + } +} + +func getObjectMeta(val *Values) metav1.ObjectMeta { + labels := map[string]string{"name": "etcd", "instance": val.EtcdName} + for key, value := range val.Labels { + labels[key] = value + } + + annotations := map[string]string{ + "gardener.cloud/owned-by": fmt.Sprintf("%s/%s", val.EtcdNameSpace, val.EtcdName), + "gardener.cloud/owner-type": "etcd", + } + + if val.Annotations != nil { + for key, value := range val.Annotations { + annotations[key] = value + } + } + + ownerRefs := []metav1.OwnerReference{ + { + APIVersion: druidv1alpha1.GroupVersion.String(), + Kind: "Etcd", + Name: val.EtcdName, + UID: val.EtcdUID, + Controller: pointer.BoolPtr(true), + BlockOwnerDeletion: pointer.BoolPtr(true), + }, + } + + return metav1.ObjectMeta{ + Name: val.EtcdMainStsName, + Namespace: val.EtcdNameSpace, + Labels: labels, + Annotations: annotations, + OwnerReferences: ownerRefs, + } +} diff --git a/pkg/component/etcd/statefulset/statefulset_suite_test.go b/pkg/component/etcd/statefulset/statefulset_suite_test.go new file mode 100644 index 000000000..ca290b2e6 --- /dev/null +++ b/pkg/component/etcd/statefulset/statefulset_suite_test.go @@ -0,0 +1,27 @@ +// Copyright (c) 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package statefulset + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestService(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Statefulset Component Suite") +} diff --git a/pkg/component/etcd/statefulset/statefulset_test.go b/pkg/component/etcd/statefulset/statefulset_test.go new file mode 100644 index 000000000..e096c1a11 --- /dev/null +++ b/pkg/component/etcd/statefulset/statefulset_test.go @@ -0,0 +1,716 @@ +// Copyright (c) 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package statefulset_test + +import ( + "context" + "fmt" + "time" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/pkg/client/kubernetes" + "github.com/gardener/etcd-druid/pkg/common" + . "github.com/gardener/etcd-druid/pkg/component/etcd/statefulset" + + druidutils "github.com/gardener/etcd-druid/pkg/utils" + "github.com/gardener/gardener/pkg/operation/botanist/component" + kutil "github.com/gardener/gardener/pkg/utils/kubernetes" + . "github.com/gardener/gardener/pkg/utils/test/matchers" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +var ( + backupRestore = "backup-restore" + deltaSnapshotPeriod = metav1.Duration{ + Duration: 300 * time.Second, + } + garbageCollectionPeriod = metav1.Duration{ + Duration: 43200 * time.Second, + } + clientPort int32 = 2379 + serverPort int32 = 2380 + backupPort int32 = 8080 + uid = "a9b8c7d6e5f4" + imageEtcd = "eu.gcr.io/gardener-project/gardener/etcd:v3.4.13-bootstrap" + imageBR = "eu.gcr.io/gardener-project/gardener/etcdbrctl:v0.12.0" + snapshotSchedule = "0 */24 * * *" + defragSchedule = "0 */24 * * *" + container = "default.bkp" + storageCapacity = resource.MustParse("5Gi") + //defaultStorageCapacity = resource.MustParse("16Gi") + storageClass = "gardener.fast" + priorityClassName = "class_priority" + deltaSnapShotMemLimit = resource.MustParse("100Mi") + autoCompactionMode = druidv1alpha1.Periodic + autoCompactionRetention = "2m" + quota = resource.MustParse("8Gi") + //provider = druidv1alpha1.StorageProvider("Local") + prefix = "/tmp" + volumeClaimTemplateName = "etcd-main" + garbageCollectionPolicy = druidv1alpha1.GarbageCollectionPolicy(druidv1alpha1.GarbageCollectionPolicyExponential) + metricsBasic = druidv1alpha1.Basic + //maxBackups = 7 + etcdSnapshotTimeout = metav1.Duration{ + Duration: 10 * time.Minute, + } + etcdDefragTimeout = metav1.Duration{ + Duration: 10 * time.Minute, + } + etcdConnectionTimeout = metav1.Duration{ + Duration: 5 * time.Minute, + } + ownerName = "owner.foo.example.com" + ownerID = "bar" + ownerCheckInterval = metav1.Duration{ + Duration: 30 * time.Second, + } + ownerCheckTimeout = metav1.Duration{ + Duration: 2 * time.Minute, + } + ownerCheckDNSCacheTTL = metav1.Duration{ + Duration: 1 * time.Minute, + } + heartbeatDuration = metav1.Duration{ + Duration: 10 * time.Second, + } +) + +var _ = Describe("Statefulset", func() { + var ( + ctx context.Context + cl client.Client + + etcd *druidv1alpha1.Etcd + namespace string + name string + + sts *appsv1.StatefulSet + + values Values + stsDeployer component.Deployer + ) + + BeforeEach(func() { + ctx = context.Background() + cl = fakeclient.NewClientBuilder().WithScheme(kubernetes.Scheme).Build() + + name = "statefulset" + namespace = "default" + quota = resource.MustParse("8Gi") + + etcd = getEtcd(name, namespace, true) + + values = GenerateValues(etcd) + + sts = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: values.EtcdMainStsName, + Namespace: values.EtcdNameSpace, + }, + } + + stsDeployer = New(cl, namespace, values) + }) + + Describe("#Deploy", func() { + Context("when statefulset does not exist", func() { + It("should create the statefulset successfully", func() { + Expect(stsDeployer.Deploy(ctx)).To(Succeed()) + + sts := &appsv1.StatefulSet{} + + Expect(cl.Get(ctx, kutil.Key(namespace, values.EtcdMainStsName), sts)).To(Succeed()) + checkStatefulset(sts, values) + + }) + }) + + Context("when statefulset exists", func() { + It("should update the statefulset successfully", func() { + Expect(cl.Create(ctx, sts)).To(Succeed()) + + Expect(stsDeployer.Deploy(ctx)).To(Succeed()) + + sts := &appsv1.StatefulSet{} + + Expect(cl.Get(ctx, kutil.Key(namespace, values.EtcdMainStsName), sts)).To(Succeed()) + checkStatefulset(sts, values) + }) + }) + }) + + Describe("#Destroy", func() { + Context("when configmap do not exist", func() { + It("should destroy successfully", func() { + Expect(stsDeployer.Destroy(ctx)).To(Succeed()) + Expect(cl.Get(ctx, client.ObjectKeyFromObject(sts), &appsv1.StatefulSet{})).To(BeNotFoundError()) + }) + }) + + Context("when configmap exist", func() { + It("should destroy successfully", func() { + Expect(cl.Create(ctx, sts)).To(Succeed()) + + Expect(stsDeployer.Destroy(ctx)).To(Succeed()) + + Expect(cl.Get(ctx, kutil.Key(namespace, sts.Name), &appsv1.StatefulSet{})).To(BeNotFoundError()) + }) + }) + }) +}) + +func checkStatefulset(sts *appsv1.StatefulSet, values Values) { + checkStsMetadata(sts.ObjectMeta.OwnerReferences, values) + + readinessProbeUrl := fmt.Sprintf("https://%s-local:%d/health", values.EtcdName, clientPort) + if int(values.Replicas) == 1 { + readinessProbeUrl = fmt.Sprintf("https://%s-local:%d/healthz", values.EtcdName, backupPort) + } + + store, err := druidutils.StorageProviderFromInfraProvider(values.BackupStore.Provider) + Expect(err).NotTo(HaveOccurred()) + Expect(*sts).To(MatchFields(IgnoreExtras, Fields{ + "ObjectMeta": MatchFields(IgnoreExtras, Fields{ + "Name": Equal(values.EtcdName), + "Namespace": Equal(values.EtcdNameSpace), + "Annotations": MatchAllKeys(Keys{ + "gardener.cloud/owned-by": Equal(fmt.Sprintf("%s/%s", values.EtcdNameSpace, values.EtcdName)), + "gardener.cloud/owner-type": Equal("etcd"), + "app": Equal("etcd-statefulset"), + "role": Equal("test"), + "instance": Equal(values.EtcdName), + }), + "Labels": MatchAllKeys(Keys{ + "name": Equal("etcd"), + "instance": Equal(values.EtcdName), + }), + }), + + "Spec": MatchFields(IgnoreExtras, Fields{ + "UpdateStrategy": MatchFields(IgnoreExtras, Fields{ + "Type": Equal(appsv1.RollingUpdateStatefulSetStrategyType), + }), + "Replicas": PointTo(Equal(int32(values.Replicas))), + "Selector": PointTo(MatchFields(IgnoreExtras, Fields{ + "MatchLabels": MatchAllKeys(Keys{ + "name": Equal("etcd"), + "instance": Equal(values.EtcdName), + }), + })), + "Template": MatchFields(IgnoreExtras, Fields{ + "ObjectMeta": MatchFields(IgnoreExtras, Fields{ + "Annotations": MatchKeys(IgnoreExtras, Keys{ + "app": Equal("etcd-statefulset"), + "role": Equal("test"), + "instance": Equal(values.EtcdName), + }), + "Labels": MatchAllKeys(Keys{ + "name": Equal("etcd"), + "instance": Equal(values.EtcdName), + }), + }), + //s.Spec.Template.Spec.HostAliases + "Spec": MatchFields(IgnoreExtras, Fields{ + "HostAliases": MatchAllElements(hostAliasIterator, Elements{ + "127.0.0.1": MatchFields(IgnoreExtras, Fields{ + "IP": Equal("127.0.0.1"), + "Hostnames": MatchAllElements(cmdIterator, Elements{ + fmt.Sprintf("%s-local", values.EtcdName): Equal(fmt.Sprintf("%s-local", values.EtcdName)), + }), + }), + }), + "Containers": MatchAllElements(containerIterator, Elements{ + common.Etcd: MatchFields(IgnoreExtras, Fields{ + "Ports": ConsistOf([]corev1.ContainerPort{ + { + Name: "server", + Protocol: corev1.ProtocolTCP, + HostPort: 0, + ContainerPort: *values.ServerPort, + }, + { + Name: "client", + Protocol: corev1.ProtocolTCP, + HostPort: 0, + ContainerPort: *values.ClientPort, + }, + }), + "Command": MatchAllElements(cmdIterator, Elements{ + "/var/etcd/bin/bootstrap.sh": Equal("/var/etcd/bin/bootstrap.sh"), + }), + "ImagePullPolicy": Equal(corev1.PullIfNotPresent), + "Image": Equal(values.EtcdImage), + "ReadinessProbe": PointTo(MatchFields(IgnoreExtras, Fields{ + "Handler": MatchFields(IgnoreExtras, Fields{ + "Exec": PointTo(MatchFields(IgnoreExtras, Fields{ + "Command": MatchAllElements(cmdIterator, Elements{ + "/usr/bin/curl": Equal("/usr/bin/curl"), + "--cert": Equal("--cert"), + "/var/etcd/ssl/client/client/tls.crt": Equal("/var/etcd/ssl/client/client/tls.crt"), + "--key": Equal("--key"), + "/var/etcd/ssl/client/client/tls.key": Equal("/var/etcd/ssl/client/client/tls.key"), + "--cacert": Equal("--cacert"), + "/var/etcd/ssl/client/ca/ca.crt": Equal("/var/etcd/ssl/client/ca/ca.crt"), + readinessProbeUrl: Equal(readinessProbeUrl), + }), + })), + }), + "InitialDelaySeconds": Equal(int32(15)), + "PeriodSeconds": Equal(int32(5)), + })), + "LivenessProbe": PointTo(MatchFields(IgnoreExtras, Fields{ + "Handler": MatchFields(IgnoreExtras, Fields{ + "Exec": PointTo(MatchFields(IgnoreExtras, Fields{ + "Command": MatchAllElements(cmdIterator, Elements{ + "/bin/sh": Equal("/bin/sh"), + "-ec": Equal("-ec"), + "ETCDCTL_API=3": Equal("ETCDCTL_API=3"), + "etcdctl": Equal("etcdctl"), + "--cert=/var/etcd/ssl/client/client/tls.crt": Equal("--cert=/var/etcd/ssl/client/client/tls.crt"), + "--key=/var/etcd/ssl/client/client/tls.key": Equal("--key=/var/etcd/ssl/client/client/tls.key"), + "--cacert=/var/etcd/ssl/client/ca/ca.crt": Equal("--cacert=/var/etcd/ssl/client/ca/ca.crt"), + fmt.Sprintf("--endpoints=https://%s-local:%d", values.EtcdName, clientPort): Equal(fmt.Sprintf("--endpoints=https://%s-local:%d", values.EtcdName, clientPort)), + "get": Equal("get"), + "foo": Equal("foo"), + "--consistency=s": Equal("--consistency=s"), + }), + })), + }), + "InitialDelaySeconds": Equal(int32(15)), + "PeriodSeconds": Equal(int32(5)), + })), + "VolumeMounts": MatchAllElements(volumeMountIterator, Elements{ + values.VolumeClaimTemplateName: MatchFields(IgnoreExtras, Fields{ + "Name": Equal(values.VolumeClaimTemplateName), + "MountPath": Equal("/var/etcd/data/"), + }), + "client-url-ca-etcd": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("client-url-ca-etcd"), + "MountPath": Equal("/var/etcd/ssl/client/ca"), + }), + "client-url-etcd-server-tls": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("client-url-etcd-server-tls"), + "MountPath": Equal("/var/etcd/ssl/client/server"), + }), + "client-url-etcd-client-tls": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("client-url-etcd-client-tls"), + "MountPath": Equal("/var/etcd/ssl/client/client"), + }), + "peer-url-ca-etcd": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("peer-url-ca-etcd"), + "MountPath": Equal("/var/etcd/ssl/peer/ca"), + }), + "peer-url-etcd-server-tls": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("peer-url-etcd-server-tls"), + "MountPath": Equal("/var/etcd/ssl/peer/server"), + }), + }), + }), + + backupRestore: MatchFields(IgnoreExtras, Fields{ + "Command": MatchAllElements(cmdIterator, Elements{ + "etcdbrctl": Equal("etcdbrctl"), + "server": Equal("server"), + "--cert=/var/etcd/ssl/client/client/tls.crt": Equal("--cert=/var/etcd/ssl/client/client/tls.crt"), + "--key=/var/etcd/ssl/client/client/tls.key": Equal("--key=/var/etcd/ssl/client/client/tls.key"), + "--cacert=/var/etcd/ssl/client/ca/ca.crt": Equal("--cacert=/var/etcd/ssl/client/ca/ca.crt"), + "--server-cert=/var/etcd/ssl/client/server/tls.crt": Equal("--server-cert=/var/etcd/ssl/client/server/tls.crt"), + "--server-key=/var/etcd/ssl/client/server/tls.key": Equal("--server-key=/var/etcd/ssl/client/server/tls.key"), + "--data-dir=/var/etcd/data/new.etcd": Equal("--data-dir=/var/etcd/data/new.etcd"), + "--insecure-transport=false": Equal("--insecure-transport=false"), + "--insecure-skip-tls-verify=false": Equal("--insecure-skip-tls-verify=false"), + "--snapstore-temp-directory=/var/etcd/data/temp": Equal("--snapstore-temp-directory=/var/etcd/data/temp"), + "--etcd-process-name=etcd": Equal("--etcd-process-name=etcd"), + fmt.Sprintf("%s=%s", "--etcd-connection-timeout", etcdConnectionTimeout.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--etcd-connection-timeout", values.LeaderElection.EtcdConnectionTimeout.Duration.String())), + "--enable-snapshot-lease-renewal=true": Equal("--enable-snapshot-lease-renewal=true"), + "--enable-member-lease-renewal=true": Equal("--enable-member-lease-renewal=true"), + "--k8s-heartbeat-duration=10s": Equal("--k8s-heartbeat-duration=10s"), + fmt.Sprintf("--defragmentation-schedule=%s", *values.DefragmentationSchedule): Equal(fmt.Sprintf("--defragmentation-schedule=%s", *values.DefragmentationSchedule)), + fmt.Sprintf("--schedule=%s", *values.FullSnapshotSchedule): Equal(fmt.Sprintf("--schedule=%s", *values.FullSnapshotSchedule)), + fmt.Sprintf("%s=%s", "--garbage-collection-policy", *values.GarbageCollectionPolicy): Equal(fmt.Sprintf("%s=%s", "--garbage-collection-policy", *values.GarbageCollectionPolicy)), + fmt.Sprintf("%s=%s", "--storage-provider", store): Equal(fmt.Sprintf("%s=%s", "--storage-provider", store)), + fmt.Sprintf("%s=%s", "--store-prefix", values.BackupStore.Prefix): Equal(fmt.Sprintf("%s=%s", "--store-prefix", values.BackupStore.Prefix)), + fmt.Sprintf("--delta-snapshot-memory-limit=%d", values.DeltaSnapshotMemoryLimit.Value()): Equal(fmt.Sprintf("--delta-snapshot-memory-limit=%d", values.DeltaSnapshotMemoryLimit.Value())), + fmt.Sprintf("--garbage-collection-policy=%s", *values.GarbageCollectionPolicy): Equal(fmt.Sprintf("--garbage-collection-policy=%s", *values.GarbageCollectionPolicy)), + fmt.Sprintf("--endpoints=https://%s-local:%d", values.EtcdName, clientPort): Equal(fmt.Sprintf("--endpoints=https://%s-local:%d", values.EtcdName, clientPort)), + fmt.Sprintf("--embedded-etcd-quota-bytes=%d", int64(values.Quota.Value())): Equal(fmt.Sprintf("--embedded-etcd-quota-bytes=%d", int64(values.Quota.Value()))), + fmt.Sprintf("%s=%s", "--delta-snapshot-period", values.DeltaSnapshotPeriod.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--delta-snapshot-period", values.DeltaSnapshotPeriod.Duration.String())), + fmt.Sprintf("%s=%s", "--garbage-collection-period", values.GarbageCollectionPeriod.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--garbage-collection-period", values.GarbageCollectionPeriod.Duration.String())), + fmt.Sprintf("%s=%s", "--auto-compaction-mode", *values.AutoCompactionMode): Equal(fmt.Sprintf("%s=%s", "--auto-compaction-mode", *values.AutoCompactionMode)), + fmt.Sprintf("%s=%s", "--auto-compaction-retention", *values.AutoCompactionRetention): Equal(fmt.Sprintf("%s=%s", "--auto-compaction-retention", *values.AutoCompactionRetention)), + fmt.Sprintf("%s=%s", "--etcd-snapshot-timeout", values.EtcdSnapshotTimeout.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--etcd-snapshot-timeout", values.EtcdSnapshotTimeout.Duration.String())), + fmt.Sprintf("%s=%s", "--etcd-defrag-timeout", values.EtcdDefragTimeout.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--etcd-defrag-timeout", values.EtcdDefragTimeout.Duration.String())), + fmt.Sprintf("%s=%s", "--owner-name", values.OwnerCheck.Name): Equal(fmt.Sprintf("%s=%s", "--owner-name", values.OwnerCheck.Name)), + fmt.Sprintf("%s=%s", "--owner-id", values.OwnerCheck.ID): Equal(fmt.Sprintf("%s=%s", "--owner-id", values.OwnerCheck.ID)), + fmt.Sprintf("%s=%s", "--owner-check-interval", values.OwnerCheck.Interval.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--owner-check-interval", values.OwnerCheck.Interval.Duration.String())), + fmt.Sprintf("%s=%s", "--owner-check-timeout", values.OwnerCheck.Timeout.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--owner-check-timeout", values.OwnerCheck.Timeout.Duration.String())), + fmt.Sprintf("%s=%s", "--owner-check-dns-cache-ttl", values.OwnerCheck.DNSCacheTTL.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--owner-check-dns-cache-ttl", values.OwnerCheck.DNSCacheTTL.Duration.String())), + fmt.Sprintf("%s=%s", "--delta-snapshot-lease-name", values.DeltaSnapLeaseName): Equal(fmt.Sprintf("%s=%s", "--delta-snapshot-lease-name", values.DeltaSnapLeaseName)), + fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", values.FullSnapLeaseName): Equal(fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", values.FullSnapLeaseName)), + }), + "Ports": ConsistOf([]corev1.ContainerPort{ + { + Name: "server", + Protocol: corev1.ProtocolTCP, + HostPort: 0, + ContainerPort: *values.BackupPort, + }, + }), + "Image": Equal(values.BackupImage), + "ImagePullPolicy": Equal(corev1.PullIfNotPresent), + "VolumeMounts": MatchElements(volumeMountIterator, IgnoreExtras, Elements{ + values.VolumeClaimTemplateName: MatchFields(IgnoreExtras, Fields{ + "Name": Equal(values.VolumeClaimTemplateName), + "MountPath": Equal("/var/etcd/data"), + }), + "etcd-config-file": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("etcd-config-file"), + "MountPath": Equal("/var/etcd/config/"), + }), + "etcd-backup": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("etcd-backup"), + "MountPath": Equal("/root/etcd-backup/"), + }), + }), + "Env": MatchElements(envIterator, IgnoreExtras, Elements{ + "STORAGE_CONTAINER": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("STORAGE_CONTAINER"), + "Value": Equal(*values.BackupStore.Container), + }), + "POD_NAME": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("POD_NAME"), + "ValueFrom": PointTo(MatchFields(IgnoreExtras, Fields{ + "FieldRef": PointTo(MatchFields(IgnoreExtras, Fields{ + "FieldPath": Equal("metadata.name"), + })), + })), + }), + "POD_NAMESPACE": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("POD_NAMESPACE"), + "ValueFrom": PointTo(MatchFields(IgnoreExtras, Fields{ + "FieldRef": PointTo(MatchFields(IgnoreExtras, Fields{ + "FieldPath": Equal("metadata.namespace"), + })), + })), + }), + "AZURE_APPLICATION_CREDENTIALS": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("AZURE_APPLICATION_CREDENTIALS"), + "Value": Equal("/root/etcd-backup"), + }), + }), + "SecurityContext": PointTo(MatchFields(IgnoreExtras, Fields{ + "Capabilities": PointTo(MatchFields(IgnoreExtras, Fields{ + "Add": ConsistOf([]corev1.Capability{ + "SYS_PTRACE", + }), + })), + })), + }), + }), + "ShareProcessNamespace": Equal(pointer.BoolPtr(true)), + "Volumes": MatchAllElements(volumeIterator, Elements{ + "etcd-config-file": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("etcd-config-file"), + "VolumeSource": MatchFields(IgnoreExtras, Fields{ + "ConfigMap": PointTo(MatchFields(IgnoreExtras, Fields{ + "LocalObjectReference": MatchFields(IgnoreExtras, Fields{ + "Name": Equal(fmt.Sprintf("etcd-bootstrap-%s", string(values.EtcdUID[:6]))), + }), + "DefaultMode": PointTo(Equal(int32(0644))), + "Items": MatchAllElements(keyIterator, Elements{ + "etcd.conf.yaml": MatchFields(IgnoreExtras, Fields{ + "Key": Equal("etcd.conf.yaml"), + "Path": Equal("etcd.conf.yaml"), + }), + }), + })), + }), + }), + "etcd-backup": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("etcd-backup"), + "VolumeSource": MatchFields(IgnoreExtras, Fields{ + "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ + "SecretName": Equal(values.BackupStore.SecretRef.Name), + })), + }), + }), + "client-url-etcd-server-tls": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("client-url-etcd-server-tls"), + "VolumeSource": MatchFields(IgnoreExtras, Fields{ + "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ + "SecretName": Equal(values.ClientUrlTLS.ServerTLSSecretRef.Name), + })), + }), + }), + "client-url-etcd-client-tls": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("client-url-etcd-client-tls"), + "VolumeSource": MatchFields(IgnoreExtras, Fields{ + "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ + "SecretName": Equal(values.ClientUrlTLS.ClientTLSSecretRef.Name), + })), + }), + }), + "client-url-ca-etcd": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("client-url-ca-etcd"), + "VolumeSource": MatchFields(IgnoreExtras, Fields{ + "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ + "SecretName": Equal(values.ClientUrlTLS.TLSCASecretRef.Name), + })), + }), + }), + "peer-url-etcd-server-tls": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("peer-url-etcd-server-tls"), + "VolumeSource": MatchFields(IgnoreExtras, Fields{ + "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ + "SecretName": Equal(values.PeerUrlTLS.ServerTLSSecretRef.Name), + })), + }), + }), + "peer-url-ca-etcd": MatchFields(IgnoreExtras, Fields{ + "Name": Equal("peer-url-ca-etcd"), + "VolumeSource": MatchFields(IgnoreExtras, Fields{ + "Secret": PointTo(MatchFields(IgnoreExtras, Fields{ + "SecretName": Equal(values.PeerUrlTLS.TLSCASecretRef.Name), + })), + }), + }), + }), + }), + }), + "VolumeClaimTemplates": MatchAllElements(pvcIterator, Elements{ + values.VolumeClaimTemplateName: MatchFields(IgnoreExtras, Fields{ + "ObjectMeta": MatchFields(IgnoreExtras, Fields{ + "Name": Equal(values.VolumeClaimTemplateName), + }), + "Spec": MatchFields(IgnoreExtras, Fields{ + "StorageClassName": PointTo(Equal(*values.StorageClass)), + "AccessModes": MatchAllElements(accessModeIterator, Elements{ + "ReadWriteOnce": Equal(corev1.ReadWriteOnce), + }), + "Resources": MatchFields(IgnoreExtras, Fields{ + "Requests": MatchKeys(IgnoreExtras, Keys{ + corev1.ResourceStorage: Equal(*values.StorageCapacity), + }), + }), + }), + }), + }), + }), + })) +} + +func checkStsMetadata(ors []metav1.OwnerReference, values Values) { + Expect(ors).To(ConsistOf(Equal(metav1.OwnerReference{ + APIVersion: druidv1alpha1.GroupVersion.String(), + Kind: "Etcd", + Name: values.EtcdName, + UID: values.EtcdUID, + Controller: pointer.BoolPtr(true), + BlockOwnerDeletion: pointer.BoolPtr(true), + }))) +} + +func getEtcd(name, namespace string, tlsEnabled bool) *druidv1alpha1.Etcd { + + instance := &druidv1alpha1.Etcd{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + UID: types.UID(uid), + }, + Spec: druidv1alpha1.EtcdSpec{ + Annotations: map[string]string{ + "app": "etcd-statefulset", + "role": "test", + "instance": name, + }, + Labels: map[string]string{ + "name": "etcd", + "instance": name, + }, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "name": "etcd", + "instance": name, + }, + }, + Replicas: 1, + StorageCapacity: &storageCapacity, + StorageClass: &storageClass, + PriorityClassName: &priorityClassName, + VolumeClaimTemplate: &volumeClaimTemplateName, + Backup: druidv1alpha1.BackupSpec{ + Image: &imageBR, + Port: pointer.Int32Ptr(backupPort), + Store: getEtcdWithABS(), + FullSnapshotSchedule: &snapshotSchedule, + GarbageCollectionPolicy: &garbageCollectionPolicy, + GarbageCollectionPeriod: &garbageCollectionPeriod, + DeltaSnapshotPeriod: &deltaSnapshotPeriod, + DeltaSnapshotMemoryLimit: &deltaSnapShotMemLimit, + EtcdSnapshotTimeout: &etcdSnapshotTimeout, + LeaderElection: &druidv1alpha1.LeaderElectionSpec{ + EtcdConnectionTimeout: &etcdConnectionTimeout, + }, + + Resources: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "cpu": parseQuantity("500m"), + "memory": parseQuantity("2Gi"), + }, + Requests: corev1.ResourceList{ + "cpu": parseQuantity("23m"), + "memory": parseQuantity("128Mi"), + }, + }, + /*Store: &druidv1alpha1.StoreSpec{ + SecretRef: &corev1.SecretReference{ + Name: "etcd-backup", + }, + Container: &container, + Provider: &provider, + Prefix: prefix, + },*/ + OwnerCheck: &druidv1alpha1.OwnerCheckSpec{ + Name: ownerName, + ID: ownerID, + Interval: &ownerCheckInterval, + Timeout: &ownerCheckTimeout, + DNSCacheTTL: &ownerCheckDNSCacheTTL, + }, + }, + Etcd: druidv1alpha1.EtcdConfig{ + Quota: "a, + Metrics: &metricsBasic, + Image: &imageEtcd, + DefragmentationSchedule: &defragSchedule, + EtcdDefragTimeout: &etcdDefragTimeout, + HeartbeatDuration: &heartbeatDuration, + Resources: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "cpu": parseQuantity("2500m"), + "memory": parseQuantity("4Gi"), + }, + Requests: corev1.ResourceList{ + "cpu": parseQuantity("500m"), + "memory": parseQuantity("1000Mi"), + }, + }, + ClientPort: pointer.Int32Ptr(clientPort), + ServerPort: pointer.Int32Ptr(serverPort), + }, + Common: druidv1alpha1.SharedConfig{ + AutoCompactionMode: &autoCompactionMode, + AutoCompactionRetention: &autoCompactionRetention, + }, + }, + } + + if tlsEnabled { + clientTlsConfig := &druidv1alpha1.TLSConfig{ + TLSCASecretRef: druidv1alpha1.SecretReference{ + SecretReference: corev1.SecretReference{ + Name: "client-url-ca-etcd", + }, + DataKey: pointer.String("ca.crt"), + }, + ClientTLSSecretRef: corev1.SecretReference{ + Name: "client-url-etcd-client-tls", + }, + ServerTLSSecretRef: corev1.SecretReference{ + Name: "client-url-etcd-server-tls", + }, + } + + peerTlsConfig := &druidv1alpha1.TLSConfig{ + TLSCASecretRef: druidv1alpha1.SecretReference{ + SecretReference: corev1.SecretReference{ + Name: "peer-url-ca-etcd", + }, + DataKey: pointer.String("ca.crt"), + }, + ServerTLSSecretRef: corev1.SecretReference{ + Name: "peer-url-etcd-server-tls", + }, + } + + instance.Spec.Etcd.ClientUrlTLS = clientTlsConfig + instance.Spec.Etcd.PeerUrlTLS = peerTlsConfig + instance.Spec.Backup.TLS = clientTlsConfig + } + return instance +} + +func parseQuantity(q string) resource.Quantity { + val, _ := resource.ParseQuantity(q) + return val +} + +func getEtcdWithABS() *druidv1alpha1.StoreSpec { + return &druidv1alpha1.StoreSpec{ + Container: &container, + Prefix: prefix, + Provider: (*druidv1alpha1.StorageProvider)(pointer.StringPtr("azure")), + SecretRef: &corev1.SecretReference{ + Name: "etcd-backup", + }, + } +} + +func volumeMountIterator(element interface{}) string { + return (element.(corev1.VolumeMount)).Name +} + +func volumeIterator(element interface{}) string { + return (element.(corev1.Volume)).Name +} + +func keyIterator(element interface{}) string { + return (element.(corev1.KeyToPath)).Key +} + +func envIterator(element interface{}) string { + return (element.(corev1.EnvVar)).Name +} + +func containerIterator(element interface{}) string { + return (element.(corev1.Container)).Name +} + +func hostAliasIterator(element interface{}) string { + return (element.(corev1.HostAlias)).IP +} + +func pvcIterator(element interface{}) string { + return (element.(corev1.PersistentVolumeClaim)).Name +} + +func accessModeIterator(element interface{}) string { + return string(element.(corev1.PersistentVolumeAccessMode)) +} + +func cmdIterator(element interface{}) string { + return element.(string) +} diff --git a/pkg/component/etcd/statefulset/values.go b/pkg/component/etcd/statefulset/values.go new file mode 100644 index 000000000..ebc904110 --- /dev/null +++ b/pkg/component/etcd/statefulset/values.go @@ -0,0 +1,125 @@ +// Copyright (c) 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package statefulset + +import ( + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/types" +) + +type Values struct { + // EtcdName is the name of the etcd resource. + EtcdName string + // EtcdNameSpace is the namespace of etcd resource + EtcdNameSpace string + // EtcdName is the UID of the etcd resource. + EtcdUID types.UID + + // EtcdMainStsName is the name of main ETCD statefulset + EtcdMainStsName string + // Replicas is the number of ETCD instance that the ETCD cluster will have + Replicas int32 + // Annotations is the annotation provided in ETCD spec + Annotations map[string]string + // Labels is the labels provided in ETCD spec + Labels map[string]string + // BackupImage is the backup restore image + BackupImage string + // EtcdImage is the etcd custom image + EtcdImage string + // priorityClassName is the Priority Class name + PriorityClassName *string + // ServiceName is the name of the peer service + ServiceName string + // ServiceAccountName is the service account name + ServiceAccountName string + Affinity *corev1.Affinity + TopologySpreadConstraints []corev1.TopologySpreadConstraint + + EtcdResources *corev1.ResourceRequirements + BackupResources *corev1.ResourceRequirements + + EtcdCommand []string + ReadinessProbeCommand []string + LivenessProbCommand []string + EtcdBackupCommand []string + + EnableClientTLS string + EnablePeerTLS string + + FailBelowRevision string + VolumeClaimTemplateName string + + FullSnapLeaseName string + DeltaSnapLeaseName string + + StorageCapacity *resource.Quantity + StorageClass *string + + DefragmentationSchedule *string + FullSnapshotSchedule *string + + EtcdSnapshotTimeout *metav1.Duration + EtcdDefragTimeout *metav1.Duration + + DeltaSnapshotMemoryLimit *resource.Quantity + + GarbageCollectionPolicy *druidv1alpha1.GarbageCollectionPolicy + GarbageCollectionPeriod *metav1.Duration + + LeaderElection *druidv1alpha1.LeaderElectionSpec + BackupStore *druidv1alpha1.StoreSpec + + EnableProfiling *bool + + DeltaSnapshotPeriod *metav1.Duration + + SnapshotCompression *druidv1alpha1.CompressionSpec + HeartbeatDuration *metav1.Duration + + // Metrics defines the level of detail for exported metrics of etcd, specify 'extensive' to include histogram metrics. + Metrics *druidv1alpha1.MetricsLevel + // Quota defines the etcd DB quota. + Quota *resource.Quantity + + // ClientUrlTLS hold the TLS configuration details for Client Communication + ClientUrlTLS *druidv1alpha1.TLSConfig + // PeerUrlTLS hold the TLS configuration details for Peer Communication + PeerUrlTLS *druidv1alpha1.TLSConfig + // BackupTLS hold the TLS configuration for communication with Backup server + BackupTLS *druidv1alpha1.TLSConfig + + //ClientServiceName is name of the etcd client service + ClientServiceName string + // ClientPort holds the client port + ClientPort *int32 + //PeerServiceName is name of the etcd peer service + PeerServiceName string + // ServerPort holds the peer port + ServerPort *int32 + BackupPort *int32 + + OwnerCheck *druidv1alpha1.OwnerCheckSpec + // AutoCompactionMode defines the auto-compaction-mode: 'periodic' or 'revision'. + AutoCompactionMode *druidv1alpha1.CompactionMode + //AutoCompactionRetention defines the auto-compaction-retention length for etcd as well as for embedded-Etcd of backup-restore sidecar. + AutoCompactionRetention *string + // ConfigMapName is the name of the configmap that holds the ETCD config + ConfigMapName string +} diff --git a/pkg/component/etcd/statefulset/values_helper.go b/pkg/component/etcd/statefulset/values_helper.go new file mode 100644 index 000000000..06992082b --- /dev/null +++ b/pkg/component/etcd/statefulset/values_helper.go @@ -0,0 +1,683 @@ +// Copyright (c) 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v.2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package statefulset + +import ( + "fmt" + "strconv" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/pkg/utils" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/utils/pointer" +) + +var ( + defaultBackupPort int32 = 8080 + defaultServerPort int32 = 2380 + defaultClientPort int32 = 2379 + defaultheartbeatDuration string = "10s" + defaultGbcPolicy string = "LimitBased" + defaultAutoCompactionRetention string = "30m" + defaultEtcdSnapshotTimeout string = "15m" + defaultEtcdDefragTimeout string = "15m" + defaultAutoCompactionMode string = "periodic" + defaultEtcdConnectionTimeout string = "5m" + defaultStorageCapacity = resource.MustParse("16Gi") + defaultLocalPrefix string = "/etc/gardener/local-backupbuckets" +) + +// GenerateValues generates `statefulset.Values` for the statefulset component with the given parameters. +func GenerateValues(etcd *druidv1alpha1.Etcd) Values { + volumeClaimTemplateName := etcd.Name + if etcd.Spec.VolumeClaimTemplate != nil && len(*etcd.Spec.VolumeClaimTemplate) != 0 { + volumeClaimTemplateName = *etcd.Spec.VolumeClaimTemplate + } + + values := Values{ + EtcdName: etcd.Name, + EtcdNameSpace: etcd.Namespace, + EtcdUID: etcd.UID, + EtcdMainStsName: utils.GetETCDMainStsName(etcd), + Replicas: etcd.Spec.Replicas, + Annotations: etcd.Spec.Annotations, + Labels: etcd.Spec.Labels, + BackupImage: *etcd.Spec.Backup.Image, + EtcdImage: *etcd.Spec.Etcd.Image, + PriorityClassName: etcd.Spec.PriorityClassName, + ServiceName: utils.GetPeerServiceName(etcd), + ServiceAccountName: utils.GetServiceAccountName(etcd), + Affinity: etcd.Spec.SchedulingConstraints.Affinity, + TopologySpreadConstraints: etcd.Spec.SchedulingConstraints.TopologySpreadConstraints, + + EtcdResources: etcd.Spec.Etcd.Resources, + BackupResources: etcd.Spec.Backup.Resources, + + VolumeClaimTemplateName: volumeClaimTemplateName, + + FullSnapLeaseName: utils.GetFullSnapshotLeaseName(etcd), + DeltaSnapLeaseName: utils.GetDeltaSnapshotLeaseName(etcd), + + StorageCapacity: etcd.Spec.StorageCapacity, + StorageClass: etcd.Spec.StorageClass, + + ClientUrlTLS: etcd.Spec.Etcd.ClientUrlTLS, + PeerUrlTLS: etcd.Spec.Etcd.PeerUrlTLS, + BackupTLS: etcd.Spec.Backup.TLS, + + LeaderElection: etcd.Spec.Backup.LeaderElection, + + BackupStore: etcd.Spec.Backup.Store, + EnableProfiling: etcd.Spec.Backup.EnableProfiling, + + DeltaSnapshotPeriod: etcd.Spec.Backup.DeltaSnapshotPeriod, + DeltaSnapshotMemoryLimit: etcd.Spec.Backup.DeltaSnapshotMemoryLimit, + + DefragmentationSchedule: etcd.Spec.Etcd.DefragmentationSchedule, + FullSnapshotSchedule: etcd.Spec.Backup.FullSnapshotSchedule, + + EtcdSnapshotTimeout: etcd.Spec.Backup.EtcdSnapshotTimeout, + EtcdDefragTimeout: etcd.Spec.Etcd.EtcdDefragTimeout, + + GarbageCollectionPolicy: etcd.Spec.Backup.GarbageCollectionPolicy, + GarbageCollectionPeriod: etcd.Spec.Backup.GarbageCollectionPeriod, + + SnapshotCompression: etcd.Spec.Backup.SnapshotCompression, + HeartbeatDuration: etcd.Spec.Etcd.HeartbeatDuration, + + Metrics: etcd.Spec.Etcd.Metrics, + Quota: etcd.Spec.Etcd.Quota, + ClientServiceName: utils.GetClientServiceName(etcd), + ClientPort: etcd.Spec.Etcd.ClientPort, + PeerServiceName: utils.GetPeerServiceName(etcd), + ServerPort: etcd.Spec.Etcd.ServerPort, + BackupPort: etcd.Spec.Backup.Port, + + OwnerCheck: etcd.Spec.Backup.OwnerCheck, + + AutoCompactionMode: etcd.Spec.Common.AutoCompactionMode, + AutoCompactionRetention: etcd.Spec.Common.AutoCompactionRetention, + ConfigMapName: utils.GetConfigmapName(etcd), + } + + values.EtcdCommand = getEtcdCommand() + values.ReadinessProbeCommand = getReadinessProbeCommand(values) + values.LivenessProbCommand = getLivenessProbeCommand(values) + values.EtcdBackupCommand = getEtcdBackupCommand(values) + + return values +} + +func getEtcdCommand() []string { + command := []string{"" + "/var/etcd/bin/bootstrap.sh"} + + return command +} + +func getReadinessProbeCommand(val Values) []string { + command := []string{"" + "/usr/bin/curl"} + + if val.ClientUrlTLS != nil { + + command = append(command, "--cert") + command = append(command, "/var/etcd/ssl/client/client/tls.crt") + command = append(command, "--key") + command = append(command, "/var/etcd/ssl/client/client/tls.key") + if dataKey := val.ClientUrlTLS.TLSCASecretRef.DataKey; dataKey != nil { + command = append(command, "--cacert") + command = append(command, "/var/etcd/ssl/client/ca/"+*dataKey) + } + + if val.Replicas == 1 { + command = append(command, fmt.Sprintf("https://%s-local:%d/healthz", val.EtcdName, pointer.Int32Deref(val.BackupPort, defaultBackupPort))) + } else { + command = append(command, fmt.Sprintf("https://%s-local:%d/health", val.EtcdName, pointer.Int32Deref(val.ClientPort, defaultClientPort))) + } + } else { + if val.Replicas == 1 { + command = append(command, fmt.Sprintf("http://%s-local:%d/healthz", val.EtcdName, pointer.Int32Deref(val.BackupPort, defaultBackupPort))) + } else { + command = append(command, fmt.Sprintf("http://%s-local:%d/health", val.EtcdName, pointer.Int32Deref(val.ClientPort, defaultClientPort))) + } + } + return command +} + +func getLivenessProbeCommand(val Values) []string { + command := []string{"" + "/bin/sh"} + command = append(command, "-ec") + command = append(command, "ETCDCTL_API=3") + command = append(command, "etcdctl") + + if val.ClientUrlTLS != nil { + + command = append(command, "--cert=/var/etcd/ssl/client/client/tls.crt") + command = append(command, "--key=/var/etcd/ssl/client/client/tls.key") + if dataKey := val.ClientUrlTLS.TLSCASecretRef.DataKey; dataKey != nil { + command = append(command, "--cacert=/var/etcd/ssl/client/ca/"+*dataKey) + } + command = append(command, fmt.Sprintf("--endpoints=https://%s-local:%d", val.EtcdName, pointer.Int32Deref(val.ClientPort, defaultClientPort))) + } else { + command = append(command, fmt.Sprintf("--endpoints=http://%s-local:%d", val.EtcdName, pointer.Int32Deref(val.ClientPort, defaultClientPort))) + } + command = append(command, "get") + command = append(command, "foo") + command = append(command, "--consistency=s") + return command +} + +func getEtcdBackupCommand(val Values) []string { + command := []string{"" + "etcdbrctl"} + command = append(command, "server") + + if val.BackupStore != nil { + command = append(command, "--enable-snapshot-lease-renewal=true") + command = append(command, "--delta-snapshot-lease-name="+val.DeltaSnapLeaseName) + command = append(command, "--full-snapshot-lease-name="+val.FullSnapLeaseName) + } + + if val.DefragmentationSchedule != nil { + command = append(command, "--defragmentation-schedule="+*val.DefragmentationSchedule) + } + + if val.FullSnapshotSchedule != nil { + command = append(command, "--schedule="+*val.FullSnapshotSchedule) + } + + if val.GarbageCollectionPolicy != nil { + + gbc := string(*val.GarbageCollectionPolicy) + command = append(command, "--garbage-collection-policy="+gbc) + + if gbc == "LimitBased" { + command = append(command, "--max-backups=7") + } + } else { + command = append(command, "--garbage-collection-policy="+defaultGbcPolicy) + command = append(command, "--max-backups=7") + } + + command = append(command, "--data-dir=/var/etcd/data/new.etcd") + + if val.BackupStore != nil { + store, _ := utils.StorageProviderFromInfraProvider(val.BackupStore.Provider) + command = append(command, "--storage-provider="+store) + command = append(command, "--store-prefix="+string(val.BackupStore.Prefix)) + } + + var quota int64 = 8 * 1024 * 1024 * 1024 // 8Gi + if val.Quota != nil { + quota = val.Quota.Value() + } + + command = append(command, "--embedded-etcd-quota-bytes="+fmt.Sprint(quota)) + + if pointer.BoolDeref(val.EnableProfiling, false) { + command = append(command, "--enable-profiling=true") + } + + if val.BackupTLS != nil { + command = append(command, "--cert=/var/etcd/ssl/client/client/tls.crt") + command = append(command, "--key=/var/etcd/ssl/client/client/tls.key") + if dataKey := val.ClientUrlTLS.TLSCASecretRef.DataKey; dataKey != nil { + command = append(command, "--cacert=/var/etcd/ssl/client/ca/"+*dataKey) + } + command = append(command, "--insecure-transport=false") + command = append(command, "--insecure-skip-tls-verify=false") + + command = append(command, fmt.Sprintf("--endpoints=https://%s-local:%d", val.EtcdName, pointer.Int32Deref(val.ClientPort, defaultClientPort))) + + command = append(command, "--server-cert=/var/etcd/ssl/client/server/tls.crt") + command = append(command, "--server-key=/var/etcd/ssl/client/server/tls.key") + } else { + command = append(command, "--insecure-transport=true") + command = append(command, "--insecure-skip-tls-verify=true") + command = append(command, fmt.Sprintf("--endpoints=http://%s-local:%d", val.EtcdName, pointer.Int32Deref(val.ClientPort, defaultClientPort))) + } + + if val.LeaderElection != nil { + if val.LeaderElection.EtcdConnectionTimeout != nil { + command = append(command, "--etcd-connection-timeout="+val.LeaderElection.EtcdConnectionTimeout.Duration.String()) + } + + if val.LeaderElection.ReelectionPeriod != nil { + command = append(command, "--reelection-period="+val.LeaderElection.ReelectionPeriod.Duration.String()) + } + } else { + command = append(command, "--etcd-connection-timeout="+defaultEtcdConnectionTimeout) + } + + if val.DeltaSnapshotPeriod != nil { + command = append(command, "--delta-snapshot-period="+val.DeltaSnapshotPeriod.Duration.String()) + } + + var deltaSnapshotMemoryLimit int64 = 100 * 1024 * 1024 // 100Mi + if val.DeltaSnapshotMemoryLimit != nil { + deltaSnapshotMemoryLimit = val.DeltaSnapshotMemoryLimit.Value() + } + + command = append(command, "--delta-snapshot-memory-limit="+fmt.Sprint(deltaSnapshotMemoryLimit)) + + if val.SnapshotCompression != nil { + if pointer.BoolPtrDerefOr(val.SnapshotCompression.Enabled, false) { + command = append(command, "--compress-snapshots="+strconv.FormatBool(pointer.BoolPtrDerefOr(val.SnapshotCompression.Enabled, false))) + } + if val.SnapshotCompression.Policy != nil { + command = append(command, "--compression-policy="+string(*val.SnapshotCompression.Policy)) + } + } + + if val.GarbageCollectionPeriod != nil { + command = append(command, "--garbage-collection-period="+val.GarbageCollectionPeriod.Duration.String()) + } + + if val.OwnerCheck != nil { + command = append(command, "--owner-name="+val.OwnerCheck.Name) + command = append(command, "--owner-id="+val.OwnerCheck.ID) + + if val.OwnerCheck.Interval != nil { + command = append(command, "--owner-check-interval="+val.OwnerCheck.Interval.Duration.String()) + } + if val.OwnerCheck.Timeout != nil { + command = append(command, "--owner-check-timeout="+val.OwnerCheck.Timeout.Duration.String()) + } + if val.OwnerCheck.DNSCacheTTL != nil { + command = append(command, "--owner-check-dns-cache-ttl="+val.OwnerCheck.DNSCacheTTL.Duration.String()) + } + } + + if val.AutoCompactionMode != nil { + command = append(command, "--auto-compaction-mode="+string(*val.AutoCompactionMode)) + } else { + command = append(command, "--auto-compaction-mode="+defaultAutoCompactionMode) + } + + if val.AutoCompactionRetention != nil { + command = append(command, "--auto-compaction-retention="+string(*val.AutoCompactionRetention)) + } else { + command = append(command, "--auto-compaction-retention="+defaultAutoCompactionRetention) + } + + if val.EtcdSnapshotTimeout != nil { + command = append(command, "--etcd-snapshot-timeout="+val.EtcdSnapshotTimeout.Duration.String()) + } else { + command = append(command, "--etcd-snapshot-timeout="+defaultEtcdSnapshotTimeout) + } + + if val.EtcdDefragTimeout != nil { + command = append(command, "--etcd-defrag-timeout="+val.EtcdDefragTimeout.Duration.String()) + } else { + command = append(command, "--etcd-defrag-timeout="+defaultEtcdDefragTimeout) + } + + command = append(command, "--snapstore-temp-directory=/var/etcd/data/temp") + command = append(command, "--enable-member-lease-renewal=true") + command = append(command, "--etcd-process-name=etcd") + + if heartBeatDuration := val.HeartbeatDuration; heartBeatDuration != nil { + command = append(command, "--k8s-heartbeat-duration="+heartBeatDuration.Duration.String()) + } else { + command = append(command, "--k8s-heartbeat-duration="+defaultheartbeatDuration) + } + + return command +} + +func getStsEnvVar(val Values) []corev1.EnvVar { + var env []corev1.EnvVar + env = append(env, getEnvVarFromFields("POD_NAME", "metadata.name")) + env = append(env, getEnvVarFromFields("POD_NAMESPACE", "metadata.namespace")) + + if val.BackupStore == nil { + env = append(env, getEnvVarFromValues("STORAGE_CONTAINER", "")) + return env + } + + storeValues := val.BackupStore + + env = append(env, getEnvVarFromValues("STORAGE_CONTAINER", *storeValues.Container)) + + provider, err := utils.StorageProviderFromInfraProvider(val.BackupStore.Provider) + if err != nil { + return env + } + + if provider == "S3" { + env = append(env, getEnvVarFromValues("AWS_APPLICATION_CREDENTIALS", "/root/etcd-backup")) + } + + if provider == "ABS" { + env = append(env, getEnvVarFromValues("AZURE_APPLICATION_CREDENTIALS", "/root/etcd-backup")) + } + + if provider == "GCS" { + env = append(env, getEnvVarFromValues("GOOGLE_APPLICATION_CREDENTIALS", "/root/.gcp/serviceaccount.json")) + } + + if provider == "Swift" { + env = append(env, getEnvVarFromValues("OPENSTACK_APPLICATION_CREDENTIALS", "/root/etcd-backup")) + } + + if provider == "OSS" { + env = append(env, getEnvVarFromValues("ALICLOUD_APPLICATION_CREDENTIALS", "/root/etcd-backup")) + } + + if provider == "ECS" { + env = append(env, getEnvVarFromSecrets("ECS_ENDPOINT", storeValues.SecretRef.Name, "endpoint")) + env = append(env, getEnvVarFromSecrets("ECS_ACCESS_KEY_ID", storeValues.SecretRef.Name, "accessKeyID")) + env = append(env, getEnvVarFromSecrets("ECS_SECRET_ACCESS_KEY", storeValues.SecretRef.Name, "secretAccessKey")) + } + + if provider == "OCS" { + env = append(env, getEnvVarFromValues("OPENSHIFT_APPLICATION_CREDENTIALS", "/root/etcd-backup")) + } + + return env +} + +func getEnvVarFromValues(name, value string) corev1.EnvVar { + return corev1.EnvVar{ + Name: name, + Value: value, + } +} + +func getEnvVarFromFields(name, fieldPath string) corev1.EnvVar { + return corev1.EnvVar{ + Name: name, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: fieldPath, + }, + }, + } +} + +func getEnvVarFromSecrets(name, secretName, secretKey string) corev1.EnvVar { + return corev1.EnvVar{ + Name: name, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secretName, + }, + Key: secretKey, + }, + }, + } +} + +func getBackupRestoreVolumeMounts(val Values) []corev1.VolumeMount { + vms := []corev1.VolumeMount{ + { + Name: "etcd-config-file", + MountPath: "/var/etcd/config/", + }, + } + + vms = append(vms, corev1.VolumeMount{ + Name: val.VolumeClaimTemplateName, + MountPath: "/var/etcd/data", + }) + + vms = append(vms, getSecretVolumeMounts(val)...) + + if val.BackupStore == nil { + return vms + } + + provider, err := utils.StorageProviderFromInfraProvider(val.BackupStore.Provider) + if err != nil { + return vms + } + + if provider == "Local" && val.BackupStore.Container != nil { + vms = append(vms, corev1.VolumeMount{ + Name: "host-storage", + MountPath: *val.BackupStore.Container, + }) + } + + if provider == "GCS" { + vms = append(vms, corev1.VolumeMount{ + Name: "etcd-backup", + MountPath: "/root/.gcp/", + }) + } else if provider == "S3" || provider == "ABS" || provider == "OSS" || provider == "Swift" || provider == "OCS" { + vms = append(vms, corev1.VolumeMount{ + Name: "etcd-backup", + MountPath: "/root/etcd-backup/", + }) + } + + return vms +} + +func getBackupRestoreVolumes(val Values) []corev1.Volume { + vs := []corev1.Volume{ + { + Name: "etcd-config-file", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: val.ConfigMapName, + }, + Items: []corev1.KeyToPath{ + { + Key: "etcd.conf.yaml", + Path: "etcd.conf.yaml", + }, + }, + DefaultMode: pointer.Int32(0644), + }, + }, + }, + } + + if val.ClientUrlTLS != nil { + vs = append(vs, corev1.Volume{ + Name: "client-url-ca-etcd", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: val.ClientUrlTLS.TLSCASecretRef.Name, + }, + }, + }, + corev1.Volume{ + Name: "client-url-etcd-server-tls", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: val.ClientUrlTLS.ServerTLSSecretRef.Name, + }, + }, + }, + corev1.Volume{ + Name: "client-url-etcd-client-tls", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: val.ClientUrlTLS.ClientTLSSecretRef.Name, + }, + }, + }) + } + + if val.PeerUrlTLS != nil { + vs = append(vs, corev1.Volume{ + Name: "peer-url-ca-etcd", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: val.PeerUrlTLS.TLSCASecretRef.Name, + }, + }, + }, + corev1.Volume{ + Name: "peer-url-etcd-server-tls", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: val.PeerUrlTLS.ServerTLSSecretRef.Name, + }, + }, + }) + } + + if val.BackupStore == nil { + return vs + } + + storeValues := val.BackupStore + provider, err := utils.StorageProviderFromInfraProvider(storeValues.Provider) + if err != nil { + return vs + } + + if provider == "Local" { + hpt := corev1.HostPathDirectory + vs = append(vs, corev1.Volume{ + Name: "host-storage", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: defaultLocalPrefix + "/" + *storeValues.Container, + Type: &hpt, + }, + }, + }) + } + + if provider == "GCS" || provider == "S3" || provider == "OSS" || provider == "ABS" || provider == "Swift" || provider == "OCS" { + vs = append(vs, corev1.Volume{ + Name: "etcd-backup", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: storeValues.SecretRef.Name, + }, + }, + }) + } + + return vs +} + +func getEtcdVolumeMounts(val Values) []corev1.VolumeMount { + vms := []corev1.VolumeMount{ + { + Name: val.VolumeClaimTemplateName, + MountPath: "/var/etcd/data/", + }, + } + + vms = append(vms, getSecretVolumeMounts(val)...) + + return vms +} + +func getSecretVolumeMounts(val Values) []corev1.VolumeMount { + vms := []corev1.VolumeMount{} + + if val.ClientUrlTLS != nil { + vms = append(vms, corev1.VolumeMount{ + Name: "client-url-ca-etcd", + MountPath: "/var/etcd/ssl/client/ca", + }, corev1.VolumeMount{ + Name: "client-url-etcd-server-tls", + MountPath: "/var/etcd/ssl/client/server", + }, corev1.VolumeMount{ + Name: "client-url-etcd-client-tls", + MountPath: "/var/etcd/ssl/client/client", + }) + } + + if val.PeerUrlTLS != nil { + vms = append(vms, corev1.VolumeMount{ + Name: "peer-url-ca-etcd", + MountPath: "/var/etcd/ssl/peer/ca", + }, corev1.VolumeMount{ + Name: "peer-url-etcd-server-tls", + MountPath: "/var/etcd/ssl/peer/server", + }) + } + + return vms +} + +func getStorageReq(val Values) corev1.ResourceRequirements { + if val.StorageCapacity != nil { + return corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: *val.StorageCapacity, + }, + } + } + + return corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: defaultStorageCapacity, + }, + } +} + +func getEtcdPorts(val Values) []corev1.ContainerPort { + ports := []corev1.ContainerPort{} + + ports = append(ports, corev1.ContainerPort{ + Name: "server", + Protocol: "TCP", + ContainerPort: pointer.Int32Deref(val.ServerPort, defaultServerPort), + }) + + ports = append(ports, corev1.ContainerPort{ + Name: "client", + Protocol: "TCP", + ContainerPort: pointer.Int32Deref(val.ClientPort, defaultClientPort), + }) + + return ports +} + +func getBackupPorts(val Values) []corev1.ContainerPort { + ports := []corev1.ContainerPort{} + + ports = append(ports, corev1.ContainerPort{ + Name: "server", + Protocol: "TCP", + ContainerPort: pointer.Int32Deref(val.BackupPort, defaultBackupPort), + }) + + return ports +} + +func getEtcdResources(val Values) corev1.ResourceRequirements { + if val.EtcdResources != nil { + return *val.EtcdResources + } + return corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("50m"), + corev1.ResourceMemory: resource.MustParse("128Mi"), + }, + } +} + +func getBackupResources(val Values) corev1.ResourceRequirements { + if val.EtcdResources != nil { + return *val.EtcdResources + } + return corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("50m"), + corev1.ResourceMemory: resource.MustParse("128Mi"), + }, + } +} diff --git a/pkg/component/etcd/values.go b/pkg/component/etcd/values.go index cc95326ba..fbf9c83ad 100644 --- a/pkg/component/etcd/values.go +++ b/pkg/component/etcd/values.go @@ -18,11 +18,13 @@ import ( "github.com/gardener/etcd-druid/pkg/component/etcd/configmap" "github.com/gardener/etcd-druid/pkg/component/etcd/lease" "github.com/gardener/etcd-druid/pkg/component/etcd/service" + "github.com/gardener/etcd-druid/pkg/component/etcd/statefulset" ) // Values contains all values relevant for deploying etcd components. type Values struct { - ConfigMap *configmap.Values - Service service.Values - Lease lease.Values + ConfigMap *configmap.Values + Service service.Values + Lease lease.Values + StatefulSet statefulset.Values } diff --git a/pkg/utils/names.go b/pkg/utils/names.go index 8ad48319a..311987e95 100644 --- a/pkg/utils/names.go +++ b/pkg/utils/names.go @@ -40,6 +40,11 @@ func GetConfigmapName(etcd *druidv1alpha1.Etcd) string { return fmt.Sprintf("etcd-bootstrap-%s", string(etcd.UID[:6])) } +// GetETCDMainStsName returns the name of the main ETCD based on the given `etcd` object. +func GetETCDMainStsName(etcd *druidv1alpha1.Etcd) string { + return etcd.Name +} + // GetCronJobName returns the legacy compaction cron job name func GetCronJobName(etcd *druidv1alpha1.Etcd) string { return fmt.Sprintf("%s-compact-backup", etcd.Name) @@ -54,3 +59,13 @@ func GetJobName(etcd *druidv1alpha1.Etcd) string { func GetOrdinalPodName(etcd *druidv1alpha1.Etcd, order int) string { return fmt.Sprintf("%s-%d", etcd.Name, order) } + +// GetDeltaSnapshotLeaseName returns the name of the delta snapshot lease based on the given `etcd` object. +func GetDeltaSnapshotLeaseName(etcd *druidv1alpha1.Etcd) string { + return fmt.Sprintf("%s-delta-snap", etcd.Name) +} + +// GetFullSnapshotLeaseName returns the name of the full snapshot lease based on the given `etcd` object. +func GetFullSnapshotLeaseName(etcd *druidv1alpha1.Etcd) string { + return fmt.Sprintf("%s-full-snap", etcd.Name) +}