diff --git a/api/v1alpha1/etcd_types.go b/api/v1alpha1/etcd_types.go index 3a0397cf7..940ec2be0 100644 --- a/api/v1alpha1/etcd_types.go +++ b/api/v1alpha1/etcd_types.go @@ -121,9 +121,9 @@ type OwnerCheckSpec struct { DNSCacheTTL *metav1.Duration `json:"dnsCacheTTL,omitempty"` } -// BackupSpec defines parametes associated with the full and delta snapshots of etcd +// BackupSpec defines parameters associated with the full and delta snapshots of etcd. type BackupSpec struct { - // Port define the port on which etcd-backup-restore server will exposed. + // Port define the port on which etcd-backup-restore server will be exposed. // +optional Port *int32 `json:"port,omitempty"` // +optional @@ -134,11 +134,11 @@ type BackupSpec struct { // Store defines the specification of object store provider for storing backups. // +optional Store *StoreSpec `json:"store,omitempty"` - // Resources defines the compute Resources required by backup-restore container. + // Resources defines compute Resources required by backup-restore container. // More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ // +optional Resources *corev1.ResourceRequirements `json:"resources,omitempty"` - // CompactionResources defines the compute Resources required by compaction job. + // CompactionResources defines compute Resources required by compaction job. // More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ // +optional CompactionResources *corev1.ResourceRequirements `json:"compactionResources,omitempty"` @@ -201,6 +201,9 @@ type EtcdConfig struct { // EtcdDefragTimeout defines the timeout duration for etcd defrag call // +optional EtcdDefragTimeout *metav1.Duration `json:"etcdDefragTimeout,omitempty"` + // HeartbeatDuration defines the duration for members to send heartbeats. The default value is 10s. + // +optional + HeartbeatDuration *metav1.Duration `json:"heartbeatDuration,omitempty"` } // SharedConfig defines parameters shared and used by Etcd as well as backup-restore sidecar. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index f94820503..d91832408 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -246,6 +246,11 @@ func (in *EtcdConfig) DeepCopyInto(out *EtcdConfig) { *out = new(metav1.Duration) **out = **in } + if in.HeartbeatDuration != nil { + in, out := &in.HeartbeatDuration, &out.HeartbeatDuration + *out = new(metav1.Duration) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdConfig. diff --git a/charts/etcd/templates/etcd-statefulset.yaml b/charts/etcd/templates/etcd-statefulset.yaml index b2906e279..4c7401a2e 100644 --- a/charts/etcd/templates/etcd-statefulset.yaml +++ b/charts/etcd/templates/etcd-statefulset.yaml @@ -210,6 +210,10 @@ spec: {{- end }} - --snapstore-temp-directory={{ .Values.backup.snapstoreTempDir }} - --etcd-process-name=etcd +{{- if .Values.etcd.heartbeatDuration }} + - --enable-member-lease-renewal=true + - --k8s-heartbeat-duration={{ .Values.etcd.heartbeatDuration }} +{{- end }} image: {{ .Values.backup.image }} imagePullPolicy: {{ .Values.backup.pullPolicy }} ports: diff --git a/charts/etcd/values.yaml b/charts/etcd/values.yaml index 683070032..dc3ff8dfb 100644 --- a/charts/etcd/values.yaml +++ b/charts/etcd/values.yaml @@ -33,6 +33,7 @@ etcd: memory: 128Mi #username: username #password: password + heartbeatDuration: 10s backup: port: 8080 diff --git a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml index e88965397..adecd4694 100644 --- a/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml +++ b/config/crd/bases/10-crd-druid.gardener.cloud_etcds.yaml @@ -48,12 +48,12 @@ spec: type: string type: object backup: - description: BackupSpec defines parametes associated with the full - and delta snapshots of etcd + description: BackupSpec defines parameters associated with the full + and delta snapshots of etcd. properties: compactionResources: - description: 'CompactionResources defines the compute Resources - required by compaction job. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'CompactionResources defines compute Resources required + by compaction job. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' properties: limits: additionalProperties: @@ -158,12 +158,12 @@ spec: type: object port: description: Port define the port on which etcd-backup-restore - server will exposed. + server will be exposed. format: int32 type: integer resources: - description: 'Resources defines the compute Resources required - by backup-restore container. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Resources defines compute Resources required by + backup-restore container. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' properties: limits: additionalProperties: @@ -293,6 +293,10 @@ spec: description: EtcdDefragTimeout defines the timeout duration for etcd defrag call type: string + heartbeatDuration: + description: HeartbeatDuration defines the duration for members + to send heartbeats. The default value is 10s. + type: string image: description: Image defines the etcd container image and tag type: string diff --git a/config/samples/druid_v1alpha1_etcd.yaml b/config/samples/druid_v1alpha1_etcd.yaml index 9507eaaf4..c72f26e59 100644 --- a/config/samples/druid_v1alpha1_etcd.yaml +++ b/config/samples/druid_v1alpha1_etcd.yaml @@ -35,6 +35,7 @@ spec: clientPort: 2379 serverPort: 2380 quota: 8Gi +# heartbeatDuration: 10s backup: port: 8080 fullSnapshotSchedule: "0 */24 * * *" diff --git a/controllers/compaction_lease_controller.go b/controllers/compaction_lease_controller.go index 29a4c7001..592cd8783 100644 --- a/controllers/compaction_lease_controller.go +++ b/controllers/compaction_lease_controller.go @@ -38,6 +38,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" controllersconfig "github.com/gardener/etcd-druid/controllers/config" "github.com/gardener/etcd-druid/pkg/common" + componentlease "github.com/gardener/etcd-druid/pkg/component/etcd/lease" druidpredicates "github.com/gardener/etcd-druid/pkg/predicate" "github.com/gardener/etcd-druid/pkg/utils" "github.com/gardener/gardener/pkg/utils/imagevector" @@ -116,7 +117,7 @@ func (lc *CompactionLeaseController) Reconcile(ctx context.Context, req ctrl.Req // Get full and delta snapshot lease to check the HolderIdentity value to take decision on compaction job fullLease := &coordinationv1.Lease{} - if err := lc.Get(ctx, kutil.Key(etcd.Namespace, getFullSnapshotLeaseName(etcd)), fullLease); err != nil { + if err := lc.Get(ctx, kutil.Key(etcd.Namespace, componentlease.GetFullSnapshotLeaseName(etcd)), fullLease); err != nil { logger.Info("Couldn't fetch full snap lease because: " + err.Error()) return ctrl.Result{ @@ -125,7 +126,7 @@ func (lc *CompactionLeaseController) Reconcile(ctx context.Context, req ctrl.Req } deltaLease := &coordinationv1.Lease{} - if err := lc.Get(ctx, kutil.Key(etcd.Namespace, getDeltaSnapshotLeaseName(etcd)), deltaLease); err != nil { + if err := lc.Get(ctx, kutil.Key(etcd.Namespace, componentlease.GetDeltaSnapshotLeaseName(etcd)), deltaLease); err != nil { logger.Info("Couldn't fetch delta snap lease because: " + err.Error()) return ctrl.Result{ @@ -494,8 +495,8 @@ func getCompactJobCommands(etcd *druidv1alpha1.Etcd) []string { command = append(command, "--data-dir=/var/etcd/data") command = append(command, "--snapstore-temp-directory=/var/etcd/data/tmp") command = append(command, "--enable-snapshot-lease-renewal=true") - command = append(command, "--full-snapshot-lease-name="+getFullSnapshotLeaseName(etcd)) - command = append(command, "--delta-snapshot-lease-name="+getDeltaSnapshotLeaseName(etcd)) + command = append(command, "--full-snapshot-lease-name="+componentlease.GetFullSnapshotLeaseName(etcd)) + command = append(command, "--delta-snapshot-lease-name="+componentlease.GetDeltaSnapshotLeaseName(etcd)) var quota int64 = DefaultETCDQuota if etcd.Spec.Etcd.Quota != nil { diff --git a/controllers/compaction_lease_controller_test.go b/controllers/compaction_lease_controller_test.go index 3aa23ae79..67c926ef1 100644 --- a/controllers/compaction_lease_controller_test.go +++ b/controllers/compaction_lease_controller_test.go @@ -19,6 +19,7 @@ import ( "time" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + componentlease "github.com/gardener/etcd-druid/pkg/component/etcd/lease" "github.com/gardener/etcd-druid/pkg/utils" "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/utils/test/matchers" @@ -402,11 +403,11 @@ func validateEtcdForCmpctJob(instance *druidv1alpha1.Etcd, j *batchv1.Job) { "Containers": MatchElements(containerIterator, IgnoreExtras, Elements{ "compact-backup": MatchFields(IgnoreExtras, Fields{ "Command": MatchElements(cmdIterator, IgnoreExtras, Elements{ - "--data-dir=/var/etcd/data": Equal("--data-dir=/var/etcd/data"), - "--snapstore-temp-directory=/var/etcd/data/tmp": Equal("--snapstore-temp-directory=/var/etcd/data/tmp"), - "--enable-snapshot-lease-renewal=true": Equal("--enable-snapshot-lease-renewal=true"), - fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", getFullSnapshotLeaseName(instance)): Equal(fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", getFullSnapshotLeaseName(instance))), - fmt.Sprintf("%s=%s", "--delta-snapshot-lease-name", getDeltaSnapshotLeaseName(instance)): Equal(fmt.Sprintf("%s=%s", "--delta-snapshot-lease-name", getDeltaSnapshotLeaseName(instance))), + "--data-dir=/var/etcd/data": Equal("--data-dir=/var/etcd/data"), + "--snapstore-temp-directory=/var/etcd/data/tmp": Equal("--snapstore-temp-directory=/var/etcd/data/tmp"), + "--enable-snapshot-lease-renewal=true": Equal("--enable-snapshot-lease-renewal=true"), + fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", componentlease.GetFullSnapshotLeaseName(instance)): Equal(fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", componentlease.GetFullSnapshotLeaseName(instance))), + fmt.Sprintf("%s=%s", "--delta-snapshot-lease-name", componentlease.GetDeltaSnapshotLeaseName(instance)): Equal(fmt.Sprintf("%s=%s", "--delta-snapshot-lease-name", componentlease.GetDeltaSnapshotLeaseName(instance))), fmt.Sprintf("%s=%s", "--store-prefix", instance.Spec.Backup.Store.Prefix): Equal(fmt.Sprintf("%s=%s", "--store-prefix", instance.Spec.Backup.Store.Prefix)), fmt.Sprintf("%s=%s", "--storage-provider", store): Equal(fmt.Sprintf("%s=%s", "--storage-provider", store)), fmt.Sprintf("%s=%s", "--store-container", *instance.Spec.Backup.Store.Container): Equal(fmt.Sprintf("%s=%s", "--store-container", *instance.Spec.Backup.Store.Container)), @@ -843,7 +844,7 @@ func fullLeaseIsCorrectlyReconciled(c client.Client, instance *druidv1alpha1.Etc ctx, cancel := context.WithTimeout(context.TODO(), timeout) defer cancel() req := types.NamespacedName{ - Name: getFullSnapshotLeaseName(instance), + Name: componentlease.GetFullSnapshotLeaseName(instance), Namespace: instance.Namespace, } @@ -861,7 +862,7 @@ func deltaLeaseIsCorrectlyReconciled(c client.Client, instance *druidv1alpha1.Et ctx, cancel := context.WithTimeout(context.TODO(), timeout) defer cancel() req := types.NamespacedName{ - Name: getDeltaSnapshotLeaseName(instance), + Name: componentlease.GetDeltaSnapshotLeaseName(instance), Namespace: instance.Namespace, } diff --git a/controllers/config/custodian.go b/controllers/config/custodian.go index 62134a832..f964fdd41 100644 --- a/controllers/config/custodian.go +++ b/controllers/config/custodian.go @@ -25,6 +25,8 @@ type EtcdCustodianController struct { } type EtcdMemberConfig struct { - // EtcdMemberUnknownThreshold is the duration after which a etcd member's state is considered `NotReady`. + // EtcdMemberNotReadyThreshold is the duration after which an etcd member's state is considered `NotReady`. EtcdMemberNotReadyThreshold time.Duration + // EtcdMemberUnknownThreshold is the duration after which an etcd member's state is considered `Unknown`. + EtcdMemberUnknownThreshold time.Duration } diff --git a/controllers/etcd_controller.go b/controllers/etcd_controller.go index 176ca8202..6952834bb 100644 --- a/controllers/etcd_controller.go +++ b/controllers/etcd_controller.go @@ -25,6 +25,8 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/pkg/common" + componentetcd "github.com/gardener/etcd-druid/pkg/component/etcd" + componentlease "github.com/gardener/etcd-druid/pkg/component/etcd/lease" druidpredicates "github.com/gardener/etcd-druid/pkg/predicate" "github.com/gardener/etcd-druid/pkg/utils" @@ -39,7 +41,6 @@ import ( "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" batchv1beta1 "k8s.io/api/batch/v1beta1" - coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" eventsv1 "k8s.io/api/events/v1" eventsv1beta1 "k8s.io/api/events/v1beta1" @@ -269,26 +270,6 @@ func (r *EtcdReconciler) reconcile(ctx context.Context, etcd *druidv1alpha1.Etcd }, err } - // It isn't necessary to reconcile delta/full leases if no store configuration is given because potential compaction - // jobs need access to the store where backups are stored. - if etcd.Spec.Backup.Store != nil { - fl, err := r.reconcileFullLease(ctx, logger, etcd) - if err != nil { - return ctrl.Result{ - Requeue: true, - }, err - } - logger.Info("Available Full Snapshot Lease: " + fl.Name) - - dl, err := r.reconcileDeltaLease(ctx, logger, etcd) - if err != nil { - return ctrl.Result{ - Requeue: true, - }, err - } - logger.Info("Available Delta Snapshot Lease: " + dl.Name) - } - // Delete any existing cronjob if required. // TODO(abdasgupta) : This is for backward compatibility towards ETCD-Druid 0.6.0. Remove it. cronJob, err := r.cleanCronJobs(ctx, logger, etcd) @@ -403,6 +384,13 @@ func (r *EtcdReconciler) delete(ctx context.Context, etcd *druidv1alpha1.Etcd) ( }, err } + leaseDeployer := componentlease.New(r.Client, etcd.Namespace, componentlease.GenerateValues(etcd)) + if err := leaseDeployer.Destroy(ctx); err != nil { + return ctrl.Result{ + Requeue: true, + }, err + } + if sets.NewString(etcd.Finalizers...).Has(FinalizerName) { logger.Info("Removing finalizer") @@ -935,81 +923,6 @@ func (r *EtcdReconciler) getStatefulSetFromEtcd(etcd *druidv1alpha1.Etcd, values return decoded, nil } -func (r *EtcdReconciler) reconcileFullLease(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) (*coordinationv1.Lease, error) { - // Get or Create fullSnapshotRevisions lease object that will help to set BackupReady condition - fullSnapshotRevisions := getFullSnapshotLeaseName(etcd) - - fullLease := &coordinationv1.Lease{} - if err := r.Get(ctx, kutil.Key(etcd.Namespace, fullSnapshotRevisions), fullLease); err != nil { - logger.Info("Couldn't fetch full snap lease " + fullSnapshotRevisions + ":" + err.Error()) - - if !apierrors.IsNotFound(err) { - return nil, err - } - - logger.Info("Creating the full snap lease " + fullSnapshotRevisions) - - fullLease = createSnapshotLease(etcd, fullSnapshotRevisions) - if err := r.Create(ctx, fullLease); err != nil { - logger.Error(err, "Full snap lease "+fullSnapshotRevisions+" couldn't be created") - return nil, err - } - } - - return fullLease, nil -} - -func getFullSnapshotLeaseName(etcd *druidv1alpha1.Etcd) string { - return fmt.Sprintf("%s-full-snap", string(etcd.Name)) -} - -func (r *EtcdReconciler) reconcileDeltaLease(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) (*coordinationv1.Lease, error) { - // Get or Create delta_snapshot_revisions lease object that will keep track of delta snapshot revisions based on which - // compaction job will be scheduled - deltaSnapshotRevisions := getDeltaSnapshotLeaseName(etcd) - - deltaLease := &coordinationv1.Lease{} - if err := r.Get(ctx, kutil.Key(etcd.Namespace, deltaSnapshotRevisions), deltaLease); err != nil { - logger.Info("Couldn't fetch delta snap lease " + deltaSnapshotRevisions + " because: " + err.Error()) - - if !apierrors.IsNotFound(err) { - return nil, err - } - - logger.Info("Creating the delta snap lease " + deltaSnapshotRevisions) - - deltaLease = createSnapshotLease(etcd, deltaSnapshotRevisions) - if err := r.Create(ctx, deltaLease); err != nil { - logger.Error(err, "Delta snap lease "+deltaSnapshotRevisions+" couldn't be created") - return nil, err - } - } - - return deltaLease, nil -} - -func getDeltaSnapshotLeaseName(etcd *druidv1alpha1.Etcd) string { - return fmt.Sprintf("%s-delta-snap", string(etcd.Name)) -} - -func createSnapshotLease(etcd *druidv1alpha1.Etcd, snapshotLeaseName string) *coordinationv1.Lease { - return &coordinationv1.Lease{ - ObjectMeta: metav1.ObjectMeta{ - Name: snapshotLeaseName, - Namespace: etcd.Namespace, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "druid.gardener.cloud/v1alpha1", - BlockOwnerDeletion: pointer.BoolPtr(true), - Controller: pointer.BoolPtr(true), - Kind: "Etcd", - Name: etcd.Name, - UID: etcd.UID, - }, - }, - }, - } -} func decodeObject(renderedChart *chartrenderer.RenderedChart, path string, object interface{}) error { if content, ok := renderedChart.Files()[path]; ok { decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader([]byte(content)), 1024) @@ -1144,7 +1057,11 @@ func (r *EtcdReconciler) reconcileRoleBinding(ctx context.Context, logger logr.L } func (r *EtcdReconciler) reconcileEtcd(ctx context.Context, logger logr.Logger, etcd *druidv1alpha1.Etcd) (*corev1.Service, *appsv1.StatefulSet, error) { - values, err := getMapFromEtcd(r.ImageVector, etcd, r.disableEtcdServiceAccountAutomount) + val := componentetcd.Values{ + Lease: componentlease.GenerateValues(etcd), + } + + values, err := getMapFromEtcd(r.ImageVector, etcd, val, r.disableEtcdServiceAccountAutomount) if err != nil { return nil, nil, err } @@ -1155,6 +1072,12 @@ func (r *EtcdReconciler) reconcileEtcd(ctx context.Context, logger logr.Logger, return nil, nil, err } + leaseDeployer := componentlease.New(r.Client, etcd.Namespace, val.Lease) + + if err := leaseDeployer.Deploy(ctx); err != nil { + return nil, nil, err + } + svc, err := r.reconcileServices(ctx, logger, etcd, renderedChart) if err != nil { return nil, nil, err @@ -1227,7 +1150,7 @@ func checkEtcdAnnotations(annotations map[string]string, etcd metav1.Object) boo } -func getMapFromEtcd(im imagevector.ImageVector, etcd *druidv1alpha1.Etcd, disableEtcdServiceAccountAutomount bool) (map[string]interface{}, error) { +func getMapFromEtcd(im imagevector.ImageVector, etcd *druidv1alpha1.Etcd, val componentetcd.Values, disableEtcdServiceAccountAutomount bool) (map[string]interface{}, error) { var statefulsetReplicas int if etcd.Spec.Replicas != 0 { statefulsetReplicas = 1 @@ -1426,13 +1349,17 @@ func getMapFromEtcd(im imagevector.ImageVector, etcd *druidv1alpha1.Etcd, disabl values["tlsCASecret"] = etcd.Spec.Etcd.TLS.TLSCASecretRef.Name } + if heartBeatDuration := etcd.Spec.Etcd.HeartbeatDuration; heartBeatDuration != nil { + values["heartbeatDuration"] = heartBeatDuration + } + if etcd.Spec.Backup.Store != nil { if values["store"], err = utils.GetStoreValues(etcd.Spec.Backup.Store); err != nil { return nil, err } - backupValues["fullSnapLeaseName"] = getFullSnapshotLeaseName(etcd) - backupValues["deltaSnapLeaseName"] = getDeltaSnapshotLeaseName(etcd) + backupValues["fullSnapLeaseName"] = val.Lease.FullSnapshotLeaseName + backupValues["deltaSnapLeaseName"] = val.Lease.DeltaSnapshotLeaseName } return values, nil diff --git a/controllers/etcd_controller_test.go b/controllers/etcd_controller_test.go index 9e131dc85..eb75fcb24 100644 --- a/controllers/etcd_controller_test.go +++ b/controllers/etcd_controller_test.go @@ -22,6 +22,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/pkg/common" + componentlease "github.com/gardener/etcd-druid/pkg/component/etcd/lease" "github.com/gardener/etcd-druid/pkg/utils" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" @@ -1065,6 +1066,8 @@ func validateEtcdWithDefaults(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSe "--etcd-connection-timeout=5m": Equal("--etcd-connection-timeout=5m"), "--snapstore-temp-directory=/var/etcd/data/temp": Equal("--snapstore-temp-directory=/var/etcd/data/temp"), "--etcd-process-name=etcd": Equal("--etcd-process-name=etcd"), + "--enable-member-lease-renewal=true": Equal("--enable-member-lease-renewal=true"), + "--k8s-heartbeat-duration=10s": Equal("--k8s-heartbeat-duration=10s"), fmt.Sprintf("--delta-snapshot-memory-limit=%d", deltaSnapShotMemLimit.Value()): Equal(fmt.Sprintf("--delta-snapshot-memory-limit=%d", deltaSnapShotMemLimit.Value())), fmt.Sprintf("--garbage-collection-policy=%s", druidv1alpha1.GarbageCollectionPolicyLimitBased): Equal(fmt.Sprintf("--garbage-collection-policy=%s", druidv1alpha1.GarbageCollectionPolicyLimitBased)), @@ -1450,6 +1453,8 @@ func validateEtcd(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev "--etcd-process-name=etcd": Equal("--etcd-process-name=etcd"), "--etcd-connection-timeout=5m": Equal("--etcd-connection-timeout=5m"), "--enable-snapshot-lease-renewal=true": Equal("--enable-snapshot-lease-renewal=true"), + "--enable-member-lease-renewal=true": Equal("--enable-member-lease-renewal=true"), + "--k8s-heartbeat-duration=10s": Equal("--k8s-heartbeat-duration=10s"), fmt.Sprintf("--defragmentation-schedule=%s", *instance.Spec.Etcd.DefragmentationSchedule): Equal(fmt.Sprintf("--defragmentation-schedule=%s", *instance.Spec.Etcd.DefragmentationSchedule)), fmt.Sprintf("--schedule=%s", *instance.Spec.Backup.FullSnapshotSchedule): Equal(fmt.Sprintf("--schedule=%s", *instance.Spec.Backup.FullSnapshotSchedule)), fmt.Sprintf("%s=%s", "--garbage-collection-policy", *instance.Spec.Backup.GarbageCollectionPolicy): Equal(fmt.Sprintf("%s=%s", "--garbage-collection-policy", *instance.Spec.Backup.GarbageCollectionPolicy)), @@ -1470,8 +1475,8 @@ func validateEtcd(instance *druidv1alpha1.Etcd, s *appsv1.StatefulSet, cm *corev fmt.Sprintf("%s=%s", "--owner-check-interval", instance.Spec.Backup.OwnerCheck.Interval.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--owner-check-interval", instance.Spec.Backup.OwnerCheck.Interval.Duration.String())), fmt.Sprintf("%s=%s", "--owner-check-timeout", instance.Spec.Backup.OwnerCheck.Timeout.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--owner-check-timeout", instance.Spec.Backup.OwnerCheck.Timeout.Duration.String())), fmt.Sprintf("%s=%s", "--owner-check-dns-cache-ttl", instance.Spec.Backup.OwnerCheck.DNSCacheTTL.Duration.String()): Equal(fmt.Sprintf("%s=%s", "--owner-check-dns-cache-ttl", instance.Spec.Backup.OwnerCheck.DNSCacheTTL.Duration.String())), - fmt.Sprintf("%s=%s", "--delta-snapshot-lease-name", getDeltaSnapshotLeaseName(instance)): Equal(fmt.Sprintf("%s=%s", "--delta-snapshot-lease-name", getDeltaSnapshotLeaseName(instance))), - fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", getFullSnapshotLeaseName(instance)): Equal(fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", getFullSnapshotLeaseName(instance))), + fmt.Sprintf("%s=%s", "--delta-snapshot-lease-name", componentlease.GetDeltaSnapshotLeaseName(instance)): Equal(fmt.Sprintf("%s=%s", "--delta-snapshot-lease-name", componentlease.GetDeltaSnapshotLeaseName(instance))), + fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", componentlease.GetFullSnapshotLeaseName(instance)): Equal(fmt.Sprintf("%s=%s", "--full-snapshot-lease-name", componentlease.GetFullSnapshotLeaseName(instance))), }), "Ports": ConsistOf([]corev1.ContainerPort{ corev1.ContainerPort{ diff --git a/main.go b/main.go index 2bbe20d6c..5719ac8e5 100644 --- a/main.go +++ b/main.go @@ -20,15 +20,12 @@ import ( "os" "time" - druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" "github.com/gardener/etcd-druid/controllers" controllersconfig "github.com/gardener/etcd-druid/controllers/config" + "github.com/gardener/etcd-druid/pkg/client/kubernetes" coordinationv1 "k8s.io/api/coordination/v1" coordinationv1beta1 "k8s.io/api/coordination/v1beta1" - "k8s.io/apimachinery/pkg/runtime" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - schemev1 "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" "k8s.io/client-go/tools/leaderelection/resourcelock" ctrl "sigs.k8s.io/controller-runtime" @@ -36,17 +33,7 @@ import ( // +kubebuilder:scaffold:imports ) -var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") -) - -func init() { - utilruntime.Must(schemev1.AddToScheme(scheme)) - utilruntime.Must(druidv1alpha1.AddToScheme(scheme)) - - // +kubebuilder:scaffold:scheme -} +var setupLog = ctrl.Log.WithName("setup") func main() { var ( @@ -67,6 +54,7 @@ func main() { disableEtcdServiceAccountAutomount bool etcdMemberNotReadyThreshold time.Duration + etcdMemberUnknownThreshold time.Duration // TODO: migrate default to `leases` in one of the next releases defaultLeaderElectionResourceLock = resourcelock.ConfigMapsLeasesResourceLock @@ -93,6 +81,7 @@ func main() { flag.BoolVar(&ignoreOperationAnnotation, "ignore-operation-annotation", true, "Ignore the operation annotation or not.") flag.DurationVar(&etcdMemberNotReadyThreshold, "etcd-member-notready-threshold", 5*time.Minute, "Threshold after which an etcd member is considered not ready if the status was unknown before.") flag.BoolVar(&disableEtcdServiceAccountAutomount, "disable-etcd-serviceaccount-automount", false, "If true then .automountServiceAccountToken will be set to false for the ServiceAccount created for etcd statefulsets.") + flag.DurationVar(&etcdMemberUnknownThreshold, "etcd-member-unknown-threshold", 1*time.Minute, "Threshold after which an etcd member is considered unknown.") flag.Parse() @@ -108,7 +97,7 @@ func main() { mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ ClientDisableCacheFor: uncachedObjects, - Scheme: scheme, + Scheme: kubernetes.Scheme, MetricsBindAddress: metricsAddr, LeaderElection: enableLeaderElection, LeaderElectionID: leaderElectionID, @@ -133,6 +122,7 @@ func main() { custodian := controllers.NewEtcdCustodian(mgr, controllersconfig.EtcdCustodianController{ EtcdMember: controllersconfig.EtcdMemberConfig{ EtcdMemberNotReadyThreshold: etcdMemberNotReadyThreshold, + EtcdMemberUnknownThreshold: etcdMemberUnknownThreshold, }, SyncPeriod: custodianSyncPeriod, }) diff --git a/pkg/client/kubernetes/types.go b/pkg/client/kubernetes/types.go new file mode 100644 index 000000000..f0df70f86 --- /dev/null +++ b/pkg/client/kubernetes/types.go @@ -0,0 +1,34 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubernetes + +import ( + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + schemev1 "k8s.io/client-go/kubernetes/scheme" +) + +var Scheme = runtime.NewScheme() + +func init() { + localSchemeBuilder := runtime.NewSchemeBuilder( + druidv1alpha1.AddToScheme, + schemev1.AddToScheme, + ) + + utilruntime.Must(localSchemeBuilder.AddToScheme(Scheme)) +} diff --git a/pkg/component/etcd/lease/lease.go b/pkg/component/etcd/lease/lease.go new file mode 100644 index 000000000..1ab50b278 --- /dev/null +++ b/pkg/component/etcd/lease/lease.go @@ -0,0 +1,107 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lease + +import ( + "context" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + + gardenercomponent "github.com/gardener/gardener/pkg/operation/botanist/component" + coordinationv1 "k8s.io/api/coordination/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type component struct { + client client.Client + namespace string + + values Values +} + +func (c *component) Deploy(ctx context.Context) error { + var ( + deltaSnapshotLease = c.emptyLease(c.values.DeltaSnapshotLeaseName) + fullSnapshotLease = c.emptyLease(c.values.FullSnapshotLeaseName) + ) + + if err := c.syncSnapshotLease(ctx, deltaSnapshotLease); err != nil { + return err + } + + if err := c.syncSnapshotLease(ctx, fullSnapshotLease); err != nil { + return err + } + + if err := c.syncMemberLeases(ctx); err != nil { + return err + } + + return nil +} + +func (c *component) Destroy(ctx context.Context) error { + var ( + deltaSnapshotLease = c.emptyLease(c.values.DeltaSnapshotLeaseName) + fullSnapshotLease = c.emptyLease(c.values.FullSnapshotLeaseName) + ) + + if err := c.deleteSnapshotLease(ctx, deltaSnapshotLease); err != nil { + return err + } + + if err := c.deleteSnapshotLease(ctx, fullSnapshotLease); err != nil { + return err + } + + if err := c.deleteAllMemberLeases(ctx); err != nil { + return err + } + + return nil +} + +// New creates a new lease deployer instance. +func New(c client.Client, namespace string, values Values) gardenercomponent.Deployer { + return &component{ + client: c, + namespace: namespace, + values: values, + } +} + +func (c *component) emptyLease(name string) *coordinationv1.Lease { + return &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: c.namespace, + }, + } +} + +func getOwnerReferences(val Values) []metav1.OwnerReference { + return []metav1.OwnerReference{ + { + APIVersion: druidv1alpha1.GroupVersion.String(), + Kind: "Etcd", + Name: val.EtcdName, + UID: val.EtcdUID, + Controller: pointer.BoolPtr(true), + BlockOwnerDeletion: pointer.BoolPtr(true), + }, + } +} diff --git a/pkg/component/etcd/lease/lease_member.go b/pkg/component/etcd/lease/lease_member.go new file mode 100644 index 000000000..f8bea489a --- /dev/null +++ b/pkg/component/etcd/lease/lease_member.go @@ -0,0 +1,102 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lease + +import ( + "context" + "fmt" + + "github.com/gardener/etcd-druid/pkg/common" + + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/gardener/gardener/pkg/controllerutils" + "github.com/gardener/gardener/pkg/utils/flow" + coordinationv1 "k8s.io/api/coordination/v1" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func (c *component) deleteAllMemberLeases(ctx context.Context) error { + labels := getMemberLeaseLabels(c.values) + + return c.client.DeleteAllOf(ctx, &coordinationv1.Lease{}, client.InNamespace(c.namespace), client.MatchingLabels(labels)) +} + +func (c *component) syncMemberLeases(ctx context.Context) error { + var ( + fns []flow.TaskFn + + labels = getMemberLeaseLabels(c.values) + prefix = c.values.EtcdName + leaseNames = sets.NewString() + ) + + // Patch or create necessary member leases. + for i := 0; i < int(c.values.Replicas); i++ { + leaseName := memberLeaseName(prefix, i) + + lease := c.emptyLease(leaseName) + fns = append(fns, func(ctx context.Context) error { + _, err := controllerutils.GetAndCreateOrMergePatch(ctx, c.client, lease, func() error { + if lease.Labels == nil { + lease.Labels = make(map[string]string) + } + for k, v := range labels { + lease.Labels[k] = v + } + lease.OwnerReferences = getOwnerReferences(c.values) + return nil + }) + return err + }) + + leaseNames = leaseNames.Insert(leaseName) + } + + leaseList := &coordinationv1.LeaseList{} + if err := c.client.List(ctx, leaseList, client.MatchingLabels(labels)); err != nil { + return err + } + + // Clean up superfluous member leases. + for _, lease := range leaseList.Items { + ls := lease + if leaseNames.Has(ls.Name) { + continue + } + fns = append(fns, func(ctx context.Context) error { + if err := c.client.Delete(ctx, &ls); client.IgnoreNotFound(err) != nil { + return err + } + return nil + }) + } + + return flow.Parallel(fns...)(ctx) +} + +// PurposeMemberLease is a constant used as a purpose for etcd member lease objects. +const PurposeMemberLease = "etcd-member-lease" + +func getMemberLeaseLabels(val Values) map[string]string { + return map[string]string{ + common.GardenerOwnedBy: val.EtcdName, + v1beta1constants.GardenerPurpose: PurposeMemberLease, + } +} + +func memberLeaseName(etcdName string, replica int) string { + return fmt.Sprintf("%s-%d", etcdName, replica) +} diff --git a/pkg/component/etcd/lease/lease_snapshot.go b/pkg/component/etcd/lease/lease_snapshot.go new file mode 100644 index 000000000..0781d56b8 --- /dev/null +++ b/pkg/component/etcd/lease/lease_snapshot.go @@ -0,0 +1,39 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lease + +import ( + "context" + + "github.com/gardener/gardener/pkg/controllerutils" + + coordinationv1 "k8s.io/api/coordination/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func (c *component) deleteSnapshotLease(ctx context.Context, lease *coordinationv1.Lease) error { + return client.IgnoreNotFound(c.client.Delete(ctx, lease)) +} + +func (c *component) syncSnapshotLease(ctx context.Context, lease *coordinationv1.Lease) error { + if !c.values.BackupEnabled { + return c.deleteSnapshotLease(ctx, lease) + } + _, err := controllerutils.GetAndCreateOrMergePatch(ctx, c.client, lease, func() error { + lease.OwnerReferences = getOwnerReferences(c.values) + return nil + }) + return err +} diff --git a/pkg/component/etcd/lease/lease_suite_test.go b/pkg/component/etcd/lease/lease_suite_test.go new file mode 100644 index 000000000..e2556eb9a --- /dev/null +++ b/pkg/component/etcd/lease/lease_suite_test.go @@ -0,0 +1,27 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lease_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestLease(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Etcd Component Suite") +} diff --git a/pkg/component/etcd/lease/lease_test.go b/pkg/component/etcd/lease/lease_test.go new file mode 100644 index 000000000..07e12afd3 --- /dev/null +++ b/pkg/component/etcd/lease/lease_test.go @@ -0,0 +1,268 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lease_test + +import ( + "context" + "fmt" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + "github.com/gardener/etcd-druid/pkg/client/kubernetes" + "github.com/gardener/etcd-druid/pkg/common" + . "github.com/gardener/etcd-druid/pkg/component/etcd/lease" + + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/gardener/gardener/pkg/operation/botanist/component" + "github.com/gardener/gardener/pkg/utils/test/matchers" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + gomegatypes "github.com/onsi/gomega/types" + coordinationv1 "k8s.io/api/coordination/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +var _ = Describe("Lease", func() { + var ( + ctx context.Context + c client.Client + + etcd *druidv1alpha1.Etcd + namespace string + name string + uid types.UID + replicas int32 + + values Values + leaseDeployer component.Deployer + ) + + BeforeEach(func() { + ctx = context.Background() + namespace = "default" + name = "etcd-test" + replicas = 3 + uid = "123" + + etcd = &druidv1alpha1.Etcd{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + UID: uid, + }, + Spec: druidv1alpha1.EtcdSpec{ + Backup: druidv1alpha1.BackupSpec{ + Store: new(druidv1alpha1.StoreSpec), + }, + Replicas: replicas, + }, + } + + c = fakeclient.NewClientBuilder().WithScheme(kubernetes.Scheme).Build() + + values = GenerateValues(etcd) + leaseDeployer = New(c, namespace, values) + }) + + Describe("#Deploy", func() { + Context("when deployment is new", func() { + It("should successfully deploy", func() { + Expect(leaseDeployer.Deploy(ctx)).To(Succeed()) + + checkMemberLeases(ctx, c, etcd) + checkSnapshotLeases(ctx, c, etcd, values) + }) + }) + + Context("when deployment is changed from 5 -> 3 replicas", func() { + BeforeEach(func() { + // Create existing leases + for _, l := range []coordinationv1.Lease{ + memberLease(etcd, 0), + memberLease(etcd, 1), + memberLease(etcd, 2), + memberLease(etcd, 3), + memberLease(etcd, 4), + } { + lease := l + Expect(c.Create(ctx, &lease)).To(Succeed()) + } + }) + + It("should successfully deploy", func() { + Expect(leaseDeployer.Deploy(ctx)).To(Succeed()) + + checkMemberLeases(ctx, c, etcd) + checkSnapshotLeases(ctx, c, etcd, values) + }) + }) + + Context("when deployment is changed from 1 -> 3 replicas", func() { + BeforeEach(func() { + // Create existing leases + for _, l := range []coordinationv1.Lease{ + memberLease(etcd, 0), + } { + lease := l + Expect(c.Create(ctx, &lease)).To(Succeed()) + } + }) + + It("should successfully deploy", func() { + Expect(leaseDeployer.Deploy(ctx)).To(Succeed()) + + checkMemberLeases(ctx, c, etcd) + checkSnapshotLeases(ctx, c, etcd, values) + }) + }) + + Context("when backup is disabled", func() { + BeforeEach(func() { + leaseDeployer = New(c, namespace, Values{ + BackupEnabled: false, + EtcdName: name, + EtcdUID: uid, + Replicas: replicas, + }) + }) + + It("should successfully deploy", func() { + // Snapshot leases might exist before, so create them here. + for _, l := range []coordinationv1.Lease{ + {ObjectMeta: metav1.ObjectMeta{Name: values.DeltaSnapshotLeaseName}}, + {ObjectMeta: metav1.ObjectMeta{Name: values.FullSnapshotLeaseName}}, + } { + lease := l + Expect(c.Create(ctx, &lease)).To(Succeed()) + } + + Expect(leaseDeployer.Deploy(ctx)).To(Succeed()) + + checkMemberLeases(ctx, c, etcd) + + _, err := getSnapshotLease(ctx, c, namespace, values.DeltaSnapshotLeaseName) + Expect(err).To(matchers.BeNotFoundError()) + _, err = getSnapshotLease(ctx, c, namespace, values.FullSnapshotLeaseName) + Expect(err).To(matchers.BeNotFoundError()) + }) + }) + }) + + Describe("#Destroy", func() { + Context("when no leases exist", func() { + It("should destroy without errors", func() { + Expect(leaseDeployer.Destroy(ctx)).To(Succeed()) + }) + }) + + Context("when leases exist", func() { + It("should destroy without errors", func() { + for _, l := range []coordinationv1.Lease{ + memberLease(etcd, 0), + memberLease(etcd, 1), + memberLease(etcd, 2), + {ObjectMeta: metav1.ObjectMeta{Name: values.DeltaSnapshotLeaseName}}, + {ObjectMeta: metav1.ObjectMeta{Name: values.FullSnapshotLeaseName}}, + } { + lease := l + Expect(c.Create(ctx, &lease)).To(Succeed()) + } + + Expect(leaseDeployer.Destroy(ctx)).To(Succeed()) + + _, err := getSnapshotLease(ctx, c, namespace, values.DeltaSnapshotLeaseName) + Expect(err).To(matchers.BeNotFoundError()) + _, err = getSnapshotLease(ctx, c, namespace, values.FullSnapshotLeaseName) + Expect(err).To(matchers.BeNotFoundError()) + + leases := &coordinationv1.LeaseList{} + Expect(c.List(ctx, leases, client.InNamespace(etcd.Namespace), client.MatchingLabels(map[string]string{ + common.GardenerOwnedBy: etcd.Name, + v1beta1constants.GardenerPurpose: "etcd-member-lease", + }))).To(Succeed()) + + Expect(leases.Items).To(BeEmpty()) + }) + }) + }) +}) + +func getSnapshotLease(ctx context.Context, c client.Client, namespace, name string) (*coordinationv1.Lease, error) { + snapshotLease := coordinationv1.Lease{} + if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, &snapshotLease); err != nil { + return nil, err + } + return &snapshotLease, nil +} + +func checkSnapshotLeases(ctx context.Context, c client.Client, etcd *druidv1alpha1.Etcd, val Values) { + deltaLease, err := getSnapshotLease(ctx, c, etcd.Namespace, val.DeltaSnapshotLeaseName) + Expect(err).NotTo(HaveOccurred()) + Expect(deltaLease).To(PointTo(matchLeaseElement(deltaLease.Name, etcd.Name, etcd.UID))) + + fullLease, err := getSnapshotLease(ctx, c, etcd.Namespace, val.FullSnapshotLeaseName) + Expect(err).NotTo(HaveOccurred()) + Expect(fullLease).To(PointTo(matchLeaseElement(fullLease.Name, etcd.Name, etcd.UID))) +} + +func checkMemberLeases(ctx context.Context, c client.Client, etcd *druidv1alpha1.Etcd) { + leases := &coordinationv1.LeaseList{} + Expect(c.List(ctx, leases, client.InNamespace(etcd.Namespace), client.MatchingLabels(map[string]string{ + common.GardenerOwnedBy: etcd.Name, + v1beta1constants.GardenerPurpose: "etcd-member-lease", + }))).To(Succeed()) + + Expect(leases.Items).To(ConsistOf(memberLeases(etcd.Name, etcd.UID, etcd.Spec.Replicas))) +} + +func memberLeases(name string, etcdUID types.UID, replicas int32) []interface{} { + var elements []interface{} + for i := 0; i < int(replicas); i++ { + elements = append(elements, matchLeaseElement(fmt.Sprintf("%s-%d", name, i), name, etcdUID)) + } + + return elements +} + +func matchLeaseElement(leaseName, etcdName string, etcdUID types.UID) gomegatypes.GomegaMatcher { + return MatchFields(IgnoreExtras, Fields{ + "ObjectMeta": MatchFields(IgnoreExtras, Fields{ + "Name": Equal(leaseName), + "OwnerReferences": ConsistOf(MatchFields(IgnoreExtras, Fields{ + "APIVersion": Equal(druidv1alpha1.GroupVersion.String()), + "Kind": Equal("Etcd"), + "Name": Equal(etcdName), + "UID": Equal(etcdUID), + "Controller": PointTo(BeTrue()), + "BlockOwnerDeletion": PointTo(BeTrue()), + })), + }), + }) +} + +func memberLease(etcd *druidv1alpha1.Etcd, replica int) coordinationv1.Lease { + return coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%d", etcd.Name, replica), + Labels: map[string]string{ + common.GardenerOwnedBy: etcd.Name, + v1beta1constants.GardenerPurpose: "etcd-member-lease", + }, + }, + } +} diff --git a/pkg/component/etcd/lease/values.go b/pkg/component/etcd/lease/values.go new file mode 100644 index 000000000..45c9c1807 --- /dev/null +++ b/pkg/component/etcd/lease/values.go @@ -0,0 +1,34 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lease + +import ( + "k8s.io/apimachinery/pkg/types" +) + +type Values struct { + // BackupEnabled specifies if the backup functionality for the etcd cluster is enabled. + BackupEnabled bool + // EtcdName is the name of the etcd resource. + EtcdName string + // EtcdName is the UID of the etcd resource. + EtcdUID types.UID + // DeltaSnapshotLeaseName is the name of the delta snapshot lease object. + DeltaSnapshotLeaseName string + // FullSnapshotLeaseName is the name of the full snapshot lease object. + FullSnapshotLeaseName string + // Replicas is the replica count of the etcd cluster. + Replicas int32 +} diff --git a/pkg/component/etcd/lease/values_helper.go b/pkg/component/etcd/lease/values_helper.go new file mode 100644 index 000000000..914e56540 --- /dev/null +++ b/pkg/component/etcd/lease/values_helper.go @@ -0,0 +1,43 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lease + +import ( + "fmt" + + druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" +) + +// GenerateValues generates `lease.Values` for the lease component with the given parameters. +func GenerateValues(etcd *druidv1alpha1.Etcd) Values { + return Values{ + BackupEnabled: etcd.Spec.Backup.Store != nil, + EtcdName: etcd.Name, + EtcdUID: etcd.UID, + DeltaSnapshotLeaseName: GetDeltaSnapshotLeaseName(etcd), + FullSnapshotLeaseName: GetFullSnapshotLeaseName(etcd), + Replicas: etcd.Spec.Replicas, + } +} + +// GetDeltaSnapshotLeaseName returns the name of the delta snapshot lease based on the given `etcd` object. +func GetDeltaSnapshotLeaseName(etcd *druidv1alpha1.Etcd) string { + return fmt.Sprintf("%s-delta-snap", etcd.Name) +} + +// GetFullSnapshotLeaseName returns the name of the full snapshot lease based on the given `etcd` object. +func GetFullSnapshotLeaseName(etcd *druidv1alpha1.Etcd) string { + return fmt.Sprintf("%s-full-snap", etcd.Name) +} diff --git a/pkg/component/etcd/values.go b/pkg/component/etcd/values.go new file mode 100644 index 000000000..4d447b033 --- /dev/null +++ b/pkg/component/etcd/values.go @@ -0,0 +1,22 @@ +// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package etcd + +import "github.com/gardener/etcd-druid/pkg/component/etcd/lease" + +// Values contains all values relevant for deploying etcd components. +type Values struct { + Lease lease.Values +} diff --git a/pkg/health/etcdmember/check_ready.go b/pkg/health/etcdmember/check_ready.go index 9b9d63d8c..b917f7ed8 100644 --- a/pkg/health/etcdmember/check_ready.go +++ b/pkg/health/etcdmember/check_ready.go @@ -16,10 +16,13 @@ package etcdmember import ( "context" - "fmt" "strings" "time" + componentlease "github.com/gardener/etcd-druid/pkg/component/etcd/lease" + + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + kutil "github.com/gardener/gardener/pkg/utils/kubernetes" "github.com/go-logr/logr" coordinationv1 "k8s.io/api/coordination/v1" @@ -49,17 +52,12 @@ func (r *readyCheck) Check(ctx context.Context, etcd druidv1alpha1.Etcd) []Resul ) leases := &coordinationv1.LeaseList{} - if err := r.cl.List(ctx, leases, client.InNamespace(etcd.Namespace), client.MatchingLabels{common.GardenerOwnedBy: etcd.Name}); err != nil { + if err := r.cl.List(ctx, leases, client.InNamespace(etcd.Namespace), client.MatchingLabels{ + common.GardenerOwnedBy: etcd.Name, v1beta1constants.GardenerPurpose: componentlease.PurposeMemberLease}); err != nil { r.logger.Error(err, "failed to get leases for etcd member readiness check") } for _, lease := range leases.Items { - leaseDurationSeconds := lease.Spec.LeaseDurationSeconds - if leaseDurationSeconds == nil { - r.logger.Error(fmt.Errorf("leaseDurationSeconds not set for lease object %s/%s", lease.Namespace, lease.Name), "Failed to perform member readiness check") - continue - } - var ( id, role = separateIdFromRole(lease.Spec.HolderIdentity) res = &result{ @@ -79,7 +77,7 @@ func (r *readyCheck) Check(ctx context.Context, etcd druidv1alpha1.Etcd) []Resul } // Check if member state must be considered as not ready - if renew.Add(time.Duration(*leaseDurationSeconds) * time.Second).Add(r.memberConfig.EtcdMemberNotReadyThreshold).Before(checkTime) { + if renew.Add(r.memberConfig.EtcdMemberUnknownThreshold).Add(r.memberConfig.EtcdMemberNotReadyThreshold).Before(checkTime) { res.status = druidv1alpha1.EtcdMemberStatusNotReady res.reason = "UnknownGracePeriodExceeded" results = append(results, res) @@ -87,7 +85,7 @@ func (r *readyCheck) Check(ctx context.Context, etcd druidv1alpha1.Etcd) []Resul } // Check if member state must be considered as unknown - if renew.Add(time.Duration(*leaseDurationSeconds) * time.Second).Before(checkTime) { + if renew.Add(r.memberConfig.EtcdMemberUnknownThreshold).Before(checkTime) { // If pod is not running or cannot be found then we deduce that the status is NotReady. ready, err := r.checkContainersAreReady(ctx, lease.Namespace, lease.Name) if (err == nil && !ready) || apierrors.IsNotFound(err) { diff --git a/pkg/health/etcdmember/check_ready_test.go b/pkg/health/etcdmember/check_ready_test.go index 2d63f5004..9b4b1a23e 100644 --- a/pkg/health/etcdmember/check_ready_test.go +++ b/pkg/health/etcdmember/check_ready_test.go @@ -20,6 +20,8 @@ import ( "fmt" "time" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + kutil "github.com/gardener/gardener/pkg/utils/kubernetes" "github.com/gardener/gardener/pkg/utils/test" "github.com/golang/mock/gomock" @@ -37,6 +39,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" controllersconfig "github.com/gardener/etcd-druid/controllers/config" "github.com/gardener/etcd-druid/pkg/common" + componentlease "github.com/gardener/etcd-druid/pkg/component/etcd/lease" . "github.com/gardener/etcd-druid/pkg/health/etcdmember" mockclient "github.com/gardener/etcd-druid/pkg/mock/controller-runtime/client" ) @@ -44,13 +47,13 @@ import ( var _ = Describe("ReadyCheck", func() { Describe("#Check", func() { var ( - ctx context.Context - mockCtrl *gomock.Controller - cl *mockclient.MockClient - leaseDurationSeconds *int32 - leaseDuration, notReadyThreshold time.Duration - now time.Time - check Checker + ctx context.Context + mockCtrl *gomock.Controller + cl *mockclient.MockClient + leaseDurationSeconds *int32 + unknownThreshold, notReadyThreshold time.Duration + now time.Time + check Checker member1Name string member1ID *string @@ -62,13 +65,13 @@ var _ = Describe("ReadyCheck", func() { ctx = context.Background() mockCtrl = gomock.NewController(GinkgoT()) cl = mockclient.NewMockClient(mockCtrl) - leaseDurationSeconds = pointer.Int32Ptr(300) - leaseDuration = time.Duration(*leaseDurationSeconds) * time.Second + unknownThreshold = 300 * time.Second notReadyThreshold = 60 * time.Second now, _ = time.Parse(time.RFC3339, "2021-06-01T00:00:00Z") check = ReadyCheck(cl, log.NullLogger{}, controllersconfig.EtcdCustodianController{ EtcdMember: controllersconfig.EtcdMemberConfig{ EtcdMemberNotReadyThreshold: notReadyThreshold, + EtcdMemberUnknownThreshold: unknownThreshold, }, }) @@ -88,16 +91,18 @@ var _ = Describe("ReadyCheck", func() { }) JustBeforeEach(func() { - cl.EXPECT().List(ctx, gomock.AssignableToTypeOf(&coordinationv1.LeaseList{}), client.InNamespace(etcd.Namespace), client.MatchingLabels{common.GardenerOwnedBy: etcd.Name}).DoAndReturn( - func(_ context.Context, leases *coordinationv1.LeaseList, _ ...client.ListOption) error { - *leases = *leasesList - return nil - }) + cl.EXPECT().List(ctx, gomock.AssignableToTypeOf(&coordinationv1.LeaseList{}), client.InNamespace(etcd.Namespace), + client.MatchingLabels{common.GardenerOwnedBy: etcd.Name, v1beta1constants.GardenerPurpose: componentlease.PurposeMemberLease}). + DoAndReturn( + func(_ context.Context, leases *coordinationv1.LeaseList, _ ...client.ListOption) error { + *leases = *leasesList + return nil + }) }) Context("when just expired", func() { BeforeEach(func() { - renewTime := metav1.NewMicroTime(now.Add(-1 * leaseDuration).Add(-1 * time.Second)) + renewTime := metav1.NewMicroTime(now.Add(-1 * unknownThreshold).Add(-1 * time.Second)) leasesList = &coordinationv1.LeaseList{ Items: []coordinationv1.Lease{ { @@ -223,8 +228,8 @@ var _ = Describe("ReadyCheck", func() { member2ID = pointer.StringPtr("2") var ( - shortExpirationTime = metav1.NewMicroTime(now.Add(-1 * leaseDuration).Add(-1 * time.Second)) - longExpirationTime = metav1.NewMicroTime(now.Add(-1 * leaseDuration).Add(-1 * time.Second).Add(-1 * notReadyThreshold)) + shortExpirationTime = metav1.NewMicroTime(now.Add(-1 * unknownThreshold).Add(-1 * time.Second)) + longExpirationTime = metav1.NewMicroTime(now.Add(-1 * unknownThreshold).Add(-1 * time.Second).Add(-1 * notReadyThreshold)) ) leasesList = &coordinationv1.LeaseList{ @@ -289,7 +294,7 @@ var _ = Describe("ReadyCheck", func() { member2ID = pointer.StringPtr("2") member3Name = "member3" member3ID = pointer.StringPtr("3") - renewTime := metav1.NewMicroTime(now.Add(-1 * leaseDuration)) + renewTime := metav1.NewMicroTime(now.Add(-1 * unknownThreshold)) leasesList = &coordinationv1.LeaseList{ Items: []coordinationv1.Lease{ { @@ -356,7 +361,7 @@ var _ = Describe("ReadyCheck", func() { BeforeEach(func() { member2Name = "member2" - renewTime := metav1.NewMicroTime(now.Add(-1 * leaseDuration)) + renewTime := metav1.NewMicroTime(now.Add(-1 * unknownThreshold)) leasesList = &coordinationv1.LeaseList{ Items: []coordinationv1.Lease{ { diff --git a/vendor/github.com/gardener/gardener/pkg/operation/botanist/component/deploy.go b/vendor/github.com/gardener/gardener/pkg/operation/botanist/component/deploy.go new file mode 100644 index 000000000..be9c45fee --- /dev/null +++ b/vendor/github.com/gardener/gardener/pkg/operation/botanist/component/deploy.go @@ -0,0 +1,131 @@ +// Copyright (c) 2020 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package component + +import "context" + +// OpDestroy creates a DeployWaiter which calls Destroy instead of create +// and WaitCleanup instead of Wait +func OpDestroy(dw ...DeployWaiter) DeployWaiter { + return &deploy{ + dw: dw, + invert: true, + destroyOnly: false, + } +} + +// OpDestroyAndWait creates a Deployer which calls Destroy instead of create +// and waits for destruction. +func OpDestroyAndWait(dw ...DeployWaiter) Deployer { + return &deploy{ + dw: dw, + invert: true, + destroyOnly: true, + } +} + +// OpWaiter creates a Deployer which calls waits for each operation. +func OpWaiter(dw ...DeployWaiter) Deployer { + return &deploy{ + dw: dw, + invert: false, + destroyOnly: true, + } +} + +// NoOp does nothing +func NoOp() DeployWaiter { return &deploy{} } + +type deploy struct { + invert bool + destroyOnly bool + dw []DeployWaiter +} + +func (d *deploy) Deploy(ctx context.Context) error { + if d.invert { + return d.Destroy(ctx) + } + + for _, dw := range d.dw { + if dw == nil { + continue + } + + if err := dw.Deploy(ctx); err != nil { + return err + } + + if d.destroyOnly { + if err := dw.Wait(ctx); err != nil { + return err + } + } + } + + return nil +} + +func (d *deploy) Destroy(ctx context.Context) error { + for _, dw := range d.dw { + if dw == nil { + continue + } + + if err := dw.Destroy(ctx); err != nil { + return err + } + + if d.destroyOnly { + if err := dw.WaitCleanup(ctx); err != nil { + return err + } + } + } + + return nil +} + +func (d *deploy) Wait(ctx context.Context) error { + if d.invert { + return d.WaitCleanup(ctx) + } + + for _, dw := range d.dw { + if dw == nil { + continue + } + + if err := dw.Wait(ctx); err != nil { + return err + } + } + + return nil +} + +func (d *deploy) WaitCleanup(ctx context.Context) error { + for _, dw := range d.dw { + if dw == nil { + continue + } + + if err := dw.WaitCleanup(ctx); err != nil { + return err + } + } + + return nil +} diff --git a/vendor/github.com/gardener/gardener/pkg/operation/botanist/component/interfaces.go b/vendor/github.com/gardener/gardener/pkg/operation/botanist/component/interfaces.go new file mode 100644 index 000000000..1e37d1c08 --- /dev/null +++ b/vendor/github.com/gardener/gardener/pkg/operation/botanist/component/interfaces.go @@ -0,0 +1,76 @@ +// Copyright (c) 2020 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package component + +import ( + "context" + + "github.com/gardener/gardener/pkg/apis/core/v1alpha1" +) + +// Deployer is used to control the life-cycle of a component. +type Deployer interface { + // Deploy a component. + Deploy(ctx context.Context) error + // Destroy already deployed component. + Destroy(ctx context.Context) error +} + +// Waiter waits for life-cycle operations of a component to finish. +type Waiter interface { + // Wait for deployment to finish and component to report ready. + Wait(ctx context.Context) error + // WaitCleanup for destruction to finish and component to be fully removed. + WaitCleanup(ctx context.Context) error +} + +// Migrator is used to control the control-plane migration operations of a component. +type Migrator interface { + Restore(ctx context.Context, shootState *v1alpha1.ShootState) error + Migrate(ctx context.Context) error +} + +// MigrateWaiter waits for the control-plane migration operations of a component to finish. +type MigrateWaiter interface { + WaitMigrate(ctx context.Context) error +} + +// MonitoringComponent exposes configuration for Prometheus as well as the AlertManager. +type MonitoringComponent interface { + // ScrapeConfigs returns the scrape configurations for Prometheus. + ScrapeConfigs() ([]string, error) + // AlertingRules returns the alerting rules configs for AlertManager (mapping file name to rule config). + AlertingRules() (map[string]string, error) +} + +// CentralMonitoringConfiguration is a function alias for returning configuration for the central monitoring. +type CentralMonitoringConfiguration func() (CentralMonitoringConfig, error) + +// CentralLoggingConfiguration is a function alias for returning configuration for the central logging. +type CentralLoggingConfiguration func() (CentralLoggingConfig, error) + +// DeployWaiter controls and waits for life-cycle operations of a component. +type DeployWaiter interface { + Deployer + Waiter +} + +// DeployMigrateWaiter controls and waits for the life-cycle and control-plane migration operations of a component. +type DeployMigrateWaiter interface { + Deployer + Migrator + MigrateWaiter + Waiter +} diff --git a/vendor/github.com/gardener/gardener/pkg/operation/botanist/component/phase.go b/vendor/github.com/gardener/gardener/pkg/operation/botanist/component/phase.go new file mode 100644 index 000000000..f974a1ddb --- /dev/null +++ b/vendor/github.com/gardener/gardener/pkg/operation/botanist/component/phase.go @@ -0,0 +1,48 @@ +// Copyright (c) 2020 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package component + +// Phase is the phase of a component. +type Phase int + +const ( + // PhaseUnknown is in an unknown component phase. + PhaseUnknown Phase = iota + // PhaseEnabled is when a component was enabled before and it's still active. + PhaseEnabled + // PhaseDisabled is when a component was disabled before and it's still disabled. + PhaseDisabled + // PhaseEnabling is when a component was disabled before, but it's being activated. + PhaseEnabling + // PhaseDisabling is when a component was enabled before, but it's being disabled. + PhaseDisabling +) + +// Done returns a completed phase. e.g. +// Enabling -> Enabled +// Disabling -> Disabled +// otherwise returns the same phase. +func (s Phase) Done() Phase { + switch s { + case PhaseEnabling: + return PhaseEnabled + case PhaseDisabling: + return PhaseDisabled + case PhaseEnabled, PhaseDisabled, PhaseUnknown: + return s + default: + return PhaseUnknown + } +} diff --git a/vendor/github.com/gardener/gardener/pkg/operation/botanist/component/types.go b/vendor/github.com/gardener/gardener/pkg/operation/botanist/component/types.go new file mode 100644 index 000000000..3f3ae896d --- /dev/null +++ b/vendor/github.com/gardener/gardener/pkg/operation/botanist/component/types.go @@ -0,0 +1,45 @@ +// Copyright (c) 2020 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package component + +// Secret is a structure that contains information about a Kubernetes secret which is managed externally. +type Secret struct { + // Name is the name of the Kubernetes secret object. + Name string + // Checksum is the checksum of the secret's data. + Checksum string + // Data is the data of the secret. + Data map[string][]byte +} + +// CentralMonitoringConfig is a structure that contains configuration for the central monitoring stack. +type CentralMonitoringConfig struct { + // ScrapeConfigs are the scrape configurations for central Prometheus. + ScrapeConfigs []string + // CAdvisorScrapeConfigMetricRelabelConfigs are metric_relabel_configs for the cadvisor scrape config job. + CAdvisorScrapeConfigMetricRelabelConfigs []string +} + +// CentralLoggingConfig is a structure that contains configuration for the central logging stack. +type CentralLoggingConfig struct { + // Filters contains the filters for specific component. + Filters string + // Parser contains the parsers for specific component. + Parsers string + // PodPrefix is the prefix of the pod name. + PodPrefix string + // UserExposed defines if the component is exposed to the end-user. + UserExposed bool +} diff --git a/vendor/k8s.io/client-go/testing/actions.go b/vendor/k8s.io/client-go/testing/actions.go new file mode 100644 index 000000000..b6b2c1f22 --- /dev/null +++ b/vendor/k8s.io/client-go/testing/actions.go @@ -0,0 +1,681 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "fmt" + "path" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" +) + +func NewRootGetAction(resource schema.GroupVersionResource, name string) GetActionImpl { + action := GetActionImpl{} + action.Verb = "get" + action.Resource = resource + action.Name = name + + return action +} + +func NewGetAction(resource schema.GroupVersionResource, namespace, name string) GetActionImpl { + action := GetActionImpl{} + action.Verb = "get" + action.Resource = resource + action.Namespace = namespace + action.Name = name + + return action +} + +func NewGetSubresourceAction(resource schema.GroupVersionResource, namespace, subresource, name string) GetActionImpl { + action := GetActionImpl{} + action.Verb = "get" + action.Resource = resource + action.Subresource = subresource + action.Namespace = namespace + action.Name = name + + return action +} + +func NewRootGetSubresourceAction(resource schema.GroupVersionResource, subresource, name string) GetActionImpl { + action := GetActionImpl{} + action.Verb = "get" + action.Resource = resource + action.Subresource = subresource + action.Name = name + + return action +} + +func NewRootListAction(resource schema.GroupVersionResource, kind schema.GroupVersionKind, opts interface{}) ListActionImpl { + action := ListActionImpl{} + action.Verb = "list" + action.Resource = resource + action.Kind = kind + labelSelector, fieldSelector, _ := ExtractFromListOptions(opts) + action.ListRestrictions = ListRestrictions{labelSelector, fieldSelector} + + return action +} + +func NewListAction(resource schema.GroupVersionResource, kind schema.GroupVersionKind, namespace string, opts interface{}) ListActionImpl { + action := ListActionImpl{} + action.Verb = "list" + action.Resource = resource + action.Kind = kind + action.Namespace = namespace + labelSelector, fieldSelector, _ := ExtractFromListOptions(opts) + action.ListRestrictions = ListRestrictions{labelSelector, fieldSelector} + + return action +} + +func NewRootCreateAction(resource schema.GroupVersionResource, object runtime.Object) CreateActionImpl { + action := CreateActionImpl{} + action.Verb = "create" + action.Resource = resource + action.Object = object + + return action +} + +func NewCreateAction(resource schema.GroupVersionResource, namespace string, object runtime.Object) CreateActionImpl { + action := CreateActionImpl{} + action.Verb = "create" + action.Resource = resource + action.Namespace = namespace + action.Object = object + + return action +} + +func NewRootCreateSubresourceAction(resource schema.GroupVersionResource, name, subresource string, object runtime.Object) CreateActionImpl { + action := CreateActionImpl{} + action.Verb = "create" + action.Resource = resource + action.Subresource = subresource + action.Name = name + action.Object = object + + return action +} + +func NewCreateSubresourceAction(resource schema.GroupVersionResource, name, subresource, namespace string, object runtime.Object) CreateActionImpl { + action := CreateActionImpl{} + action.Verb = "create" + action.Resource = resource + action.Namespace = namespace + action.Subresource = subresource + action.Name = name + action.Object = object + + return action +} + +func NewRootUpdateAction(resource schema.GroupVersionResource, object runtime.Object) UpdateActionImpl { + action := UpdateActionImpl{} + action.Verb = "update" + action.Resource = resource + action.Object = object + + return action +} + +func NewUpdateAction(resource schema.GroupVersionResource, namespace string, object runtime.Object) UpdateActionImpl { + action := UpdateActionImpl{} + action.Verb = "update" + action.Resource = resource + action.Namespace = namespace + action.Object = object + + return action +} + +func NewRootPatchAction(resource schema.GroupVersionResource, name string, pt types.PatchType, patch []byte) PatchActionImpl { + action := PatchActionImpl{} + action.Verb = "patch" + action.Resource = resource + action.Name = name + action.PatchType = pt + action.Patch = patch + + return action +} + +func NewPatchAction(resource schema.GroupVersionResource, namespace string, name string, pt types.PatchType, patch []byte) PatchActionImpl { + action := PatchActionImpl{} + action.Verb = "patch" + action.Resource = resource + action.Namespace = namespace + action.Name = name + action.PatchType = pt + action.Patch = patch + + return action +} + +func NewRootPatchSubresourceAction(resource schema.GroupVersionResource, name string, pt types.PatchType, patch []byte, subresources ...string) PatchActionImpl { + action := PatchActionImpl{} + action.Verb = "patch" + action.Resource = resource + action.Subresource = path.Join(subresources...) + action.Name = name + action.PatchType = pt + action.Patch = patch + + return action +} + +func NewPatchSubresourceAction(resource schema.GroupVersionResource, namespace, name string, pt types.PatchType, patch []byte, subresources ...string) PatchActionImpl { + action := PatchActionImpl{} + action.Verb = "patch" + action.Resource = resource + action.Subresource = path.Join(subresources...) + action.Namespace = namespace + action.Name = name + action.PatchType = pt + action.Patch = patch + + return action +} + +func NewRootUpdateSubresourceAction(resource schema.GroupVersionResource, subresource string, object runtime.Object) UpdateActionImpl { + action := UpdateActionImpl{} + action.Verb = "update" + action.Resource = resource + action.Subresource = subresource + action.Object = object + + return action +} +func NewUpdateSubresourceAction(resource schema.GroupVersionResource, subresource string, namespace string, object runtime.Object) UpdateActionImpl { + action := UpdateActionImpl{} + action.Verb = "update" + action.Resource = resource + action.Subresource = subresource + action.Namespace = namespace + action.Object = object + + return action +} + +func NewRootDeleteAction(resource schema.GroupVersionResource, name string) DeleteActionImpl { + action := DeleteActionImpl{} + action.Verb = "delete" + action.Resource = resource + action.Name = name + + return action +} + +func NewRootDeleteSubresourceAction(resource schema.GroupVersionResource, subresource string, name string) DeleteActionImpl { + action := DeleteActionImpl{} + action.Verb = "delete" + action.Resource = resource + action.Subresource = subresource + action.Name = name + + return action +} + +func NewDeleteAction(resource schema.GroupVersionResource, namespace, name string) DeleteActionImpl { + action := DeleteActionImpl{} + action.Verb = "delete" + action.Resource = resource + action.Namespace = namespace + action.Name = name + + return action +} + +func NewDeleteSubresourceAction(resource schema.GroupVersionResource, subresource, namespace, name string) DeleteActionImpl { + action := DeleteActionImpl{} + action.Verb = "delete" + action.Resource = resource + action.Subresource = subresource + action.Namespace = namespace + action.Name = name + + return action +} + +func NewRootDeleteCollectionAction(resource schema.GroupVersionResource, opts interface{}) DeleteCollectionActionImpl { + action := DeleteCollectionActionImpl{} + action.Verb = "delete-collection" + action.Resource = resource + labelSelector, fieldSelector, _ := ExtractFromListOptions(opts) + action.ListRestrictions = ListRestrictions{labelSelector, fieldSelector} + + return action +} + +func NewDeleteCollectionAction(resource schema.GroupVersionResource, namespace string, opts interface{}) DeleteCollectionActionImpl { + action := DeleteCollectionActionImpl{} + action.Verb = "delete-collection" + action.Resource = resource + action.Namespace = namespace + labelSelector, fieldSelector, _ := ExtractFromListOptions(opts) + action.ListRestrictions = ListRestrictions{labelSelector, fieldSelector} + + return action +} + +func NewRootWatchAction(resource schema.GroupVersionResource, opts interface{}) WatchActionImpl { + action := WatchActionImpl{} + action.Verb = "watch" + action.Resource = resource + labelSelector, fieldSelector, resourceVersion := ExtractFromListOptions(opts) + action.WatchRestrictions = WatchRestrictions{labelSelector, fieldSelector, resourceVersion} + + return action +} + +func ExtractFromListOptions(opts interface{}) (labelSelector labels.Selector, fieldSelector fields.Selector, resourceVersion string) { + var err error + switch t := opts.(type) { + case metav1.ListOptions: + labelSelector, err = labels.Parse(t.LabelSelector) + if err != nil { + panic(fmt.Errorf("invalid selector %q: %v", t.LabelSelector, err)) + } + fieldSelector, err = fields.ParseSelector(t.FieldSelector) + if err != nil { + panic(fmt.Errorf("invalid selector %q: %v", t.FieldSelector, err)) + } + resourceVersion = t.ResourceVersion + default: + panic(fmt.Errorf("expect a ListOptions %T", opts)) + } + if labelSelector == nil { + labelSelector = labels.Everything() + } + if fieldSelector == nil { + fieldSelector = fields.Everything() + } + return labelSelector, fieldSelector, resourceVersion +} + +func NewWatchAction(resource schema.GroupVersionResource, namespace string, opts interface{}) WatchActionImpl { + action := WatchActionImpl{} + action.Verb = "watch" + action.Resource = resource + action.Namespace = namespace + labelSelector, fieldSelector, resourceVersion := ExtractFromListOptions(opts) + action.WatchRestrictions = WatchRestrictions{labelSelector, fieldSelector, resourceVersion} + + return action +} + +func NewProxyGetAction(resource schema.GroupVersionResource, namespace, scheme, name, port, path string, params map[string]string) ProxyGetActionImpl { + action := ProxyGetActionImpl{} + action.Verb = "get" + action.Resource = resource + action.Namespace = namespace + action.Scheme = scheme + action.Name = name + action.Port = port + action.Path = path + action.Params = params + return action +} + +type ListRestrictions struct { + Labels labels.Selector + Fields fields.Selector +} +type WatchRestrictions struct { + Labels labels.Selector + Fields fields.Selector + ResourceVersion string +} + +type Action interface { + GetNamespace() string + GetVerb() string + GetResource() schema.GroupVersionResource + GetSubresource() string + Matches(verb, resource string) bool + + // DeepCopy is used to copy an action to avoid any risk of accidental mutation. Most people never need to call this + // because the invocation logic deep copies before calls to storage and reactors. + DeepCopy() Action +} + +type GenericAction interface { + Action + GetValue() interface{} +} + +type GetAction interface { + Action + GetName() string +} + +type ListAction interface { + Action + GetListRestrictions() ListRestrictions +} + +type CreateAction interface { + Action + GetObject() runtime.Object +} + +type UpdateAction interface { + Action + GetObject() runtime.Object +} + +type DeleteAction interface { + Action + GetName() string +} + +type DeleteCollectionAction interface { + Action + GetListRestrictions() ListRestrictions +} + +type PatchAction interface { + Action + GetName() string + GetPatchType() types.PatchType + GetPatch() []byte +} + +type WatchAction interface { + Action + GetWatchRestrictions() WatchRestrictions +} + +type ProxyGetAction interface { + Action + GetScheme() string + GetName() string + GetPort() string + GetPath() string + GetParams() map[string]string +} + +type ActionImpl struct { + Namespace string + Verb string + Resource schema.GroupVersionResource + Subresource string +} + +func (a ActionImpl) GetNamespace() string { + return a.Namespace +} +func (a ActionImpl) GetVerb() string { + return a.Verb +} +func (a ActionImpl) GetResource() schema.GroupVersionResource { + return a.Resource +} +func (a ActionImpl) GetSubresource() string { + return a.Subresource +} +func (a ActionImpl) Matches(verb, resource string) bool { + // Stay backwards compatible. + if !strings.Contains(resource, "/") { + return strings.EqualFold(verb, a.Verb) && + strings.EqualFold(resource, a.Resource.Resource) + } + + parts := strings.SplitN(resource, "/", 2) + topresource, subresource := parts[0], parts[1] + + return strings.EqualFold(verb, a.Verb) && + strings.EqualFold(topresource, a.Resource.Resource) && + strings.EqualFold(subresource, a.Subresource) +} +func (a ActionImpl) DeepCopy() Action { + ret := a + return ret +} + +type GenericActionImpl struct { + ActionImpl + Value interface{} +} + +func (a GenericActionImpl) GetValue() interface{} { + return a.Value +} + +func (a GenericActionImpl) DeepCopy() Action { + return GenericActionImpl{ + ActionImpl: a.ActionImpl.DeepCopy().(ActionImpl), + // TODO this is wrong, but no worse than before + Value: a.Value, + } +} + +type GetActionImpl struct { + ActionImpl + Name string +} + +func (a GetActionImpl) GetName() string { + return a.Name +} + +func (a GetActionImpl) DeepCopy() Action { + return GetActionImpl{ + ActionImpl: a.ActionImpl.DeepCopy().(ActionImpl), + Name: a.Name, + } +} + +type ListActionImpl struct { + ActionImpl + Kind schema.GroupVersionKind + Name string + ListRestrictions ListRestrictions +} + +func (a ListActionImpl) GetKind() schema.GroupVersionKind { + return a.Kind +} + +func (a ListActionImpl) GetListRestrictions() ListRestrictions { + return a.ListRestrictions +} + +func (a ListActionImpl) DeepCopy() Action { + return ListActionImpl{ + ActionImpl: a.ActionImpl.DeepCopy().(ActionImpl), + Kind: a.Kind, + Name: a.Name, + ListRestrictions: ListRestrictions{ + Labels: a.ListRestrictions.Labels.DeepCopySelector(), + Fields: a.ListRestrictions.Fields.DeepCopySelector(), + }, + } +} + +type CreateActionImpl struct { + ActionImpl + Name string + Object runtime.Object +} + +func (a CreateActionImpl) GetObject() runtime.Object { + return a.Object +} + +func (a CreateActionImpl) DeepCopy() Action { + return CreateActionImpl{ + ActionImpl: a.ActionImpl.DeepCopy().(ActionImpl), + Name: a.Name, + Object: a.Object.DeepCopyObject(), + } +} + +type UpdateActionImpl struct { + ActionImpl + Object runtime.Object +} + +func (a UpdateActionImpl) GetObject() runtime.Object { + return a.Object +} + +func (a UpdateActionImpl) DeepCopy() Action { + return UpdateActionImpl{ + ActionImpl: a.ActionImpl.DeepCopy().(ActionImpl), + Object: a.Object.DeepCopyObject(), + } +} + +type PatchActionImpl struct { + ActionImpl + Name string + PatchType types.PatchType + Patch []byte +} + +func (a PatchActionImpl) GetName() string { + return a.Name +} + +func (a PatchActionImpl) GetPatch() []byte { + return a.Patch +} + +func (a PatchActionImpl) GetPatchType() types.PatchType { + return a.PatchType +} + +func (a PatchActionImpl) DeepCopy() Action { + patch := make([]byte, len(a.Patch)) + copy(patch, a.Patch) + return PatchActionImpl{ + ActionImpl: a.ActionImpl.DeepCopy().(ActionImpl), + Name: a.Name, + PatchType: a.PatchType, + Patch: patch, + } +} + +type DeleteActionImpl struct { + ActionImpl + Name string +} + +func (a DeleteActionImpl) GetName() string { + return a.Name +} + +func (a DeleteActionImpl) DeepCopy() Action { + return DeleteActionImpl{ + ActionImpl: a.ActionImpl.DeepCopy().(ActionImpl), + Name: a.Name, + } +} + +type DeleteCollectionActionImpl struct { + ActionImpl + ListRestrictions ListRestrictions +} + +func (a DeleteCollectionActionImpl) GetListRestrictions() ListRestrictions { + return a.ListRestrictions +} + +func (a DeleteCollectionActionImpl) DeepCopy() Action { + return DeleteCollectionActionImpl{ + ActionImpl: a.ActionImpl.DeepCopy().(ActionImpl), + ListRestrictions: ListRestrictions{ + Labels: a.ListRestrictions.Labels.DeepCopySelector(), + Fields: a.ListRestrictions.Fields.DeepCopySelector(), + }, + } +} + +type WatchActionImpl struct { + ActionImpl + WatchRestrictions WatchRestrictions +} + +func (a WatchActionImpl) GetWatchRestrictions() WatchRestrictions { + return a.WatchRestrictions +} + +func (a WatchActionImpl) DeepCopy() Action { + return WatchActionImpl{ + ActionImpl: a.ActionImpl.DeepCopy().(ActionImpl), + WatchRestrictions: WatchRestrictions{ + Labels: a.WatchRestrictions.Labels.DeepCopySelector(), + Fields: a.WatchRestrictions.Fields.DeepCopySelector(), + ResourceVersion: a.WatchRestrictions.ResourceVersion, + }, + } +} + +type ProxyGetActionImpl struct { + ActionImpl + Scheme string + Name string + Port string + Path string + Params map[string]string +} + +func (a ProxyGetActionImpl) GetScheme() string { + return a.Scheme +} + +func (a ProxyGetActionImpl) GetName() string { + return a.Name +} + +func (a ProxyGetActionImpl) GetPort() string { + return a.Port +} + +func (a ProxyGetActionImpl) GetPath() string { + return a.Path +} + +func (a ProxyGetActionImpl) GetParams() map[string]string { + return a.Params +} + +func (a ProxyGetActionImpl) DeepCopy() Action { + params := map[string]string{} + for k, v := range a.Params { + params[k] = v + } + return ProxyGetActionImpl{ + ActionImpl: a.ActionImpl.DeepCopy().(ActionImpl), + Scheme: a.Scheme, + Name: a.Name, + Port: a.Port, + Path: a.Path, + Params: params, + } +} diff --git a/vendor/k8s.io/client-go/testing/fake.go b/vendor/k8s.io/client-go/testing/fake.go new file mode 100644 index 000000000..3ab9c1b07 --- /dev/null +++ b/vendor/k8s.io/client-go/testing/fake.go @@ -0,0 +1,220 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "fmt" + "sync" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + restclient "k8s.io/client-go/rest" +) + +// Fake implements client.Interface. Meant to be embedded into a struct to get +// a default implementation. This makes faking out just the method you want to +// test easier. +type Fake struct { + sync.RWMutex + actions []Action // these may be castable to other types, but "Action" is the minimum + + // ReactionChain is the list of reactors that will be attempted for every + // request in the order they are tried. + ReactionChain []Reactor + // WatchReactionChain is the list of watch reactors that will be attempted + // for every request in the order they are tried. + WatchReactionChain []WatchReactor + // ProxyReactionChain is the list of proxy reactors that will be attempted + // for every request in the order they are tried. + ProxyReactionChain []ProxyReactor + + Resources []*metav1.APIResourceList +} + +// Reactor is an interface to allow the composition of reaction functions. +type Reactor interface { + // Handles indicates whether or not this Reactor deals with a given + // action. + Handles(action Action) bool + // React handles the action and returns results. It may choose to + // delegate by indicated handled=false. + React(action Action) (handled bool, ret runtime.Object, err error) +} + +// WatchReactor is an interface to allow the composition of watch functions. +type WatchReactor interface { + // Handles indicates whether or not this Reactor deals with a given + // action. + Handles(action Action) bool + // React handles a watch action and returns results. It may choose to + // delegate by indicating handled=false. + React(action Action) (handled bool, ret watch.Interface, err error) +} + +// ProxyReactor is an interface to allow the composition of proxy get +// functions. +type ProxyReactor interface { + // Handles indicates whether or not this Reactor deals with a given + // action. + Handles(action Action) bool + // React handles a watch action and returns results. It may choose to + // delegate by indicating handled=false. + React(action Action) (handled bool, ret restclient.ResponseWrapper, err error) +} + +// ReactionFunc is a function that returns an object or error for a given +// Action. If "handled" is false, then the test client will ignore the +// results and continue to the next ReactionFunc. A ReactionFunc can describe +// reactions on subresources by testing the result of the action's +// GetSubresource() method. +type ReactionFunc func(action Action) (handled bool, ret runtime.Object, err error) + +// WatchReactionFunc is a function that returns a watch interface. If +// "handled" is false, then the test client will ignore the results and +// continue to the next ReactionFunc. +type WatchReactionFunc func(action Action) (handled bool, ret watch.Interface, err error) + +// ProxyReactionFunc is a function that returns a ResponseWrapper interface +// for a given Action. If "handled" is false, then the test client will +// ignore the results and continue to the next ProxyReactionFunc. +type ProxyReactionFunc func(action Action) (handled bool, ret restclient.ResponseWrapper, err error) + +// AddReactor appends a reactor to the end of the chain. +func (c *Fake) AddReactor(verb, resource string, reaction ReactionFunc) { + c.ReactionChain = append(c.ReactionChain, &SimpleReactor{verb, resource, reaction}) +} + +// PrependReactor adds a reactor to the beginning of the chain. +func (c *Fake) PrependReactor(verb, resource string, reaction ReactionFunc) { + c.ReactionChain = append([]Reactor{&SimpleReactor{verb, resource, reaction}}, c.ReactionChain...) +} + +// AddWatchReactor appends a reactor to the end of the chain. +func (c *Fake) AddWatchReactor(resource string, reaction WatchReactionFunc) { + c.Lock() + defer c.Unlock() + c.WatchReactionChain = append(c.WatchReactionChain, &SimpleWatchReactor{resource, reaction}) +} + +// PrependWatchReactor adds a reactor to the beginning of the chain. +func (c *Fake) PrependWatchReactor(resource string, reaction WatchReactionFunc) { + c.Lock() + defer c.Unlock() + c.WatchReactionChain = append([]WatchReactor{&SimpleWatchReactor{resource, reaction}}, c.WatchReactionChain...) +} + +// AddProxyReactor appends a reactor to the end of the chain. +func (c *Fake) AddProxyReactor(resource string, reaction ProxyReactionFunc) { + c.ProxyReactionChain = append(c.ProxyReactionChain, &SimpleProxyReactor{resource, reaction}) +} + +// PrependProxyReactor adds a reactor to the beginning of the chain. +func (c *Fake) PrependProxyReactor(resource string, reaction ProxyReactionFunc) { + c.ProxyReactionChain = append([]ProxyReactor{&SimpleProxyReactor{resource, reaction}}, c.ProxyReactionChain...) +} + +// Invokes records the provided Action and then invokes the ReactionFunc that +// handles the action if one exists. defaultReturnObj is expected to be of the +// same type a normal call would return. +func (c *Fake) Invokes(action Action, defaultReturnObj runtime.Object) (runtime.Object, error) { + c.Lock() + defer c.Unlock() + + actionCopy := action.DeepCopy() + c.actions = append(c.actions, action.DeepCopy()) + for _, reactor := range c.ReactionChain { + if !reactor.Handles(actionCopy) { + continue + } + + handled, ret, err := reactor.React(actionCopy) + if !handled { + continue + } + + return ret, err + } + + return defaultReturnObj, nil +} + +// InvokesWatch records the provided Action and then invokes the ReactionFunc +// that handles the action if one exists. +func (c *Fake) InvokesWatch(action Action) (watch.Interface, error) { + c.Lock() + defer c.Unlock() + + actionCopy := action.DeepCopy() + c.actions = append(c.actions, action.DeepCopy()) + for _, reactor := range c.WatchReactionChain { + if !reactor.Handles(actionCopy) { + continue + } + + handled, ret, err := reactor.React(actionCopy) + if !handled { + continue + } + + return ret, err + } + + return nil, fmt.Errorf("unhandled watch: %#v", action) +} + +// InvokesProxy records the provided Action and then invokes the ReactionFunc +// that handles the action if one exists. +func (c *Fake) InvokesProxy(action Action) restclient.ResponseWrapper { + c.Lock() + defer c.Unlock() + + actionCopy := action.DeepCopy() + c.actions = append(c.actions, action.DeepCopy()) + for _, reactor := range c.ProxyReactionChain { + if !reactor.Handles(actionCopy) { + continue + } + + handled, ret, err := reactor.React(actionCopy) + if !handled || err != nil { + continue + } + + return ret + } + + return nil +} + +// ClearActions clears the history of actions called on the fake client. +func (c *Fake) ClearActions() { + c.Lock() + defer c.Unlock() + + c.actions = make([]Action, 0) +} + +// Actions returns a chronologically ordered slice fake actions called on the +// fake client. +func (c *Fake) Actions() []Action { + c.RLock() + defer c.RUnlock() + fa := make([]Action, len(c.actions)) + copy(fa, c.actions) + return fa +} diff --git a/vendor/k8s.io/client-go/testing/fixture.go b/vendor/k8s.io/client-go/testing/fixture.go new file mode 100644 index 000000000..fe7f0cd32 --- /dev/null +++ b/vendor/k8s.io/client-go/testing/fixture.go @@ -0,0 +1,571 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "fmt" + "reflect" + "sort" + "strings" + "sync" + + jsonpatch "github.com/evanphx/json-patch" + + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/json" + "k8s.io/apimachinery/pkg/util/strategicpatch" + "k8s.io/apimachinery/pkg/watch" + restclient "k8s.io/client-go/rest" +) + +// ObjectTracker keeps track of objects. It is intended to be used to +// fake calls to a server by returning objects based on their kind, +// namespace and name. +type ObjectTracker interface { + // Add adds an object to the tracker. If object being added + // is a list, its items are added separately. + Add(obj runtime.Object) error + + // Get retrieves the object by its kind, namespace and name. + Get(gvr schema.GroupVersionResource, ns, name string) (runtime.Object, error) + + // Create adds an object to the tracker in the specified namespace. + Create(gvr schema.GroupVersionResource, obj runtime.Object, ns string) error + + // Update updates an existing object in the tracker in the specified namespace. + Update(gvr schema.GroupVersionResource, obj runtime.Object, ns string) error + + // List retrieves all objects of a given kind in the given + // namespace. Only non-List kinds are accepted. + List(gvr schema.GroupVersionResource, gvk schema.GroupVersionKind, ns string) (runtime.Object, error) + + // Delete deletes an existing object from the tracker. If object + // didn't exist in the tracker prior to deletion, Delete returns + // no error. + Delete(gvr schema.GroupVersionResource, ns, name string) error + + // Watch watches objects from the tracker. Watch returns a channel + // which will push added / modified / deleted object. + Watch(gvr schema.GroupVersionResource, ns string) (watch.Interface, error) +} + +// ObjectScheme abstracts the implementation of common operations on objects. +type ObjectScheme interface { + runtime.ObjectCreater + runtime.ObjectTyper +} + +// ObjectReaction returns a ReactionFunc that applies core.Action to +// the given tracker. +func ObjectReaction(tracker ObjectTracker) ReactionFunc { + return func(action Action) (bool, runtime.Object, error) { + ns := action.GetNamespace() + gvr := action.GetResource() + // Here and below we need to switch on implementation types, + // not on interfaces, as some interfaces are identical + // (e.g. UpdateAction and CreateAction), so if we use them, + // updates and creates end up matching the same case branch. + switch action := action.(type) { + + case ListActionImpl: + obj, err := tracker.List(gvr, action.GetKind(), ns) + return true, obj, err + + case GetActionImpl: + obj, err := tracker.Get(gvr, ns, action.GetName()) + return true, obj, err + + case CreateActionImpl: + objMeta, err := meta.Accessor(action.GetObject()) + if err != nil { + return true, nil, err + } + if action.GetSubresource() == "" { + err = tracker.Create(gvr, action.GetObject(), ns) + } else { + // TODO: Currently we're handling subresource creation as an update + // on the enclosing resource. This works for some subresources but + // might not be generic enough. + err = tracker.Update(gvr, action.GetObject(), ns) + } + if err != nil { + return true, nil, err + } + obj, err := tracker.Get(gvr, ns, objMeta.GetName()) + return true, obj, err + + case UpdateActionImpl: + objMeta, err := meta.Accessor(action.GetObject()) + if err != nil { + return true, nil, err + } + err = tracker.Update(gvr, action.GetObject(), ns) + if err != nil { + return true, nil, err + } + obj, err := tracker.Get(gvr, ns, objMeta.GetName()) + return true, obj, err + + case DeleteActionImpl: + err := tracker.Delete(gvr, ns, action.GetName()) + if err != nil { + return true, nil, err + } + return true, nil, nil + + case PatchActionImpl: + obj, err := tracker.Get(gvr, ns, action.GetName()) + if err != nil { + return true, nil, err + } + + old, err := json.Marshal(obj) + if err != nil { + return true, nil, err + } + + // reset the object in preparation to unmarshal, since unmarshal does not guarantee that fields + // in obj that are removed by patch are cleared + value := reflect.ValueOf(obj) + value.Elem().Set(reflect.New(value.Type().Elem()).Elem()) + + switch action.GetPatchType() { + case types.JSONPatchType: + patch, err := jsonpatch.DecodePatch(action.GetPatch()) + if err != nil { + return true, nil, err + } + modified, err := patch.Apply(old) + if err != nil { + return true, nil, err + } + + if err = json.Unmarshal(modified, obj); err != nil { + return true, nil, err + } + case types.MergePatchType: + modified, err := jsonpatch.MergePatch(old, action.GetPatch()) + if err != nil { + return true, nil, err + } + + if err := json.Unmarshal(modified, obj); err != nil { + return true, nil, err + } + case types.StrategicMergePatchType: + mergedByte, err := strategicpatch.StrategicMergePatch(old, action.GetPatch(), obj) + if err != nil { + return true, nil, err + } + if err = json.Unmarshal(mergedByte, obj); err != nil { + return true, nil, err + } + default: + return true, nil, fmt.Errorf("PatchType is not supported") + } + + if err = tracker.Update(gvr, obj, ns); err != nil { + return true, nil, err + } + + return true, obj, nil + + default: + return false, nil, fmt.Errorf("no reaction implemented for %s", action) + } + } +} + +type tracker struct { + scheme ObjectScheme + decoder runtime.Decoder + lock sync.RWMutex + objects map[schema.GroupVersionResource]map[types.NamespacedName]runtime.Object + // The value type of watchers is a map of which the key is either a namespace or + // all/non namespace aka "" and its value is list of fake watchers. + // Manipulations on resources will broadcast the notification events into the + // watchers' channel. Note that too many unhandled events (currently 100, + // see apimachinery/pkg/watch.DefaultChanSize) will cause a panic. + watchers map[schema.GroupVersionResource]map[string][]*watch.RaceFreeFakeWatcher +} + +var _ ObjectTracker = &tracker{} + +// NewObjectTracker returns an ObjectTracker that can be used to keep track +// of objects for the fake clientset. Mostly useful for unit tests. +func NewObjectTracker(scheme ObjectScheme, decoder runtime.Decoder) ObjectTracker { + return &tracker{ + scheme: scheme, + decoder: decoder, + objects: make(map[schema.GroupVersionResource]map[types.NamespacedName]runtime.Object), + watchers: make(map[schema.GroupVersionResource]map[string][]*watch.RaceFreeFakeWatcher), + } +} + +func (t *tracker) List(gvr schema.GroupVersionResource, gvk schema.GroupVersionKind, ns string) (runtime.Object, error) { + // Heuristic for list kind: original kind + List suffix. Might + // not always be true but this tracker has a pretty limited + // understanding of the actual API model. + listGVK := gvk + listGVK.Kind = listGVK.Kind + "List" + // GVK does have the concept of "internal version". The scheme recognizes + // the runtime.APIVersionInternal, but not the empty string. + if listGVK.Version == "" { + listGVK.Version = runtime.APIVersionInternal + } + + list, err := t.scheme.New(listGVK) + if err != nil { + return nil, err + } + + if !meta.IsListType(list) { + return nil, fmt.Errorf("%q is not a list type", listGVK.Kind) + } + + t.lock.RLock() + defer t.lock.RUnlock() + + objs, ok := t.objects[gvr] + if !ok { + return list, nil + } + + matchingObjs, err := filterByNamespace(objs, ns) + if err != nil { + return nil, err + } + if err := meta.SetList(list, matchingObjs); err != nil { + return nil, err + } + return list.DeepCopyObject(), nil +} + +func (t *tracker) Watch(gvr schema.GroupVersionResource, ns string) (watch.Interface, error) { + t.lock.Lock() + defer t.lock.Unlock() + + fakewatcher := watch.NewRaceFreeFake() + + if _, exists := t.watchers[gvr]; !exists { + t.watchers[gvr] = make(map[string][]*watch.RaceFreeFakeWatcher) + } + t.watchers[gvr][ns] = append(t.watchers[gvr][ns], fakewatcher) + return fakewatcher, nil +} + +func (t *tracker) Get(gvr schema.GroupVersionResource, ns, name string) (runtime.Object, error) { + errNotFound := errors.NewNotFound(gvr.GroupResource(), name) + + t.lock.RLock() + defer t.lock.RUnlock() + + objs, ok := t.objects[gvr] + if !ok { + return nil, errNotFound + } + + matchingObj, ok := objs[types.NamespacedName{Namespace: ns, Name: name}] + if !ok { + return nil, errNotFound + } + + // Only one object should match in the tracker if it works + // correctly, as Add/Update methods enforce kind/namespace/name + // uniqueness. + obj := matchingObj.DeepCopyObject() + if status, ok := obj.(*metav1.Status); ok { + if status.Status != metav1.StatusSuccess { + return nil, &errors.StatusError{ErrStatus: *status} + } + } + + return obj, nil +} + +func (t *tracker) Add(obj runtime.Object) error { + if meta.IsListType(obj) { + return t.addList(obj, false) + } + objMeta, err := meta.Accessor(obj) + if err != nil { + return err + } + gvks, _, err := t.scheme.ObjectKinds(obj) + if err != nil { + return err + } + + if partial, ok := obj.(*metav1.PartialObjectMetadata); ok && len(partial.TypeMeta.APIVersion) > 0 { + gvks = []schema.GroupVersionKind{partial.TypeMeta.GroupVersionKind()} + } + + if len(gvks) == 0 { + return fmt.Errorf("no registered kinds for %v", obj) + } + for _, gvk := range gvks { + // NOTE: UnsafeGuessKindToResource is a heuristic and default match. The + // actual registration in apiserver can specify arbitrary route for a + // gvk. If a test uses such objects, it cannot preset the tracker with + // objects via Add(). Instead, it should trigger the Create() function + // of the tracker, where an arbitrary gvr can be specified. + gvr, _ := meta.UnsafeGuessKindToResource(gvk) + // Resource doesn't have the concept of "__internal" version, just set it to "". + if gvr.Version == runtime.APIVersionInternal { + gvr.Version = "" + } + + err := t.add(gvr, obj, objMeta.GetNamespace(), false) + if err != nil { + return err + } + } + return nil +} + +func (t *tracker) Create(gvr schema.GroupVersionResource, obj runtime.Object, ns string) error { + return t.add(gvr, obj, ns, false) +} + +func (t *tracker) Update(gvr schema.GroupVersionResource, obj runtime.Object, ns string) error { + return t.add(gvr, obj, ns, true) +} + +func (t *tracker) getWatches(gvr schema.GroupVersionResource, ns string) []*watch.RaceFreeFakeWatcher { + watches := []*watch.RaceFreeFakeWatcher{} + if t.watchers[gvr] != nil { + if w := t.watchers[gvr][ns]; w != nil { + watches = append(watches, w...) + } + if ns != metav1.NamespaceAll { + if w := t.watchers[gvr][metav1.NamespaceAll]; w != nil { + watches = append(watches, w...) + } + } + } + return watches +} + +func (t *tracker) add(gvr schema.GroupVersionResource, obj runtime.Object, ns string, replaceExisting bool) error { + t.lock.Lock() + defer t.lock.Unlock() + + gr := gvr.GroupResource() + + // To avoid the object from being accidentally modified by caller + // after it's been added to the tracker, we always store the deep + // copy. + obj = obj.DeepCopyObject() + + newMeta, err := meta.Accessor(obj) + if err != nil { + return err + } + + // Propagate namespace to the new object if hasn't already been set. + if len(newMeta.GetNamespace()) == 0 { + newMeta.SetNamespace(ns) + } + + if ns != newMeta.GetNamespace() { + msg := fmt.Sprintf("request namespace does not match object namespace, request: %q object: %q", ns, newMeta.GetNamespace()) + return errors.NewBadRequest(msg) + } + + _, ok := t.objects[gvr] + if !ok { + t.objects[gvr] = make(map[types.NamespacedName]runtime.Object) + } + + namespacedName := types.NamespacedName{Namespace: newMeta.GetNamespace(), Name: newMeta.GetName()} + if _, ok = t.objects[gvr][namespacedName]; ok { + if replaceExisting { + for _, w := range t.getWatches(gvr, ns) { + // To avoid the object from being accidentally modified by watcher + w.Modify(obj.DeepCopyObject()) + } + t.objects[gvr][namespacedName] = obj + return nil + } + return errors.NewAlreadyExists(gr, newMeta.GetName()) + } + + if replaceExisting { + // Tried to update but no matching object was found. + return errors.NewNotFound(gr, newMeta.GetName()) + } + + t.objects[gvr][namespacedName] = obj + + for _, w := range t.getWatches(gvr, ns) { + // To avoid the object from being accidentally modified by watcher + w.Add(obj.DeepCopyObject()) + } + + return nil +} + +func (t *tracker) addList(obj runtime.Object, replaceExisting bool) error { + list, err := meta.ExtractList(obj) + if err != nil { + return err + } + errs := runtime.DecodeList(list, t.decoder) + if len(errs) > 0 { + return errs[0] + } + for _, obj := range list { + if err := t.Add(obj); err != nil { + return err + } + } + return nil +} + +func (t *tracker) Delete(gvr schema.GroupVersionResource, ns, name string) error { + t.lock.Lock() + defer t.lock.Unlock() + + objs, ok := t.objects[gvr] + if !ok { + return errors.NewNotFound(gvr.GroupResource(), name) + } + + namespacedName := types.NamespacedName{Namespace: ns, Name: name} + obj, ok := objs[namespacedName] + if !ok { + return errors.NewNotFound(gvr.GroupResource(), name) + } + + delete(objs, namespacedName) + for _, w := range t.getWatches(gvr, ns) { + w.Delete(obj.DeepCopyObject()) + } + return nil +} + +// filterByNamespace returns all objects in the collection that +// match provided namespace. Empty namespace matches +// non-namespaced objects. +func filterByNamespace(objs map[types.NamespacedName]runtime.Object, ns string) ([]runtime.Object, error) { + var res []runtime.Object + + for _, obj := range objs { + acc, err := meta.Accessor(obj) + if err != nil { + return nil, err + } + if ns != "" && acc.GetNamespace() != ns { + continue + } + res = append(res, obj) + } + + // Sort res to get deterministic order. + sort.Slice(res, func(i, j int) bool { + acc1, _ := meta.Accessor(res[i]) + acc2, _ := meta.Accessor(res[j]) + if acc1.GetNamespace() != acc2.GetNamespace() { + return acc1.GetNamespace() < acc2.GetNamespace() + } + return acc1.GetName() < acc2.GetName() + }) + return res, nil +} + +func DefaultWatchReactor(watchInterface watch.Interface, err error) WatchReactionFunc { + return func(action Action) (bool, watch.Interface, error) { + return true, watchInterface, err + } +} + +// SimpleReactor is a Reactor. Each reaction function is attached to a given verb,resource tuple. "*" in either field matches everything for that value. +// For instance, *,pods matches all verbs on pods. This allows for easier composition of reaction functions +type SimpleReactor struct { + Verb string + Resource string + + Reaction ReactionFunc +} + +func (r *SimpleReactor) Handles(action Action) bool { + verbCovers := r.Verb == "*" || r.Verb == action.GetVerb() + if !verbCovers { + return false + } + + return resourceCovers(r.Resource, action) +} + +func (r *SimpleReactor) React(action Action) (bool, runtime.Object, error) { + return r.Reaction(action) +} + +// SimpleWatchReactor is a WatchReactor. Each reaction function is attached to a given resource. "*" matches everything for that value. +// For instance, *,pods matches all verbs on pods. This allows for easier composition of reaction functions +type SimpleWatchReactor struct { + Resource string + + Reaction WatchReactionFunc +} + +func (r *SimpleWatchReactor) Handles(action Action) bool { + return resourceCovers(r.Resource, action) +} + +func (r *SimpleWatchReactor) React(action Action) (bool, watch.Interface, error) { + return r.Reaction(action) +} + +// SimpleProxyReactor is a ProxyReactor. Each reaction function is attached to a given resource. "*" matches everything for that value. +// For instance, *,pods matches all verbs on pods. This allows for easier composition of reaction functions. +type SimpleProxyReactor struct { + Resource string + + Reaction ProxyReactionFunc +} + +func (r *SimpleProxyReactor) Handles(action Action) bool { + return resourceCovers(r.Resource, action) +} + +func (r *SimpleProxyReactor) React(action Action) (bool, restclient.ResponseWrapper, error) { + return r.Reaction(action) +} + +func resourceCovers(resource string, action Action) bool { + if resource == "*" { + return true + } + + if resource == action.GetResource().Resource { + return true + } + + if index := strings.Index(resource, "/"); index != -1 && + resource[:index] == action.GetResource().Resource && + resource[index+1:] == action.GetSubresource() { + return true + } + + return false +} diff --git a/vendor/k8s.io/client-go/testing/interface.go b/vendor/k8s.io/client-go/testing/interface.go new file mode 100644 index 000000000..266c6ba3f --- /dev/null +++ b/vendor/k8s.io/client-go/testing/interface.go @@ -0,0 +1,66 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + restclient "k8s.io/client-go/rest" +) + +type FakeClient interface { + // Tracker gives access to the ObjectTracker internal to the fake client. + Tracker() ObjectTracker + + // AddReactor appends a reactor to the end of the chain. + AddReactor(verb, resource string, reaction ReactionFunc) + + // PrependReactor adds a reactor to the beginning of the chain. + PrependReactor(verb, resource string, reaction ReactionFunc) + + // AddWatchReactor appends a reactor to the end of the chain. + AddWatchReactor(resource string, reaction WatchReactionFunc) + + // PrependWatchReactor adds a reactor to the beginning of the chain. + PrependWatchReactor(resource string, reaction WatchReactionFunc) + + // AddProxyReactor appends a reactor to the end of the chain. + AddProxyReactor(resource string, reaction ProxyReactionFunc) + + // PrependProxyReactor adds a reactor to the beginning of the chain. + PrependProxyReactor(resource string, reaction ProxyReactionFunc) + + // Invokes records the provided Action and then invokes the ReactionFunc that + // handles the action if one exists. defaultReturnObj is expected to be of the + // same type a normal call would return. + Invokes(action Action, defaultReturnObj runtime.Object) (runtime.Object, error) + + // InvokesWatch records the provided Action and then invokes the ReactionFunc + // that handles the action if one exists. + InvokesWatch(action Action) (watch.Interface, error) + + // InvokesProxy records the provided Action and then invokes the ReactionFunc + // that handles the action if one exists. + InvokesProxy(action Action) restclient.ResponseWrapper + + // ClearActions clears the history of actions called on the fake client. + ClearActions() + + // Actions returns a chronologically ordered slice fake actions called on the + // fake client. + Actions() []Action +} diff --git a/vendor/modules.txt b/vendor/modules.txt index d33805c70..7df79b355 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -84,6 +84,7 @@ github.com/gardener/gardener/pkg/gardenlet/apis/config/helper github.com/gardener/gardener/pkg/gardenlet/apis/config/v1alpha1 github.com/gardener/gardener/pkg/logger github.com/gardener/gardener/pkg/mock/controller-runtime/client +github.com/gardener/gardener/pkg/operation/botanist/component github.com/gardener/gardener/pkg/resourcemanager/controller/garbagecollector/references github.com/gardener/gardener/pkg/utils github.com/gardener/gardener/pkg/utils/context @@ -681,6 +682,7 @@ k8s.io/client-go/plugin/pkg/client/auth/gcp k8s.io/client-go/rest k8s.io/client-go/rest/watch k8s.io/client-go/restmapper +k8s.io/client-go/testing k8s.io/client-go/third_party/forked/golang/template k8s.io/client-go/tools/auth k8s.io/client-go/tools/cache @@ -808,6 +810,7 @@ sigs.k8s.io/controller-runtime/pkg/certwatcher sigs.k8s.io/controller-runtime/pkg/client sigs.k8s.io/controller-runtime/pkg/client/apiutil sigs.k8s.io/controller-runtime/pkg/client/config +sigs.k8s.io/controller-runtime/pkg/client/fake sigs.k8s.io/controller-runtime/pkg/cluster sigs.k8s.io/controller-runtime/pkg/config sigs.k8s.io/controller-runtime/pkg/config/v1alpha1 diff --git a/vendor/sigs.k8s.io/controller-runtime/pkg/client/fake/client.go b/vendor/sigs.k8s.io/controller-runtime/pkg/client/fake/client.go new file mode 100644 index 000000000..1f0efc82b --- /dev/null +++ b/vendor/sigs.k8s.io/controller-runtime/pkg/client/fake/client.go @@ -0,0 +1,751 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + "strconv" + "strings" + "sync" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + utilrand "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/testing" + + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/internal/objectutil" +) + +type versionedTracker struct { + testing.ObjectTracker + scheme *runtime.Scheme +} + +type fakeClient struct { + tracker versionedTracker + scheme *runtime.Scheme + schemeWriteLock sync.Mutex +} + +var _ client.WithWatch = &fakeClient{} + +const ( + maxNameLength = 63 + randomLength = 5 + maxGeneratedNameLength = maxNameLength - randomLength +) + +// NewFakeClient creates a new fake client for testing. +// You can choose to initialize it with a slice of runtime.Object. +// +// Deprecated: Please use NewClientBuilder instead. +func NewFakeClient(initObjs ...runtime.Object) client.WithWatch { + return NewClientBuilder().WithRuntimeObjects(initObjs...).Build() +} + +// NewFakeClientWithScheme creates a new fake client with the given scheme +// for testing. +// You can choose to initialize it with a slice of runtime.Object. +// +// Deprecated: Please use NewClientBuilder instead. +func NewFakeClientWithScheme(clientScheme *runtime.Scheme, initObjs ...runtime.Object) client.WithWatch { + return NewClientBuilder().WithScheme(clientScheme).WithRuntimeObjects(initObjs...).Build() +} + +// NewClientBuilder returns a new builder to create a fake client. +func NewClientBuilder() *ClientBuilder { + return &ClientBuilder{} +} + +// ClientBuilder builds a fake client. +type ClientBuilder struct { + scheme *runtime.Scheme + initObject []client.Object + initLists []client.ObjectList + initRuntimeObjects []runtime.Object +} + +// WithScheme sets this builder's internal scheme. +// If not set, defaults to client-go's global scheme.Scheme. +func (f *ClientBuilder) WithScheme(scheme *runtime.Scheme) *ClientBuilder { + f.scheme = scheme + return f +} + +// WithObjects can be optionally used to initialize this fake client with client.Object(s). +func (f *ClientBuilder) WithObjects(initObjs ...client.Object) *ClientBuilder { + f.initObject = append(f.initObject, initObjs...) + return f +} + +// WithLists can be optionally used to initialize this fake client with client.ObjectList(s). +func (f *ClientBuilder) WithLists(initLists ...client.ObjectList) *ClientBuilder { + f.initLists = append(f.initLists, initLists...) + return f +} + +// WithRuntimeObjects can be optionally used to initialize this fake client with runtime.Object(s). +func (f *ClientBuilder) WithRuntimeObjects(initRuntimeObjs ...runtime.Object) *ClientBuilder { + f.initRuntimeObjects = append(f.initRuntimeObjects, initRuntimeObjs...) + return f +} + +// Build builds and returns a new fake client. +func (f *ClientBuilder) Build() client.WithWatch { + if f.scheme == nil { + f.scheme = scheme.Scheme + } + + tracker := versionedTracker{ObjectTracker: testing.NewObjectTracker(f.scheme, scheme.Codecs.UniversalDecoder()), scheme: f.scheme} + for _, obj := range f.initObject { + if err := tracker.Add(obj); err != nil { + panic(fmt.Errorf("failed to add object %v to fake client: %w", obj, err)) + } + } + for _, obj := range f.initLists { + if err := tracker.Add(obj); err != nil { + panic(fmt.Errorf("failed to add list %v to fake client: %w", obj, err)) + } + } + for _, obj := range f.initRuntimeObjects { + if err := tracker.Add(obj); err != nil { + panic(fmt.Errorf("failed to add runtime object %v to fake client: %w", obj, err)) + } + } + return &fakeClient{ + tracker: tracker, + scheme: f.scheme, + } +} + +const trackerAddResourceVersion = "999" + +func (t versionedTracker) Add(obj runtime.Object) error { + var objects []runtime.Object + if meta.IsListType(obj) { + var err error + objects, err = meta.ExtractList(obj) + if err != nil { + return err + } + } else { + objects = []runtime.Object{obj} + } + for _, obj := range objects { + accessor, err := meta.Accessor(obj) + if err != nil { + return fmt.Errorf("failed to get accessor for object: %w", err) + } + if accessor.GetResourceVersion() == "" { + // We use a "magic" value of 999 here because this field + // is parsed as uint and and 0 is already used in Update. + // As we can't go lower, go very high instead so this can + // be recognized + accessor.SetResourceVersion(trackerAddResourceVersion) + } + + obj, err = convertFromUnstructuredIfNecessary(t.scheme, obj) + if err != nil { + return err + } + if err := t.ObjectTracker.Add(obj); err != nil { + return err + } + } + + return nil +} + +func (t versionedTracker) Create(gvr schema.GroupVersionResource, obj runtime.Object, ns string) error { + accessor, err := meta.Accessor(obj) + if err != nil { + return fmt.Errorf("failed to get accessor for object: %v", err) + } + if accessor.GetName() == "" { + return apierrors.NewInvalid( + obj.GetObjectKind().GroupVersionKind().GroupKind(), + accessor.GetName(), + field.ErrorList{field.Required(field.NewPath("metadata.name"), "name is required")}) + } + if accessor.GetResourceVersion() != "" { + return apierrors.NewBadRequest("resourceVersion can not be set for Create requests") + } + accessor.SetResourceVersion("1") + obj, err = convertFromUnstructuredIfNecessary(t.scheme, obj) + if err != nil { + return err + } + if err := t.ObjectTracker.Create(gvr, obj, ns); err != nil { + accessor.SetResourceVersion("") + return err + } + + return nil +} + +// convertFromUnstructuredIfNecessary will convert *unstructured.Unstructured for a GVK that is recocnized +// by the schema into the whatever the schema produces with New() for said GVK. +// This is required because the tracker unconditionally saves on manipulations, but it's List() implementation +// tries to assign whatever it finds into a ListType it gets from schema.New() - Thus we have to ensure +// we save as the very same type, otherwise subsequent List requests will fail. +func convertFromUnstructuredIfNecessary(s *runtime.Scheme, o runtime.Object) (runtime.Object, error) { + u, isUnstructured := o.(*unstructured.Unstructured) + if !isUnstructured || !s.Recognizes(u.GroupVersionKind()) { + return o, nil + } + + typed, err := s.New(u.GroupVersionKind()) + if err != nil { + return nil, fmt.Errorf("scheme recognizes %s but failed to produce an object for it: %w", u.GroupVersionKind().String(), err) + } + + unstructuredSerialized, err := json.Marshal(u) + if err != nil { + return nil, fmt.Errorf("failed to serialize %T: %w", unstructuredSerialized, err) + } + if err := json.Unmarshal(unstructuredSerialized, typed); err != nil { + return nil, fmt.Errorf("failed to unmarshal the content of %T into %T: %w", u, typed, err) + } + + return typed, nil +} + +func (t versionedTracker) Update(gvr schema.GroupVersionResource, obj runtime.Object, ns string) error { + accessor, err := meta.Accessor(obj) + if err != nil { + return fmt.Errorf("failed to get accessor for object: %v", err) + } + + if accessor.GetName() == "" { + return apierrors.NewInvalid( + obj.GetObjectKind().GroupVersionKind().GroupKind(), + accessor.GetName(), + field.ErrorList{field.Required(field.NewPath("metadata.name"), "name is required")}) + } + + gvk := obj.GetObjectKind().GroupVersionKind() + if gvk.Empty() { + gvk, err = apiutil.GVKForObject(obj, t.scheme) + if err != nil { + return err + } + } + + oldObject, err := t.ObjectTracker.Get(gvr, ns, accessor.GetName()) + if err != nil { + // If the resource is not found and the resource allows create on update, issue a + // create instead. + if apierrors.IsNotFound(err) && allowsCreateOnUpdate(gvk) { + return t.Create(gvr, obj, ns) + } + return err + } + + oldAccessor, err := meta.Accessor(oldObject) + if err != nil { + return err + } + + // If the new object does not have the resource version set and it allows unconditional update, + // default it to the resource version of the existing resource + if accessor.GetResourceVersion() == "" && allowsUnconditionalUpdate(gvk) { + accessor.SetResourceVersion(oldAccessor.GetResourceVersion()) + } + if accessor.GetResourceVersion() != oldAccessor.GetResourceVersion() { + return apierrors.NewConflict(gvr.GroupResource(), accessor.GetName(), errors.New("object was modified")) + } + if oldAccessor.GetResourceVersion() == "" { + oldAccessor.SetResourceVersion("0") + } + intResourceVersion, err := strconv.ParseUint(oldAccessor.GetResourceVersion(), 10, 64) + if err != nil { + return fmt.Errorf("can not convert resourceVersion %q to int: %v", oldAccessor.GetResourceVersion(), err) + } + intResourceVersion++ + accessor.SetResourceVersion(strconv.FormatUint(intResourceVersion, 10)) + if !accessor.GetDeletionTimestamp().IsZero() && len(accessor.GetFinalizers()) == 0 { + return t.ObjectTracker.Delete(gvr, accessor.GetNamespace(), accessor.GetName()) + } + obj, err = convertFromUnstructuredIfNecessary(t.scheme, obj) + if err != nil { + return err + } + return t.ObjectTracker.Update(gvr, obj, ns) +} + +func (c *fakeClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object) error { + gvr, err := getGVRFromObject(obj, c.scheme) + if err != nil { + return err + } + o, err := c.tracker.Get(gvr, key.Namespace, key.Name) + if err != nil { + return err + } + + gvk, err := apiutil.GVKForObject(obj, c.scheme) + if err != nil { + return err + } + ta, err := meta.TypeAccessor(o) + if err != nil { + return err + } + ta.SetKind(gvk.Kind) + ta.SetAPIVersion(gvk.GroupVersion().String()) + + j, err := json.Marshal(o) + if err != nil { + return err + } + decoder := scheme.Codecs.UniversalDecoder() + zero(obj) + _, _, err = decoder.Decode(j, nil, obj) + return err +} + +func (c *fakeClient) Watch(ctx context.Context, list client.ObjectList, opts ...client.ListOption) (watch.Interface, error) { + gvk, err := apiutil.GVKForObject(list, c.scheme) + if err != nil { + return nil, err + } + + if strings.HasSuffix(gvk.Kind, "List") { + gvk.Kind = gvk.Kind[:len(gvk.Kind)-4] + } + + listOpts := client.ListOptions{} + listOpts.ApplyOptions(opts) + + gvr, _ := meta.UnsafeGuessKindToResource(gvk) + return c.tracker.Watch(gvr, listOpts.Namespace) +} + +func (c *fakeClient) List(ctx context.Context, obj client.ObjectList, opts ...client.ListOption) error { + gvk, err := apiutil.GVKForObject(obj, c.scheme) + if err != nil { + return err + } + + originalKind := gvk.Kind + + if strings.HasSuffix(gvk.Kind, "List") { + gvk.Kind = gvk.Kind[:len(gvk.Kind)-4] + } + + if _, isUnstructuredList := obj.(*unstructured.UnstructuredList); isUnstructuredList && !c.scheme.Recognizes(gvk) { + // We need to register the ListKind with UnstructuredList: + // https://github.com/kubernetes/kubernetes/blob/7b2776b89fb1be28d4e9203bdeec079be903c103/staging/src/k8s.io/client-go/dynamic/fake/simple.go#L44-L51 + c.schemeWriteLock.Lock() + c.scheme.AddKnownTypeWithName(gvk.GroupVersion().WithKind(gvk.Kind+"List"), &unstructured.UnstructuredList{}) + c.schemeWriteLock.Unlock() + } + + listOpts := client.ListOptions{} + listOpts.ApplyOptions(opts) + + gvr, _ := meta.UnsafeGuessKindToResource(gvk) + o, err := c.tracker.List(gvr, gvk, listOpts.Namespace) + if err != nil { + return err + } + + ta, err := meta.TypeAccessor(o) + if err != nil { + return err + } + ta.SetKind(originalKind) + ta.SetAPIVersion(gvk.GroupVersion().String()) + + j, err := json.Marshal(o) + if err != nil { + return err + } + decoder := scheme.Codecs.UniversalDecoder() + zero(obj) + _, _, err = decoder.Decode(j, nil, obj) + if err != nil { + return err + } + + if listOpts.LabelSelector != nil { + objs, err := meta.ExtractList(obj) + if err != nil { + return err + } + filteredObjs, err := objectutil.FilterWithLabels(objs, listOpts.LabelSelector) + if err != nil { + return err + } + err = meta.SetList(obj, filteredObjs) + if err != nil { + return err + } + } + return nil +} + +func (c *fakeClient) Scheme() *runtime.Scheme { + return c.scheme +} + +func (c *fakeClient) RESTMapper() meta.RESTMapper { + // TODO: Implement a fake RESTMapper. + return nil +} + +func (c *fakeClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + createOptions := &client.CreateOptions{} + createOptions.ApplyOptions(opts) + + for _, dryRunOpt := range createOptions.DryRun { + if dryRunOpt == metav1.DryRunAll { + return nil + } + } + + gvr, err := getGVRFromObject(obj, c.scheme) + if err != nil { + return err + } + accessor, err := meta.Accessor(obj) + if err != nil { + return err + } + + if accessor.GetName() == "" && accessor.GetGenerateName() != "" { + base := accessor.GetGenerateName() + if len(base) > maxGeneratedNameLength { + base = base[:maxGeneratedNameLength] + } + accessor.SetName(fmt.Sprintf("%s%s", base, utilrand.String(randomLength))) + } + + return c.tracker.Create(gvr, obj, accessor.GetNamespace()) +} + +func (c *fakeClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + gvr, err := getGVRFromObject(obj, c.scheme) + if err != nil { + return err + } + accessor, err := meta.Accessor(obj) + if err != nil { + return err + } + delOptions := client.DeleteOptions{} + delOptions.ApplyOptions(opts) + + // Check the ResourceVersion if that Precondition was specified. + if delOptions.Preconditions != nil && delOptions.Preconditions.ResourceVersion != nil { + name := accessor.GetName() + dbObj, err := c.tracker.Get(gvr, accessor.GetNamespace(), name) + if err != nil { + return err + } + oldAccessor, err := meta.Accessor(dbObj) + if err != nil { + return err + } + actualRV := oldAccessor.GetResourceVersion() + expectRV := *delOptions.Preconditions.ResourceVersion + if actualRV != expectRV { + msg := fmt.Sprintf( + "the ResourceVersion in the precondition (%s) does not match the ResourceVersion in record (%s). "+ + "The object might have been modified", + expectRV, actualRV) + return apierrors.NewConflict(gvr.GroupResource(), name, errors.New(msg)) + } + } + + return c.deleteObject(gvr, accessor) +} + +func (c *fakeClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { + gvk, err := apiutil.GVKForObject(obj, c.scheme) + if err != nil { + return err + } + + dcOptions := client.DeleteAllOfOptions{} + dcOptions.ApplyOptions(opts) + + gvr, _ := meta.UnsafeGuessKindToResource(gvk) + o, err := c.tracker.List(gvr, gvk, dcOptions.Namespace) + if err != nil { + return err + } + + objs, err := meta.ExtractList(o) + if err != nil { + return err + } + filteredObjs, err := objectutil.FilterWithLabels(objs, dcOptions.LabelSelector) + if err != nil { + return err + } + for _, o := range filteredObjs { + accessor, err := meta.Accessor(o) + if err != nil { + return err + } + err = c.deleteObject(gvr, accessor) + if err != nil { + return err + } + } + return nil +} + +func (c *fakeClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + updateOptions := &client.UpdateOptions{} + updateOptions.ApplyOptions(opts) + + for _, dryRunOpt := range updateOptions.DryRun { + if dryRunOpt == metav1.DryRunAll { + return nil + } + } + + gvr, err := getGVRFromObject(obj, c.scheme) + if err != nil { + return err + } + accessor, err := meta.Accessor(obj) + if err != nil { + return err + } + return c.tracker.Update(gvr, obj, accessor.GetNamespace()) +} + +func (c *fakeClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + patchOptions := &client.PatchOptions{} + patchOptions.ApplyOptions(opts) + + for _, dryRunOpt := range patchOptions.DryRun { + if dryRunOpt == metav1.DryRunAll { + return nil + } + } + + gvr, err := getGVRFromObject(obj, c.scheme) + if err != nil { + return err + } + accessor, err := meta.Accessor(obj) + if err != nil { + return err + } + data, err := patch.Data(obj) + if err != nil { + return err + } + + reaction := testing.ObjectReaction(c.tracker) + handled, o, err := reaction(testing.NewPatchAction(gvr, accessor.GetNamespace(), accessor.GetName(), patch.Type(), data)) + if err != nil { + return err + } + if !handled { + panic("tracker could not handle patch method") + } + + gvk, err := apiutil.GVKForObject(obj, c.scheme) + if err != nil { + return err + } + ta, err := meta.TypeAccessor(o) + if err != nil { + return err + } + ta.SetKind(gvk.Kind) + ta.SetAPIVersion(gvk.GroupVersion().String()) + + j, err := json.Marshal(o) + if err != nil { + return err + } + decoder := scheme.Codecs.UniversalDecoder() + zero(obj) + _, _, err = decoder.Decode(j, nil, obj) + return err +} + +func (c *fakeClient) Status() client.StatusWriter { + return &fakeStatusWriter{client: c} +} + +func (c *fakeClient) deleteObject(gvr schema.GroupVersionResource, accessor metav1.Object) error { + old, err := c.tracker.Get(gvr, accessor.GetNamespace(), accessor.GetName()) + if err == nil { + oldAccessor, err := meta.Accessor(old) + if err == nil { + if len(oldAccessor.GetFinalizers()) > 0 { + now := metav1.Now() + oldAccessor.SetDeletionTimestamp(&now) + return c.tracker.Update(gvr, old, accessor.GetNamespace()) + } + } + } + + //TODO: implement propagation + return c.tracker.Delete(gvr, accessor.GetNamespace(), accessor.GetName()) +} + +func getGVRFromObject(obj runtime.Object, scheme *runtime.Scheme) (schema.GroupVersionResource, error) { + gvk, err := apiutil.GVKForObject(obj, scheme) + if err != nil { + return schema.GroupVersionResource{}, err + } + gvr, _ := meta.UnsafeGuessKindToResource(gvk) + return gvr, nil +} + +type fakeStatusWriter struct { + client *fakeClient +} + +func (sw *fakeStatusWriter) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + // TODO(droot): This results in full update of the obj (spec + status). Need + // a way to update status field only. + return sw.client.Update(ctx, obj, opts...) +} + +func (sw *fakeStatusWriter) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + // TODO(droot): This results in full update of the obj (spec + status). Need + // a way to update status field only. + return sw.client.Patch(ctx, obj, patch, opts...) +} + +func allowsUnconditionalUpdate(gvk schema.GroupVersionKind) bool { + switch gvk.Group { + case "apps": + switch gvk.Kind { + case "ControllerRevision", "DaemonSet", "Deployment", "ReplicaSet", "StatefulSet": + return true + } + case "autoscaling": + switch gvk.Kind { + case "HorizontalPodAutoscaler": + return true + } + case "batch": + switch gvk.Kind { + case "CronJob", "Job": + return true + } + case "certificates": + switch gvk.Kind { + case "Certificates": + return true + } + case "flowcontrol": + switch gvk.Kind { + case "FlowSchema", "PriorityLevelConfiguration": + return true + } + case "networking": + switch gvk.Kind { + case "Ingress", "IngressClass", "NetworkPolicy": + return true + } + case "policy": + switch gvk.Kind { + case "PodSecurityPolicy": + return true + } + case "rbac": + switch gvk.Kind { + case "ClusterRole", "ClusterRoleBinding", "Role", "RoleBinding": + return true + } + case "scheduling": + switch gvk.Kind { + case "PriorityClass": + return true + } + case "settings": + switch gvk.Kind { + case "PodPreset": + return true + } + case "storage": + switch gvk.Kind { + case "StorageClass": + return true + } + case "": + switch gvk.Kind { + case "ConfigMap", "Endpoint", "Event", "LimitRange", "Namespace", "Node", + "PersistentVolume", "PersistentVolumeClaim", "Pod", "PodTemplate", + "ReplicationController", "ResourceQuota", "Secret", "Service", + "ServiceAccount", "EndpointSlice": + return true + } + } + + return false +} + +func allowsCreateOnUpdate(gvk schema.GroupVersionKind) bool { + switch gvk.Group { + case "coordination": + switch gvk.Kind { + case "Lease": + return true + } + case "node": + switch gvk.Kind { + case "RuntimeClass": + return true + } + case "rbac": + switch gvk.Kind { + case "ClusterRole", "ClusterRoleBinding", "Role", "RoleBinding": + return true + } + case "": + switch gvk.Kind { + case "Endpoint", "Event", "LimitRange", "Service": + return true + } + } + + return false +} + +// zero zeros the value of a pointer. +func zero(x interface{}) { + if x == nil { + return + } + res := reflect.ValueOf(x).Elem() + res.Set(reflect.Zero(res.Type())) +} diff --git a/vendor/sigs.k8s.io/controller-runtime/pkg/client/fake/doc.go b/vendor/sigs.k8s.io/controller-runtime/pkg/client/fake/doc.go new file mode 100644 index 000000000..7d680690d --- /dev/null +++ b/vendor/sigs.k8s.io/controller-runtime/pkg/client/fake/doc.go @@ -0,0 +1,39 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Package fake provides a fake client for testing. + +A fake client is backed by its simple object store indexed by GroupVersionResource. +You can create a fake client with optional objects. + + client := NewFakeClientWithScheme(scheme, initObjs...) // initObjs is a slice of runtime.Object + +You can invoke the methods defined in the Client interface. + +When in doubt, it's almost always better not to use this package and instead use +envtest.Environment with a real client and API server. + +WARNING: ⚠️ Current Limitations / Known Issues with the fake Client ⚠️ +- This client does not have a way to inject specific errors to test handled vs. unhandled errors. +- There is some support for sub resources which can cause issues with tests if you're trying to update + e.g. metadata and status in the same reconcile. +- No OpeanAPI validation is performed when creating or updating objects. +- ObjectMeta's `Generation` and `ResourceVersion` don't behave properly, Patch or Update +operations that rely on these fields will fail, or give false positives. + +*/ +package fake