diff --git a/docs/usage/registry-cache/configuration.md b/docs/usage/registry-cache/configuration.md index 458adac9..0564793e 100644 --- a/docs/usage/registry-cache/configuration.md +++ b/docs/usage/registry-cache/configuration.md @@ -92,6 +92,8 @@ The `providerConfig.caches[].garbageCollection.ttl` field is the time to live of The `providerConfig.caches[].secretReferenceName` is the name of the reference for the Secret containing the upstream registry credentials. To cache images from a private registry, credentials to the upstream registry should be supplied. For more details, see [How to provide credentials for upstream registry](upstream-credentials.md#how-to-provide-credentials-for-upstream-registry). +The `providerConfig.caches[].highAvailability` defines if the StatefulSet is scaled to 2 replicas. See [Increase the cache disk size](#high-availability) for more information. + > **Note**: It is only possible to provide one set of credentials for one private upstream registry. ## Garbage Collection @@ -139,7 +141,9 @@ There is always the option to remove the cache from the Shoot spec and to readd ## High Аvailability -The registry cache runs with a single replica. This fact may lead to concerns for the high availability such as "What happens when the registry cache is down? Does containerd fail to pull the image?". As outlined in the [How does it work? section](#how-does-it-work), containerd is configured to fall back to the upstream registry if it fails to pull the image from the registry cache. Hence, when the registry cache is unavailable, the containerd's image pull operations are not affected because containerd falls back to image pull from the upstream registry. +Per default the registry cache runs with a single replica. This fact may lead to concerns for the high availability such as "What happens when the registry cache is down? Does containerd fail to pull the image?". As outlined in the [How does it work? section](#how-does-it-work), containerd is configured to fall back to the upstream registry if it fails to pull the image from the registry cache. Hence, when the registry cache is unavailable, the containerd's image pull operations are not affected because containerd falls back to image pull from the upstream registry. + +In special cases where this is not enough (for example when using the registry cache with a proxy) it is possible to set `providerConfig.caches[].highAvailability` to `true`, this will add the label `high-availability-config.resources.gardener.cloud/type=server` and scale the statefulset to 2 replicas. The `topologySpreadConstraints` is added according to the cluster configuration. See also [High Availability of Deployed Components](https://gardener.cloud/docs/gardener/high-availability/#system-components). ## Possible Pitfalls diff --git a/hack/api-reference/registry.md b/hack/api-reference/registry.md index 0ea8d807..850e47cf 100644 --- a/hack/api-reference/registry.md +++ b/hack/api-reference/registry.md @@ -129,6 +129,18 @@ string

SecretReferenceName is the name of the reference for the Secret containing the upstream registry credentials.

+ + +highAvailability
+ +bool + + + +(Optional) +

HighAvailability defines if the StatefulSet is scaled with the High Availability feature.

+ +

RegistryCacheStatus diff --git a/pkg/apis/registry/types.go b/pkg/apis/registry/types.go index d469db59..cf4933cb 100644 --- a/pkg/apis/registry/types.go +++ b/pkg/apis/registry/types.go @@ -38,6 +38,8 @@ type RegistryCache struct { GarbageCollection *GarbageCollection // SecretReferenceName is the name of the reference for the Secret containing the upstream registry credentials SecretReferenceName *string + // HighAvailability defines if the StatefulSet is scaled with the [High Availability](https://gardener.cloud/docs/gardener/high-availability/#system-components) feature. + HighAvailability *bool } // Volume contains settings for the registry cache volume. diff --git a/pkg/apis/registry/v1alpha3/types.go b/pkg/apis/registry/v1alpha3/types.go index 74792cef..33570773 100644 --- a/pkg/apis/registry/v1alpha3/types.go +++ b/pkg/apis/registry/v1alpha3/types.go @@ -43,6 +43,9 @@ type RegistryCache struct { // SecretReferenceName is the name of the reference for the Secret containing the upstream registry credentials. // +optional SecretReferenceName *string `json:"secretReferenceName,omitempty"` + // HighAvailability defines if the StatefulSet is scaled with the [High Availability](https://gardener.cloud/docs/gardener/high-availability/#system-components) feature. + // +optional + HighAvailability *bool `json:"highAvailability,omitempty"` } // Volume contains settings for the registry cache volume. diff --git a/pkg/apis/registry/v1alpha3/zz_generated.conversion.go b/pkg/apis/registry/v1alpha3/zz_generated.conversion.go index 4801e3e6..596cb11e 100644 --- a/pkg/apis/registry/v1alpha3/zz_generated.conversion.go +++ b/pkg/apis/registry/v1alpha3/zz_generated.conversion.go @@ -113,6 +113,7 @@ func autoConvert_v1alpha3_RegistryCache_To_registry_RegistryCache(in *RegistryCa out.Volume = (*registry.Volume)(unsafe.Pointer(in.Volume)) out.GarbageCollection = (*registry.GarbageCollection)(unsafe.Pointer(in.GarbageCollection)) out.SecretReferenceName = (*string)(unsafe.Pointer(in.SecretReferenceName)) + out.HighAvailability = (*bool)(unsafe.Pointer(in.HighAvailability)) return nil } @@ -127,6 +128,7 @@ func autoConvert_registry_RegistryCache_To_v1alpha3_RegistryCache(in *registry.R out.Volume = (*Volume)(unsafe.Pointer(in.Volume)) out.GarbageCollection = (*GarbageCollection)(unsafe.Pointer(in.GarbageCollection)) out.SecretReferenceName = (*string)(unsafe.Pointer(in.SecretReferenceName)) + out.HighAvailability = (*bool)(unsafe.Pointer(in.HighAvailability)) return nil } diff --git a/pkg/apis/registry/v1alpha3/zz_generated.deepcopy.go b/pkg/apis/registry/v1alpha3/zz_generated.deepcopy.go index 204e5317..f9c56d2a 100644 --- a/pkg/apis/registry/v1alpha3/zz_generated.deepcopy.go +++ b/pkg/apis/registry/v1alpha3/zz_generated.deepcopy.go @@ -52,6 +52,11 @@ func (in *RegistryCache) DeepCopyInto(out *RegistryCache) { *out = new(string) **out = **in } + if in.HighAvailability != nil { + in, out := &in.HighAvailability, &out.HighAvailability + *out = new(bool) + **out = **in + } return } diff --git a/pkg/apis/registry/zz_generated.deepcopy.go b/pkg/apis/registry/zz_generated.deepcopy.go index 0e57aa7d..35770c83 100644 --- a/pkg/apis/registry/zz_generated.deepcopy.go +++ b/pkg/apis/registry/zz_generated.deepcopy.go @@ -52,6 +52,11 @@ func (in *RegistryCache) DeepCopyInto(out *RegistryCache) { *out = new(string) **out = **in } + if in.HighAvailability != nil { + in, out := &in.HighAvailability, &out.HighAvailability + *out = new(bool) + **out = **in + } return } diff --git a/pkg/component/registrycaches/registry_caches.go b/pkg/component/registrycaches/registry_caches.go index e85dd498..4c461f47 100644 --- a/pkg/component/registrycaches/registry_caches.go +++ b/pkg/component/registrycaches/registry_caches.go @@ -17,6 +17,7 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" + "github.com/gardener/gardener/pkg/apis/resources/v1alpha1" "github.com/gardener/gardener/pkg/client/kubernetes" "github.com/gardener/gardener/pkg/component" "github.com/gardener/gardener/pkg/utils" @@ -104,8 +105,8 @@ func (r *registryCaches) Deploy(ctx context.Context) error { secretName, secret = managedresources.NewSecret(r.client, r.namespace, managedResourceName, data, false) managedResource = managedresources.NewForShoot(r.client, r.namespace, managedResourceName, constants.Origin, keepObjects). - WithSecretRef(secretName). - DeletePersistentVolumeClaims(true) + WithSecretRef(secretName). + DeletePersistentVolumeClaims(true) ) if err := secret.Reconcile(ctx); err != nil { @@ -257,11 +258,16 @@ func (r *registryCaches) computeResourcesDataForRegistryCache(ctx context.Contex }, } + additionalStatefulSetLabels := map[string]string{} + if cache.HighAvailability != nil && *cache.HighAvailability { + additionalStatefulSetLabels[v1alpha1.HighAvailabilityConfigType] = v1alpha1.HighAvailabilityConfigTypeServer + } + statefulSet := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: metav1.NamespaceSystem, - Labels: getLabels(name, upstreamLabel), + Labels: utils.MergeStringMaps(getLabels(name, upstreamLabel), additionalStatefulSetLabels), }, Spec: appsv1.StatefulSetSpec{ ServiceName: service.Name, diff --git a/pkg/component/registrycaches/registry_caches_test.go b/pkg/component/registrycaches/registry_caches_test.go index ff71a01a..ea66eca4 100644 --- a/pkg/component/registrycaches/registry_caches_test.go +++ b/pkg/component/registrycaches/registry_caches_test.go @@ -198,13 +198,18 @@ status: ` } - statefulSetYAMLFor = func(name, upstream, size, configSecretName string, storageClassName *string) string { + statefulSetYAMLFor = func(name, upstream, size, configSecretName string, storageClassName *string, ha bool) string { out := `apiVersion: apps/v1 kind: StatefulSet metadata: creationTimestamp: null labels: - app: ` + name + ` + app: ` + name + if ha { + out += ` + ` + resourcesv1alpha1.HighAvailabilityConfigType + `: ` + resourcesv1alpha1.HighAvailabilityConfigTypeServer + } + out += ` upstream-host: ` + upstream + ` name: ` + name + ` namespace: kube-system @@ -379,11 +384,11 @@ status: {} expectedManifests := []string{ configSecretYAMLFor(dockerConfigSecretName, "registry-docker-io", "docker.io", configYAMLFor("https://registry-1.docker.io", "336h0m0s", "", "")), serviceYAMLFor("registry-docker-io", "docker.io", "https://registry-1.docker.io"), - statefulSetYAMLFor("registry-docker-io", "docker.io", "10Gi", dockerConfigSecretName, nil), + statefulSetYAMLFor("registry-docker-io", "docker.io", "10Gi", dockerConfigSecretName, nil, false), vpaYAMLFor("registry-docker-io"), configSecretYAMLFor(arConfigSecretName, "registry-europe-docker-pkg-dev", "europe-docker.pkg.dev", configYAMLFor("https://europe-docker.pkg.dev", "0s", "", "")), serviceYAMLFor("registry-europe-docker-pkg-dev", "europe-docker.pkg.dev", "https://europe-docker.pkg.dev"), - statefulSetYAMLFor("registry-europe-docker-pkg-dev", "europe-docker.pkg.dev", "20Gi", arConfigSecretName, ptr.To("premium")), + statefulSetYAMLFor("registry-europe-docker-pkg-dev", "europe-docker.pkg.dev", "20Gi", arConfigSecretName, ptr.To("premium"), false), vpaYAMLFor("registry-europe-docker-pkg-dev"), } Expect(manifests).To(ConsistOf(expectedManifests)) @@ -411,10 +416,43 @@ status: {} expectedManifests := []string{ configSecretYAMLFor(dockerConfigSecretName, "registry-docker-io", "docker.io", configYAMLFor("https://registry-1.docker.io", "336h0m0s", "", "")), serviceYAMLFor("registry-docker-io", "docker.io", "https://registry-1.docker.io"), - statefulSetYAMLFor("registry-docker-io", "docker.io", "10Gi", dockerConfigSecretName, nil), + statefulSetYAMLFor("registry-docker-io", "docker.io", "10Gi", dockerConfigSecretName, nil, false), + configSecretYAMLFor(arConfigSecretName, "registry-europe-docker-pkg-dev", "europe-docker.pkg.dev", configYAMLFor("https://europe-docker.pkg.dev", "0s", "", "")), + serviceYAMLFor("registry-europe-docker-pkg-dev", "europe-docker.pkg.dev", "https://europe-docker.pkg.dev"), + statefulSetYAMLFor("registry-europe-docker-pkg-dev", "europe-docker.pkg.dev", "20Gi", arConfigSecretName, ptr.To("premium"), false), + } + Expect(manifests).To(ConsistOf(expectedManifests)) + }) + }) + + Context("when HA is enabled", func() { + BeforeEach(func() { + values.Caches[0].HighAvailability = ptr.To(true) + values.Caches[1].HighAvailability = ptr.To(true) + }) + + It("should successfully deploy the resources", func() { + Expect(registryCaches.Deploy(ctx)).To(Succeed()) + + Expect(c.Get(ctx, client.ObjectKeyFromObject(managedResource), managedResource)).To(Succeed()) + managedResourceSecret.Name = managedResource.Spec.SecretRefs[0].Name + Expect(c.Get(ctx, client.ObjectKeyFromObject(managedResourceSecret), managedResourceSecret)).To(Succeed()) + + manifests, err := test.ExtractManifestsFromManagedResourceData(managedResourceSecret.Data) + Expect(err).NotTo(HaveOccurred()) + Expect(manifests).To(HaveLen(8)) + + dockerConfigSecretName := "registry-docker-io-config-2935d46f" + arConfigSecretName := "registry-europe-docker-pkg-dev-config-245e2638" + expectedManifests := []string{ + configSecretYAMLFor(dockerConfigSecretName, "registry-docker-io", "docker.io", configYAMLFor("https://registry-1.docker.io", "336h0m0s", "", "")), + serviceYAMLFor("registry-docker-io", "docker.io", "https://registry-1.docker.io"), + statefulSetYAMLFor("registry-docker-io", "docker.io", "10Gi", dockerConfigSecretName, nil, true), + vpaYAMLFor("registry-docker-io"), configSecretYAMLFor(arConfigSecretName, "registry-europe-docker-pkg-dev", "europe-docker.pkg.dev", configYAMLFor("https://europe-docker.pkg.dev", "0s", "", "")), serviceYAMLFor("registry-europe-docker-pkg-dev", "europe-docker.pkg.dev", "https://europe-docker.pkg.dev"), - statefulSetYAMLFor("registry-europe-docker-pkg-dev", "europe-docker.pkg.dev", "20Gi", arConfigSecretName, ptr.To("premium")), + statefulSetYAMLFor("registry-europe-docker-pkg-dev", "europe-docker.pkg.dev", "20Gi", arConfigSecretName, ptr.To("premium"), true), + vpaYAMLFor("registry-europe-docker-pkg-dev"), } Expect(manifests).To(ConsistOf(expectedManifests)) }) @@ -480,11 +518,11 @@ status: {} expectedManifests := []string{ configSecretYAMLFor(dockerConfigSecretName, "registry-docker-io", "docker.io", configYAMLFor("https://registry-1.docker.io", "336h0m0s", "docker-user", "s3cret")), serviceYAMLFor("registry-docker-io", "docker.io", "https://registry-1.docker.io"), - statefulSetYAMLFor("registry-docker-io", "docker.io", "10Gi", dockerConfigSecretName, nil), + statefulSetYAMLFor("registry-docker-io", "docker.io", "10Gi", dockerConfigSecretName, nil, false), vpaYAMLFor("registry-docker-io"), configSecretYAMLFor(arConfigSecretName, "registry-europe-docker-pkg-dev", "europe-docker.pkg.dev", configYAMLFor("https://europe-docker.pkg.dev", "0s", "ar-user", `{"foo":"bar"}`)), serviceYAMLFor("registry-europe-docker-pkg-dev", "europe-docker.pkg.dev", "https://europe-docker.pkg.dev"), - statefulSetYAMLFor("registry-europe-docker-pkg-dev", "europe-docker.pkg.dev", "20Gi", arConfigSecretName, ptr.To("premium")), + statefulSetYAMLFor("registry-europe-docker-pkg-dev", "europe-docker.pkg.dev", "20Gi", arConfigSecretName, ptr.To("premium"), false), vpaYAMLFor("registry-europe-docker-pkg-dev"), } Expect(manifests).To(ConsistOf(expectedManifests))