diff --git a/webhosting-operator/config/policy/controlplane/etcd-main.yaml b/webhosting-operator/config/policy/controlplane/etcd-main.yaml new file mode 100644 index 00000000..385d9459 --- /dev/null +++ b/webhosting-operator/config/policy/controlplane/etcd-main.yaml @@ -0,0 +1,31 @@ +apiVersion: kyverno.io/v1 +kind: Policy +metadata: + name: etcd-main + namespace: shoot--fb28d21f90--sharding +spec: + failurePolicy: Ignore + rules: + # set static requests/limits on etcd-main to ensure similar evaluation environment between load test runs + - name: resources + match: + any: + - resources: + kinds: + - Pod + selector: + matchLabels: + instance: etcd-main + name: etcd + mutate: + patchStrategicMerge: + spec: + containers: + - name: etcd + resources: + requests: + cpu: 1000m + memory: 3Gi + limits: + cpu: 2000m + memory: 6Gi diff --git a/webhosting-operator/config/policy/controlplane/kube-apiserver-scale.yaml b/webhosting-operator/config/policy/controlplane/kube-apiserver-scale.yaml new file mode 100644 index 00000000..d36ddc77 --- /dev/null +++ b/webhosting-operator/config/policy/controlplane/kube-apiserver-scale.yaml @@ -0,0 +1,25 @@ +apiVersion: kyverno.io/v1 +kind: Policy +metadata: + name: kube-apiserver-scale + namespace: shoot--fb28d21f90--sharding +spec: + failurePolicy: Ignore + # schema validation doesn't seem to work in combination with the /scale subresource, disable it for now + schemaValidation: false + rules: + # set static replicas on kube-apiserver to ensure similar evaluation environment between load test runs + - name: replicas + match: + any: + - resources: + # mutate scale requests by HPA + kinds: + - Deployment/scale + # the Scale subresource doesn't have the original resource's labels -> we have to match by name + names: + - kube-apiserver + mutate: + patchStrategicMerge: + spec: + replicas: 4 diff --git a/webhosting-operator/config/policy/controlplane/kube-apiserver.yaml b/webhosting-operator/config/policy/controlplane/kube-apiserver.yaml new file mode 100644 index 00000000..891b8352 --- /dev/null +++ b/webhosting-operator/config/policy/controlplane/kube-apiserver.yaml @@ -0,0 +1,55 @@ +apiVersion: kyverno.io/v1 +kind: Policy +metadata: + name: kube-apiserver + namespace: shoot--fb28d21f90--sharding +spec: + failurePolicy: Ignore + rules: + # set static replicas on kube-apiserver to ensure similar evaluation environment between load test runs + # if the cluster is hibernated (spec.replicas=0), this rule is skipped + - name: replicas + match: + any: + - resources: + kinds: + - Deployment + selector: + matchLabels: + app: kubernetes + role: apiserver + preconditions: + all: + # Only patch spec.replicas if the control plane is not hibernated, i.e., if spec.replicas>=1. + # NB: gardenlet deploys kube-apiserver with spec.replicas=null which is defaulted after the policy webhook call + # to spec.replicas=1. Hence, treat spec.replicas=null the same way as spec.replicas=1. + - key: "{{ request.object.spec.replicas || `1` }}" + operator: GreaterThan + value: 0 + mutate: + patchStrategicMerge: + spec: + replicas: 4 + # set static requests/limits on kube-apiserver to ensure similar evaluation environment between load test runs + - name: resources + match: + any: + - resources: + kinds: + - Pod + selector: + matchLabels: + app: kubernetes + role: apiserver + mutate: + patchStrategicMerge: + spec: + containers: + - name: kube-apiserver + resources: + requests: + cpu: 1000m + memory: 4Gi + limits: + cpu: 2000m + memory: 8Gi diff --git a/webhosting-operator/config/policy/controlplane/policy-kube-controller-manager.yaml b/webhosting-operator/config/policy/controlplane/kube-controller-manager.yaml similarity index 100% rename from webhosting-operator/config/policy/controlplane/policy-kube-controller-manager.yaml rename to webhosting-operator/config/policy/controlplane/kube-controller-manager.yaml diff --git a/webhosting-operator/config/policy/controlplane/kustomization.yaml b/webhosting-operator/config/policy/controlplane/kustomization.yaml index eaff9569..f854e40e 100644 --- a/webhosting-operator/config/policy/controlplane/kustomization.yaml +++ b/webhosting-operator/config/policy/controlplane/kustomization.yaml @@ -2,7 +2,10 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization # This kustomization contains policies for manipulating shoot control plane components. -# If the seed doesn't have kyverno installed, you can use the patch file alternatively. +# For this to work, kyverno needs to be installed on the seed cluster. resources: -- policy-kube-controller-manager.yaml +- etcd-main.yaml +- kube-apiserver.yaml +- kube-apiserver-scale.yaml +- kube-controller-manager.yaml diff --git a/webhosting-operator/config/policy/controlplane/patch_kube-controller-manager_ratelimiter.yaml b/webhosting-operator/config/policy/controlplane/patch_kube-controller-manager_ratelimiter.yaml deleted file mode 100644 index 957b466e..00000000 --- a/webhosting-operator/config/policy/controlplane/patch_kube-controller-manager_ratelimiter.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# patch rate limiter settings of kube-controller-manager in shoot control plane, use with: -# k -n shoot--fb28d21f90--sharding patch deploy kube-controller-manager --type json --patch-file config/patch_kube-controller-manager_ratelimiter.yaml -- op: add - path: /spec/template/spec/containers/0/command/- - value: "--kube-api-qps=800" -- op: add - path: /spec/template/spec/containers/0/command/- - value: "--kube-api-burst=1000"