feat/fix: enhance cert-manager integration for metrics endpoints (fol…

…low-up to PR kubernetes-sigs#4243) This commit is a follow-up to PR kubernetes-sigs#4243, which introduced support for using cert-manager certificates for securing the metrics endpoint and ServiceMonitor. Key enhancements: - Added support for configuring certificate integration via a Kustomize patch. - Introduced configurable flags for greater flexibility in customization. - (fix)Updated the patch logic to append volumes and arguments without overwriting existing configurations, ensuring seamless integration. These improvements enhance usability and adaptability while maintaining compatibility with the initial implementation. As the feature has not yet been released, this update ensures a polished and user-friendly integration for upcoming releases.
camilamacedo86 · Dec 1, 2024 · 6114658 · 6114658
1 parent 781e93f
commit 6114658
Show file tree

Hide file tree

Showing 44 changed files with 775 additions and 424 deletions.
diff --git a/.github/workflows/test-e2e-samples.yml b/.github/workflows/test-e2e-samples.yml
@@ -41,7 +41,7 @@ jobs:
         run: |
           KUSTOMIZATION_FILE_PATH="testdata/project-v4/config/default/kustomization.yaml"
           sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH
-          sed -i '55,182s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '57,184s/^#//' $KUSTOMIZATION_FILE_PATH
           cd testdata/project-v4/
           go mod tidy
 
@@ -82,8 +82,8 @@ jobs:
           sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH
           # Uncomment only ValidatingWebhookConfiguration
           # from cert-manager replaces
-          sed -i '55,121s/^#//' $KUSTOMIZATION_FILE_PATH
-          sed -i '153,182s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '57,123s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '155,184s/^#//' $KUSTOMIZATION_FILE_PATH
           cd testdata/project-v4-with-plugins/
           go mod tidy
 
@@ -122,7 +122,7 @@ jobs:
         run: |
           KUSTOMIZATION_FILE_PATH="testdata/project-v4-multigroup/config/default/kustomization.yaml"
           sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH
-          sed -i '55,182s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '57,185s/^#//' $KUSTOMIZATION_FILE_PATH
           cd testdata/project-v4-multigroup
           go mod tidy
 

diff --git a/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go b/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go
@@ -74,6 +74,9 @@ func main() {
 	/*
 	 */
 	var metricsAddr string
+	var certDir string
+	var certName string
+	var certKey string
 	var enableLeaderElection bool
 	var probeAddr string
 	var secureMetrics bool
@@ -87,6 +90,10 @@ func main() {
 			"Enabling this will ensure there is only one active controller manager.")
 	flag.BoolVar(&secureMetrics, "metrics-secure", true,
 		"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
+	flag.StringVar(&certDir, "cert-dir", "",
+		"The directory of TLS certificate to use for verifying HTTPS connections for the metrics server.")
+	flag.StringVar(&certName, "cert-name", "", "CertName is the server certificate name. Defaults to tls.crt")
+	flag.StringVar(&certKey, "cert-key", "", "KeyName is the server key name. Defaults to tls.key")
 	flag.BoolVar(&enableHTTP2, "enable-http2", false,
 		"If set, HTTP/2 will be enabled for the metrics and webhook servers")
 	opts := zap.Options{
@@ -133,15 +140,18 @@ func main() {
 		// https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization
 		metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
 
-		// TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically
-		// generate self-signed certificates for the metrics server. While convenient for development and testing,
-		// this setup is not recommended for production.
+		// If CertDir, CertName, and KeyName are not set, controller-runtime generates self-signed certificates
+		// for the metrics server, suitable for development but not recommended for production.
+		// To use cert-manager-managed certificates, enable [METRICS WITH CERTMANAGER] in
+		// config/default/kustomization.yaml and specify CertDir, CertName, and KeyName accordingly.
+		if certDir != "" {
+			metricsServerOptions.CertDir = certDir
+		}
 
-		// TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
-		// you can uncomment the following lines to use the certificate managed by cert-manager.
-		// metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
-		// metricsServerOptions.CertName = "tls.crt"
-		// metricsServerOptions.KeyName = "tls.key"
+		if certName != "" && certKey != "" {
+			metricsServerOptions.CertName = certName
+			metricsServerOptions.KeyName = certKey
+		}
 
 	}
 

diff --git a/...c/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml b/...c/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml
@@ -1,21 +1,43 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: controller-manager
-  namespace: system
-  labels:
-    app.kubernetes.io/name: project
-    app.kubernetes.io/managed-by: kustomize
-spec:
-  template:
-    spec:
-      containers:
-      - name: manager
-        volumeMounts:
-        - mountPath: /tmp/k8s-metrics-server/metrics-certs
-          name: metrics-certs
-          readOnly: true
-      volumes:
-      - name: metrics-certs
-        secret:
-          secretName: metrics-server-cert
+# This patch adds the args and volumes to allow the manager to use the metrics-server certs
+# Ensure the volumeMounts field exists by creating it if missing
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts
+  value: []
+# Add the volume mount for the serving certificates
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts/-
+  value:
+    mountPath: /tmp/k8s-metrics-server/serving-certs
+    name: metrics-certs
+    readOnly: true
+# Add the cert-dir argument
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --cert-dir=/tmp/k8s-metrics-server/serving-certs
+# Add the cert-name argument
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --cert-name=/tmp/k8s-metrics-server/serving-certs/tls.crt
+# Add the cert-key argument
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --cert-key=/tmp/k8s-metrics-server/serving-certs/tls.key
+# Ensure the volumes field exists by creating it if missing
+- op: add
+  path: /spec/template/spec/volumes
+  value: []
+# Add the volume for the serving certificates
+- op: add
+  path: /spec/template/spec/volumes/-
+  value:
+    name: metrics-certs
+    secret:
+      secretName: metrics-server-cert
+      optional: false
+      items:
+        - key: ca.crt
+          path: ca.crt
+        - key: tls.crt
+          path: tls.crt
+        - key: tls.key
+          path: tls.key
diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml
@@ -18,8 +18,7 @@ resources:
 - ../crd
 - ../rbac
 - ../manager
-# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
-# crd/kustomization.yaml
+# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK]
 - ../webhook
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
 - ../certmanager
@@ -45,10 +44,17 @@ patches:
 # [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line.
 # This patch will protect the metrics with certmanager self-signed certs.
 - path: certmanager_metrics_manager_patch.yaml
+  target:
+    kind: Deployment
+    name: controller-manager
+    namespace: system
 
-# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
-# crd/kustomization.yaml
+# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK]
 - path: manager_webhook_patch.yaml
+  target:
+    kind: Deployment
+    name: controller-manager
+    namespace: system
 
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
 # Uncomment the following replacements to add the cert-manager CA injection annotations

diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_metrics_patch.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_metrics_patch.yaml
@@ -2,3 +2,6 @@
 - op: add
   path: /spec/template/spec/containers/0/args/0
   value: --metrics-bind-address=:8443
+- op: add
+  path: /spec/template/spec/containers/0/args/0
+  value: --metrics-secure=true
diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_webhook_patch.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_webhook_patch.yaml
@@ -1,26 +1,37 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: controller-manager
-  namespace: system
-  labels:
-    app.kubernetes.io/name: project
-    app.kubernetes.io/managed-by: kustomize
-spec:
-  template:
-    spec:
-      containers:
-      - name: manager
-        ports:
-        - containerPort: 9443
-          name: webhook-server
-          protocol: TCP
-        volumeMounts:
-        - mountPath: /tmp/k8s-webhook-server/serving-certs
-          name: cert
-          readOnly: true
-      volumes:
-      - name: cert
-        secret:
-          defaultMode: 420
-          secretName: webhook-server-cert
+# This patch adds the args and volumes to allow the manager to use the webhook-server certs
+# Ensure the ports field exists in the container
+- op: add
+  path: /spec/template/spec/containers/0/ports
+  value: []
+# Add the webhook-server port if it does not already exist
+- op: add
+  path: /spec/template/spec/containers/0/ports/-
+  value:
+    containerPort: 9443
+    name: webhook-server
+    protocol: TCP
+
+# Ensure the volumeMounts field exists in the container
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts
+  value: []
+# Add the serving-cert volume mount if it does not already exist
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts/-
+  value:
+    mountPath: /tmp/k8s-webhook-server/serving-certs
+    name: cert
+    readOnly: true
+
+# Ensure the volumes field exists in the pod spec
+- op: add
+  path: /spec/template/spec/volumes
+  value: []
+# Add the cert volume if it does not already exist
+- op: add
+  path: /spec/template/spec/volumes/-
+  value:
+    name: cert
+    secret:
+      defaultMode: 420
+      secretName: webhook-server-cert
diff --git a/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml b/docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml
@@ -4115,6 +4115,7 @@ spec:
     spec:
       containers:
       - args:
+        - --metrics-secure=true
         - --metrics-bind-address=:8443
         - --leader-elect
         - --health-probe-bind-address=:8081

diff --git a/docs/book/src/getting-started/testdata/project/cmd/main.go b/docs/book/src/getting-started/testdata/project/cmd/main.go
@@ -54,6 +54,9 @@ func init() {
 
 func main() {
 	var metricsAddr string
+	var certDir string
+	var certName string
+	var certKey string
 	var enableLeaderElection bool
 	var probeAddr string
 	var secureMetrics bool
@@ -67,6 +70,10 @@ func main() {
 			"Enabling this will ensure there is only one active controller manager.")
 	flag.BoolVar(&secureMetrics, "metrics-secure", true,
 		"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
+	flag.StringVar(&certDir, "cert-dir", "",
+		"The directory of TLS certificate to use for verifying HTTPS connections for the metrics server.")
+	flag.StringVar(&certName, "cert-name", "", "CertName is the server certificate name. Defaults to tls.crt")
+	flag.StringVar(&certKey, "cert-key", "", "KeyName is the server key name. Defaults to tls.key")
 	flag.BoolVar(&enableHTTP2, "enable-http2", false,
 		"If set, HTTP/2 will be enabled for the metrics and webhook servers")
 	opts := zap.Options{
@@ -113,15 +120,18 @@ func main() {
 		// https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization
 		metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
 
-		// TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically
-		// generate self-signed certificates for the metrics server. While convenient for development and testing,
-		// this setup is not recommended for production.
-
-		// TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
-		// you can uncomment the following lines to use the certificate managed by cert-manager.
-		// metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
-		// metricsServerOptions.CertName = "tls.crt"
-		// metricsServerOptions.KeyName = "tls.key"
+		// If CertDir, CertName, and KeyName are not set, controller-runtime generates self-signed certificates
+		// for the metrics server, suitable for development but not recommended for production.
+		// To use cert-manager-managed certificates, enable [METRICS WITH CERTMANAGER] in
+		// config/default/kustomization.yaml and specify CertDir, CertName, and KeyName accordingly.
+		if certDir != "" {
+			metricsServerOptions.CertDir = certDir
+		}
+
+		if certName != "" && certKey != "" {
+			metricsServerOptions.CertName = certName
+			metricsServerOptions.KeyName = certKey
+		}
 
 	}
 

diff --git a/...rc/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml b/...rc/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml
@@ -1,21 +1,43 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: controller-manager
-  namespace: system
-  labels:
-    app.kubernetes.io/name: project
-    app.kubernetes.io/managed-by: kustomize
-spec:
-  template:
-    spec:
-      containers:
-      - name: manager
-        volumeMounts:
-        - mountPath: /tmp/k8s-metrics-server/metrics-certs
-          name: metrics-certs
-          readOnly: true
-      volumes:
-      - name: metrics-certs
-        secret:
-          secretName: metrics-server-cert
+# This patch adds the args and volumes to allow the manager to use the metrics-server certs
+# Ensure the volumeMounts field exists by creating it if missing
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts
+  value: []
+# Add the volume mount for the serving certificates
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts/-
+  value:
+    mountPath: /tmp/k8s-metrics-server/serving-certs
+    name: metrics-certs
+    readOnly: true
+# Add the cert-dir argument
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --cert-dir=/tmp/k8s-metrics-server/serving-certs
+# Add the cert-name argument
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --cert-name=/tmp/k8s-metrics-server/serving-certs/tls.crt
+# Add the cert-key argument
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --cert-key=/tmp/k8s-metrics-server/serving-certs/tls.key
+# Ensure the volumes field exists by creating it if missing
+- op: add
+  path: /spec/template/spec/volumes
+  value: []
+# Add the volume for the serving certificates
+- op: add
+  path: /spec/template/spec/volumes/-
+  value:
+    name: metrics-certs
+    secret:
+      secretName: metrics-server-cert
+      optional: false
+      items:
+        - key: ca.crt
+          path: ca.crt
+        - key: tls.crt
+          path: tls.crt
+        - key: tls.key
+          path: tls.key
diff --git a/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml b/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml
@@ -18,8 +18,7 @@ resources:
 - ../crd
 - ../rbac
 - ../manager
-# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
-# crd/kustomization.yaml
+# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK]
 #- ../webhook
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
 #- ../certmanager
@@ -45,10 +44,17 @@ patches:
 # [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line.
 # This patch will protect the metrics with certmanager self-signed certs.
 #- path: certmanager_metrics_manager_patch.yaml
+#  target:
+#    kind: Deployment
+#    name: controller-manager
+#    namespace: system
 
-# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
-# crd/kustomization.yaml
+# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK]
 #- path: manager_webhook_patch.yaml
+#  target:
+#    kind: Deployment
+#    name: controller-manager
+#    namespace: system
 
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
 # Uncomment the following replacements to add the cert-manager CA injection annotations