feat/fix: enhance cert-manager integration for metrics endpoints (fol…

…low-up to PR kubernetes-sigs#4243)

This commit is a follow-up to PR kubernetes-sigs#4243, which introduced support for using cert-manager certificates for securing the metrics endpoint and ServiceMonitor.

Key enhancements:
- Added support for configuring certificate integration via a Kustomize patch.
- Introduced configurable flags for greater flexibility in customization.
- (fix)Updated the patch logic to append volumes and arguments without overwriting existing configurations, ensuring seamless integration.

These improvements enhance usability and adaptability while maintaining compatibility with the initial implementation. As the feature has not yet been released, this update ensures a polished and user-friendly integration for upcoming releases.

.github/workflows/test-e2e-samples.yml

@@ -41,7 +41,7 @@ jobs:
         run: |
           KUSTOMIZATION_FILE_PATH="testdata/project-v4/config/default/kustomization.yaml"
           sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH
-          sed -i '55,182s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '57,184s/^#//' $KUSTOMIZATION_FILE_PATH
           cd testdata/project-v4/
           go mod tidy
 
@@ -82,8 +82,8 @@ jobs:
           sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH
           # Uncomment only ValidatingWebhookConfiguration
           # from cert-manager replaces
-          sed -i '55,121s/^#//' $KUSTOMIZATION_FILE_PATH
-          sed -i '153,182s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '57,123s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '155,184s/^#//' $KUSTOMIZATION_FILE_PATH
           cd testdata/project-v4-with-plugins/
           go mod tidy
 
@@ -122,7 +122,7 @@ jobs:
         run: |
           KUSTOMIZATION_FILE_PATH="testdata/project-v4-multigroup/config/default/kustomization.yaml"
           sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH
-          sed -i '55,182s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '57,185s/^#//' $KUSTOMIZATION_FILE_PATH
           cd testdata/project-v4-multigroup
           go mod tidy
 

docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go

@@ -74,6 +74,9 @@ func main() {
 	/*
 	 */
 	var metricsAddr string
+	var certDir string
+	var certName string
+	var certKey string
 	var enableLeaderElection bool
 	var probeAddr string
 	var secureMetrics bool
@@ -87,6 +90,10 @@ func main() {
 			"Enabling this will ensure there is only one active controller manager.")
 	flag.BoolVar(&secureMetrics, "metrics-secure", true,
 		"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
+	flag.StringVar(&certDir, "cert-dir", "",
+		"The directory of TLS certificate to use for verifying HTTPS connections for the metrics server.")
+	flag.StringVar(&certName, "cert-name", "", "CertName is the server certificate name. Defaults to tls.crt")
+	flag.StringVar(&certKey, "cert-key", "", "KeyName is the server key name. Defaults to tls.key")
 	flag.BoolVar(&enableHTTP2, "enable-http2", false,
 		"If set, HTTP/2 will be enabled for the metrics and webhook servers")
 	opts := zap.Options{
@@ -135,13 +142,18 @@ func main() {
 
 		// TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically
 		// generate self-signed certificates for the metrics server. While convenient for development and testing,
-		// this setup is not recommended for production.
+		// this setup is not recommended for production. Not that, if cert-manager is enabled
+		// in config/default/kustomization.yaml, you can use the certificate managed by cert-manager by
+		// specifying the CertDir, CertName, and KeyName by uncommenting [METRICS WITH CERTMANGER]
 
-		// TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
-		// you can uncomment the following lines to use the certificate managed by cert-manager.
-		// metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
-		// metricsServerOptions.CertName = "tls.crt"
-		// metricsServerOptions.KeyName = "tls.key"
+		if certDir != "" {
+			metricsServerOptions.CertDir = certDir
+		}
+
+		if certName != "" && certKey != "" {
+			metricsServerOptions.CertName = certName
+			metricsServerOptions.KeyName = certKey
+		}
 
 	}
 

docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml

@@ -1,21 +1,35 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: controller-manager
-  namespace: system
-  labels:
-    app.kubernetes.io/name: project
-    app.kubernetes.io/managed-by: kustomize
-spec:
-  template:
-    spec:
-      containers:
-      - name: manager
-        volumeMounts:
-        - mountPath: /tmp/k8s-metrics-server/metrics-certs
-          name: metrics-certs
-          readOnly: true
-      volumes:
-      - name: metrics-certs
-        secret:
-          secretName: metrics-server-cert
+# This patch adds the args and volumes to allow the manager to use the metrics-server certs
+# Ensure the volumeMounts field exists by creating it if missing
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts
+  value: []
+# Add the volume mount for the serving certificates
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts/-
+  value:
+    mountPath: /tmp/k8s-metrics-server/metrics-certs
+    name: metrics-certs
+    readOnly: true
+# Add the cert-dir argument
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --cert-dir=/tmp/k8s-metrics-server/metrics-certs
+# Add the cert-name argument
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --cert-name=/tmp/k8s-metrics-server/metrics-certs/tls.cert
+# Add the cert-key argument
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --cert-key=/tmp/k8s-metrics-server/metrics-certs/tls.key
+# Ensure the volumes field exists by creating it if missing
+- op: add
+  path: /spec/template/spec/volumes
+  value: []
+# Add the volume for the serving certificates
+- op: add
+  path: /spec/template/spec/volumes/-
+  value:
+    name: metrics-certs
+    secret:
+      secretName: metrics-server-cert

docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml

@@ -45,6 +45,8 @@ patches:
 # [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line.
 # This patch will protect the metrics with certmanager self-signed certs.
 - path: certmanager_metrics_manager_patch.yaml
+  target:
+    kind: Deployment
 
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml

docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_metrics_patch.yaml

@@ -2,3 +2,6 @@
 - op: add
   path: /spec/template/spec/containers/0/args/0
   value: --metrics-bind-address=:8443
+- op: add
+  path: /spec/template/spec/containers/0/args/0
+  value: --secure-metrics=true

docs/book/src/cronjob-tutorial/testdata/project/dist/install.yaml

@@ -4115,6 +4115,7 @@ spec:
     spec:
       containers:
       - args:
+        - --secure-metrics=true
         - --metrics-bind-address=:8443
         - --leader-elect
         - --health-probe-bind-address=:8081

docs/book/src/getting-started/testdata/project/cmd/main.go

@@ -54,6 +54,9 @@ func init() {
 
 func main() {
 	var metricsAddr string
+	var certDir string
+	var certName string
+	var certKey string
 	var enableLeaderElection bool
 	var probeAddr string
 	var secureMetrics bool
@@ -67,6 +70,10 @@ func main() {
 			"Enabling this will ensure there is only one active controller manager.")
 	flag.BoolVar(&secureMetrics, "metrics-secure", true,
 		"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
+	flag.StringVar(&certDir, "cert-dir", "",
+		"The directory of TLS certificate to use for verifying HTTPS connections for the metrics server.")
+	flag.StringVar(&certName, "cert-name", "", "CertName is the server certificate name. Defaults to tls.crt")
+	flag.StringVar(&certKey, "cert-key", "", "KeyName is the server key name. Defaults to tls.key")
 	flag.BoolVar(&enableHTTP2, "enable-http2", false,
 		"If set, HTTP/2 will be enabled for the metrics and webhook servers")
 	opts := zap.Options{
@@ -115,13 +122,18 @@ func main() {
 
 		// TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically
 		// generate self-signed certificates for the metrics server. While convenient for development and testing,
-		// this setup is not recommended for production.
-
-		// TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
-		// you can uncomment the following lines to use the certificate managed by cert-manager.
-		// metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
-		// metricsServerOptions.CertName = "tls.crt"
-		// metricsServerOptions.KeyName = "tls.key"
+		// this setup is not recommended for production. Not that, if cert-manager is enabled
+		// in config/default/kustomization.yaml, you can use the certificate managed by cert-manager by
+		// specifying the CertDir, CertName, and KeyName by uncommenting [METRICS WITH CERTMANGER]
+
+		if certDir != "" {
+			metricsServerOptions.CertDir = certDir
+		}
+
+		if certName != "" && certKey != "" {
+			metricsServerOptions.CertName = certName
+			metricsServerOptions.KeyName = certKey
+		}
 
 	}
 

docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml

@@ -1,21 +1,35 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: controller-manager
-  namespace: system
-  labels:
-    app.kubernetes.io/name: project
-    app.kubernetes.io/managed-by: kustomize
-spec:
-  template:
-    spec:
-      containers:
-      - name: manager
-        volumeMounts:
-        - mountPath: /tmp/k8s-metrics-server/metrics-certs
-          name: metrics-certs
-          readOnly: true
-      volumes:
-      - name: metrics-certs
-        secret:
-          secretName: metrics-server-cert
+# This patch adds the args and volumes to allow the manager to use the metrics-server certs
+# Ensure the volumeMounts field exists by creating it if missing
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts
+  value: []
+# Add the volume mount for the serving certificates
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts/-
+  value:
+    mountPath: /tmp/k8s-metrics-server/metrics-certs
+    name: metrics-certs
+    readOnly: true
+# Add the cert-dir argument
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --cert-dir=/tmp/k8s-metrics-server/metrics-certs
+# Add the cert-name argument
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --cert-name=/tmp/k8s-metrics-server/metrics-certs/tls.cert
+# Add the cert-key argument
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --cert-key=/tmp/k8s-metrics-server/metrics-certs/tls.key
+# Ensure the volumes field exists by creating it if missing
+- op: add
+  path: /spec/template/spec/volumes
+  value: []
+# Add the volume for the serving certificates
+- op: add
+  path: /spec/template/spec/volumes/-
+  value:
+    name: metrics-certs
+    secret:
+      secretName: metrics-server-cert

docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml

@@ -45,6 +45,8 @@ patches:
 # [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line.
 # This patch will protect the metrics with certmanager self-signed certs.
 #- path: certmanager_metrics_manager_patch.yaml
+#  target:
+#    kind: Deployment
 
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml

docs/book/src/getting-started/testdata/project/config/default/manager_metrics_patch.yaml

@@ -2,3 +2,6 @@
 - op: add
   path: /spec/template/spec/containers/0/args/0
   value: --metrics-bind-address=:8443
+- op: add
+  path: /spec/template/spec/containers/0/args/0
+  value: --secure-metrics=true

docs/book/src/getting-started/testdata/project/dist/install.yaml

@@ -418,6 +418,7 @@ spec:
     spec:
       containers:
       - args:
+        - --secure-metrics=true
         - --metrics-bind-address=:8443
         - --leader-elect
         - --health-probe-bind-address=:8081

docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go

@@ -73,6 +73,9 @@ func main() {
 	/*
 	 */
 	var metricsAddr string
+	var certDir string
+	var certName string
+	var certKey string
 	var enableLeaderElection bool
 	var probeAddr string
 	var secureMetrics bool
@@ -86,6 +89,10 @@ func main() {
 			"Enabling this will ensure there is only one active controller manager.")
 	flag.BoolVar(&secureMetrics, "metrics-secure", true,
 		"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
+	flag.StringVar(&certDir, "cert-dir", "",
+		"The directory of TLS certificate to use for verifying HTTPS connections for the metrics server.")
+	flag.StringVar(&certName, "cert-name", "", "CertName is the server certificate name. Defaults to tls.crt")
+	flag.StringVar(&certKey, "cert-key", "", "KeyName is the server key name. Defaults to tls.key")
 	flag.BoolVar(&enableHTTP2, "enable-http2", false,
 		"If set, HTTP/2 will be enabled for the metrics and webhook servers")
 	opts := zap.Options{
@@ -134,13 +141,18 @@ func main() {
 
 		// TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically
 		// generate self-signed certificates for the metrics server. While convenient for development and testing,
-		// this setup is not recommended for production.
+		// this setup is not recommended for production. Not that, if cert-manager is enabled
+		// in config/default/kustomization.yaml, you can use the certificate managed by cert-manager by
+		// specifying the CertDir, CertName, and KeyName by uncommenting [METRICS WITH CERTMANGER]
 
-		// TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
-		// you can uncomment the following lines to use the certificate managed by cert-manager.
-		// metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
-		// metricsServerOptions.CertName = "tls.crt"
-		// metricsServerOptions.KeyName = "tls.key"
+		if certDir != "" {
+			metricsServerOptions.CertDir = certDir
+		}
+
+		if certName != "" && certKey != "" {
+			metricsServerOptions.CertName = certName
+			metricsServerOptions.KeyName = certKey
+		}
 
 	}
 

docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml

@@ -1,21 +1,35 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: controller-manager
-  namespace: system
-  labels:
-    app.kubernetes.io/name: project
-    app.kubernetes.io/managed-by: kustomize
-spec:
-  template:
-    spec:
-      containers:
-      - name: manager
-        volumeMounts:
-        - mountPath: /tmp/k8s-metrics-server/metrics-certs
-          name: metrics-certs
-          readOnly: true
-      volumes:
-      - name: metrics-certs
-        secret:
-          secretName: metrics-server-cert
+# This patch adds the args and volumes to allow the manager to use the metrics-server certs
+# Ensure the volumeMounts field exists by creating it if missing
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts
+  value: []
+# Add the volume mount for the serving certificates
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts/-
+  value:
+    mountPath: /tmp/k8s-metrics-server/metrics-certs
+    name: metrics-certs
+    readOnly: true
+# Add the cert-dir argument
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --cert-dir=/tmp/k8s-metrics-server/metrics-certs
+# Add the cert-name argument
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --cert-name=/tmp/k8s-metrics-server/metrics-certs/tls.cert
+# Add the cert-key argument
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --cert-key=/tmp/k8s-metrics-server/metrics-certs/tls.key
+# Ensure the volumes field exists by creating it if missing
+- op: add
+  path: /spec/template/spec/volumes
+  value: []
+# Add the volume for the serving certificates
+- op: add
+  path: /spec/template/spec/volumes/-
+  value:
+    name: metrics-certs
+    secret:
+      secretName: metrics-server-cert

docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml

@@ -45,6 +45,8 @@ patches:
 # [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line.
 # This patch will protect the metrics with certmanager self-signed certs.
 - path: certmanager_metrics_manager_patch.yaml
+  target:
+    kind: Deployment
 
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml