RHOAIENG-15772: tests(odh-nbc): write auditlogs from envtest tests to…

… disk upon request (#451)

* RHOAIENG-15772: tests(odh-nbc): write auditlogs from envtest tests to disk upon request

This helps with debugging because it's then possible to do e.g.

```
cat /tmp/audit.log | jq | ...
```

and investigate what happened when a test was running.

* fixup logging and order and few comments

* fixup readme and tweaks

components/odh-notebook-controller/README.md

@@ -91,7 +91,7 @@ The following environment variables are used to enable additional debug options
 | Environment variable   | Description                                                                                                                                  |
 |------------------------|----------------------------------------------------------------------------------------------------------------------------------------------|
 | DEBUG_WRITE_KUBECONFIG | Writes a Kubeconfig file to disk. It can be used with `kubectl` or `k9s` to examine the envtest cluster when test is paused on a breakpoint. |
-|                        |                                                                                                                                              |
+| DEBUG_WRITE_AUDITLOG   | Writes kube-apiserver auditlogs to disk. The config is in `envtest-audit-policy.yaml`, set the namespace of interest there.                  |
 
 ### Run locally
 

components/odh-notebook-controller/controllers/suite_test.go

@@ -92,6 +92,9 @@ var _ = BeforeSuite(func() {
 	// https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/envtest#Environment.Start
 	By("Bootstrapping test environment")
 	envTest = &envtest.Environment{
+		ControlPlane: envtest.ControlPlane{
+			APIServer: &envtest.APIServer{},
+		},
 		CRDInstallOptions: envtest.CRDInstallOptions{
 			Paths:              []string{filepath.Join("..", "config", "crd", "external")},
 			ErrorIfPathMissing: true,
@@ -102,13 +105,27 @@ var _ = BeforeSuite(func() {
 			IgnoreErrorIfPathMissing: false,
 		},
 	}
+	if auditLogPath, found := os.LookupEnv("DEBUG_WRITE_AUDITLOG"); found {
+		envTest.ControlPlane.APIServer.Configure().
+			// https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/#log-backend
+			Append("audit-log-maxage", "1").
+			Append("audit-log-maxbackup", "5").
+			Append("audit-log-maxsize", "100"). // in MiB
+			Append("audit-log-format", "json").
+			Append("audit-policy-file", filepath.Join("..", "envtest-audit-policy.yaml")).
+			Append("audit-log-path", auditLogPath)
+		GinkgoT().Logf("DEBUG_WRITE_AUDITLOG is set, writing `envtest-audit-policy.yaml` auditlog to %s", auditLogPath)
+	} else {
+		GinkgoT().Logf("DEBUG_WRITE_AUDITLOG environment variable was not provided")
+	}
 
 	var err error
 	cfg, err = envTest.Start()
 	Expect(err).NotTo(HaveOccurred())
 	Expect(cfg).NotTo(BeNil())
 
 	if kubeconfigPath, found := os.LookupEnv("DEBUG_WRITE_KUBECONFIG"); found {
+		// https://github.com/rancher/fleet/blob/main/integrationtests/utils/kubeconfig.go
 		user := envtest.User{Name: "MasterOfTheSystems", Groups: []string{"system:masters"}}
 		authedUser, err := envTest.ControlPlane.AddUser(user, nil)
 		Expect(err).NotTo(HaveOccurred())

components/odh-notebook-controller/envtest-audit-policy.yaml

@@ -0,0 +1,16 @@
+# https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/#audit-policy
+# This is extremely verbose kube-apiserver logging that may be enabled for debugging of envtest-based tests
+---
+apiVersion: audit.k8s.io/v1
+kind: Policy
+rules:
+  # Log all requests in `developer` namespace at the RequestResponse (maximum verbosity) level.
+  - level: RequestResponse
+    namespaces: ["developer"]
+
+# Use jq to analyze the log file this produces. For example:
+
+# jq 'select((.objectRef.apiGroup == "dscinitialization.opendatahub.io"
+#              or .objectRef.apiGroup == "datasciencecluster.opendatahub.io")
+#            and .user.username != "system:serviceaccount:redhat-ods-operator:redhat-ods-operator-controller-manager"
+#            and .verb != "get" and .verb != "watch" and .verb != "list")' < /tmp/kube-apiserver-audit.log