Add customizeKserveConfigMap action

controllers/components/kserve/kserve_controller.go

@@ -79,13 +79,12 @@ func NewComponentReconciler(ctx context.Context, mgr ctrl.Manager) error {
 		//
 		Watches(&extv1.CustomResourceDefinition{}).
 
-		// TODO should watch these as they may be created by features stuff? Owned by FT?
-		// Not sure where to import all of them from
-		// Watches(&servingv1beta1.KnativeServing{}).
-		// Watches(&maistrav1.ServiceMeshMember{}).
-		// Watches(&EnvoyFilter{}).
-		// Watches(&AuthorizationPolicy{}).
-		// Watches(&Gateway{}).
+		// TODO dynamic watch these if kinds exist?
+		// WatchesGVK(gvk.KnativeServing, enqueueRequestsForChildFT(mgr.GetClient())).
+		// WatchesGVK(gvk.ServiceMeshMember, enqueueRequestsForChildFT(mgr.GetClient())).
+		// WatchesGVK(gvk.EnvoyFilter, enqueueRequestsForChildFT(mgr.GetClient())).
+		// WatchesGVK(gvk.AuthorizationPolicy, enqueueRequestsForChildFT(mgr.GetClient())).
+		// WatchesGVK(gvk.Gateway, enqueueRequestsForChildFT(mgr.GetClient())).
 
 		// actions
 		WithAction(initialize).
@@ -105,11 +104,11 @@ func NewComponentReconciler(ctx context.Context, mgr ctrl.Manager) error {
 			kustomize.WithLabel(labels.ODH.Component(componentName), "true"),
 			kustomize.WithLabel(labels.K8SCommon.PartOf, componentName),
 		)).
+		WithAction(customizeKserveConfigMap).
 		WithAction(deploy.NewAction(
 			deploy.WithFieldOwner(componentsv1.KserveInstanceName),
 			deploy.WithLabel(labels.ComponentPartOf, componentsv1.KserveInstanceName),
 		)).
-		WithAction(setupKserveConfig).
 		WithAction(security.NewUpdatePodSecurityRoleBindingAction(serviceAccounts)).
 		WithAction(updatestatus.NewAction(
 			updatestatus.WithSelectorLabel(labels.ComponentPartOf, componentsv1.KserveInstanceName),

controllers/components/kserve/kserve_controller_actions.go

@@ -7,12 +7,16 @@ import (
 	"strings"
 
 	operatorv1 "github.com/openshift/api/operator/v1"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
 	componentsv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/components/v1"
+	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster/gvk"
 	odhtypes "github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/types"
 	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy"
 	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature"
+	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/metadata/labels"
 )
 
 func initialize(ctx context.Context, rr *odhtypes.ReconciliationRequest) error {
@@ -149,26 +153,30 @@ func removeServerlessFeatures(ctx context.Context, rr *odhtypes.ReconciliationRe
 	return serverlessFeatures.Delete(ctx, rr.Client)
 }
 
-func setupKserveConfig(ctx context.Context, rr *odhtypes.ReconciliationRequest) error {
+func customizeKserveConfigMap(ctx context.Context, rr *odhtypes.ReconciliationRequest) error {
 	k, ok := rr.Instance.(*componentsv1.Kserve)
 	if !ok {
 		return fmt.Errorf("resource instance %v is not a componentsv1.Kserve)", rr.Instance)
 	}
 
 	logger := logf.FromContext(ctx)
-	cli := rr.Client
 
-	// as long as Kserve.Serving is not 'Removed', we will setup the dependencies
+	kserveConfigMap := corev1.ConfigMap{}
+	cmidx, err := getIndexedResource(rr.Resources, &kserveConfigMap, gvk.ConfigMap, kserveConfigMapName)
+	if err != nil {
+		return err
+	}
+
 	switch k.Spec.Serving.ManagementState {
 	case operatorv1.Managed, operatorv1.Unmanaged:
 		if k.Spec.DefaultDeploymentMode == "" {
 			// if the default mode is empty in the DSC, assume mode is "Serverless" since k.Serving is Managed
-			if err := setDefaultDeploymentMode(ctx, cli, &rr.DSCI.Spec, componentsv1.Serverless); err != nil {
+			if err := setDefaultDeploymentMode(&kserveConfigMap, componentsv1.Serverless); err != nil {
 				return err
 			}
 		} else {
 			// if the default mode is explicitly specified, respect that
-			if err := setDefaultDeploymentMode(ctx, cli, &rr.DSCI.Spec, k.Spec.DefaultDeploymentMode); err != nil {
+			if err := setDefaultDeploymentMode(&kserveConfigMap, k.Spec.DefaultDeploymentMode); err != nil {
 				return err
 			}
 		}
@@ -177,11 +185,35 @@ func setupKserveConfig(ctx context.Context, rr *odhtypes.ReconciliationRequest)
 			return errors.New("setting defaultdeployment mode as Serverless is incompatible with having Serving 'Removed'")
 		}
 		if k.Spec.DefaultDeploymentMode == "" {
-			logger.Info("Serving is removed, Kserve will default to rawdeployment")
+			logger.Info("Serving is removed, Kserve will default to RawDeployment")
 		}
-		if err := setDefaultDeploymentMode(ctx, cli, &rr.DSCI.Spec, componentsv1.RawDeployment); err != nil {
+		if err := setDefaultDeploymentMode(&kserveConfigMap, componentsv1.RawDeployment); err != nil {
 			return err
 		}
 	}
+
+	err = replaceResourceAtIndex(rr.Resources, cmidx, &kserveConfigMap)
+	if err != nil {
+		return err
+	}
+
+	kserveConfigHash, err := hashConfigMap(&kserveConfigMap)
+	if err != nil {
+		return err
+	}
+
+	kserveDeployment := appsv1.Deployment{}
+	deployidx, err := getIndexedResource(rr.Resources, &kserveDeployment, gvk.Deployment, "kserve-controller-manager")
+	if err != nil {
+		return err
+	}
+
+	kserveDeployment.Spec.Template.Annotations[labels.ODHAppPrefix+"/KserveConfigHash"] = kserveConfigHash
+
+	err = replaceResourceAtIndex(rr.Resources, deployidx, &kserveDeployment)
+	if err != nil {
+		return err
+	}
+
 	return nil
 }

controllers/components/kserve/kserve_support.go

@@ -10,6 +10,9 @@ import (
 	"github.com/hashicorp/go-multierror"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/event"
@@ -24,7 +27,6 @@ import (
 	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature/manifest"
 	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature/serverless"
 	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature/servicemesh"
-	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/metadata/labels"
 	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/resources"
 )
 
@@ -213,19 +215,10 @@ func removeServiceMeshConfigurations(ctx context.Context, cli client.Client, own
 	return serviceMeshInitializer.Delete(ctx, cli)
 }
 
-func setDefaultDeploymentMode(ctx context.Context, cli client.Client, dscispec *dsciv1.DSCInitializationSpec, defaultmode componentsv1.DefaultDeploymentMode) error {
-	inferenceServiceConfigMap := &corev1.ConfigMap{}
-	err := cli.Get(ctx, client.ObjectKey{
-		Namespace: dscispec.ApplicationsNamespace,
-		Name:      kserveConfigMapName,
-	}, inferenceServiceConfigMap)
-	if err != nil {
-		return fmt.Errorf("error getting configmap %v: %w", kserveConfigMapName, err)
-	}
-
+func setDefaultDeploymentMode(inferenceServiceConfigMap *corev1.ConfigMap, defaultmode componentsv1.DefaultDeploymentMode) error {
 	// set data.deploy.defaultDeploymentMode to the model specified in the Kserve spec
 	var deployData map[string]interface{}
-	if err = json.Unmarshal([]byte(inferenceServiceConfigMap.Data["deploy"]), &deployData); err != nil {
+	if err := json.Unmarshal([]byte(inferenceServiceConfigMap.Data["deploy"]), &deployData); err != nil {
 		return fmt.Errorf("error retrieving value for key 'deploy' from configmap %s. %w", kserveConfigMapName, err)
 	}
 	modeFound := deployData["defaultDeploymentMode"]
@@ -251,30 +244,52 @@ func setDefaultDeploymentMode(ctx context.Context, cli client.Client, dscispec *
 			return fmt.Errorf("could not set values in configmap %s. %w", kserveConfigMapName, err)
 		}
 		inferenceServiceConfigMap.Data["ingress"] = string(ingressDataBytes)
+	}
 
-		if err = cli.Update(ctx, inferenceServiceConfigMap); err != nil {
-			return fmt.Errorf("could not set default deployment mode for Kserve. %w", err)
-		}
+	return nil
+}
 
-		// Restart the pod if configmap is updated so that kserve boots with the correct value
-		podList := &corev1.PodList{}
-		listOpts := []client.ListOption{
-			client.InNamespace(dscispec.ApplicationsNamespace),
-			client.MatchingLabels{
-				labels.ODH.Component(componentName): "true",
-				"control-plane":                     "kserve-controller-manager",
-			},
-		}
-		if err := cli.List(ctx, podList, listOpts...); err != nil {
-			return fmt.Errorf("failed to list pods: %w", err)
-		}
-		for _, pod := range podList.Items {
-			pod := pod
-			if err := cli.Delete(ctx, &pod); err != nil {
-				return fmt.Errorf("failed to delete pod %s: %w", pod.Name, err)
-			}
+func getIndexedResource(rs []unstructured.Unstructured, obj any, g schema.GroupVersionKind, name string) (int, error) {
+	var idx = -1
+	for i, r := range rs {
+		if r.GroupVersionKind() == g && r.GetName() == name {
+			idx = i
+			break
 		}
 	}
 
+	if idx == -1 {
+		return -1, fmt.Errorf("could not find %T with name %v in resources list", obj, name)
+	}
+
+	err := runtime.DefaultUnstructuredConverter.FromUnstructured(rs[idx].Object, obj)
+	if err != nil {
+		return idx, fmt.Errorf("failed converting to %T from Unstructured.Object: %v", obj, rs[idx].Object)
+	}
+
+	return idx, nil
+}
+
+func replaceResourceAtIndex(rs []unstructured.Unstructured, idx int, obj any) error {
+	u, err := resources.ToUnstructured(obj)
+	if err != nil {
+		return err
+	}
+
+	rs[idx] = *u
 	return nil
 }
+
+func hashConfigMap(cm *corev1.ConfigMap) (string, error) {
+	u, err := resources.ToUnstructured(cm)
+	if err != nil {
+		return "", err
+	}
+
+	h, err := resources.Hash(u)
+	if err != nil {
+		return "", err
+	}
+
+	return base64.RawURLEncoding.EncodeToString(h), nil
+}

pkg/cluster/gvk/gvk.go

@@ -168,4 +168,22 @@ var (
 		Version: "v1",
 		Kind:    "ServiceMeshMember",
 	}
+
+	EnvoyFilter = schema.GroupVersionKind{
+		Group:   "networking.istio.io",
+		Version: "v1alpha3",
+		Kind:    "EnvoyFilter",
+	}
+
+	AuthorizationPolicy = schema.GroupVersionKind{
+		Group:   "security.istio.io",
+		Version: "v1",
+		Kind:    "AuthorizationPolicy",
+	}
+
+	Gateway = schema.GroupVersionKind{
+		Group:   "networking.istio.io",
+		Version: "v1beta1",
+		Kind:    "Gateway",
+	}
 )