From df278784a87b0c7f012bc5002b5648bb606bf60c Mon Sep 17 00:00:00 2001 From: Gerard Ryan Date: Fri, 15 Nov 2024 02:22:21 +0000 Subject: [PATCH] Add customizeKserveConfigMap action --- .../components/kserve/kserve_controller.go | 15 ++-- .../kserve/kserve_controller_actions.go | 46 +++++++++-- .../components/kserve/kserve_support.go | 79 +++++++++++-------- pkg/cluster/gvk/gvk.go | 18 +++++ 4 files changed, 111 insertions(+), 47 deletions(-) diff --git a/controllers/components/kserve/kserve_controller.go b/controllers/components/kserve/kserve_controller.go index 9797b614867..75e16934791 100644 --- a/controllers/components/kserve/kserve_controller.go +++ b/controllers/components/kserve/kserve_controller.go @@ -79,13 +79,12 @@ func NewComponentReconciler(ctx context.Context, mgr ctrl.Manager) error { // Watches(&extv1.CustomResourceDefinition{}). - // TODO should watch these as they may be created by features stuff? Owned by FT? - // Not sure where to import all of them from - // Watches(&servingv1beta1.KnativeServing{}). - // Watches(&maistrav1.ServiceMeshMember{}). - // Watches(&EnvoyFilter{}). - // Watches(&AuthorizationPolicy{}). - // Watches(&Gateway{}). + // TODO dynamic watch these if kinds exist? + // WatchesGVK(gvk.KnativeServing, enqueueRequestsForChildFT(mgr.GetClient())). + // WatchesGVK(gvk.ServiceMeshMember, enqueueRequestsForChildFT(mgr.GetClient())). + // WatchesGVK(gvk.EnvoyFilter, enqueueRequestsForChildFT(mgr.GetClient())). + // WatchesGVK(gvk.AuthorizationPolicy, enqueueRequestsForChildFT(mgr.GetClient())). + // WatchesGVK(gvk.Gateway, enqueueRequestsForChildFT(mgr.GetClient())). // actions WithAction(initialize). @@ -105,11 +104,11 @@ func NewComponentReconciler(ctx context.Context, mgr ctrl.Manager) error { kustomize.WithLabel(labels.ODH.Component(componentName), "true"), kustomize.WithLabel(labels.K8SCommon.PartOf, componentName), )). + WithAction(customizeKserveConfigMap). WithAction(deploy.NewAction( deploy.WithFieldOwner(componentsv1.KserveInstanceName), deploy.WithLabel(labels.ComponentPartOf, componentsv1.KserveInstanceName), )). - WithAction(setupKserveConfig). WithAction(security.NewUpdatePodSecurityRoleBindingAction(serviceAccounts)). WithAction(updatestatus.NewAction( updatestatus.WithSelectorLabel(labels.ComponentPartOf, componentsv1.KserveInstanceName), diff --git a/controllers/components/kserve/kserve_controller_actions.go b/controllers/components/kserve/kserve_controller_actions.go index 814df22d8bb..8b7a01931a2 100644 --- a/controllers/components/kserve/kserve_controller_actions.go +++ b/controllers/components/kserve/kserve_controller_actions.go @@ -7,12 +7,16 @@ import ( "strings" operatorv1 "github.com/openshift/api/operator/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" logf "sigs.k8s.io/controller-runtime/pkg/log" componentsv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/components/v1" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster/gvk" odhtypes "github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/types" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/metadata/labels" ) func initialize(ctx context.Context, rr *odhtypes.ReconciliationRequest) error { @@ -149,26 +153,30 @@ func removeServerlessFeatures(ctx context.Context, rr *odhtypes.ReconciliationRe return serverlessFeatures.Delete(ctx, rr.Client) } -func setupKserveConfig(ctx context.Context, rr *odhtypes.ReconciliationRequest) error { +func customizeKserveConfigMap(ctx context.Context, rr *odhtypes.ReconciliationRequest) error { k, ok := rr.Instance.(*componentsv1.Kserve) if !ok { return fmt.Errorf("resource instance %v is not a componentsv1.Kserve)", rr.Instance) } logger := logf.FromContext(ctx) - cli := rr.Client - // as long as Kserve.Serving is not 'Removed', we will setup the dependencies + kserveConfigMap := corev1.ConfigMap{} + cmidx, err := getIndexedResource(rr.Resources, &kserveConfigMap, gvk.ConfigMap, kserveConfigMapName) + if err != nil { + return err + } + switch k.Spec.Serving.ManagementState { case operatorv1.Managed, operatorv1.Unmanaged: if k.Spec.DefaultDeploymentMode == "" { // if the default mode is empty in the DSC, assume mode is "Serverless" since k.Serving is Managed - if err := setDefaultDeploymentMode(ctx, cli, &rr.DSCI.Spec, componentsv1.Serverless); err != nil { + if err := setDefaultDeploymentMode(&kserveConfigMap, componentsv1.Serverless); err != nil { return err } } else { // if the default mode is explicitly specified, respect that - if err := setDefaultDeploymentMode(ctx, cli, &rr.DSCI.Spec, k.Spec.DefaultDeploymentMode); err != nil { + if err := setDefaultDeploymentMode(&kserveConfigMap, k.Spec.DefaultDeploymentMode); err != nil { return err } } @@ -177,11 +185,35 @@ func setupKserveConfig(ctx context.Context, rr *odhtypes.ReconciliationRequest) return errors.New("setting defaultdeployment mode as Serverless is incompatible with having Serving 'Removed'") } if k.Spec.DefaultDeploymentMode == "" { - logger.Info("Serving is removed, Kserve will default to rawdeployment") + logger.Info("Serving is removed, Kserve will default to RawDeployment") } - if err := setDefaultDeploymentMode(ctx, cli, &rr.DSCI.Spec, componentsv1.RawDeployment); err != nil { + if err := setDefaultDeploymentMode(&kserveConfigMap, componentsv1.RawDeployment); err != nil { return err } } + + err = replaceResourceAtIndex(rr.Resources, cmidx, &kserveConfigMap) + if err != nil { + return err + } + + kserveConfigHash, err := hashConfigMap(&kserveConfigMap) + if err != nil { + return err + } + + kserveDeployment := appsv1.Deployment{} + deployidx, err := getIndexedResource(rr.Resources, &kserveDeployment, gvk.Deployment, "kserve-controller-manager") + if err != nil { + return err + } + + kserveDeployment.Spec.Template.Annotations[labels.ODHAppPrefix+"/KserveConfigHash"] = kserveConfigHash + + err = replaceResourceAtIndex(rr.Resources, deployidx, &kserveDeployment) + if err != nil { + return err + } + return nil } diff --git a/controllers/components/kserve/kserve_support.go b/controllers/components/kserve/kserve_support.go index 52795107bb9..7fd762b2f5f 100644 --- a/controllers/components/kserve/kserve_support.go +++ b/controllers/components/kserve/kserve_support.go @@ -10,6 +10,9 @@ import ( "github.com/hashicorp/go-multierror" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" @@ -24,7 +27,6 @@ import ( "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature/manifest" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature/serverless" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature/servicemesh" - "github.com/opendatahub-io/opendatahub-operator/v2/pkg/metadata/labels" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/resources" ) @@ -213,19 +215,10 @@ func removeServiceMeshConfigurations(ctx context.Context, cli client.Client, own return serviceMeshInitializer.Delete(ctx, cli) } -func setDefaultDeploymentMode(ctx context.Context, cli client.Client, dscispec *dsciv1.DSCInitializationSpec, defaultmode componentsv1.DefaultDeploymentMode) error { - inferenceServiceConfigMap := &corev1.ConfigMap{} - err := cli.Get(ctx, client.ObjectKey{ - Namespace: dscispec.ApplicationsNamespace, - Name: kserveConfigMapName, - }, inferenceServiceConfigMap) - if err != nil { - return fmt.Errorf("error getting configmap %v: %w", kserveConfigMapName, err) - } - +func setDefaultDeploymentMode(inferenceServiceConfigMap *corev1.ConfigMap, defaultmode componentsv1.DefaultDeploymentMode) error { // set data.deploy.defaultDeploymentMode to the model specified in the Kserve spec var deployData map[string]interface{} - if err = json.Unmarshal([]byte(inferenceServiceConfigMap.Data["deploy"]), &deployData); err != nil { + if err := json.Unmarshal([]byte(inferenceServiceConfigMap.Data["deploy"]), &deployData); err != nil { return fmt.Errorf("error retrieving value for key 'deploy' from configmap %s. %w", kserveConfigMapName, err) } modeFound := deployData["defaultDeploymentMode"] @@ -251,30 +244,52 @@ func setDefaultDeploymentMode(ctx context.Context, cli client.Client, dscispec * return fmt.Errorf("could not set values in configmap %s. %w", kserveConfigMapName, err) } inferenceServiceConfigMap.Data["ingress"] = string(ingressDataBytes) + } - if err = cli.Update(ctx, inferenceServiceConfigMap); err != nil { - return fmt.Errorf("could not set default deployment mode for Kserve. %w", err) - } + return nil +} - // Restart the pod if configmap is updated so that kserve boots with the correct value - podList := &corev1.PodList{} - listOpts := []client.ListOption{ - client.InNamespace(dscispec.ApplicationsNamespace), - client.MatchingLabels{ - labels.ODH.Component(componentName): "true", - "control-plane": "kserve-controller-manager", - }, - } - if err := cli.List(ctx, podList, listOpts...); err != nil { - return fmt.Errorf("failed to list pods: %w", err) - } - for _, pod := range podList.Items { - pod := pod - if err := cli.Delete(ctx, &pod); err != nil { - return fmt.Errorf("failed to delete pod %s: %w", pod.Name, err) - } +func getIndexedResource(rs []unstructured.Unstructured, obj any, g schema.GroupVersionKind, name string) (int, error) { + var idx = -1 + for i, r := range rs { + if r.GroupVersionKind() == g && r.GetName() == name { + idx = i + break } } + if idx == -1 { + return -1, fmt.Errorf("could not find %T with name %v in resources list", obj, name) + } + + err := runtime.DefaultUnstructuredConverter.FromUnstructured(rs[idx].Object, obj) + if err != nil { + return idx, fmt.Errorf("failed converting to %T from Unstructured.Object: %v", obj, rs[idx].Object) + } + + return idx, nil +} + +func replaceResourceAtIndex(rs []unstructured.Unstructured, idx int, obj any) error { + u, err := resources.ToUnstructured(obj) + if err != nil { + return err + } + + rs[idx] = *u return nil } + +func hashConfigMap(cm *corev1.ConfigMap) (string, error) { + u, err := resources.ToUnstructured(cm) + if err != nil { + return "", err + } + + h, err := resources.Hash(u) + if err != nil { + return "", err + } + + return base64.RawURLEncoding.EncodeToString(h), nil +} diff --git a/pkg/cluster/gvk/gvk.go b/pkg/cluster/gvk/gvk.go index a22e95fa698..903b8c21138 100644 --- a/pkg/cluster/gvk/gvk.go +++ b/pkg/cluster/gvk/gvk.go @@ -168,4 +168,22 @@ var ( Version: "v1", Kind: "ServiceMeshMember", } + + EnvoyFilter = schema.GroupVersionKind{ + Group: "networking.istio.io", + Version: "v1alpha3", + Kind: "EnvoyFilter", + } + + AuthorizationPolicy = schema.GroupVersionKind{ + Group: "security.istio.io", + Version: "v1", + Kind: "AuthorizationPolicy", + } + + Gateway = schema.GroupVersionKind{ + Group: "networking.istio.io", + Version: "v1beta1", + Kind: "Gateway", + } )