From ee8ac4e3a749946e6656758c448ef2b8967b9b9d Mon Sep 17 00:00:00 2001 From: Wen Zhou Date: Tue, 19 Nov 2024 15:52:02 +0100 Subject: [PATCH] feat: add support for modelmeshserving as component - add new modelcontroller component: do not show in DSC.components - e2e on modelcontroller wont run in parallel but after others are done - if modelmesh is enabled, or kserve (later) is enabled, modelcontroller should be managed Signed-off-by: Wen Zhou --- README.md | 2 +- apis/components/v1/modelcontroller_types.go | 78 +++++ apis/components/v1/modelmeshserving_types.go | 54 ++-- apis/components/v1/zz_generated.deepcopy.go | 127 +++++++- .../v1/datasciencecluster_types.go | 5 +- ...nents.opendatahub.io_modelcontrollers.yaml | 163 ++++++++++ ...ents.opendatahub.io_modelmeshservings.yaml | 44 ++- ...er.opendatahub.io_datascienceclusters.yaml | 5 +- ...atahub-operator.clusterserviceversion.yaml | 13 +- .../modelmeshserving/modelmeshserving.go | 179 ----------- .../modelmeshserving/zz_generated.deepcopy.go | 39 --- ...nents.opendatahub.io_modelcontrollers.yaml | 157 ++++++++++ ...ents.opendatahub.io_modelmeshservings.yaml | 44 ++- ...er.opendatahub.io_datascienceclusters.yaml | 5 +- config/crd/kustomization.yaml | 1 + ...atahub-operator.clusterserviceversion.yaml | 5 +- config/rbac/role.yaml | 3 + .../modelcontroller/modelcontroller.go | 59 ++++ .../modelcontroller_actions.go | 84 ++++++ .../modelcontroller_controller.go | 89 ++++++ .../modelmeshserving/modelmeshserving.go | 60 ++++ .../modelmeshserving_actions.go | 82 +++++ .../modelmeshserving_controller.go | 85 ++++-- .../datasciencecluster_controller.go | 133 ++------- .../datasciencecluster/kubebuilder_rbac.go | 7 +- controllers/dscinitialization/suite_test.go | 2 + controllers/webhook/webhook_suite_test.go | 5 +- docs/api-overview.md | 150 ++++++++-- main.go | 19 +- pkg/upgrade/upgrade.go | 5 +- tests/e2e/controller_test.go | 4 + tests/e2e/helper_test.go | 7 +- tests/e2e/modelcontroller_test.go | 263 ++++++++++++++++ tests/e2e/modelmeshserving_test.go | 280 ++++++++++++++++++ tests/e2e/odh_manager_test.go | 12 + 35 files changed, 1834 insertions(+), 436 deletions(-) create mode 100644 apis/components/v1/modelcontroller_types.go create mode 100644 bundle/manifests/components.opendatahub.io_modelcontrollers.yaml delete mode 100644 components/modelmeshserving/modelmeshserving.go delete mode 100644 components/modelmeshserving/zz_generated.deepcopy.go create mode 100644 config/crd/bases/components.opendatahub.io_modelcontrollers.yaml create mode 100644 controllers/components/modelcontroller/modelcontroller.go create mode 100644 controllers/components/modelcontroller/modelcontroller_actions.go create mode 100644 controllers/components/modelcontroller/modelcontroller_controller.go create mode 100644 controllers/components/modelmeshserving/modelmeshserving.go create mode 100644 controllers/components/modelmeshserving/modelmeshserving_actions.go create mode 100644 tests/e2e/modelcontroller_test.go create mode 100644 tests/e2e/modelmeshserving_test.go diff --git a/README.md b/README.md index 801328202f3..d9bb63b6f8b 100644 --- a/README.md +++ b/README.md @@ -412,7 +412,7 @@ Example commands to run test suite for the dashboard `component` only, with the make run-nowebhook ``` ```shell -make e2e-test -e OPERATOR_NAMESPACE= -e E2E_TEST_FLAGS="--test-operator-controller=false --test-webhook=false --test-component=dashboard,trustyai" +make e2e-test -e OPERATOR_NAMESPACE= -e E2E_TEST_FLAGS="--test-operator-controller=false --test-webhook=false --test-component=dashboard" ``` diff --git a/apis/components/v1/modelcontroller_types.go b/apis/components/v1/modelcontroller_types.go new file mode 100644 index 00000000000..72e03306298 --- /dev/null +++ b/apis/components/v1/modelcontroller_types.go @@ -0,0 +1,78 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "github.com/opendatahub-io/opendatahub-operator/v2/apis/components" + operatorv1 "github.com/openshift/api/operator/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + ModelControllerComponentName = "odh-model-controller" // shared by kserve and modelmeshserving + // value should match whats set in the XValidation below + ModelControllerInstanceName = "default-modelcontroller" + ModelControllerKind = "ModelController" +) + +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster +// +kubebuilder:validation:XValidation:rule="self.metadata.name == 'default-modelcontroller'",message="ModelController name must be default-modelcontroller" +// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status`,description="Ready" +// +kubebuilder:printcolumn:name="Reason",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].reason`,description="Reason" + +// ModelController is the Schema for the modelcontroller API, it is a shared component between kserve and modelmeshserving +type ModelController struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ModelControllerSpec `json:"spec,omitempty"` + Status ModelControllerStatus `json:"status,omitempty"` +} + +// ModelControllerSpec defines the desired state of ModelController +type ModelControllerSpec struct { + components.DevFlagsSpec `json:",inline"` + ModelMeshServing operatorv1.ManagementState `json:"modelMeshServing,omitempty"` + Kserve operatorv1.ManagementState `json:"kserve,omitempty"` +} + +// ModelControllerStatus defines the observed state of ModelController +type ModelControllerStatus struct { + components.Status `json:",inline"` +} + +// +kubebuilder:object:root=true +// ModelControllerList contains a list of ModelController +type ModelControllerList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ModelController `json:"items"` +} + +func init() { + SchemeBuilder.Register(&ModelController{}, &ModelControllerList{}) +} + +func (c *ModelController) GetDevFlags() *components.DevFlags { return nil } + +func (c *ModelController) GetStatus() *components.Status { + return &c.Status.Status +} diff --git a/apis/components/v1/modelmeshserving_types.go b/apis/components/v1/modelmeshserving_types.go index f945869bfc4..79b16985616 100644 --- a/apis/components/v1/modelmeshserving_types.go +++ b/apis/components/v1/modelmeshserving_types.go @@ -21,26 +21,21 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! -// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. - -// ModelMeshServingSpec defines the desired state of ModelMeshServing -type ModelMeshServingSpec struct { - // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster - // Important: Run "make" to regenerate code after modifying this file - - // Foo is an example field of ModelMeshServing. Edit modelmeshserving_types.go to remove/update - Foo string `json:"foo,omitempty"` -} +const ( + ModelMeshServingComponentName = "model-mesh" + // value should match whats set in the XValidation below + ModelMeshServingInstanceName = "default-modelmesh" + ModelMeshServingKind = "ModelMeshServing" +) -// ModelMeshServingStatus defines the observed state of ModelMeshServing -type ModelMeshServingStatus struct { - components.Status `json:",inline"` -} +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. // +kubebuilder:object:root=true // +kubebuilder:subresource:status // +kubebuilder:resource:scope=Cluster +// +kubebuilder:validation:XValidation:rule="self.metadata.name == 'default-modelmesh'",message="ModelMeshServing name must be default-modelmesh" +// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status`,description="Ready" +// +kubebuilder:printcolumn:name="Reason",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].reason`,description="Reason" // ModelMeshServing is the Schema for the modelmeshservings API type ModelMeshServing struct { @@ -51,16 +46,21 @@ type ModelMeshServing struct { Status ModelMeshServingStatus `json:"status,omitempty"` } -func (c *ModelMeshServing) GetDevFlags() *components.DevFlags { - return nil +// ModelMeshServingSpec defines the desired state of ModelMeshServing +type ModelMeshServingSpec struct { + ModelMeshServingCommonSpec `json:",inline"` } -func (c *ModelMeshServing) GetStatus() *components.Status { - return &c.Status.Status +type ModelMeshServingCommonSpec struct { + components.DevFlagsSpec `json:",inline"` } -// +kubebuilder:object:root=true +// ModelMeshServingStatus defines the observed state of ModelMeshServing +type ModelMeshServingStatus struct { + components.Status `json:",inline"` +} +// +kubebuilder:object:root=true // ModelMeshServingList contains a list of ModelMeshServing type ModelMeshServingList struct { metav1.TypeMeta `json:",inline"` @@ -71,3 +71,17 @@ type ModelMeshServingList struct { func init() { SchemeBuilder.Register(&ModelMeshServing{}, &ModelMeshServingList{}) } + +func (c *ModelMeshServing) GetDevFlags() *components.DevFlags { + return c.Spec.DevFlags +} +func (c *ModelMeshServing) GetStatus() *components.Status { + return &c.Status.Status +} + +// DSCModelMeshServing contains all the configuration exposed in DSC instance for ModelMeshServing component +type DSCModelMeshServing struct { + components.ManagementSpec `json:",inline"` + // configuration fields common across components + ModelMeshServingCommonSpec `json:",inline"` +} diff --git a/apis/components/v1/zz_generated.deepcopy.go b/apis/components/v1/zz_generated.deepcopy.go index ae2d20b9829..fc1fb7e263b 100644 --- a/apis/components/v1/zz_generated.deepcopy.go +++ b/apis/components/v1/zz_generated.deepcopy.go @@ -165,6 +165,23 @@ func (in *DSCKueue) DeepCopy() *DSCKueue { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DSCModelMeshServing) DeepCopyInto(out *DSCModelMeshServing) { + *out = *in + out.ManagementSpec = in.ManagementSpec + in.ModelMeshServingCommonSpec.DeepCopyInto(&out.ModelMeshServingCommonSpec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DSCModelMeshServing. +func (in *DSCModelMeshServing) DeepCopy() *DSCModelMeshServing { + if in == nil { + return nil + } + out := new(DSCModelMeshServing) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DSCModelRegistry) DeepCopyInto(out *DSCModelRegistry) { *out = *in @@ -659,12 +676,103 @@ func (in *KueueStatus) DeepCopy() *KueueStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModelController) DeepCopyInto(out *ModelController) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelController. +func (in *ModelController) DeepCopy() *ModelController { + if in == nil { + return nil + } + out := new(ModelController) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ModelController) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModelControllerList) DeepCopyInto(out *ModelControllerList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ModelController, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelControllerList. +func (in *ModelControllerList) DeepCopy() *ModelControllerList { + if in == nil { + return nil + } + out := new(ModelControllerList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ModelControllerList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModelControllerSpec) DeepCopyInto(out *ModelControllerSpec) { + *out = *in + in.DevFlagsSpec.DeepCopyInto(&out.DevFlagsSpec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelControllerSpec. +func (in *ModelControllerSpec) DeepCopy() *ModelControllerSpec { + if in == nil { + return nil + } + out := new(ModelControllerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModelControllerStatus) DeepCopyInto(out *ModelControllerStatus) { + *out = *in + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelControllerStatus. +func (in *ModelControllerStatus) DeepCopy() *ModelControllerStatus { + if in == nil { + return nil + } + out := new(ModelControllerStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ModelMeshServing) DeepCopyInto(out *ModelMeshServing) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } @@ -686,6 +794,22 @@ func (in *ModelMeshServing) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModelMeshServingCommonSpec) DeepCopyInto(out *ModelMeshServingCommonSpec) { + *out = *in + in.DevFlagsSpec.DeepCopyInto(&out.DevFlagsSpec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelMeshServingCommonSpec. +func (in *ModelMeshServingCommonSpec) DeepCopy() *ModelMeshServingCommonSpec { + if in == nil { + return nil + } + out := new(ModelMeshServingCommonSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ModelMeshServingList) DeepCopyInto(out *ModelMeshServingList) { *out = *in @@ -721,6 +845,7 @@ func (in *ModelMeshServingList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ModelMeshServingSpec) DeepCopyInto(out *ModelMeshServingSpec) { *out = *in + in.ModelMeshServingCommonSpec.DeepCopyInto(&out.ModelMeshServingCommonSpec) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelMeshServingSpec. diff --git a/apis/datasciencecluster/v1/datasciencecluster_types.go b/apis/datasciencecluster/v1/datasciencecluster_types.go index f31be96c96c..aa58ddd2b91 100644 --- a/apis/datasciencecluster/v1/datasciencecluster_types.go +++ b/apis/datasciencecluster/v1/datasciencecluster_types.go @@ -28,7 +28,6 @@ import ( "github.com/opendatahub-io/opendatahub-operator/v2/components" "github.com/opendatahub-io/opendatahub-operator/v2/components/codeflare" "github.com/opendatahub-io/opendatahub-operator/v2/components/kserve" - "github.com/opendatahub-io/opendatahub-operator/v2/components/modelmeshserving" "github.com/opendatahub-io/opendatahub-operator/v2/components/workbenches" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster" ) @@ -48,8 +47,7 @@ type Components struct { Workbenches workbenches.Workbenches `json:"workbenches,omitempty"` // ModelMeshServing component configuration. - // Does not support enabled Kserve at the same time - ModelMeshServing modelmeshserving.ModelMeshServing `json:"modelmeshserving,omitempty"` + ModelMeshServing componentsv1.DSCModelMeshServing `json:"modelmeshserving,omitempty"` // DataServicePipeline component configuration. // Require OpenShift Pipelines Operator to be installed before enable component @@ -57,7 +55,6 @@ type Components struct { // Kserve component configuration. // Require OpenShift Serverless and OpenShift Service Mesh Operators to be installed before enable component - // Does not support enabled ModelMeshServing at the same time Kserve kserve.Kserve `json:"kserve,omitempty"` // Kueue component configuration. diff --git a/bundle/manifests/components.opendatahub.io_modelcontrollers.yaml b/bundle/manifests/components.opendatahub.io_modelcontrollers.yaml new file mode 100644 index 00000000000..443f728fa4a --- /dev/null +++ b/bundle/manifests/components.opendatahub.io_modelcontrollers.yaml @@ -0,0 +1,163 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.1 + creationTimestamp: null + name: modelcontrollers.components.opendatahub.io +spec: + group: components.opendatahub.io + names: + kind: ModelController + listKind: ModelControllerList + plural: modelcontrollers + singular: modelcontroller + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Ready + jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - description: Reason + jsonPath: .status.conditions[?(@.type=="Ready")].reason + name: Reason + type: string + name: v1 + schema: + openAPIV3Schema: + description: ModelController is the Schema for the modelcontroller API, it + is a shared component between kserve and modelmeshserving + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ModelControllerSpec defines the desired state of ModelController + properties: + devFlags: + description: Add developer fields + properties: + manifests: + description: List of custom manifests for the given component + items: + properties: + contextDir: + default: manifests + description: contextDir is the relative path to the folder + containing manifests in a repository, default value "manifests" + type: string + sourcePath: + default: "" + description: 'sourcePath is the subpath within contextDir + where kustomize builds start. Examples include any sub-folder + or path: `base`, `overlays/dev`, `default`, `odh` etc.' + type: string + uri: + default: "" + description: uri is the URI point to a git repo with tag/branch. + e.g. https://github.com/org/repo/tarball/ + type: string + type: object + type: array + type: object + kserve: + pattern: ^(Managed|Unmanaged|Force|Removed)$ + type: string + modelMeshServing: + pattern: ^(Managed|Unmanaged|Force|Removed)$ + type: string + type: object + status: + description: ModelControllerStatus defines the observed state of ModelController + properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + observedGeneration: + format: int64 + type: integer + phase: + type: string + type: object + type: object + x-kubernetes-validations: + - message: ModelController name must be default-modelcontroller + rule: self.metadata.name == 'default-modelcontroller' + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: null + storedVersions: null diff --git a/bundle/manifests/components.opendatahub.io_modelmeshservings.yaml b/bundle/manifests/components.opendatahub.io_modelmeshservings.yaml index 38a1b8ec765..83a844ea618 100644 --- a/bundle/manifests/components.opendatahub.io_modelmeshservings.yaml +++ b/bundle/manifests/components.opendatahub.io_modelmeshservings.yaml @@ -14,7 +14,16 @@ spec: singular: modelmeshserving scope: Cluster versions: - - name: v1 + - additionalPrinterColumns: + - description: Ready + jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - description: Reason + jsonPath: .status.conditions[?(@.type=="Ready")].reason + name: Reason + type: string + name: v1 schema: openAPIV3Schema: description: ModelMeshServing is the Schema for the modelmeshservings API @@ -39,10 +48,32 @@ spec: spec: description: ModelMeshServingSpec defines the desired state of ModelMeshServing properties: - foo: - description: Foo is an example field of ModelMeshServing. Edit modelmeshserving_types.go - to remove/update - type: string + devFlags: + description: Add developer fields + properties: + manifests: + description: List of custom manifests for the given component + items: + properties: + contextDir: + default: manifests + description: contextDir is the relative path to the folder + containing manifests in a repository, default value "manifests" + type: string + sourcePath: + default: "" + description: 'sourcePath is the subpath within contextDir + where kustomize builds start. Examples include any sub-folder + or path: `base`, `overlays/dev`, `default`, `odh` etc.' + type: string + uri: + default: "" + description: uri is the URI point to a git repo with tag/branch. + e.g. https://github.com/org/repo/tarball/ + type: string + type: object + type: array + type: object type: object status: description: ModelMeshServingStatus defines the observed state of ModelMeshServing @@ -110,6 +141,9 @@ spec: type: string type: object type: object + x-kubernetes-validations: + - message: ModelMeshServing name must be default-modelmesh + rule: self.metadata.name == 'default-modelmesh' served: true storage: true subresources: diff --git a/bundle/manifests/datasciencecluster.opendatahub.io_datascienceclusters.yaml b/bundle/manifests/datasciencecluster.opendatahub.io_datascienceclusters.yaml index da6d20509c6..1b68d81c1de 100644 --- a/bundle/manifests/datasciencecluster.opendatahub.io_datascienceclusters.yaml +++ b/bundle/manifests/datasciencecluster.opendatahub.io_datascienceclusters.yaml @@ -191,7 +191,6 @@ spec: description: |- Kserve component configuration. Require OpenShift Serverless and OpenShift Service Mesh Operators to be installed before enable component - Does not support enabled ModelMeshServing at the same time properties: defaultDeploymentMode: description: |- @@ -351,9 +350,7 @@ spec: type: string type: object modelmeshserving: - description: |- - ModelMeshServing component configuration. - Does not support enabled Kserve at the same time + description: ModelMeshServing component configuration. properties: devFlags: description: Add developer fields diff --git a/bundle/manifests/opendatahub-operator.clusterserviceversion.yaml b/bundle/manifests/opendatahub-operator.clusterserviceversion.yaml index f413d22c831..62bd69ef46f 100644 --- a/bundle/manifests/opendatahub-operator.clusterserviceversion.yaml +++ b/bundle/manifests/opendatahub-operator.clusterserviceversion.yaml @@ -103,15 +103,16 @@ metadata: categories: AI/Machine Learning, Big Data certified: "False" containerImage: quay.io/opendatahub/opendatahub-operator:v2.19.0 - createdAt: "2024-11-19T13:21:31Z" + createdAt: "2024-11-19T14:08:15Z" olm.skipRange: '>=1.0.0 <2.19.0' operators.operatorframework.io/builder: operator-sdk-v1.31.0 operators.operatorframework.io/internal-objects: '["featuretrackers.features.opendatahub.io", "codeflares.components.opendatahub.io", "dashboards.components.opendatahub.io", "datasciencepipelines.components.opendatahub.io", "kserves.components.opendatahub.io", "kueues.components.opendatahub.io", "modelmeshservings.components.opendatahub.io", - "modelregistries.components.opendatahub.io", "rays.components.opendatahub.io", - "trainingoperators.components.opendatahub.io", "trustyais.components.opendatahub.io", "workbenches.components.opendatahub.io"]' + "modelcontrollers.components.opendatahub.io", "modelregistries.components.opendatahub.io", + "rays.components.opendatahub.io", "trainingoperators.components.opendatahub.io", + "trustyais.components.opendatahub.io", "workbenches.components.opendatahub.io"]' operators.operatorframework.io/project_layout: go.kubebuilder.io/v3 repository: https://github.com/opendatahub-io/opendatahub-operator name: opendatahub-operator.v2.19.0 @@ -193,6 +194,9 @@ spec: kind: Kueue name: kueues.components.opendatahub.io version: v1 + - kind: ModelController + name: modelcontrollers.components.opendatahub.io + version: v1 - description: ModelMeshServing is the Schema for the modelmeshservings API displayName: Model Mesh Serving kind: ModelMeshServing @@ -424,6 +428,7 @@ spec: - datasciencepipelines - kserves - kueues + - modelcontrollers - modelmeshservings - modelregistries - rays @@ -445,6 +450,7 @@ spec: - datasciencepipelines/finalizers - kserves/finalizers - kueues/finalizers + - modelcontrollers/finalizers - modelmeshservings/finalizers - modelregistries/finalizers - rays/finalizers @@ -461,6 +467,7 @@ spec: - datasciencepipelines/status - kserves/status - kueues/status + - modelcontrollers/status - modelmeshservings/status - modelregistries/status - rays/status diff --git a/components/modelmeshserving/modelmeshserving.go b/components/modelmeshserving/modelmeshserving.go deleted file mode 100644 index cb1d07b7838..00000000000 --- a/components/modelmeshserving/modelmeshserving.go +++ /dev/null @@ -1,179 +0,0 @@ -// Package modelmeshserving provides utility functions to config MoModelMesh, a general-purpose model serving management/routing layer -// +groupName=datasciencecluster.opendatahub.io -package modelmeshserving - -import ( - "context" - "fmt" - "path/filepath" - "strings" - - "github.com/go-logr/logr" - operatorv1 "github.com/openshift/api/operator/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/log" - - dsciv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" - "github.com/opendatahub-io/opendatahub-operator/v2/components" - "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster" - "github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy" -) - -var ( - ComponentName = "model-mesh" - Path = deploy.DefaultManifestPath + "/" + ComponentName + "/overlays/odh" - DependentComponentName = "odh-model-controller" - DependentPath = deploy.DefaultManifestPath + "/" + DependentComponentName + "/base" -) - -// Verifies that Dashboard implements ComponentInterface. -var _ components.ComponentInterface = (*ModelMeshServing)(nil) - -// ModelMeshServing struct holds the configuration for the ModelMeshServing component. -// +kubebuilder:object:generate=true -type ModelMeshServing struct { - components.Component `json:""` -} - -func (m *ModelMeshServing) Init(ctx context.Context, _ cluster.Platform) error { - log := logf.FromContext(ctx).WithName(ComponentName) - - var imageParamMap = map[string]string{ - "odh-mm-rest-proxy": "RELATED_IMAGE_ODH_MM_REST_PROXY_IMAGE", - "odh-modelmesh-runtime-adapter": "RELATED_IMAGE_ODH_MODELMESH_RUNTIME_ADAPTER_IMAGE", - "odh-modelmesh": "RELATED_IMAGE_ODH_MODELMESH_IMAGE", - "odh-modelmesh-controller": "RELATED_IMAGE_ODH_MODELMESH_CONTROLLER_IMAGE", - "odh-model-controller": "RELATED_IMAGE_ODH_MODEL_CONTROLLER_IMAGE", - } - - // odh-model-controller to use - var dependentImageParamMap = map[string]string{ - "odh-model-controller": "RELATED_IMAGE_ODH_MODEL_CONTROLLER_IMAGE", - } - - // Update image parameters - if err := deploy.ApplyParams(Path, imageParamMap); err != nil { - log.Error(err, "failed to update image", "path", Path) - } - - // Update image parameters for odh-model-controller - if err := deploy.ApplyParams(DependentPath, dependentImageParamMap); err != nil { - log.Error(err, "failed to update image", "path", DependentPath) - } - - return nil -} - -func (m *ModelMeshServing) OverrideManifests(ctx context.Context, _ cluster.Platform) error { - // Go through each manifest and set the overlays if defined - for _, subcomponent := range m.DevFlags.Manifests { - if strings.Contains(subcomponent.URI, DependentComponentName) { - // Download subcomponent - if err := deploy.DownloadManifests(ctx, DependentComponentName, subcomponent); err != nil { - return err - } - // If overlay is defined, update paths - defaultKustomizePath := "base" - if subcomponent.SourcePath != "" { - defaultKustomizePath = subcomponent.SourcePath - } - DependentPath = filepath.Join(deploy.DefaultManifestPath, DependentComponentName, defaultKustomizePath) - } - - if strings.Contains(subcomponent.URI, ComponentName) { - // Download subcomponent - if err := deploy.DownloadManifests(ctx, ComponentName, subcomponent); err != nil { - return err - } - // If overlay is defined, update paths - defaultKustomizePath := "overlays/odh" - if subcomponent.SourcePath != "" { - defaultKustomizePath = subcomponent.SourcePath - } - Path = filepath.Join(deploy.DefaultManifestPath, ComponentName, defaultKustomizePath) - } - } - return nil -} - -func (m *ModelMeshServing) GetComponentName() string { - return ComponentName -} - -func (m *ModelMeshServing) ReconcileComponent(ctx context.Context, - cli client.Client, - l logr.Logger, - owner metav1.Object, - dscispec *dsciv1.DSCInitializationSpec, - platform cluster.Platform, - _ bool, -) error { - enabled := m.GetManagementState() == operatorv1.Managed - monitoringEnabled := dscispec.Monitoring.ManagementState == operatorv1.Managed - - // Update Default rolebinding - if enabled { - if m.DevFlags != nil { - // Download manifests and update paths - if err := m.OverrideManifests(ctx, platform); err != nil { - return err - } - } - - if err := cluster.UpdatePodSecurityRolebinding(ctx, cli, dscispec.ApplicationsNamespace, - "modelmesh", - "modelmesh-controller", - "odh-prometheus-operator", - "prometheus-custom"); err != nil { - return err - } - } - - if err := deploy.DeployManifestsFromPath(ctx, cli, owner, Path, dscispec.ApplicationsNamespace, ComponentName, enabled); err != nil { - return fmt.Errorf("failed to apply manifests from %s : %w", Path, err) - } - l.WithValues("Path", Path).Info("apply manifests done for modelmesh") - // For odh-model-controller - if enabled { - if err := cluster.UpdatePodSecurityRolebinding(ctx, cli, dscispec.ApplicationsNamespace, - "odh-model-controller"); err != nil { - return err - } - } - if err := deploy.DeployManifestsFromPath(ctx, cli, owner, DependentPath, dscispec.ApplicationsNamespace, m.GetComponentName(), enabled); err != nil { - // explicitly ignore error if error contains keywords "spec.selector" and "field is immutable" and return all other error. - if !strings.Contains(err.Error(), "spec.selector") || !strings.Contains(err.Error(), "field is immutable") { - return err - } - } - - l.WithValues("Path", DependentPath).Info("apply manifests done for odh-model-controller") - - if enabled { - if err := cluster.WaitForDeploymentAvailable(ctx, cli, ComponentName, dscispec.ApplicationsNamespace, 20, 2); err != nil { - return fmt.Errorf("deployment for %s is not ready to server: %w", ComponentName, err) - } - } - - // CloudService Monitoring handling - if platform == cluster.ManagedRhods { - // first model-mesh rules - if err := m.UpdatePrometheusConfig(cli, l, enabled && monitoringEnabled, ComponentName); err != nil { - return err - } - // then odh-model-controller rules - if err := m.UpdatePrometheusConfig(cli, l, enabled && monitoringEnabled, DependentComponentName); err != nil { - return err - } - if err := deploy.DeployManifestsFromPath(ctx, cli, owner, - filepath.Join(deploy.DefaultManifestPath, "monitoring", "prometheus", "apps"), - dscispec.Monitoring.Namespace, - "prometheus", true); err != nil { - return err - } - l.Info("updating SRE monitoring done") - } - - return nil -} diff --git a/components/modelmeshserving/zz_generated.deepcopy.go b/components/modelmeshserving/zz_generated.deepcopy.go deleted file mode 100644 index fee91980836..00000000000 --- a/components/modelmeshserving/zz_generated.deepcopy.go +++ /dev/null @@ -1,39 +0,0 @@ -//go:build !ignore_autogenerated - -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by controller-gen. DO NOT EDIT. - -package modelmeshserving - -import () - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ModelMeshServing) DeepCopyInto(out *ModelMeshServing) { - *out = *in - in.Component.DeepCopyInto(&out.Component) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelMeshServing. -func (in *ModelMeshServing) DeepCopy() *ModelMeshServing { - if in == nil { - return nil - } - out := new(ModelMeshServing) - in.DeepCopyInto(out) - return out -} diff --git a/config/crd/bases/components.opendatahub.io_modelcontrollers.yaml b/config/crd/bases/components.opendatahub.io_modelcontrollers.yaml new file mode 100644 index 00000000000..f0f22e08eac --- /dev/null +++ b/config/crd/bases/components.opendatahub.io_modelcontrollers.yaml @@ -0,0 +1,157 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.1 + name: modelcontrollers.components.opendatahub.io +spec: + group: components.opendatahub.io + names: + kind: ModelController + listKind: ModelControllerList + plural: modelcontrollers + singular: modelcontroller + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Ready + jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - description: Reason + jsonPath: .status.conditions[?(@.type=="Ready")].reason + name: Reason + type: string + name: v1 + schema: + openAPIV3Schema: + description: ModelController is the Schema for the modelcontroller API, it + is a shared component between kserve and modelmeshserving + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ModelControllerSpec defines the desired state of ModelController + properties: + devFlags: + description: Add developer fields + properties: + manifests: + description: List of custom manifests for the given component + items: + properties: + contextDir: + default: manifests + description: contextDir is the relative path to the folder + containing manifests in a repository, default value "manifests" + type: string + sourcePath: + default: "" + description: 'sourcePath is the subpath within contextDir + where kustomize builds start. Examples include any sub-folder + or path: `base`, `overlays/dev`, `default`, `odh` etc.' + type: string + uri: + default: "" + description: uri is the URI point to a git repo with tag/branch. + e.g. https://github.com/org/repo/tarball/ + type: string + type: object + type: array + type: object + kserve: + pattern: ^(Managed|Unmanaged|Force|Removed)$ + type: string + modelMeshServing: + pattern: ^(Managed|Unmanaged|Force|Removed)$ + type: string + type: object + status: + description: ModelControllerStatus defines the observed state of ModelController + properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + observedGeneration: + format: int64 + type: integer + phase: + type: string + type: object + type: object + x-kubernetes-validations: + - message: ModelController name must be default-modelcontroller + rule: self.metadata.name == 'default-modelcontroller' + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/components.opendatahub.io_modelmeshservings.yaml b/config/crd/bases/components.opendatahub.io_modelmeshservings.yaml index 32d346c00e0..ee679f92e48 100644 --- a/config/crd/bases/components.opendatahub.io_modelmeshservings.yaml +++ b/config/crd/bases/components.opendatahub.io_modelmeshservings.yaml @@ -14,7 +14,16 @@ spec: singular: modelmeshserving scope: Cluster versions: - - name: v1 + - additionalPrinterColumns: + - description: Ready + jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - description: Reason + jsonPath: .status.conditions[?(@.type=="Ready")].reason + name: Reason + type: string + name: v1 schema: openAPIV3Schema: description: ModelMeshServing is the Schema for the modelmeshservings API @@ -39,10 +48,32 @@ spec: spec: description: ModelMeshServingSpec defines the desired state of ModelMeshServing properties: - foo: - description: Foo is an example field of ModelMeshServing. Edit modelmeshserving_types.go - to remove/update - type: string + devFlags: + description: Add developer fields + properties: + manifests: + description: List of custom manifests for the given component + items: + properties: + contextDir: + default: manifests + description: contextDir is the relative path to the folder + containing manifests in a repository, default value "manifests" + type: string + sourcePath: + default: "" + description: 'sourcePath is the subpath within contextDir + where kustomize builds start. Examples include any sub-folder + or path: `base`, `overlays/dev`, `default`, `odh` etc.' + type: string + uri: + default: "" + description: uri is the URI point to a git repo with tag/branch. + e.g. https://github.com/org/repo/tarball/ + type: string + type: object + type: array + type: object type: object status: description: ModelMeshServingStatus defines the observed state of ModelMeshServing @@ -110,6 +141,9 @@ spec: type: string type: object type: object + x-kubernetes-validations: + - message: ModelMeshServing name must be default-modelmesh + rule: self.metadata.name == 'default-modelmesh' served: true storage: true subresources: diff --git a/config/crd/bases/datasciencecluster.opendatahub.io_datascienceclusters.yaml b/config/crd/bases/datasciencecluster.opendatahub.io_datascienceclusters.yaml index 41d1c76d658..a60e16bc28a 100644 --- a/config/crd/bases/datasciencecluster.opendatahub.io_datascienceclusters.yaml +++ b/config/crd/bases/datasciencecluster.opendatahub.io_datascienceclusters.yaml @@ -191,7 +191,6 @@ spec: description: |- Kserve component configuration. Require OpenShift Serverless and OpenShift Service Mesh Operators to be installed before enable component - Does not support enabled ModelMeshServing at the same time properties: defaultDeploymentMode: description: |- @@ -351,9 +350,7 @@ spec: type: string type: object modelmeshserving: - description: |- - ModelMeshServing component configuration. - Does not support enabled Kserve at the same time + description: ModelMeshServing component configuration. properties: devFlags: description: Add developer fields diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 5c60c7cb8e3..3f0ba2e3ff5 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -7,6 +7,7 @@ resources: - bases/features.opendatahub.io_featuretrackers.yaml - bases/components.opendatahub.io_dashboards.yaml - bases/components.opendatahub.io_workbenches.yaml +- bases/components.opendatahub.io_modelcontrollers.yaml - bases/components.opendatahub.io_modelmeshservings.yaml - bases/components.opendatahub.io_datasciencepipelines.yaml - bases/components.opendatahub.io_kserves.yaml diff --git a/config/manifests/bases/opendatahub-operator.clusterserviceversion.yaml b/config/manifests/bases/opendatahub-operator.clusterserviceversion.yaml index d3e84983875..7ba06083565 100644 --- a/config/manifests/bases/opendatahub-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/opendatahub-operator.clusterserviceversion.yaml @@ -13,8 +13,9 @@ metadata: "codeflares.components.opendatahub.io", "dashboards.components.opendatahub.io", "datasciencepipelines.components.opendatahub.io", "kserves.components.opendatahub.io", "kueues.components.opendatahub.io", "modelmeshservings.components.opendatahub.io", - "modelregistries.components.opendatahub.io", "rays.components.opendatahub.io", - "trainingoperators.components.opendatahub.io", "trustyais.components.opendatahub.io", "workbenches.components.opendatahub.io"]' + "modelcontrollers.components.opendatahub.io", "modelregistries.components.opendatahub.io", + "rays.components.opendatahub.io", "trainingoperators.components.opendatahub.io", + "trustyais.components.opendatahub.io", "workbenches.components.opendatahub.io"]' repository: https://github.com/opendatahub-io/opendatahub-operator name: opendatahub-operator.v2.19.0 namespace: placeholder diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index c342df935d8..f2461ee701f 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -176,6 +176,7 @@ rules: - datasciencepipelines - kserves - kueues + - modelcontrollers - modelmeshservings - modelregistries - rays @@ -197,6 +198,7 @@ rules: - datasciencepipelines/finalizers - kserves/finalizers - kueues/finalizers + - modelcontrollers/finalizers - modelmeshservings/finalizers - modelregistries/finalizers - rays/finalizers @@ -213,6 +215,7 @@ rules: - datasciencepipelines/status - kserves/status - kueues/status + - modelcontrollers/status - modelmeshservings/status - modelregistries/status - rays/status diff --git a/controllers/components/modelcontroller/modelcontroller.go b/controllers/components/modelcontroller/modelcontroller.go new file mode 100644 index 00000000000..e15d42abc34 --- /dev/null +++ b/controllers/components/modelcontroller/modelcontroller.go @@ -0,0 +1,59 @@ +package modelcontroller + +import ( + "fmt" + + operatorv1 "github.com/openshift/api/operator/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + componentsv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/components/v1" + dscv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/datasciencecluster/v1" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster" + odhdeploy "github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/metadata/annotations" +) + +const ( + ComponentName = componentsv1.ModelControllerComponentName +) + +// for DSC to get compoment ModelController's CR. +func GetComponentCR(dsc *dscv1.DataScienceCluster) *componentsv1.ModelController { + // we do not handle Force and Unmanaged cases here + mcAnnotations := make(map[string]string) + if dsc.Spec.Components.ModelMeshServing.ManagementState == operatorv1.Managed { // || dsc.Spec.Components.Kserve.ManagementState == operatorv1.Managed{ + mcAnnotations[annotations.ManagementStateAnnotation] = string(operatorv1.Managed) + } else { + mcAnnotations[annotations.ManagementStateAnnotation] = string(operatorv1.Removed) + } + + return &componentsv1.ModelController{ + TypeMeta: metav1.TypeMeta{ + Kind: componentsv1.ModelControllerKind, + APIVersion: componentsv1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: componentsv1.ModelControllerInstanceName, + Annotations: mcAnnotations, + }, + Spec: componentsv1.ModelControllerSpec{ + // ModelControllerCommonSpec: dsc.Spec.Components.ModelController.ModelControllerCommonSpec, + ModelMeshServing: dsc.Spec.Components.ModelMeshServing.ManagementState, + Kserve: dsc.Spec.Components.Kserve.ManagementState, + }, + } +} + +// Init for set images. +func Init(platform cluster.Platform) error { + DefaultPath := odhdeploy.DefaultManifestPath + "/" + ComponentName + "/base" + var imageParamMap = map[string]string{ + "odh-model-controller": "RELATED_IMAGE_ODH_MODEL_CONTROLLER_IMAGE", + } + // Update image parameters + if err := odhdeploy.ApplyParams(DefaultPath, imageParamMap); err != nil { + return fmt.Errorf("failed to update images on path %s: %w", DefaultPath, err) + } + + return nil +} diff --git a/controllers/components/modelcontroller/modelcontroller_actions.go b/controllers/components/modelcontroller/modelcontroller_actions.go new file mode 100644 index 00000000000..eec1fed4580 --- /dev/null +++ b/controllers/components/modelcontroller/modelcontroller_actions.go @@ -0,0 +1,84 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package modelcontroller + +import ( + "context" + "fmt" + "strings" + + operatorv1 "github.com/openshift/api/operator/v1" + + componentsv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/components/v1" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster" + odhtypes "github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/types" + odhdeploy "github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy" +) + +func initialize(ctx context.Context, rr *odhtypes.ReconciliationRequest) error { + // early exist + _, ok := rr.Instance.(*componentsv1.ModelController) + if !ok { + return fmt.Errorf("resource instance %v is not a componentsv1.ModelController)", rr.Instance) + } + rr.Manifests = []odhtypes.ManifestInfo{} + rr.Manifests = append(rr.Manifests, odhtypes.ManifestInfo{ + Path: odhdeploy.DefaultManifestPath, + ContextDir: ComponentName, + SourcePath: "base", + }) + return nil +} + +func devFlags(ctx context.Context, rr *odhtypes.ReconciliationRequest) error { + mc, ok := rr.Instance.(*componentsv1.ModelController) + if !ok { + return fmt.Errorf("resource instance %v is not a componentsv1.ModelController)", rr.Instance) + } + + if mc.Spec.DevFlags == nil { + return nil + } + + // Get ModelMeshServing if it is enabled and has devlfags + if rr.DSC.Spec.Components.ModelMeshServing.ManagementSpec.ManagementState == operatorv1.Managed || len(rr.DSC.Spec.Components.ModelMeshServing.DevFlags.Manifests) != 0 { + for _, subcomponent := range rr.DSC.Spec.Components.ModelMeshServing.DevFlags.Manifests { + if strings.Contains(subcomponent.URI, ComponentName) { + // Download odh-model-controller + if err := odhdeploy.DownloadManifests(ctx, ComponentName, subcomponent); err != nil { + return err + } + // If overlay is defined, update paths + if subcomponent.SourcePath != "" { + rr.Manifests[0].SourcePath = subcomponent.SourcePath + } + } + } + } + // TODO: Get Kserve which can override ModelMeshServing devflags, or just do checks first then download once + + // TODO: Implement devflags logmode logic + return nil +} + +func updatePodSecurityRolebinding(ctx context.Context, rr *odhtypes.ReconciliationRequest) error { + if err := cluster.UpdatePodSecurityRolebinding(ctx, rr.Client, rr.DSCI.Spec.ApplicationsNamespace, + "odh-model-controller"); err != nil { + return fmt.Errorf("failed to update PodSecurityRolebinding for %s: %w", ComponentName, err) + } + return nil +} diff --git a/controllers/components/modelcontroller/modelcontroller_controller.go b/controllers/components/modelcontroller/modelcontroller_controller.go new file mode 100644 index 00000000000..cc551517d68 --- /dev/null +++ b/controllers/components/modelcontroller/modelcontroller_controller.go @@ -0,0 +1,89 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package modelcontroller + +import ( + "context" + + templatev1 "github.com/openshift/api/template/v1" + promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + rbacv1 "k8s.io/api/rbac/v1" + extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + + componentsv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/components/v1" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/updatestatus" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/predicates/resources" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/metadata/labels" +) + +func NewComponentReconciler(ctx context.Context, mgr ctrl.Manager) error { + _, err := reconciler.ComponentReconcilerFor( + mgr, + componentsv1.ModelControllerInstanceName, + &componentsv1.ModelController{}, + ). + // customized Owns() for Component with new predicates + Owns(&corev1.ConfigMap{}). + Owns(&corev1.ServiceAccount{}). + Owns(&promv1.ServiceMonitor{}). + Owns(&networkingv1.NetworkPolicy{}). + Owns(&rbacv1.Role{}). + Owns(&rbacv1.ClusterRole{}). + Owns(&rbacv1.RoleBinding{}). + Owns(&rbacv1.ClusterRoleBinding{}). + Owns(&corev1.Service{}). + Owns(&admissionregistrationv1.ValidatingWebhookConfiguration{}). + Owns(&templatev1.Template{}). + Owns(&appsv1.Deployment{}, builder.WithPredicates(resources.NewDeploymentPredicate())). + Owns(&componentsv1.ModelMeshServing{}). // watch ModelMeshServing + Owns(&componentsv1.Kserve{}). // watch Kserve + Watches(&extv1.CustomResourceDefinition{}). // call ForLabel() + new predicates + // Add ModelController specific actions + WithAction(initialize). + WithAction(devFlags). + WithAction(updatePodSecurityRolebinding). + WithAction(kustomize.NewAction( + kustomize.WithCache(render.DefaultCachingKeyFn), + kustomize.WithLabel(labels.ODH.Component(ComponentName), "true"), + kustomize.WithLabel(labels.K8SCommon.PartOf, ComponentName), + )). + WithAction(deploy.NewAction( + deploy.WithCache(), + deploy.WithFieldOwner(componentsv1.ModelControllerInstanceName), + deploy.WithLabel(labels.ComponentPartOf, componentsv1.ModelControllerInstanceName), + )). + WithAction(updatestatus.NewAction( + updatestatus.WithSelectorLabel(labels.ComponentPartOf, componentsv1.ModelControllerInstanceName), + )). + Build(ctx) // include GenerationChangedPredicate no need set in each Owns() above + + if err != nil { + return err // no need customize error, it is done in the caller main + } + + return nil +} diff --git a/controllers/components/modelmeshserving/modelmeshserving.go b/controllers/components/modelmeshserving/modelmeshserving.go new file mode 100644 index 00000000000..ca222263556 --- /dev/null +++ b/controllers/components/modelmeshserving/modelmeshserving.go @@ -0,0 +1,60 @@ +package modelmeshserving + +import ( + "fmt" + + operatorv1 "github.com/openshift/api/operator/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + componentsv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/components/v1" + dscv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/datasciencecluster/v1" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster" + odhdeploy "github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/metadata/annotations" +) + +const ( + ComponentName = componentsv1.ModelMeshServingComponentName +) + +// for DSC to get compoment ModelMeshServing's CR. +func GetComponentCR(dsc *dscv1.DataScienceCluster) *componentsv1.ModelMeshServing { + mmAnnotations := make(map[string]string) + switch dsc.Spec.Components.ModelMeshServing.ManagementState { + case operatorv1.Managed, operatorv1.Removed: + mmAnnotations[annotations.ManagementStateAnnotation] = string(dsc.Spec.Components.ModelMeshServing.ManagementState) + default: // Force and Unmanaged case for unknown values, we do not support these yet + mmAnnotations[annotations.ManagementStateAnnotation] = "Unknown" + } + + return &componentsv1.ModelMeshServing{ + TypeMeta: metav1.TypeMeta{ + Kind: componentsv1.ModelMeshServingKind, + APIVersion: componentsv1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: componentsv1.ModelMeshServingInstanceName, + Annotations: mmAnnotations, + }, + Spec: componentsv1.ModelMeshServingSpec{ + ModelMeshServingCommonSpec: dsc.Spec.Components.ModelMeshServing.ModelMeshServingCommonSpec, + }, + } +} + +// Init for set images. +func Init(platform cluster.Platform) error { + DefaultPath := odhdeploy.DefaultManifestPath + "/" + ComponentName + "/overlays/odh" + imageParamMap := map[string]string{ + "odh-mm-rest-proxy": "RELATED_IMAGE_ODH_MM_REST_PROXY_IMAGE", + "odh-modelmesh-runtime-adapter": "RELATED_IMAGE_ODH_MODELMESH_RUNTIME_ADAPTER_IMAGE", + "odh-modelmesh": "RELATED_IMAGE_ODH_MODELMESH_IMAGE", + "odh-modelmesh-controller": "RELATED_IMAGE_ODH_MODELMESH_CONTROLLER_IMAGE", + } + // Update image parameters + if err := odhdeploy.ApplyParams(DefaultPath, imageParamMap); err != nil { + return fmt.Errorf("failed to update images on path %s: %w", DefaultPath, err) + } + + return nil +} diff --git a/controllers/components/modelmeshserving/modelmeshserving_actions.go b/controllers/components/modelmeshserving/modelmeshserving_actions.go new file mode 100644 index 00000000000..f2ebdd11015 --- /dev/null +++ b/controllers/components/modelmeshserving/modelmeshserving_actions.go @@ -0,0 +1,82 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package modelmeshserving + +import ( + "context" + "fmt" + "strings" + + componentsv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/components/v1" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster" + odhtypes "github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/types" + odhdeploy "github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy" +) + +func initialize(ctx context.Context, rr *odhtypes.ReconciliationRequest) error { + // early exist + _, ok := rr.Instance.(*componentsv1.ModelMeshServing) + if !ok { + return fmt.Errorf("resource instance %v is not a componentsv1.ModelMeshServing)", rr.Instance) + } + rr.Manifests = []odhtypes.ManifestInfo{} + // setup Manifets[0] for modelmeshserving + rr.Manifests = append(rr.Manifests, odhtypes.ManifestInfo{ + Path: odhdeploy.DefaultManifestPath, + ContextDir: ComponentName, + SourcePath: "overlays/odh", + }) + return nil +} + +func devFlags(ctx context.Context, rr *odhtypes.ReconciliationRequest) error { + mm, ok := rr.Instance.(*componentsv1.ModelMeshServing) + if !ok { + return fmt.Errorf("resource instance %v is not a componentsv1.ModelMeshServing)", rr.Instance) + } + + if mm.Spec.DevFlags == nil { + return nil + } + // Implement devflags support logic + // If dev flags are set, update default manifests path + if len(mm.Spec.DevFlags.Manifests) != 0 { + for _, subcomponent := range mm.Spec.DevFlags.Manifests { + if strings.Contains(subcomponent.URI, ComponentName) { + // Download modelmeshserving + if err := odhdeploy.DownloadManifests(ctx, ComponentName, subcomponent); err != nil { + return err + } + // If overlay is defined, update paths + if subcomponent.SourcePath != "" { + rr.Manifests[0].SourcePath = subcomponent.SourcePath + } + } + } + } + // TODO: Implement devflags logmode logic + return nil +} + +func updatePodSecurityRolebinding(ctx context.Context, rr *odhtypes.ReconciliationRequest) error { + if err := cluster.UpdatePodSecurityRolebinding(ctx, rr.Client, rr.DSCI.Spec.ApplicationsNamespace, + "modelmesh", + "modelmesh-controller"); err != nil { + return fmt.Errorf("failed to update PodSecurityRolebinding for %s: %w", ComponentName, err) + } + return nil +} diff --git a/controllers/components/modelmeshserving/modelmeshserving_controller.go b/controllers/components/modelmeshserving/modelmeshserving_controller.go index 686d1856c10..cade0cddded 100644 --- a/controllers/components/modelmeshserving/modelmeshserving_controller.go +++ b/controllers/components/modelmeshserving/modelmeshserving_controller.go @@ -19,40 +19,67 @@ package modelmeshserving import ( "context" - "k8s.io/apimachinery/pkg/runtime" + promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + rbacv1 "k8s.io/api/rbac/v1" + extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/builder" componentsv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/components/v1" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/updatestatus" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/predicates/resources" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/metadata/labels" ) -// ModelMeshServingReconciler reconciles a ModelMeshServing object. -type ModelMeshServingReconciler struct { - client.Client - Scheme *runtime.Scheme -} +func NewComponentReconciler(ctx context.Context, mgr ctrl.Manager) error { + _, err := reconciler.ComponentReconcilerFor( + mgr, + componentsv1.ModelMeshServingInstanceName, + &componentsv1.ModelMeshServing{}, + ). + // customized Owns() for Component with new predicates + Owns(&corev1.ConfigMap{}). + Owns(&corev1.ServiceAccount{}). + Owns(&promv1.ServiceMonitor{}). + Owns(&networkingv1.NetworkPolicy{}). + Owns(&admissionregistrationv1.ValidatingWebhookConfiguration{}). + Owns(&corev1.Service{}). + Owns(&rbacv1.Role{}). + Owns(&rbacv1.ClusterRole{}). + Owns(&rbacv1.RoleBinding{}). + Owns(&rbacv1.ClusterRoleBinding{}). + Owns(&appsv1.Deployment{}, builder.WithPredicates(resources.NewDeploymentPredicate())). + Watches(&extv1.CustomResourceDefinition{}). // call ForLabel() + new predicates + // Add ModelMeshServing specific actions + WithAction(initialize). + WithAction(devFlags). + WithAction(updatePodSecurityRolebinding). + WithAction(kustomize.NewAction( + kustomize.WithCache(render.DefaultCachingKeyFn), + kustomize.WithLabel(labels.ODH.Component(ComponentName), "true"), + kustomize.WithLabel(labels.K8SCommon.PartOf, ComponentName), + )). + WithAction(deploy.NewAction( + deploy.WithCache(), + deploy.WithFieldOwner(componentsv1.ModelMeshServingInstanceName), + deploy.WithLabel(labels.ComponentPartOf, componentsv1.ModelMeshServingInstanceName), + )). + WithAction(updatestatus.NewAction( + updatestatus.WithSelectorLabel(labels.ComponentPartOf, componentsv1.ModelMeshServingInstanceName), + )). + Build(ctx) // include GenerationChangedPredicate no need set in each Owns() above -// Reconcile is part of the main kubernetes reconciliation loop which aims to -// move the current state of the cluster closer to the desired state. -// TODO(user): Modify the Reconcile function to compare the state specified by -// the ModelMeshServing object against the actual cluster state, and then -// perform operations to make the cluster state reflect the state specified by -// the user. -// -// For more details, check Reconcile and its Result here: -// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.12.2/pkg/reconcile -func (r *ModelMeshServingReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - _ = log.FromContext(ctx) - - // TODO(user): your logic here - - return ctrl.Result{}, nil -} + if err != nil { + return err // no need customize error, it is done in the caller main + } -// SetupWithManager sets up the controller with the Manager. -func (r *ModelMeshServingReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&componentsv1.ModelMeshServing{}). - Complete(r) + return nil } diff --git a/controllers/datasciencecluster/datasciencecluster_controller.go b/controllers/datasciencecluster/datasciencecluster_controller.go index 494a69c8fda..045ba3cc1e1 100644 --- a/controllers/datasciencecluster/datasciencecluster_controller.go +++ b/controllers/datasciencecluster/datasciencecluster_controller.go @@ -21,25 +21,19 @@ import ( "context" "errors" "fmt" - "strings" "time" "github.com/go-logr/logr" "github.com/hashicorp/go-multierror" - buildv1 "github.com/openshift/api/build/v1" - imagev1 "github.com/openshift/api/image/v1" operatorv1 "github.com/openshift/api/operator/v1" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - networkingv1 "k8s.io/api/networking/v1" - rbacv1 "k8s.io/api/rbac/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" k8serr "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" - apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" @@ -55,6 +49,8 @@ import ( dashboardctrl "github.com/opendatahub-io/opendatahub-operator/v2/controllers/components/dashboard" datasciencepipelinesctrl "github.com/opendatahub-io/opendatahub-operator/v2/controllers/components/datasciencepipelines" kueuectrl "github.com/opendatahub-io/opendatahub-operator/v2/controllers/components/kueue" + mcctrl "github.com/opendatahub-io/opendatahub-operator/v2/controllers/components/modelcontroller" + mmctrl "github.com/opendatahub-io/opendatahub-operator/v2/controllers/components/modelmeshserving" modelregistryctrl "github.com/opendatahub-io/opendatahub-operator/v2/controllers/components/modelregistry" rayctrl "github.com/opendatahub-io/opendatahub-operator/v2/controllers/components/ray" trainingoperatorctrl "github.com/opendatahub-io/opendatahub-operator/v2/controllers/components/trainingoperator" @@ -289,6 +285,24 @@ func (r *DataScienceClusterReconciler) Reconcile(ctx context.Context, req ctrl.R componentErrors = multierror.Append(componentErrors, err) } + // Deploy ModelMeshServing + if instance, err = r.ReconcileComponent(ctx, instance, componentsv1.ModelMeshServingComponentName, func() (error, bool) { + mm := mmctrl.GetComponentCR(instance) + return r.apply(ctx, instance, mm), instance.Spec.Components.ModelMeshServing.ManagementState == operatorv1.Managed + }); err != nil { + componentErrors = multierror.Append(componentErrors, err) + } + + // Deploy ModelController + if instance, err = r.ReconcileComponent(ctx, instance, componentsv1.ModelControllerComponentName, func() (error, bool) { + mc := mcctrl.GetComponentCR(instance) + err := r.apply(ctx, instance, mc) + isManaged := instance.Spec.Components.Kserve.ManagementState == operatorv1.Managed || instance.Spec.Components.ModelMeshServing.ManagementState == operatorv1.Managed + return err, isManaged + }); err != nil { + componentErrors = multierror.Append(componentErrors, err) + } + // Process errors for components if componentErrors != nil { log.Info("DataScienceCluster Deployment Incomplete.") @@ -401,7 +415,7 @@ var configMapPredicates = predicate.Funcs{ } // Do not reconcile on kserver's inferenceservice-config CM updates, for rawdeployment namespace := e.ObjectNew.GetNamespace() - if e.ObjectNew.GetName() == "inferenceservice-config" && (namespace == "redhat-ods-applications" || namespace == "opendatahub") { //nolint:goconst + if e.ObjectNew.GetName() == "inferenceservice-config" && (namespace == "redhat-ods-applications" || namespace == "opendatahub") { return false } return true @@ -449,108 +463,27 @@ var componentDeploymentPredicates = predicate.Funcs{ }, } -// a workaround for 2.5 due to odh-model-controller serivceaccount keeps updates with label. -var saPredicates = predicate.Funcs{ - UpdateFunc: func(e event.UpdateEvent) bool { - namespace := e.ObjectNew.GetNamespace() - if e.ObjectNew.GetName() == "odh-model-controller" && (namespace == "redhat-ods-applications" || namespace == "opendatahub") { - return false - } - return true - }, -} - -// a workaround for 2.5 due to modelmesh-servingruntime.serving.kserve.io keeps updates. -var modelMeshwebhookPredicates = predicate.Funcs{ - UpdateFunc: func(e event.UpdateEvent) bool { - return e.ObjectNew.GetName() != "modelmesh-servingruntime.serving.kserve.io" - }, -} - -var modelMeshRolePredicates = predicate.Funcs{ - UpdateFunc: func(e event.UpdateEvent) bool { - notAllowedNames := []string{"leader-election-role", "proxy-role", "metrics-reader", "kserve-prometheus-k8s", "odh-model-controller-role"} - for _, notallowedName := range notAllowedNames { - if e.ObjectNew.GetName() == notallowedName { - return false - } - } - return true - }, -} - -// a workaround for modelmesh and kserve both create same odh-model-controller NWP. -var networkpolicyPredicates = predicate.Funcs{ - UpdateFunc: func(e event.UpdateEvent) bool { - return e.ObjectNew.GetName() != "odh-model-controller" - }, -} - -var modelMeshRBPredicates = predicate.Funcs{ - UpdateFunc: func(e event.UpdateEvent) bool { - notAllowedNames := []string{"leader-election-rolebinding", "proxy-rolebinding", "odh-model-controller-rolebinding-opendatahub"} - for _, notallowedName := range notAllowedNames { - if e.ObjectNew.GetName() == notallowedName { - return false - } - } - return true - }, -} - -// ignore label updates if it is from application namespace. -var modelMeshGeneralPredicates = predicate.Funcs{ - UpdateFunc: func(e event.UpdateEvent) bool { - if strings.Contains(e.ObjectNew.GetName(), "odh-model-controller") || strings.Contains(e.ObjectNew.GetName(), "kserve") { - return false - } - return true - }, -} - // SetupWithManager sets up the controller with the Manager. func (r *DataScienceClusterReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&dscv1.DataScienceCluster{}). - Owns(&corev1.Namespace{}). - Owns(&corev1.Secret{}). + // Owns(&corev1.Namespace{}). + // Owns(&corev1.Secret{}). Owns( &corev1.ConfigMap{}, builder.WithPredicates(configMapPredicates), ). - Owns( - &networkingv1.NetworkPolicy{}, - builder.WithPredicates(networkpolicyPredicates), - ). - Owns( - &rbacv1.Role{}, - builder.WithPredicates(predicate.Or(predicate.GenerationChangedPredicate{}, modelMeshRolePredicates))). - Owns( - &rbacv1.RoleBinding{}, - builder.WithPredicates(predicate.Or(predicate.GenerationChangedPredicate{}, modelMeshRBPredicates))). - Owns( - &rbacv1.ClusterRole{}, - builder.WithPredicates(predicate.Or(predicate.GenerationChangedPredicate{}, modelMeshRolePredicates))). - Owns( - &rbacv1.ClusterRoleBinding{}, - builder.WithPredicates(predicate.Or(predicate.GenerationChangedPredicate{}, modelMeshRBPredicates))). Owns( &appsv1.Deployment{}, builder.WithPredicates(componentDeploymentPredicates)). - Owns(&corev1.PersistentVolumeClaim{}). - Owns( - &corev1.Service{}, - builder.WithPredicates(predicate.Or(predicate.GenerationChangedPredicate{}, modelMeshGeneralPredicates))). - Owns(&appsv1.StatefulSet{}). - Owns(&imagev1.ImageStream{}). - Owns(&buildv1.BuildConfig{}). - Owns(&apiregistrationv1.APIService{}). - Owns(&networkingv1.Ingress{}). + // Owns(&corev1.PersistentVolumeClaim{}). + // Owns(&appsv1.StatefulSet{}). + // Owns(&imagev1.ImageStream{}). + // Owns(&buildv1.BuildConfig{}). + // Owns(&apiregistrationv1.APIService{}). + // Owns(&networkingv1.Ingress{}). Owns(&admissionregistrationv1.MutatingWebhookConfiguration{}). - Owns( - &admissionregistrationv1.ValidatingWebhookConfiguration{}, - builder.WithPredicates(modelMeshwebhookPredicates), - ). + Owns(&admissionregistrationv1.ValidatingWebhookConfiguration{}). // components CRs Owns(&componentsv1.Dashboard{}). Owns(&componentsv1.Ray{}). @@ -559,10 +492,8 @@ func (r *DataScienceClusterReconciler) SetupWithManager(ctx context.Context, mgr Owns(&componentsv1.Kueue{}). Owns(&componentsv1.TrainingOperator{}). Owns(&componentsv1.DataSciencePipelines{}). - Owns( - &corev1.ServiceAccount{}, - builder.WithPredicates(saPredicates), - ). + Owns(&componentsv1.ModelMeshServing{}). + Owns(&componentsv1.ModelController{}). Watches( &dsciv1.DSCInitialization{}, handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, a client.Object) []reconcile.Request { diff --git a/controllers/datasciencecluster/kubebuilder_rbac.go b/controllers/datasciencecluster/kubebuilder_rbac.go index e9e29ffbd3a..80eda748e98 100644 --- a/controllers/datasciencecluster/kubebuilder_rbac.go +++ b/controllers/datasciencecluster/kubebuilder_rbac.go @@ -217,7 +217,7 @@ package datasciencecluster // +kubebuilder:rbac:groups=components.opendatahub.io,resources=trainingoperators/status,verbs=get;update;patch // +kubebuilder:rbac:groups=components.opendatahub.io,resources=trainingoperators/finalizers,verbs=update -// TODO: ModelMesh +// ModelMeshServing // +kubebuilder:rbac:groups=components.opendatahub.io,resources=modelmeshservings,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=components.opendatahub.io,resources=modelmeshservings/status,verbs=get;update;patch // +kubebuilder:rbac:groups=components.opendatahub.io,resources=modelmeshservings/finalizers,verbs=update @@ -226,3 +226,8 @@ package datasciencecluster // +kubebuilder:rbac:groups=components.opendatahub.io,resources=trustyais,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=components.opendatahub.io,resources=trustyais/status,verbs=get;update;patch // +kubebuilder:rbac:groups=components.opendatahub.io,resources=trustyais/finalizers,verbs=update + +// ModelController +// +kubebuilder:rbac:groups=components.opendatahub.io,resources=modelcontrollers,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=components.opendatahub.io,resources=modelcontrollers/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=components.opendatahub.io,resources=modelcontrollers/finalizers,verbs=update diff --git a/controllers/dscinitialization/suite_test.go b/controllers/dscinitialization/suite_test.go index 618b9a62e91..4f86de593ea 100644 --- a/controllers/dscinitialization/suite_test.go +++ b/controllers/dscinitialization/suite_test.go @@ -23,6 +23,7 @@ import ( "time" routev1 "github.com/openshift/api/route/v1" + templatev1 "github.com/openshift/api/template/v1" userv1 "github.com/openshift/api/user/v1" ofapi "github.com/operator-framework/api/pkg/operators/v1alpha1" ofapiv2 "github.com/operator-framework/api/pkg/operators/v2" @@ -117,6 +118,7 @@ var _ = BeforeSuite(func() { utilruntime.Must(routev1.Install(testScheme)) utilruntime.Must(userv1.Install(testScheme)) utilruntime.Must(monitoringv1.AddToScheme(testScheme)) + utilruntime.Must(templatev1.Install(testScheme)) // +kubebuilder:scaffold:scheme k8sClient, err = client.New(cfg, client.Options{Scheme: testScheme}) diff --git a/controllers/webhook/webhook_suite_test.go b/controllers/webhook/webhook_suite_test.go index e6a282c4fc1..dffb7996588 100644 --- a/controllers/webhook/webhook_suite_test.go +++ b/controllers/webhook/webhook_suite_test.go @@ -47,7 +47,6 @@ import ( componentsold "github.com/opendatahub-io/opendatahub-operator/v2/components" "github.com/opendatahub-io/opendatahub-operator/v2/components/codeflare" "github.com/opendatahub-io/opendatahub-operator/v2/components/kserve" - "github.com/opendatahub-io/opendatahub-operator/v2/components/modelmeshserving" "github.com/opendatahub-io/opendatahub-operator/v2/components/workbenches" modelregistry2 "github.com/opendatahub-io/opendatahub-operator/v2/controllers/components/modelregistry" "github.com/opendatahub-io/opendatahub-operator/v2/controllers/webhook" @@ -269,8 +268,8 @@ func newDSC(name string, namespace string) *dscv1.DataScienceCluster { ManagementState: operatorv1.Removed, }, }, - ModelMeshServing: modelmeshserving.ModelMeshServing{ - Component: componentsold.Component{ + ModelMeshServing: componentsv1.DSCModelMeshServing{ + ManagementSpec: components.ManagementSpec{ ManagementState: operatorv1.Removed, }, }, diff --git a/docs/api-overview.md b/docs/api-overview.md index f1078eff10d..0e9d1386aed 100644 --- a/docs/api-overview.md +++ b/docs/api-overview.md @@ -21,6 +21,8 @@ Package v1 contains API Schema definitions for the components v1 API group - [KserveList](#kservelist) - [Kueue](#kueue) - [KueueList](#kueuelist) +- [ModelController](#modelcontroller) +- [ModelControllerList](#modelcontrollerlist) - [ModelMeshServing](#modelmeshserving) - [ModelMeshServingList](#modelmeshservinglist) - [ModelRegistry](#modelregistry) @@ -154,6 +156,23 @@ DSCKueue contains all the configuration exposed in DSC instance for Kueue compon +_Appears in:_ +- [Components](#components) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `managementState` _[ManagementState](#managementstate)_ | Set to one of the following values:

- "Managed" : the operator is actively managing the component and trying to keep it active.
It will only upgrade the component if it is safe to do so

- "Removed" : the operator is actively managing the component and will not install it,
or if it is installed, the operator will try to remove it | | Enum: [Managed Removed]
| +| `devFlags` _[DevFlags](#devflags)_ | Add developer fields | | | + + +#### DSCModelMeshServing + + + +DSCModelMeshServing contains all the configuration exposed in DSC instance for ModelMeshServing component + + + _Appears in:_ - [Components](#components) @@ -605,6 +624,84 @@ _Appears in:_ | `observedGeneration` _integer_ | | | | +#### ModelController + + + +ModelController is the Schema for the modelcontroller API, it is a shared component between kserve and modelmeshserving + + + +_Appears in:_ +- [ModelControllerList](#modelcontrollerlist) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `components.opendatahub.io/v1` | | | +| `kind` _string_ | `ModelController` | | | +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[ModelControllerSpec](#modelcontrollerspec)_ | | | | +| `status` _[ModelControllerStatus](#modelcontrollerstatus)_ | | | | + + +#### ModelControllerList + + + +ModelControllerList contains a list of ModelController + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `components.opendatahub.io/v1` | | | +| `kind` _string_ | `ModelControllerList` | | | +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#listmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `items` _[ModelController](#modelcontroller) array_ | | | | + + +#### ModelControllerSpec + + + +ModelControllerSpec defines the desired state of ModelController + + + +_Appears in:_ +- [ModelController](#modelcontroller) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `devFlags` _[DevFlags](#devflags)_ | Add developer fields | | | +| `modelMeshServing` _[ManagementState](#managementstate)_ | | | | +| `kserve` _[ManagementState](#managementstate)_ | | | | + + +#### ModelControllerStatus + + + +ModelControllerStatus defines the observed state of ModelController + + + +_Appears in:_ +- [ModelController](#modelcontroller) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `phase` _string_ | | | | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#condition-v1-meta) array_ | | | | +| `observedGeneration` _integer_ | | | | + + #### ModelMeshServing @@ -627,6 +724,23 @@ _Appears in:_ | `status` _[ModelMeshServingStatus](#modelmeshservingstatus)_ | | | | +#### ModelMeshServingCommonSpec + + + + + + + +_Appears in:_ +- [DSCModelMeshServing](#dscmodelmeshserving) +- [ModelMeshServingSpec](#modelmeshservingspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `devFlags` _[DevFlags](#devflags)_ | Add developer fields | | | + + #### ModelMeshServingList @@ -660,7 +774,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `foo` _string_ | Foo is an example field of ModelMeshServing. Edit modelmeshserving_types.go to remove/update | | | +| `devFlags` _[DevFlags](#devflags)_ | Add developer fields | | | #### ModelMeshServingStatus @@ -1173,7 +1287,6 @@ Component struct defines the basis for each OpenDataHub component configuration. _Appears in:_ - [CodeFlare](#codeflare) - [Kserve](#kserve) -- [ModelMeshServing](#modelmeshserving) - [Workbenches](#workbenches) | Field | Description | Default | Validation | @@ -1216,6 +1329,7 @@ _Appears in:_ - [DSCDashboard](#dscdashboard) - [DSCDataSciencePipelines](#dscdatasciencepipelines) - [DSCKueue](#dsckueue) +- [DSCModelMeshServing](#dscmodelmeshserving) - [DSCModelRegistry](#dscmodelregistry) - [DSCRay](#dscray) - [DSCTrainingOperator](#dsctrainingoperator) @@ -1226,6 +1340,9 @@ _Appears in:_ - [DataSciencePipelinesSpec](#datasciencepipelinesspec) - [KueueCommonSpec](#kueuecommonspec) - [KueueSpec](#kueuespec) +- [ModelControllerSpec](#modelcontrollerspec) +- [ModelMeshServingCommonSpec](#modelmeshservingcommonspec) +- [ModelMeshServingSpec](#modelmeshservingspec) - [ModelRegistryCommonSpec](#modelregistrycommonspec) - [ModelRegistrySpec](#modelregistryspec) - [RayCommonSpec](#raycommonspec) @@ -1253,6 +1370,7 @@ _Appears in:_ - [DSCDashboard](#dscdashboard) - [DSCDataSciencePipelines](#dscdatasciencepipelines) - [DSCKueue](#dsckueue) +- [DSCModelMeshServing](#dscmodelmeshserving) - [DSCModelRegistry](#dscmodelregistry) - [DSCRay](#dscray) - [DSCTrainingOperator](#dsctrainingoperator) @@ -1279,6 +1397,7 @@ _Appears in:_ - [DataSciencePipelinesStatus](#datasciencepipelinesstatus) - [KserveStatus](#kservestatus) - [KueueStatus](#kueuestatus) +- [ModelControllerStatus](#modelcontrollerstatus) - [ModelMeshServingStatus](#modelmeshservingstatus) - [ModelRegistryStatus](#modelregistrystatus) - [RayStatus](#raystatus) @@ -1337,29 +1456,6 @@ _Appears in:_ -## datasciencecluster.opendatahub.io/modelmeshserving - -Package modelmeshserving provides utility functions to config MoModelMesh, a general-purpose model serving management/routing layer - - - -#### ModelMeshServing - - - -ModelMeshServing struct holds the configuration for the ModelMeshServing component. - - - -_Appears in:_ -- [Components](#components) - -| Field | Description | Default | Validation | -| --- | --- | --- | --- | -| `Component` _[Component](#component)_ | | | | - - - ## datasciencecluster.opendatahub.io/v1 @@ -1436,9 +1532,9 @@ _Appears in:_ | --- | --- | --- | --- | | `dashboard` _[DSCDashboard](#dscdashboard)_ | Dashboard component configuration. | | | | `workbenches` _[Workbenches](#workbenches)_ | Workbenches component configuration. | | | -| `modelmeshserving` _[ModelMeshServing](#modelmeshserving)_ | ModelMeshServing component configuration.
Does not support enabled Kserve at the same time | | | +| `modelmeshserving` _[DSCModelMeshServing](#dscmodelmeshserving)_ | ModelMeshServing component configuration. | | | | `datasciencepipelines` _[DSCDataSciencePipelines](#dscdatasciencepipelines)_ | DataServicePipeline component configuration.
Require OpenShift Pipelines Operator to be installed before enable component | | | -| `kserve` _[Kserve](#kserve)_ | Kserve component configuration.
Require OpenShift Serverless and OpenShift Service Mesh Operators to be installed before enable component
Does not support enabled ModelMeshServing at the same time | | | +| `kserve` _[Kserve](#kserve)_ | Kserve component configuration.
Require OpenShift Serverless and OpenShift Service Mesh Operators to be installed before enable component | | | | `kueue` _[DSCKueue](#dsckueue)_ | Kueue component configuration. | | | | `codeflare` _[CodeFlare](#codeflare)_ | CodeFlare component configuration.
If CodeFlare Operator has been installed in the cluster, it should be uninstalled first before enabled component. | | | | `ray` _[DSCRay](#dscray)_ | Ray component configuration. | | | diff --git a/main.go b/main.go index 16e61f416c2..ac0e6b3c88f 100644 --- a/main.go +++ b/main.go @@ -31,6 +31,7 @@ import ( operatorv1 "github.com/openshift/api/operator/v1" routev1 "github.com/openshift/api/route/v1" securityv1 "github.com/openshift/api/security/v1" + templatev1 "github.com/openshift/api/template/v1" userv1 "github.com/openshift/api/user/v1" ofapiv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1" ofapiv2 "github.com/operator-framework/api/pkg/operators/v2" @@ -66,6 +67,8 @@ import ( dashboardctrl "github.com/opendatahub-io/opendatahub-operator/v2/controllers/components/dashboard" datasciencepipelinesctrl "github.com/opendatahub-io/opendatahub-operator/v2/controllers/components/datasciencepipelines" kueuectrl "github.com/opendatahub-io/opendatahub-operator/v2/controllers/components/kueue" + mcctrl "github.com/opendatahub-io/opendatahub-operator/v2/controllers/components/modelcontroller" + mmctrl "github.com/opendatahub-io/opendatahub-operator/v2/controllers/components/modelmeshserving" modelregistryctrl "github.com/opendatahub-io/opendatahub-operator/v2/controllers/components/modelregistry" rayctrl "github.com/opendatahub-io/opendatahub-operator/v2/controllers/components/ray" trainingoperatorctrl "github.com/opendatahub-io/opendatahub-operator/v2/controllers/components/trainingoperator" @@ -116,6 +119,7 @@ func init() { //nolint:gochecknoinits utilruntime.Must(operatorv1.Install(scheme)) utilruntime.Must(consolev1.AddToScheme(scheme)) utilruntime.Must(securityv1.Install(scheme)) + utilruntime.Must(templatev1.Install(scheme)) } func initComponents(_ context.Context, p cluster.Platform) error { @@ -139,10 +143,15 @@ func initComponents(_ context.Context, p cluster.Platform) error { if err := datasciencepipelinesctrl.Init(p); err != nil { multiErr = multierror.Append(multiErr, err) } - if err := kueuectrl.Init(p); err != nil { multiErr = multierror.Append(multiErr, err) } + if err := mmctrl.Init(p); err != nil { + return err + } + if err := mmctrl.Init(p); err != nil { + return err + } return multiErr.ErrorOrNil() } @@ -448,6 +457,14 @@ func CreateComponentReconcilers(ctx context.Context, mgr manager.Manager) error } if err := modelregistryctrl.NewComponentReconciler(ctx, mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "ModelRegistryReconciler") + } + + if err := mmctrl.NewComponentReconciler(ctx, mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "ModelMeshServingReconciler") + return err + } + if err := mcctrl.NewComponentReconciler(ctx, mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "ModelControllerReconciler") return err } if err := trustyaictrl.NewComponentReconciler(ctx, mgr); err != nil { diff --git a/pkg/upgrade/upgrade.go b/pkg/upgrade/upgrade.go index 84d1e214c63..630288d4040 100644 --- a/pkg/upgrade/upgrade.go +++ b/pkg/upgrade/upgrade.go @@ -34,7 +34,6 @@ import ( componentsold "github.com/opendatahub-io/opendatahub-operator/v2/components" "github.com/opendatahub-io/opendatahub-operator/v2/components/codeflare" "github.com/opendatahub-io/opendatahub-operator/v2/components/kserve" - "github.com/opendatahub-io/opendatahub-operator/v2/components/modelmeshserving" "github.com/opendatahub-io/opendatahub-operator/v2/components/workbenches" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster/gvk" @@ -69,8 +68,8 @@ func CreateDefaultDSC(ctx context.Context, cli client.Client) error { Workbenches: workbenches.Workbenches{ Component: componentsold.Component{ManagementState: operatorv1.Managed}, }, - ModelMeshServing: modelmeshserving.ModelMeshServing{ - Component: componentsold.Component{ManagementState: operatorv1.Managed}, + ModelMeshServing: componentsv1.DSCModelMeshServing{ + ManagementSpec: components.ManagementSpec{ManagementState: operatorv1.Managed}, }, DataSciencePipelines: componentsv1.DSCDataSciencePipelines{ ManagementSpec: components.ManagementSpec{ManagementState: operatorv1.Managed}, diff --git a/tests/e2e/controller_test.go b/tests/e2e/controller_test.go index e526676ebad..37365a63604 100644 --- a/tests/e2e/controller_test.go +++ b/tests/e2e/controller_test.go @@ -46,6 +46,7 @@ var ( "kueue": kueueTestSuite, "trainingoperator": trainingoperatorTestSuite, "datasciencepipelienes": dataSciencePipelinesTestSuite, + "modelmesh": modelMeshServingTestSuite, } ) @@ -165,6 +166,9 @@ func TestOdhOperator(t *testing.T) { t.Run("validate installation of "+k+" component", v) } + // Run test on modelcontroller + t.Run("validate installation of modelcontroller component", modelControllerTestSuite) + // Run deletion if skipDeletion is not set if !testOpts.skipDeletion { // this is a negative test case, since by using the positive CM('true'), even CSV gets deleted which leaves no operator pod in prow diff --git a/tests/e2e/helper_test.go b/tests/e2e/helper_test.go index 6d4337f7eda..1281ad8e6fa 100644 --- a/tests/e2e/helper_test.go +++ b/tests/e2e/helper_test.go @@ -29,7 +29,6 @@ import ( componentsold "github.com/opendatahub-io/opendatahub-operator/v2/components" "github.com/opendatahub-io/opendatahub-operator/v2/components/codeflare" "github.com/opendatahub-io/opendatahub-operator/v2/components/kserve" - "github.com/opendatahub-io/opendatahub-operator/v2/components/modelmeshserving" "github.com/opendatahub-io/opendatahub-operator/v2/components/workbenches" "github.com/opendatahub-io/opendatahub-operator/v2/controllers/components/modelregistry" ) @@ -125,9 +124,9 @@ func setupDSCInstance(name string) *dscv1.DataScienceCluster { ManagementState: operatorv1.Removed, }, }, - ModelMeshServing: modelmeshserving.ModelMeshServing{ - Component: componentsold.Component{ - ManagementState: operatorv1.Removed, + ModelMeshServing: componentsv1.DSCModelMeshServing{ + ManagementSpec: components.ManagementSpec{ + ManagementState: operatorv1.Managed, }, }, DataSciencePipelines: componentsv1.DSCDataSciencePipelines{ diff --git a/tests/e2e/modelcontroller_test.go b/tests/e2e/modelcontroller_test.go new file mode 100644 index 00000000000..632dc67b62e --- /dev/null +++ b/tests/e2e/modelcontroller_test.go @@ -0,0 +1,263 @@ +package e2e_test + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + operatorv1 "github.com/openshift/api/operator/v1" + "github.com/stretchr/testify/require" + autoscalingv1 "k8s.io/api/autoscaling/v1" + k8serr "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + + componentsv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/components/v1" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/metadata/labels" +) + +type ModelControllerTestCtx struct { + testCtx *testContext + testModelControllerInstance componentsv1.ModelController +} + +func modelControllerTestSuite(t *testing.T) { + t.Helper() + + mcCtx := ModelControllerTestCtx{} + var err error + mcCtx.testCtx, err = NewTestContext() + require.NoError(t, err) + + testCtx := mcCtx.testCtx + + t.Run(testCtx.testDsc.Name, func(t *testing.T) { + // creation + t.Run("Creation of ModelController CR", func(t *testing.T) { + err = mcCtx.testModelControllerCreation() + require.NoError(t, err, "error creating ModelController CR") + }) + + t.Run("Validate Ownerrefrences exist", func(t *testing.T) { + err = mcCtx.testOwnerReferences() + require.NoError(t, err, "error getting all ModelController's Ownerrefrences") + }) + + t.Run("Validate ModelController Ready", func(t *testing.T) { + err = mcCtx.validateModelControllerReady() + require.NoError(t, err, "ModelController instance is not Ready") + }) + + // reconcile + t.Run("Validate Controller reconcile", func(t *testing.T) { + err = mcCtx.testUpdateOnModelControllerResources() + require.NoError(t, err, "error testing updates for ModelController's managed resources") + }) + + t.Run("Validate Disabling ModelMeshServing Component if ModelController is removed", func(t *testing.T) { + err = mcCtx.testUpdateModelControllerComponentDisabled() + require.NoError(t, err, "error testing modemeshserving component enabled field") + }) + }) +} + +func (tc *ModelControllerTestCtx) testModelControllerCreation() error { + // force to set modelmesh to managed, // TODO: kserve maybe later + tc.testCtx.testDsc.Spec.Components.ModelMeshServing.ManagementState = operatorv1.Managed + + err := tc.testCtx.wait(func(ctx context.Context) (bool, error) { + existingModelControllerList := &componentsv1.ModelControllerList{} + + if err := tc.testCtx.customClient.List(ctx, existingModelControllerList); err != nil { + return false, err + } + + switch { + case len(existingModelControllerList.Items) == 1: + tc.testModelControllerInstance = existingModelControllerList.Items[0] + return true, nil + case len(existingModelControllerList.Items) > 1: + return false, fmt.Errorf( + "unexpected ModelController CR instances. Expected 1 , Found %v instance", len(existingModelControllerList.Items)) + default: + return false, nil + } + }) + + if err != nil { + return fmt.Errorf("unable to find ModelController CR instance: %w", err) + } + + return nil +} + +func (tc *ModelControllerTestCtx) testOwnerReferences() error { + if len(tc.testModelControllerInstance.OwnerReferences) != 1 { + return errors.New("expect CR has ownerreferences set") + } + + // Test ModelController CR ownerref + if tc.testModelControllerInstance.OwnerReferences[0].Kind != dscKind { + return fmt.Errorf("expected ownerreference DataScienceCluster not found. Got ownereferrence: %v", + tc.testModelControllerInstance.OwnerReferences[0].Kind) + } + + // Test ModelController resources + appDeployments, err := tc.testCtx.kubeClient.AppsV1().Deployments(tc.testCtx.applicationsNamespace).List(tc.testCtx.ctx, metav1.ListOptions{ + LabelSelector: labels.ODH.Component(componentsv1.ModelControllerComponentName), + }) + if err != nil { + return fmt.Errorf("error listing component deployments %w", err) + } + // test any one deployment for ownerreference + if len(appDeployments.Items) != 0 && appDeployments.Items[0].OwnerReferences[0].Kind != componentsv1.ModelControllerKind { + return fmt.Errorf("expected ownerreference not found. Got ownereferrence: %v", + appDeployments.Items[0].OwnerReferences) + } + + return nil +} + +// Verify ModelController instance is in Ready phase when modelmeshserving (or kserve) deployments are up and running. +func (tc *ModelControllerTestCtx) validateModelControllerReady() error { + err := wait.PollUntilContextTimeout(tc.testCtx.ctx, generalRetryInterval, componentReadyTimeout, true, func(ctx context.Context) (bool, error) { + key := types.NamespacedName{Name: tc.testModelControllerInstance.Name} + mc := &componentsv1.ModelController{} + + err := tc.testCtx.customClient.Get(ctx, key, mc) + if err != nil { + return false, err + } + return mc.Status.Phase == readyStatus, nil + }) + + if err != nil { + return fmt.Errorf("error waiting Ready state for ModelController %v: %w", tc.testModelControllerInstance.Name, err) + } + + return nil +} + +func (tc *ModelControllerTestCtx) testUpdateOnModelControllerResources() error { + // Test Updating ModelController Replicas + + appDeployments, err := tc.testCtx.kubeClient.AppsV1().Deployments(tc.testCtx.applicationsNamespace).List(tc.testCtx.ctx, metav1.ListOptions{ + LabelSelector: labels.ComponentPartOf + "=" + tc.testModelControllerInstance.Name, + }) + if err != nil { + return err + } + + if len(appDeployments.Items) != 1 { + return fmt.Errorf("error getting deployment for component %s", tc.testModelControllerInstance.Name) + } + + const expectedReplica int32 = 2 // from 1 to 2 + + testDeployment := appDeployments.Items[0] + patchedReplica := &autoscalingv1.Scale{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDeployment.Name, + Namespace: testDeployment.Namespace, + }, + Spec: autoscalingv1.ScaleSpec{ + Replicas: expectedReplica, + }, + Status: autoscalingv1.ScaleStatus{}, + } + updatedDep, err := tc.testCtx.kubeClient.AppsV1().Deployments(tc.testCtx.applicationsNamespace).UpdateScale(tc.testCtx.ctx, + testDeployment.Name, patchedReplica, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("error patching component resources : %w", err) + } + if updatedDep.Spec.Replicas != patchedReplica.Spec.Replicas { + return fmt.Errorf("failed to patch replicas : expect to be %v but got %v", patchedReplica.Spec.Replicas, updatedDep.Spec.Replicas) + } + + // Sleep for 20 seconds to allow the operator to reconcile + // we expect it should not revert back to original value because of AllowList + time.Sleep(2 * generalRetryInterval) + reconciledDep, err := tc.testCtx.kubeClient.AppsV1().Deployments(tc.testCtx.applicationsNamespace).Get(tc.testCtx.ctx, testDeployment.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("error getting component resource after reconcile: %w", err) + } + if *reconciledDep.Spec.Replicas != expectedReplica { + return fmt.Errorf("failed to revert back replicas : expect to be %v but got %v", expectedReplica, *reconciledDep.Spec.Replicas) + } + + return nil +} + +func (tc *ModelControllerTestCtx) testUpdateModelControllerComponentDisabled() error { + // Test Updating ModelMeshServing to be disabled then ModelController should be removed + var mcDeploymentName string + + if tc.testCtx.testDsc.Spec.Components.ModelMeshServing.ManagementState == operatorv1.Managed { // TODO: need to update this when kserve is added + appDeployments, err := tc.testCtx.kubeClient.AppsV1().Deployments(tc.testCtx.applicationsNamespace).List(tc.testCtx.ctx, metav1.ListOptions{ + LabelSelector: labels.ODH.Component(componentsv1.ModelControllerComponentName), + }) + if err != nil { + return fmt.Errorf("error getting enabled component %v", componentsv1.ModelControllerComponentName) + } + if len(appDeployments.Items) > 0 { + mcDeploymentName = appDeployments.Items[0].Name + if appDeployments.Items[0].Status.ReadyReplicas == 0 { + return fmt.Errorf("error getting enabled component: %s its deployment 'ReadyReplicas'", mcDeploymentName) + } + } + } else { + return errors.New("modelmeshserving spec should be in 'enabled: true' state in order to perform modelcontroller test") + } + + // Disable component modelmeshserving + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + // refresh DSC instance in case it was updated during the reconcile + err := tc.testCtx.customClient.Get(tc.testCtx.ctx, types.NamespacedName{Name: tc.testCtx.testDsc.Name}, tc.testCtx.testDsc) + if err != nil { + return fmt.Errorf("error getting resource %w", err) + } + // Disable the Component + tc.testCtx.testDsc.Spec.Components.ModelMeshServing.ManagementState = operatorv1.Removed + + // Try to update + err = tc.testCtx.customClient.Update(tc.testCtx.ctx, tc.testCtx.testDsc) + // Return err itself here (not wrapped inside another error) + // so that RetryOnConflict can identify it correctly. + if err != nil { + return fmt.Errorf("error updating component from 'enabled: true' to 'enabled: false': %w", err) + } + + return nil + }) + if err != nil { + return fmt.Errorf("error after retry %w", err) + } + + if err = tc.testCtx.wait(func(ctx context.Context) (bool, error) { + // Verify ModelController CR is deleted + mc := &componentsv1.ModelController{} + err = tc.testCtx.customClient.Get(ctx, client.ObjectKey{Name: tc.testModelControllerInstance.Name}, mc) + return k8serr.IsNotFound(err), nil + }); err != nil { + return fmt.Errorf("component modemeshserving is disabled, should not get the ModelController CR %v", tc.testModelControllerInstance.Name) + } + + // Sleep for 20 seconds to allow the operator to reconcile + time.Sleep(2 * generalRetryInterval) + _, err = tc.testCtx.kubeClient.AppsV1().Deployments(tc.testCtx.applicationsNamespace).Get(tc.testCtx.ctx, mcDeploymentName, metav1.GetOptions{}) + if err != nil { + if k8serr.IsNotFound(err) { + return nil // correct result: should not find deployment after we disable it already + } + return fmt.Errorf("error getting component resource after reconcile: %w", err) + } + return fmt.Errorf("component %v is disabled, should not get its deployment %v from NS %v any more", + componentsv1.ModelControllerKind, + mcDeploymentName, + tc.testCtx.applicationsNamespace) +} diff --git a/tests/e2e/modelmeshserving_test.go b/tests/e2e/modelmeshserving_test.go new file mode 100644 index 00000000000..404123adb49 --- /dev/null +++ b/tests/e2e/modelmeshserving_test.go @@ -0,0 +1,280 @@ +package e2e_test + +import ( + "context" + "errors" + "fmt" + "reflect" + "testing" + "time" + + operatorv1 "github.com/openshift/api/operator/v1" + "github.com/stretchr/testify/require" + autoscalingv1 "k8s.io/api/autoscaling/v1" + k8serr "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + + componentsv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/components/v1" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/metadata/labels" +) + +type ModelMeshServingTestCtx struct { + testCtx *testContext + testModelMeshServingInstance componentsv1.ModelMeshServing +} + +func modelMeshServingTestSuite(t *testing.T) { + t.Helper() + + mmCtx := ModelMeshServingTestCtx{} + var err error + mmCtx.testCtx, err = NewTestContext() + require.NoError(t, err) + + testCtx := mmCtx.testCtx + + t.Run(testCtx.testDsc.Name, func(t *testing.T) { + // creation + t.Run("Creation of ModelMeshServing CR", func(t *testing.T) { + err = mmCtx.testModelMeshServingCreation() + require.NoError(t, err, "error creating ModelMeshServing CR") + }) + + t.Run("Validate ModelMeshServing instance", func(t *testing.T) { + err = mmCtx.validateModelMeshServing() + require.NoError(t, err, "error validating ModelMeshServing instance") + }) + + t.Run("Validate Ownerrefrences exist", func(t *testing.T) { + err = mmCtx.testOwnerReferences() + require.NoError(t, err, "error getting all ModelMeshServing's Ownerrefrences") + }) + + t.Run("Validate ModelMeshServing Ready", func(t *testing.T) { + err = mmCtx.validateModelMeshServingReady() + require.NoError(t, err, "ModelMeshServing instance is not Ready") + }) + + // reconcile + t.Run("Validate Controller reconcile", func(t *testing.T) { + err = mmCtx.testUpdateOnModelMeshServingResources() + require.NoError(t, err, "error testing updates for ModelMeshServing's managed resources") + }) + + t.Run("Validate Disabling ModelMeshServing Component", func(t *testing.T) { + err = mmCtx.testUpdateModelMeshServingComponentDisabled() + require.NoError(t, err, "error testing modemeshserving component enabled field") + }) + }) +} + +func (tc *ModelMeshServingTestCtx) testModelMeshServingCreation() error { + if tc.testCtx.testDsc.Spec.Components.ModelMeshServing.ManagementState != operatorv1.Managed { + return nil + } + + err := tc.testCtx.wait(func(ctx context.Context) (bool, error) { + existingModelMeshServingList := &componentsv1.ModelMeshServingList{} + + if err := tc.testCtx.customClient.List(ctx, existingModelMeshServingList); err != nil { + return false, err + } + + switch { + case len(existingModelMeshServingList.Items) == 1: + tc.testModelMeshServingInstance = existingModelMeshServingList.Items[0] + return true, nil + case len(existingModelMeshServingList.Items) > 1: + return false, fmt.Errorf( + "unexpected ModelMeshServing CR instances. Expected 1 , Found %v instance", len(existingModelMeshServingList.Items)) + default: + return false, nil + } + }) + + if err != nil { + return fmt.Errorf("unable to find ModelMeshServing CR instance: %w", err) + } + + return nil +} + +func (tc *ModelMeshServingTestCtx) validateModelMeshServing() error { + // ModelMeshServing spec should match the spec of ModelMeshServing component in DSC + if !reflect.DeepEqual(tc.testCtx.testDsc.Spec.Components.ModelMeshServing.ModelMeshServingCommonSpec, tc.testModelMeshServingInstance.Spec.ModelMeshServingCommonSpec) { + err := fmt.Errorf("expected .spec for ModelMeshServing %v, got %v", + tc.testCtx.testDsc.Spec.Components.ModelMeshServing.ModelMeshServingCommonSpec, tc.testModelMeshServingInstance.Spec.ModelMeshServingCommonSpec) + return err + } + return nil +} + +func (tc *ModelMeshServingTestCtx) testOwnerReferences() error { + if len(tc.testModelMeshServingInstance.OwnerReferences) != 1 { + return errors.New("expect CR has ownerreferences set") + } + + // Test ModelMeshServing CR ownerref + if tc.testModelMeshServingInstance.OwnerReferences[0].Kind != dscKind { + return fmt.Errorf("expected ownerreference DataScienceCluster not found. Got ownereferrence: %v", + tc.testModelMeshServingInstance.OwnerReferences[0].Kind) + } + + // Test ModelMeshServing resources + appDeployments, err := tc.testCtx.kubeClient.AppsV1().Deployments(tc.testCtx.applicationsNamespace).List(tc.testCtx.ctx, metav1.ListOptions{ + LabelSelector: labels.ODH.Component(componentsv1.ModelMeshServingComponentName), + }) + if err != nil { + return fmt.Errorf("error listing component deployments %w", err) + } + // test any one deployment for ownerreference + if len(appDeployments.Items) != 0 && appDeployments.Items[0].OwnerReferences[0].Kind != componentsv1.ModelMeshServingKind { + return fmt.Errorf("expected ownerreference not found. Got ownereferrence: %v", + appDeployments.Items[0].OwnerReferences) + } + + return nil +} + +// Verify ModelMeshServing instance is in Ready phase when modelmeshserving deployments are up and running. +func (tc *ModelMeshServingTestCtx) validateModelMeshServingReady() error { + err := wait.PollUntilContextTimeout(tc.testCtx.ctx, generalRetryInterval, componentReadyTimeout, true, func(ctx context.Context) (bool, error) { + key := types.NamespacedName{Name: tc.testModelMeshServingInstance.Name} + mm := &componentsv1.ModelMeshServing{} + + err := tc.testCtx.customClient.Get(ctx, key, mm) + if err != nil { + return false, err + } + return mm.Status.Phase == readyStatus, nil + }) + + if err != nil { + return fmt.Errorf("error waiting Ready state for ModelMeshServing %v: %w", tc.testModelMeshServingInstance.Name, err) + } + + return nil +} + +func (tc *ModelMeshServingTestCtx) testUpdateOnModelMeshServingResources() error { + // Test Updating ModelMeshServing Replicas + + appDeployments, err := tc.testCtx.kubeClient.AppsV1().Deployments(tc.testCtx.applicationsNamespace).List(tc.testCtx.ctx, metav1.ListOptions{ + LabelSelector: labels.ComponentPartOf + "=" + tc.testModelMeshServingInstance.Name, + }) + if err != nil { + return err + } + + if len(appDeployments.Items) != 1 { + return fmt.Errorf("error getting deployment for component %s", tc.testModelMeshServingInstance.Name) + } + + const expectedReplica int32 = 2 // from 1 to 2 + + testDeployment := appDeployments.Items[0] + patchedReplica := &autoscalingv1.Scale{ + ObjectMeta: metav1.ObjectMeta{ + Name: testDeployment.Name, + Namespace: testDeployment.Namespace, + }, + Spec: autoscalingv1.ScaleSpec{ + Replicas: expectedReplica, + }, + Status: autoscalingv1.ScaleStatus{}, + } + updatedDep, err := tc.testCtx.kubeClient.AppsV1().Deployments(tc.testCtx.applicationsNamespace).UpdateScale(tc.testCtx.ctx, + testDeployment.Name, patchedReplica, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("error patching component resources : %w", err) + } + if updatedDep.Spec.Replicas != patchedReplica.Spec.Replicas { + return fmt.Errorf("failed to patch replicas : expect to be %v but got %v", patchedReplica.Spec.Replicas, updatedDep.Spec.Replicas) + } + + // Sleep for 20 seconds to allow the operator to reconcile + // we expect it should not revert back to original value because of AllowList + time.Sleep(2 * generalRetryInterval) + reconciledDep, err := tc.testCtx.kubeClient.AppsV1().Deployments(tc.testCtx.applicationsNamespace).Get(tc.testCtx.ctx, testDeployment.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("error getting component resource after reconcile: %w", err) + } + if *reconciledDep.Spec.Replicas != expectedReplica { + return fmt.Errorf("failed to revert back replicas : expect to be %v but got %v", expectedReplica, *reconciledDep.Spec.Replicas) + } + + return nil +} + +func (tc *ModelMeshServingTestCtx) testUpdateModelMeshServingComponentDisabled() error { + // Test Updating ModelMeshServing to be disabled + var mmDeploymentName string + + if tc.testCtx.testDsc.Spec.Components.ModelMeshServing.ManagementState == operatorv1.Managed { + appDeployments, err := tc.testCtx.kubeClient.AppsV1().Deployments(tc.testCtx.applicationsNamespace).List(tc.testCtx.ctx, metav1.ListOptions{ + LabelSelector: labels.ODH.Component(componentsv1.ModelMeshServingComponentName), + }) + if err != nil { + return fmt.Errorf("error getting enabled component %v", componentsv1.ModelMeshServingComponentName) + } + if len(appDeployments.Items) > 0 { + mmDeploymentName = appDeployments.Items[0].Name + if appDeployments.Items[0].Status.ReadyReplicas == 0 { + return fmt.Errorf("error getting enabled component: %s its deployment 'ReadyReplicas'", mmDeploymentName) + } + } + } else { + return errors.New("modelmeshserving spec should be in 'enabled: true' state in order to perform test") + } + + // Disable component ModelMeshServing + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + // refresh DSC instance in case it was updated during the reconcile + err := tc.testCtx.customClient.Get(tc.testCtx.ctx, types.NamespacedName{Name: tc.testCtx.testDsc.Name}, tc.testCtx.testDsc) + if err != nil { + return fmt.Errorf("error getting resource %w", err) + } + // Disable the Component + tc.testCtx.testDsc.Spec.Components.ModelMeshServing.ManagementState = operatorv1.Removed + + // Try to update + err = tc.testCtx.customClient.Update(tc.testCtx.ctx, tc.testCtx.testDsc) + // Return err itself here (not wrapped inside another error) + // so that RetryOnConflict can identify it correctly. + if err != nil { + return fmt.Errorf("error updating component from 'enabled: true' to 'enabled: false': %w", err) + } + + return nil + }) + if err != nil { + return fmt.Errorf("error after retry %w", err) + } + + if err = tc.testCtx.wait(func(ctx context.Context) (bool, error) { + // Verify ModeMeshServing CR is deleted + mm := &componentsv1.ModelMeshServing{} + err = tc.testCtx.customClient.Get(ctx, client.ObjectKey{Name: tc.testModelMeshServingInstance.Name}, mm) + return k8serr.IsNotFound(err), nil + }); err != nil { + return fmt.Errorf("component modemeshserving is disabled, should not get the ModelMeshServing CR %v", tc.testModelMeshServingInstance.Name) + } + + // Sleep for 20 seconds to allow the operator to reconcile + time.Sleep(2 * generalRetryInterval) + _, err = tc.testCtx.kubeClient.AppsV1().Deployments(tc.testCtx.applicationsNamespace).Get(tc.testCtx.ctx, mmDeploymentName, metav1.GetOptions{}) + if err != nil { + if k8serr.IsNotFound(err) { + return nil // correct result: should not find deployment after we disable it already + } + return fmt.Errorf("error getting component resource after reconcile: %w", err) + } + return fmt.Errorf("component %v is disabled, should not get its deployment %v from NS %v any more", + componentsv1.ModelMeshServingKind, + mmDeploymentName, + tc.testCtx.applicationsNamespace) +} diff --git a/tests/e2e/odh_manager_test.go b/tests/e2e/odh_manager_test.go index 8e702117d77..c969aa54ca1 100644 --- a/tests/e2e/odh_manager_test.go +++ b/tests/e2e/odh_manager_test.go @@ -80,4 +80,16 @@ func (tc *testContext) validateOwnedCRDs(t *testing.T) { require.NoErrorf(t, tc.validateCRD("datasciencepipelines.components.opendatahub.io"), "error in validating CRD : datasciencepipelines.components.opendatahub.io") }) + + t.Run("Validate ModelMeshServing CRD", func(t *testing.T) { + t.Parallel() + require.NoErrorf(t, tc.validateCRD("modelmeshservings.components.opendatahub.io"), + "error in validating CRD : modelmeshservings.components.opendatahub.io") + }) + + t.Run("Validate ModelController CRD", func(t *testing.T) { + t.Parallel() + require.NoErrorf(t, tc.validateCRD("modelcontrollers.components.opendatahub.io"), + "error in validating CRD : modelcontrollers.components.opendatahub.io") + }) }