From ff749b31be270fafde66784357bba26458701e3d Mon Sep 17 00:00:00 2001 From: Martin Schuppert Date: Fri, 4 Aug 2023 11:32:23 +0200 Subject: [PATCH] Create placementapi route and svc endpoint overrides Creates the route for the placementapi, also allows to customize the route via override. Generats the service override for the env with what is configured in the externalEndpoints, or specified in the service template override. Depends-On: https://github.com/openstack-k8s-operators/lib-common/pull/313 Depends-On: https://github.com/openstack-k8s-operators/keystone-operator/pull/289 Depends-On: https://github.com/openstack-k8s-operators/placement-operator/pull/48 Jira: OSP-26690 --- ....openstack.org_openstackcontrolplanes.yaml | 127 ++++++++++++++++++ .../v1beta1/openstackcontrolplane_types.go | 5 + apis/core/v1beta1/zz_generated.deepcopy.go | 1 + ....openstack.org_openstackcontrolplanes.yaml | 127 ++++++++++++++++++ ...nstack-operator.clusterserviceversion.yaml | 18 +++ ...controlplane_galera_network_isolation.yaml | 10 +- ...ne_galera_network_isolation_3replicas.yaml | 10 +- ...enstackcontrolplane_network_isolation.yaml | 10 +- ...ckcontrolplane_network_isolation_ceph.yaml | 10 +- pkg/openstack/placement.go | 64 +++++++++ 10 files changed, 362 insertions(+), 20 deletions(-) diff --git a/apis/bases/core.openstack.org_openstackcontrolplanes.yaml b/apis/bases/core.openstack.org_openstackcontrolplanes.yaml index 8099261d1..e917218ba 100644 --- a/apis/bases/core.openstack.org_openstackcontrolplanes.yaml +++ b/apis/bases/core.openstack.org_openstackcontrolplanes.yaml @@ -8069,6 +8069,133 @@ spec: type: object placement: properties: + apiOverride: + properties: + externalEndpoints: + items: + properties: + endpoint: + default: internal + enum: + - internal + - public + type: string + ipAddressPool: + minLength: 1 + type: string + loadBalancerIPs: + items: + type: string + type: array + sharedIP: + default: true + type: boolean + sharedIPKey: + default: "" + type: string + required: + - ipAddressPool + type: object + type: array + route: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + alternateBackends: + items: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + maxItems: 3 + type: array + host: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + path: + pattern: ^/ + type: string + port: + properties: + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - targetPort + type: object + subdomain: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + tls: + properties: + caCertificate: + type: string + certificate: + type: string + destinationCACertificate: + type: string + insecureEdgeTerminationPolicy: + type: string + key: + type: string + termination: + enum: + - edge + - reencrypt + - passthrough + type: string + required: + - termination + type: object + to: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + wildcardPolicy: + enum: + - None + - Subdomain + - "" + type: string + type: object + type: object + type: object enabled: default: true type: boolean diff --git a/apis/core/v1beta1/openstackcontrolplane_types.go b/apis/core/v1beta1/openstackcontrolplane_types.go index 3c3a3eaed..3b08af2bf 100644 --- a/apis/core/v1beta1/openstackcontrolplane_types.go +++ b/apis/core/v1beta1/openstackcontrolplane_types.go @@ -222,6 +222,11 @@ type PlacementSection struct { //+operator-sdk:csv:customresourcedefinitions:type=spec // Template - Overrides to use when creating the Placement API Template placementv1.PlacementAPISpec `json:"template,omitempty"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // APIOverride, provides the ability to override the generated manifest of several child resources. + APIOverride Override `json:"apiOverride,omitempty"` } // GlanceSection defines the desired state of Glance service diff --git a/apis/core/v1beta1/zz_generated.deepcopy.go b/apis/core/v1beta1/zz_generated.deepcopy.go index 55d1b8162..cac7f1fc5 100644 --- a/apis/core/v1beta1/zz_generated.deepcopy.go +++ b/apis/core/v1beta1/zz_generated.deepcopy.go @@ -548,6 +548,7 @@ func (in *OvnSection) DeepCopy() *OvnSection { func (in *PlacementSection) DeepCopyInto(out *PlacementSection) { *out = *in in.Template.DeepCopyInto(&out.Template) + in.APIOverride.DeepCopyInto(&out.APIOverride) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlacementSection. diff --git a/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml b/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml index 8099261d1..e917218ba 100644 --- a/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml +++ b/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml @@ -8069,6 +8069,133 @@ spec: type: object placement: properties: + apiOverride: + properties: + externalEndpoints: + items: + properties: + endpoint: + default: internal + enum: + - internal + - public + type: string + ipAddressPool: + minLength: 1 + type: string + loadBalancerIPs: + items: + type: string + type: array + sharedIP: + default: true + type: boolean + sharedIPKey: + default: "" + type: string + required: + - ipAddressPool + type: object + type: array + route: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + alternateBackends: + items: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + maxItems: 3 + type: array + host: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + path: + pattern: ^/ + type: string + port: + properties: + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - targetPort + type: object + subdomain: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + tls: + properties: + caCertificate: + type: string + certificate: + type: string + destinationCACertificate: + type: string + insecureEdgeTerminationPolicy: + type: string + key: + type: string + termination: + enum: + - edge + - reencrypt + - passthrough + type: string + required: + - termination + type: object + to: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + wildcardPolicy: + enum: + - None + - Subdomain + - "" + type: string + type: object + type: object + type: object enabled: default: true type: boolean diff --git a/config/manifests/bases/openstack-operator.clusterserviceversion.yaml b/config/manifests/bases/openstack-operator.clusterserviceversion.yaml index e1e8dfd42..d69ba1dee 100644 --- a/config/manifests/bases/openstack-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/openstack-operator.clusterserviceversion.yaml @@ -248,6 +248,24 @@ spec: path: placement.enabled x-descriptors: - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: IPAddressPool expose VIP via MetalLB on the IPAddressPool + displayName: IPAddress Pool + path: placement.externalEndpoints[0].ipAddressPool + - description: LoadBalancerIPs, request given IPs from the pool if available. + Using a list to allow dual stack (IPv4/IPv6) support + displayName: Load Balancer IPs + path: placement.externalEndpoints[0].loadBalancerIPs + - description: SharedIP if true, VIP/VIPs get shared with multiple services + displayName: Shared IP + path: placement.externalEndpoints[0].sharedIP + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: SharedIPKey specifies the sharing key which gets set as the annotation + on the LoadBalancer service. Services which share the same VIP must have + the same SharedIPKey. Defaults to the IPAddressPool if SharedIP is true, + but no SharedIPKey specified. + displayName: Shared IPKey + path: placement.externalEndpoints[0].sharedIPKey - description: Template - Overrides to use when creating the Placement API displayName: Template path: placement.template diff --git a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml index da21ff67c..5a7070522 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml @@ -154,14 +154,14 @@ spec: ovn-encap-type: "geneve" networkAttachment: tenant placement: + externalEndpoints: + - endpoint: internal + ipAddressPool: internalapi + loadBalancerIPs: + - 172.17.0.80 template: databaseInstance: openstack secret: osp-secret - externalEndpoints: - - endpoint: internal - ipAddressPool: internalapi - loadBalancerIPs: - - 172.17.0.80 rabbitmq: templates: rabbitmq: diff --git a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation_3replicas.yaml b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation_3replicas.yaml index 9e58f6c39..e60fb143d 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation_3replicas.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation_3replicas.yaml @@ -154,14 +154,14 @@ spec: ovn-encap-type: "geneve" networkAttachment: tenant placement: + externalEndpoints: + - endpoint: internal + ipAddressPool: internalapi + loadBalancerIPs: + - 172.17.0.80 template: databaseInstance: openstack secret: osp-secret - externalEndpoints: - - endpoint: internal - ipAddressPool: internalapi - loadBalancerIPs: - - 172.17.0.80 rabbitmq: templates: rabbitmq: diff --git a/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml b/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml index e8fc93701..36cb9e125 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml @@ -142,14 +142,14 @@ spec: ovn-encap-type: "geneve" networkAttachment: tenant placement: + externalEndpoints: + - endpoint: internal + ipAddressPool: internalapi + loadBalancerIPs: + - 172.17.0.80 template: databaseInstance: openstack secret: osp-secret - externalEndpoints: - - endpoint: internal - ipAddressPool: internalapi - loadBalancerIPs: - - 172.17.0.80 rabbitmq: templates: rabbitmq: diff --git a/config/samples/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml b/config/samples/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml index 2e59199b1..759a882a1 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane_network_isolation_ceph.yaml @@ -199,14 +199,14 @@ spec: ovn-encap-type: "geneve" networkAttachment: tenant placement: + externalEndpoints: + - endpoint: internal + ipAddressPool: internalapi + loadBalancerIPs: + - 172.17.0.80 template: databaseInstance: openstack secret: osp-secret - externalEndpoints: - - endpoint: internal - ipAddressPool: internalapi - loadBalancerIPs: - - 172.17.0.80 rabbitmq: templates: rabbitmq: diff --git a/pkg/openstack/placement.go b/pkg/openstack/placement.go index 398a052c1..fb4af21cd 100644 --- a/pkg/openstack/placement.go +++ b/pkg/openstack/placement.go @@ -5,13 +5,17 @@ import ( "fmt" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/endpoint" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" corev1beta1 "github.com/openstack-k8s-operators/openstack-operator/apis/core/v1beta1" placementv1 "github.com/openstack-k8s-operators/placement-operator/api/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" ctrl "sigs.k8s.io/controller-runtime" ) @@ -32,9 +36,47 @@ func ReconcilePlacementAPI(ctx context.Context, instance *corev1beta1.OpenStackC return ctrl.Result{}, nil } + // Create service overrides to pass into the service CR + // and expose the public endpoint using a route per default. + // Any trailing path will be added on the service-operator level. + var endpoints = map[service.Endpoint]endpoint.Data{ + service.EndpointPublic: {}, + service.EndpointInternal: {}, + } + serviceOverrides := []service.OverrideSpec{} + serviceDetails := []ServiceDetails{} + for endpointType := range endpoints { + + sd := ServiceDetails{ + ServiceName: placementAPI.Name, + Namespace: instance.Namespace, + Endpoint: endpointType, + ExternalEndpoints: instance.Spec.Placement.APIOverride.ExternalEndpoints, + ServiceOverrideSpec: instance.Spec.Placement.Template.Override.Service, + RouteOverrideSpec: instance.Spec.Placement.APIOverride.Route, + } + + svcOverride, ctrlResult, err := sd.CreateRouteAndServiceOverride(ctx, instance, helper, endpointType) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + serviceDetails = append( + serviceDetails, + sd, + ) + if svcOverride != nil { + serviceOverrides = append(serviceOverrides, *svcOverride) + } + instance.Status.Conditions.MarkTrue(corev1beta1.OpenStackControlPlaneServiceOverrideReadyCondition, corev1beta1.OpenStackControlPlaneServiceOverrideReadyMessage) + } + helper.GetLogger().Info("Reconciling PlacementAPI", "PlacementAPI.Namespace", instance.Namespace, "PlacementAPI.Name", "placement") op, err := controllerutil.CreateOrPatch(ctx, helper.GetClient(), placementAPI, func() error { instance.Spec.Placement.Template.DeepCopyInto(&placementAPI.Spec) + placementAPI.Spec.Override.Service = serviceOverrides if placementAPI.Spec.Secret == "" { placementAPI.Spec.Secret = instance.Spec.Secret } @@ -74,6 +116,28 @@ func ReconcilePlacementAPI(ctx context.Context, instance *corev1beta1.OpenStackC corev1beta1.OpenStackControlPlanePlacementAPIReadyRunningMessage)) } + for _, sd := range serviceDetails { + // Add the service CR to the ownerRef list of the route to prevent the route being deleted + // before the service is deleted. Otherwise this can result cleanup issues which require + // the endpoint to be reachable. + // If ALL objects in the list have been deleted, this object will be garbage collected. + // https://github.com/kubernetes/apimachinery/blob/15d95c0b2af3f4fcf46dce24105e5fbb9379af5a/pkg/apis/meta/v1/types.go#L240-L247 + scheme := runtime.NewScheme() + gvk := schema.GroupVersionKind{ + Group: placementv1.GroupVersion.Group, + Version: placementv1.GroupVersion.Version, + Kind: placementAPI.Kind, + } + + // Add the GVK to the scheme + scheme.AddKnownTypeWithName(gvk, &placementv1.PlacementAPI{}) + + err = sd.AddOwnerRef(ctx, helper, placementAPI, scheme) + if err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{}, nil }