diff --git a/apis/bases/core.openstack.org_openstackcontrolplanes.yaml b/apis/bases/core.openstack.org_openstackcontrolplanes.yaml index 8bd155512..f27024429 100644 --- a/apis/bases/core.openstack.org_openstackcontrolplanes.yaml +++ b/apis/bases/core.openstack.org_openstackcontrolplanes.yaml @@ -3929,6 +3929,133 @@ spec: enabled: default: true type: boolean + externalEndpoints: + items: + properties: + endpoint: + default: internal + enum: + - internal + - public + type: string + ipAddressPool: + minLength: 1 + type: string + loadBalancerIPs: + items: + type: string + type: array + sharedIP: + default: true + type: boolean + sharedIPKey: + default: "" + type: string + required: + - ipAddressPool + type: object + type: array + override: + properties: + route: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + alternateBackends: + items: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + maxItems: 3 + type: array + host: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + path: + pattern: ^/ + type: string + port: + properties: + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - targetPort + type: object + subdomain: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + tls: + properties: + caCertificate: + type: string + certificate: + type: string + destinationCACertificate: + type: string + insecureEdgeTerminationPolicy: + type: string + key: + type: string + termination: + enum: + - edge + - reencrypt + - passthrough + type: string + required: + - termination + type: object + to: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + wildcardPolicy: + enum: + - None + - Subdomain + - "" + type: string + type: object + type: object + type: object template: properties: adminProject: @@ -3962,32 +4089,6 @@ spec: additionalProperties: type: string type: object - externalEndpoints: - items: - properties: - endpoint: - enum: - - internal - - public - type: string - ipAddressPool: - minLength: 1 - type: string - loadBalancerIPs: - items: - type: string - type: array - sharedIP: - default: true - type: boolean - sharedIPKey: - default: "" - type: string - required: - - endpoint - - ipAddressPool - type: object - type: array memcachedInstance: default: memcached type: string @@ -3999,6 +4100,123 @@ spec: additionalProperties: type: string type: object + override: + properties: + service: + items: + properties: + endpoint: + enum: + - internal + - public + type: string + endpointURL: + type: string + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + allocateLoadBalancerNodePorts: + type: boolean + clusterIP: + type: string + clusterIPs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalIPs: + items: + type: string + type: array + externalName: + type: string + externalTrafficPolicy: + type: string + healthCheckNodePort: + format: int32 + type: integer + internalTrafficPolicy: + type: string + ipFamilies: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ipFamilyPolicy: + type: string + loadBalancerClass: + type: string + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + ports: + items: + properties: + appProtocol: + type: string + name: + type: string + nodePort: + format: int32 + type: integer + port: + format: int32 + type: integer + protocol: + default: TCP + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + - protocol + x-kubernetes-list-type: map + publishNotReadyAddresses: + type: boolean + selector: + additionalProperties: + type: string + type: object + x-kubernetes-map-type: atomic + sessionAffinity: + type: string + sessionAffinityConfig: + properties: + clientIP: + properties: + timeoutSeconds: + format: int32 + type: integer + type: object + type: object + type: + type: string + type: object + required: + - endpoint + type: object + type: array + type: object passwordSelectors: default: admin: AdminPassword @@ -7341,6 +7559,12 @@ spec: type: object externalEndpoint: properties: + endpoint: + default: internal + enum: + - internal + - public + type: string ipAddressPool: minLength: 1 type: string diff --git a/apis/core/v1beta1/openstackcontrolplane_types.go b/apis/core/v1beta1/openstackcontrolplane_types.go index f86e57c74..2bd966672 100644 --- a/apis/core/v1beta1/openstackcontrolplane_types.go +++ b/apis/core/v1beta1/openstackcontrolplane_types.go @@ -28,6 +28,8 @@ import ( ironicv1 "github.com/openstack-k8s-operators/ironic-operator/api/v1beta1" keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/route" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" "github.com/openstack-k8s-operators/lib-common/modules/common/util" "github.com/openstack-k8s-operators/lib-common/modules/storage" manilav1 "github.com/openstack-k8s-operators/manila-operator/api/v1beta1" @@ -186,6 +188,22 @@ type KeystoneSection struct { //+operator-sdk:csv:customresourcedefinitions:type=spec // Template - Overrides to use when creating the Keystone service Template keystonev1.KeystoneAPISpec `json:"template,omitempty"` + + // ExternalEndpoints, expose a VIP using a pre-created IPAddressPool + ExternalEndpoints []MetalLBConfig `json:"externalEndpoints,omitempty"` + + // +kubebuilder:validation:Optional + // Override, provides the ability to override the generated manifest of several child resources. + Override KeystoneOverrideSpec `json:"override,omitempty"` +} + +// KeystoneOverrideSpec to override the generated manifest of several child resources. +type KeystoneOverrideSpec struct { + // +kubebuilder:validation:Optional + Route *route.OverrideSpec `json:"route,omitempty"` + + // Override configuration for the Service created to serve traffic to the cluster. + //Service []service.OverrideSpec `json:"service,omitempty"` } // PlacementSection defines the desired state of Placement service @@ -301,6 +319,12 @@ type RabbitmqTemplate struct { // MetalLBConfig to configure the MetalLB loadbalancer service type MetalLBConfig struct { + // +kubebuilder:validation:Optional + // +kubebuilder:validation:Enum=internal;public + // +kubebuilder:default=internal + // Endpoint, OpenStack endpoint this service maps to + Endpoint service.Endpoint `json:"endpoint"` + // +kubebuilder:validation:Required // +kubebuilder:validation:MinLength=1 //+operator-sdk:csv:customresourcedefinitions:type=spec diff --git a/apis/core/v1beta1/zz_generated.deepcopy.go b/apis/core/v1beta1/zz_generated.deepcopy.go index 1514626cd..dd345f2ba 100644 --- a/apis/core/v1beta1/zz_generated.deepcopy.go +++ b/apis/core/v1beta1/zz_generated.deepcopy.go @@ -24,6 +24,7 @@ package v1beta1 import ( memcachedv1beta1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/route" "github.com/openstack-k8s-operators/lib-common/modules/storage" apiv1beta1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" ovn_operatorapiv1beta1 "github.com/openstack-k8s-operators/ovn-operator/api/v1beta1" @@ -164,10 +165,38 @@ func (in *IronicSection) DeepCopy() *IronicSection { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KeystoneOverrideSpec) DeepCopyInto(out *KeystoneOverrideSpec) { + *out = *in + if in.Route != nil { + in, out := &in.Route, &out.Route + *out = new(route.OverrideSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KeystoneOverrideSpec. +func (in *KeystoneOverrideSpec) DeepCopy() *KeystoneOverrideSpec { + if in == nil { + return nil + } + out := new(KeystoneOverrideSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KeystoneSection) DeepCopyInto(out *KeystoneSection) { *out = *in in.Template.DeepCopyInto(&out.Template) + if in.ExternalEndpoints != nil { + in, out := &in.ExternalEndpoints, &out.ExternalEndpoints + *out = make([]MetalLBConfig, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + in.Override.DeepCopyInto(&out.Override) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KeystoneSection. diff --git a/apis/go.mod b/apis/go.mod index 4936e34fc..1ee0bcb36 100644 --- a/apis/go.mod +++ b/apis/go.mod @@ -100,3 +100,11 @@ require ( // mschuppert: map to latest commit from release-4.13 tag // must consistent within modules and service operators replace github.com/openshift/api => github.com/openshift/api v0.0.0-20230414143018-3367bc7e6ac7 //allow-merging + +replace github.com/openstack-k8s-operators/lib-common/modules/common => github.com/stuggi/lib-common/modules/common v0.0.0-20230727160754-e3ded3465918 + +//replace github.com/openstack-k8s-operators/lib-common/modules/common => /home/mschuppe/src/github.com/openstack-k8s-operators/lib-common/modules/common + +replace github.com/openstack-k8s-operators/keystone-operator/api => github.com/stuggi/keystone-operator/api v0.0.0-20230728050316-ba4f40b8bcce + +//replace github.com/openstack-k8s-operators/keystone-operator/api => /home/mschuppe/src/github.com/openstack-k8s-operators/keystone-operator/api diff --git a/apis/go.sum b/apis/go.sum index 1f55e3ea5..cbb25a6ac 100644 --- a/apis/go.sum +++ b/apis/go.sum @@ -138,10 +138,6 @@ github.com/openstack-k8s-operators/infra-operator/apis v0.1.0 h1:cavVlTrKeW2xcyt github.com/openstack-k8s-operators/infra-operator/apis v0.1.0/go.mod h1:t1xmsiZDqM3wXcLMqgHp7/iilK8ozuOkydV4Vi2Qibk= github.com/openstack-k8s-operators/ironic-operator/api v0.1.0 h1:Bm0mo2iRT16CG/WNj3stuheDGQDDZY3CGVv52AFDH1w= github.com/openstack-k8s-operators/ironic-operator/api v0.1.0/go.mod h1:slFWxIEbjO2tkyPR6GoKI0pIDgctKsPVB+Wg6D+CbQ0= -github.com/openstack-k8s-operators/keystone-operator/api v0.1.0 h1:p98vKnS4KzdgU/+vrVKFY3y9n9v1Z6cpo4JvbTNRxlM= -github.com/openstack-k8s-operators/keystone-operator/api v0.1.0/go.mod h1:LNJJdteQG4E2fhWDerE+f8S2/ephEJg8yBkH1eqYYOo= -github.com/openstack-k8s-operators/lib-common/modules/common v0.1.0 h1:F1iYRBwa0cZ2VHw8Zs4frqSWQ1B/tiCuSwH/DuHb8VM= -github.com/openstack-k8s-operators/lib-common/modules/common v0.1.0/go.mod h1:3hAC5Ce0AOSt85BqD6DgTKNkJHmpXwqbwL8mVWRJQqo= github.com/openstack-k8s-operators/lib-common/modules/openstack v0.1.0 h1:mMeJvCQfZmakssvMyHjzp/ngxKysETDj9GJYhRwydzg= github.com/openstack-k8s-operators/lib-common/modules/openstack v0.1.0/go.mod h1:+paEFOL5IlJzhg9fy7/1+HSErVkWUgUj1ORLFwgvxnI= github.com/openstack-k8s-operators/lib-common/modules/storage v0.1.0 h1:rHn0wlwBBggRl65gWDniF97XW+1XB2+4PsGZS2RIJ5E= @@ -195,6 +191,10 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stuggi/keystone-operator/api v0.0.0-20230728050316-ba4f40b8bcce h1:DV9n0l8DNZ/XDDVX5diXWKeDEc9IfJ4ATNLjHbaOAEo= +github.com/stuggi/keystone-operator/api v0.0.0-20230728050316-ba4f40b8bcce/go.mod h1:2U40N+wdR8XG0une0JNirnHbvuwsJ4QlGVGvaD8tsIE= +github.com/stuggi/lib-common/modules/common v0.0.0-20230727160754-e3ded3465918 h1:ocXtOjRXQw48VotIIvotFrjan0BkQgzk9sQ9Xr33u/g= +github.com/stuggi/lib-common/modules/common v0.0.0-20230727160754-e3ded3465918/go.mod h1:2fZFojsJsTA4MfGy50JEwbXaIP7Hr7h7x8hbhlMBedY= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= diff --git a/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml b/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml index 8bd155512..f27024429 100644 --- a/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml +++ b/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml @@ -3929,6 +3929,133 @@ spec: enabled: default: true type: boolean + externalEndpoints: + items: + properties: + endpoint: + default: internal + enum: + - internal + - public + type: string + ipAddressPool: + minLength: 1 + type: string + loadBalancerIPs: + items: + type: string + type: array + sharedIP: + default: true + type: boolean + sharedIPKey: + default: "" + type: string + required: + - ipAddressPool + type: object + type: array + override: + properties: + route: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + alternateBackends: + items: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + maxItems: 3 + type: array + host: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + path: + pattern: ^/ + type: string + port: + properties: + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - targetPort + type: object + subdomain: + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + tls: + properties: + caCertificate: + type: string + certificate: + type: string + destinationCACertificate: + type: string + insecureEdgeTerminationPolicy: + type: string + key: + type: string + termination: + enum: + - edge + - reencrypt + - passthrough + type: string + required: + - termination + type: object + to: + properties: + kind: + enum: + - Service + - "" + type: string + name: + type: string + weight: + format: int32 + maximum: 256 + minimum: 0 + type: integer + type: object + wildcardPolicy: + enum: + - None + - Subdomain + - "" + type: string + type: object + type: object + type: object template: properties: adminProject: @@ -3962,32 +4089,6 @@ spec: additionalProperties: type: string type: object - externalEndpoints: - items: - properties: - endpoint: - enum: - - internal - - public - type: string - ipAddressPool: - minLength: 1 - type: string - loadBalancerIPs: - items: - type: string - type: array - sharedIP: - default: true - type: boolean - sharedIPKey: - default: "" - type: string - required: - - endpoint - - ipAddressPool - type: object - type: array memcachedInstance: default: memcached type: string @@ -3999,6 +4100,123 @@ spec: additionalProperties: type: string type: object + override: + properties: + service: + items: + properties: + endpoint: + enum: + - internal + - public + type: string + endpointURL: + type: string + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + allocateLoadBalancerNodePorts: + type: boolean + clusterIP: + type: string + clusterIPs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalIPs: + items: + type: string + type: array + externalName: + type: string + externalTrafficPolicy: + type: string + healthCheckNodePort: + format: int32 + type: integer + internalTrafficPolicy: + type: string + ipFamilies: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ipFamilyPolicy: + type: string + loadBalancerClass: + type: string + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + ports: + items: + properties: + appProtocol: + type: string + name: + type: string + nodePort: + format: int32 + type: integer + port: + format: int32 + type: integer + protocol: + default: TCP + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + - protocol + x-kubernetes-list-type: map + publishNotReadyAddresses: + type: boolean + selector: + additionalProperties: + type: string + type: object + x-kubernetes-map-type: atomic + sessionAffinity: + type: string + sessionAffinityConfig: + properties: + clientIP: + properties: + timeoutSeconds: + format: int32 + type: integer + type: object + type: object + type: + type: string + type: object + required: + - endpoint + type: object + type: array + type: object passwordSelectors: default: admin: AdminPassword @@ -7341,6 +7559,12 @@ spec: type: object externalEndpoint: properties: + endpoint: + default: internal + enum: + - internal + - public + type: string ipAddressPool: minLength: 1 type: string diff --git a/config/manifests/bases/openstack-operator.clusterserviceversion.yaml b/config/manifests/bases/openstack-operator.clusterserviceversion.yaml index 3734864f9..f7f922ef0 100644 --- a/config/manifests/bases/openstack-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/openstack-operator.clusterserviceversion.yaml @@ -117,6 +117,24 @@ spec: path: keystone.enabled x-descriptors: - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: IPAddressPool expose VIP via MetalLB on the IPAddressPool + displayName: IPAddress Pool + path: keystone.externalEndpoints[0].ipAddressPool + - description: LoadBalancerIPs, request given IPs from the pool if available. + Using a list to allow dual stack (IPv4/IPv6) support + displayName: Load Balancer IPs + path: keystone.externalEndpoints[0].loadBalancerIPs + - description: SharedIP if true, VIP/VIPs get shared with multiple services + displayName: Shared IP + path: keystone.externalEndpoints[0].sharedIP + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: SharedIPKey specifies the sharing key which gets set as the annotation + on the LoadBalancer service. Services which share the same VIP must have + the same SharedIPKey. Defaults to the IPAddressPool if SharedIP is true, + but no SharedIPKey specified. + displayName: Shared IPKey + path: keystone.externalEndpoints[0].sharedIPKey - description: Template - Overrides to use when creating the Keystone service displayName: Template path: keystone.template diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 32ad844b0..e43290c9e 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -331,6 +331,18 @@ rules: - list - update - watch +- apiGroups: + - route.openshift.io + resources: + - routes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - security.openshift.io resourceNames: diff --git a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml index e42a54a66..e082b1c94 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml @@ -54,14 +54,14 @@ spec: networkAttachments: - storage keystone: + externalEndpoints: + - endpoint: internal + ipAddressPool: internalapi + loadBalancerIPs: + - 172.17.0.80 template: databaseInstance: openstack secret: osp-secret - externalEndpoints: - - endpoint: internal - ipAddressPool: internalapi - loadBalancerIPs: - - 172.17.0.80 mariadb: enabled: false templates: diff --git a/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml b/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml index ed3a7f5f3..8473f0ccd 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml @@ -54,14 +54,14 @@ spec: networkAttachments: - storage keystone: + externalEndpoints: + - endpoint: internal + ipAddressPool: internalapi + loadBalancerIPs: + - 172.17.0.80 template: databaseInstance: openstack secret: osp-secret - externalEndpoints: - - endpoint: internal - ipAddressPool: internalapi - loadBalancerIPs: - - 172.17.0.80 mariadb: templates: openstack: diff --git a/controllers/core/openstackcontrolplane_controller.go b/controllers/core/openstackcontrolplane_controller.go index 582809af4..7a61e55d2 100644 --- a/controllers/core/openstackcontrolplane_controller.go +++ b/controllers/core/openstackcontrolplane_controller.go @@ -20,6 +20,7 @@ import ( "context" "fmt" + routev1 "github.com/openshift/api/route/v1" cinderv1 "github.com/openstack-k8s-operators/cinder-operator/api/v1beta1" glancev1 "github.com/openstack-k8s-operators/glance-operator/api/v1beta1" heatv1 "github.com/openstack-k8s-operators/heat-operator/api/v1beta1" @@ -83,6 +84,7 @@ type OpenStackControlPlaneReconciler struct { //+kubebuilder:rbac:groups=network.openstack.org,resources=dnsmasqs,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=telemetry.openstack.org,resources=ceilometercentrals,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=swift.openstack.org,resources=swifts,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch;create;update;patch;delete; // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -313,5 +315,6 @@ func (r *OpenStackControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager) err Owns(&ironicv1.Ironic{}). Owns(&horizonv1.Horizon{}). Owns(&telemetryv1.CeilometerCentral{}). + Owns(&routev1.Route{}). Complete(r) } diff --git a/go.mod b/go.mod index d4d30184e..074b42a93 100644 --- a/go.mod +++ b/go.mod @@ -73,7 +73,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/openshift/api v3.9.0+incompatible // indirect + github.com/openshift/api v3.9.0+incompatible github.com/openstack-k8s-operators/lib-common/modules/openstack v0.1.0 //indirect github.com/openstack-k8s-operators/lib-common/modules/storage v0.1.0 //indirect github.com/pkg/errors v0.9.1 // indirect @@ -101,7 +101,7 @@ require ( k8s.io/component-base v0.26.7 //indirect k8s.io/klog/v2 v2.100.1 // indirect k8s.io/kube-openapi v0.0.0-20230515203736-54b630e78af5 //indirect - k8s.io/utils v0.0.0-20230711102312-30195339c3c7 //indirect + k8s.io/utils v0.0.0-20230711102312-30195339c3c7 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd //indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect @@ -112,3 +112,11 @@ replace github.com/openstack-k8s-operators/openstack-operator/apis => ./apis // mschuppert: map to latest commit from release-4.13 tag // must consistent within modules and service operators replace github.com/openshift/api => github.com/openshift/api v0.0.0-20230414143018-3367bc7e6ac7 //allow-merging + +replace github.com/openstack-k8s-operators/lib-common/modules/common => github.com/stuggi/lib-common/modules/common v0.0.0-20230727160754-e3ded3465918 + +//replace github.com/openstack-k8s-operators/lib-common/modules/common => /home/mschuppe/src/github.com/openstack-k8s-operators/lib-common/modules/common + +replace github.com/openstack-k8s-operators/keystone-operator/api => github.com/stuggi/keystone-operator/api v0.0.0-20230728050316-ba4f40b8bcce + +//replace github.com/openstack-k8s-operators/keystone-operator/api => /home/mschuppe/src/github.com/openstack-k8s-operators/keystone-operator/api diff --git a/go.sum b/go.sum index 84908cdd9..34a1d1030 100644 --- a/go.sum +++ b/go.sum @@ -147,10 +147,6 @@ github.com/openstack-k8s-operators/infra-operator/apis v0.1.0 h1:cavVlTrKeW2xcyt github.com/openstack-k8s-operators/infra-operator/apis v0.1.0/go.mod h1:t1xmsiZDqM3wXcLMqgHp7/iilK8ozuOkydV4Vi2Qibk= github.com/openstack-k8s-operators/ironic-operator/api v0.1.0 h1:Bm0mo2iRT16CG/WNj3stuheDGQDDZY3CGVv52AFDH1w= github.com/openstack-k8s-operators/ironic-operator/api v0.1.0/go.mod h1:slFWxIEbjO2tkyPR6GoKI0pIDgctKsPVB+Wg6D+CbQ0= -github.com/openstack-k8s-operators/keystone-operator/api v0.1.0 h1:p98vKnS4KzdgU/+vrVKFY3y9n9v1Z6cpo4JvbTNRxlM= -github.com/openstack-k8s-operators/keystone-operator/api v0.1.0/go.mod h1:LNJJdteQG4E2fhWDerE+f8S2/ephEJg8yBkH1eqYYOo= -github.com/openstack-k8s-operators/lib-common/modules/common v0.1.0 h1:F1iYRBwa0cZ2VHw8Zs4frqSWQ1B/tiCuSwH/DuHb8VM= -github.com/openstack-k8s-operators/lib-common/modules/common v0.1.0/go.mod h1:3hAC5Ce0AOSt85BqD6DgTKNkJHmpXwqbwL8mVWRJQqo= github.com/openstack-k8s-operators/lib-common/modules/openstack v0.1.0 h1:mMeJvCQfZmakssvMyHjzp/ngxKysETDj9GJYhRwydzg= github.com/openstack-k8s-operators/lib-common/modules/openstack v0.1.0/go.mod h1:+paEFOL5IlJzhg9fy7/1+HSErVkWUgUj1ORLFwgvxnI= github.com/openstack-k8s-operators/lib-common/modules/storage v0.1.0 h1:rHn0wlwBBggRl65gWDniF97XW+1XB2+4PsGZS2RIJ5E= @@ -212,6 +208,10 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stuggi/keystone-operator/api v0.0.0-20230728050316-ba4f40b8bcce h1:DV9n0l8DNZ/XDDVX5diXWKeDEc9IfJ4ATNLjHbaOAEo= +github.com/stuggi/keystone-operator/api v0.0.0-20230728050316-ba4f40b8bcce/go.mod h1:2U40N+wdR8XG0une0JNirnHbvuwsJ4QlGVGvaD8tsIE= +github.com/stuggi/lib-common/modules/common v0.0.0-20230727160754-e3ded3465918 h1:ocXtOjRXQw48VotIIvotFrjan0BkQgzk9sQ9Xr33u/g= +github.com/stuggi/lib-common/modules/common v0.0.0-20230727160754-e3ded3465918/go.mod h1:2fZFojsJsTA4MfGy50JEwbXaIP7Hr7h7x8hbhlMBedY= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= diff --git a/main.go b/main.go index 31cf5a972..490f14136 100644 --- a/main.go +++ b/main.go @@ -60,6 +60,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + routev1 "github.com/openshift/api/route/v1" clientv1 "github.com/openstack-k8s-operators/openstack-operator/apis/client/v1beta1" corev1 "github.com/openstack-k8s-operators/openstack-operator/apis/core/v1beta1" clientcontrollers "github.com/openstack-k8s-operators/openstack-operator/controllers/client" @@ -97,6 +98,7 @@ func init() { utilruntime.Must(telemetryv1.AddToScheme(scheme)) utilruntime.Must(swiftv1.AddToScheme(scheme)) utilruntime.Must(clientv1.AddToScheme(scheme)) + utilruntime.Must(routev1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } diff --git a/pkg/openstack/common.go b/pkg/openstack/common.go index 760a87c0a..de998d0e3 100644 --- a/pkg/openstack/common.go +++ b/pkg/openstack/common.go @@ -2,8 +2,15 @@ package openstack import ( "context" + "strings" + "time" + "github.com/openstack-k8s-operators/lib-common/modules/common/endpoint" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" + "github.com/openstack-k8s-operators/lib-common/modules/common/util" + corev1 "github.com/openstack-k8s-operators/openstack-operator/apis/core/v1beta1" + k8s_corev1 "k8s.io/api/core/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -27,3 +34,119 @@ func EnsureDeleted(ctx context.Context, helper *helper.Helper, obj client.Object return ctrl.Result{}, nil } + +// GetExternalEnpointDetailsForEndpoint - returns the MetalLBData for the endpoint, if not specified nil. +func GetExternalEnpointDetailsForEndpoint( + externalEndpoints []corev1.MetalLBConfig, + endpt service.Endpoint, +) *corev1.MetalLBConfig { + for _, metallbcfg := range externalEndpoints { + if metallbcfg.Endpoint == endpt { + return metallbcfg.DeepCopy() + } + } + + return nil +} + +// ServiceDetails - service details to create service.Service with overrides +type ServiceDetails struct { + Name string + Namespace string + ServiceLabels map[string]string + ServiceProtocol k8s_corev1.Protocol + Endpoint service.Endpoint + EndpointData endpoint.Data + ExternalEndpoints []corev1.MetalLBConfig + OverrideSpec []service.OverrideSpec + exportLabels map[string]string +} + +// GetExportLabels - returns the labels added set on the service as Labels and Selector +func (sd *ServiceDetails) GetExportLabels() map[string]string { + return sd.exportLabels +} + +// CreateEndpointServiceOverride - +func (sd *ServiceDetails) CreateEndpointServiceOverride() (*service.Service, error) { + sd.exportLabels = util.MergeStringMaps( + sd.ServiceLabels, + map[string]string{ + string(sd.Endpoint): "true", + }, + ) + + // initialize the service override if not specified via the CR + svcOverride := service.GetOverrideSpecForEndpoint(sd.OverrideSpec, sd.Endpoint).DeepCopy() + if svcOverride == nil { + svcOverride = &service.OverrideSpec{} + svcOverride.Endpoint = sd.Endpoint + } + + // get MetalLB config data for the endpoint + metalLBData := GetExternalEnpointDetailsForEndpoint(sd.ExternalEndpoints, sd.Endpoint) + + // Create service definitions for either either MetalLB, or generic service + var svcDef *k8s_corev1.Service + if metalLBData != nil { + // create MetalLB service definition + annotations := map[string]string{ + service.MetalLBAddressPoolAnnotation: metalLBData.IPAddressPool, + } + if len(metalLBData.LoadBalancerIPs) > 0 { + annotations[service.MetalLBLoadBalancerIPs] = strings.Join(metalLBData.LoadBalancerIPs, ",") + } + if metalLBData.SharedIP { + if metalLBData.SharedIPKey == "" { + annotations[service.MetalLBAllowSharedIPAnnotation] = metalLBData.IPAddressPool + } else { + annotations[service.MetalLBAllowSharedIPAnnotation] = metalLBData.SharedIPKey + } + } + + svcDef = service.MetalLBService(&service.MetalLBServiceDetails{ + Name: sd.Name, + Namespace: sd.Namespace, + Annotations: annotations, + Labels: sd.exportLabels, + Selector: sd.ServiceLabels, + Port: service.GenericServicePort{ + Name: sd.Name, + Port: sd.EndpointData.Port, + Protocol: sd.ServiceProtocol, + }, + }) + } else { + // create generic ClusterIP service definition + svcDef = service.GenericService(&service.GenericServiceDetails{ + Name: sd.Name, + Namespace: sd.Namespace, + Labels: sd.exportLabels, + Selector: sd.ServiceLabels, + Port: service.GenericServicePort{ + Name: sd.Name, + Port: sd.EndpointData.Port, + Protocol: sd.ServiceProtocol, + }}) + } + + // Create the service + svc, err := service.NewService( + svcDef, + sd.exportLabels, + time.Duration(5)*time.Second, + svcOverride, + ) + if err != nil { + return nil, err + } + + // add annotation to register service name in dnsmasq if it is a LoadBalancer service + if svc.GetServiceType() != k8s_corev1.ServiceTypeLoadBalancer { + svc.AddAnnotation(map[string]string{ + service.AnnotationHostnameKey: svc.GetServiceHostname(), + }) + } + + return svc, nil +} diff --git a/pkg/openstack/keystone.go b/pkg/openstack/keystone.go index b84846c9b..30b3703cc 100644 --- a/pkg/openstack/keystone.go +++ b/pkg/openstack/keystone.go @@ -3,6 +3,8 @@ package openstack import ( "context" "fmt" + "net/url" + "time" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" @@ -11,12 +13,20 @@ import ( keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" corev1beta1 "github.com/openstack-k8s-operators/openstack-operator/apis/core/v1beta1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" + + "github.com/openstack-k8s-operators/lib-common/modules/common" + "github.com/openstack-k8s-operators/lib-common/modules/common/endpoint" + "github.com/openstack-k8s-operators/lib-common/modules/common/route" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" ) // ReconcileKeystoneAPI - func ReconcileKeystoneAPI(ctx context.Context, instance *corev1beta1.OpenStackControlPlane, helper *helper.Helper) (ctrl.Result, error) { + keystoneAPI := &keystonev1.KeystoneAPI{ ObjectMeta: metav1.ObjectMeta{ Name: "keystone", //FIXME (keystone doesn't seem to work unless named "keystone") @@ -32,9 +42,121 @@ func ReconcileKeystoneAPI(ctx context.Context, instance *corev1beta1.OpenStackCo return ctrl.Result{}, nil } + spec := instance.Spec.Keystone.DeepCopy() + serviceOverrides := []service.OverrideSpec{} + + // expose the public endpoint using a route + var endpoints = map[service.Endpoint]endpoint.Data{ + service.EndpointPublic: { + Port: keystonev1.KeystonePublicPort, + }, + service.EndpointInternal: { + Port: keystonev1.KeystoneInternalPort, + }, + } + + // Set the serviceLabel which is used for the selector to of the service->pod + serviceLabels := map[string]string{ + common.AppSelector: keystonev1.ServiceName, + } + + // for each endpoint create an override to be passed to the service operator + for endpointType, data := range endpoints { + + endpointName := keystonev1.ServiceName + "-" + string(endpointType) + + // create service.Service with overrides if any + sd := ServiceDetails{ + Name: endpointName, + Namespace: instance.Namespace, + ServiceLabels: serviceLabels, + ServiceProtocol: corev1.ProtocolTCP, + Endpoint: endpointType, + EndpointData: data, + ExternalEndpoints: spec.ExternalEndpoints, + OverrideSpec: spec.Template.Override.Service, + } + svc, err := sd.CreateEndpointServiceOverride() + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ExposeServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ExposeServiceReadyErrorMessage, + err.Error())) + + return ctrl.Result{}, err + } + // create service overrides - end + + // the hostname is either ther route or the service dns name + var hostname string + + // Create the route if it is public endpoint and the service type is ClusterIP + if endpointType == service.EndpointPublic && svc.GetServiceType() == corev1.ServiceTypeClusterIP { + // Create the route + // TODO TLS + route, err := route.NewRoute( + route.GenericRoute(&route.GenericRouteDetails{ + Name: endpointName, + Namespace: helper.GetBeforeObject().GetNamespace(), + Labels: serviceLabels, + ServiceName: endpointName, + TargetPortName: endpointName, + }), + sd.GetExportLabels(), + time.Duration(5)*time.Second, + spec.Override.Route, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ExposeServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ExposeServiceReadyErrorMessage, + err.Error())) + + return ctrl.Result{}, err + } + + ctrlResult, err := route.CreateOrPatch(ctx, helper) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + // create route - end + + hostname = route.GetHostname() + } else { + hostname = svc.GetServiceHostnamePort() + } + + // create the final service override spec with + svcOverride := service.OverrideSpec{ + Endpoint: endpointType, + EmbeddedLabelsAnnotations: &service.EmbeddedLabelsAnnotations{ + Annotations: svc.GetAnnotations(), + Labels: svc.GetLabels(), + }, + Spec: svc.GetSpec(), + } + // Do not include data.Path in parsing check because %(project_id)s + // is invalid without being encoded, but they should not be encoded in the actual endpoint + apiEndpoint, err := url.Parse(endpoint.EndptProtocol(data.Protocol) + hostname) + if err != nil { + return ctrl.Result{}, err + } + + svcOverride.EndpointURL = pointer.String(apiEndpoint.String() + data.Path) + + serviceOverrides = append(serviceOverrides, svcOverride) + } + helper.GetLogger().Info("Reconciling KeystoneAPI", "KeystoneAPI.Namespace", instance.Namespace, "KeystoneAPI.Name", "keystone") op, err := controllerutil.CreateOrPatch(ctx, helper.GetClient(), keystoneAPI, func() error { instance.Spec.Keystone.Template.DeepCopyInto(&keystoneAPI.Spec) + keystoneAPI.Spec.Override.Service = serviceOverrides if keystoneAPI.Spec.Secret == "" { keystoneAPI.Spec.Secret = instance.Spec.Secret }