From b866397731e25b13290e1752162487b8a0e7d8cd Mon Sep 17 00:00:00 2001 From: Jaromir Wysoglad Date: Mon, 11 Sep 2023 03:59:52 -0400 Subject: [PATCH] Add aodh to autoscaling --- .pre-commit-config.yaml | 1 + .../telemetry.openstack.org_autoscalings.yaml | 117 ++++ .../telemetry.openstack.org_ceilometers.yaml | 10 + .../telemetry.openstack.org_telemetries.yaml | 108 +++ api/v1beta1/autoscaling_types.go | 116 ++++ api/v1beta1/autoscaling_webhook.go | 106 +++ api/v1beta1/telemetry_types.go | 32 +- api/v1beta1/telemetry_webhook.go | 30 +- api/v1beta1/zz_generated.deepcopy.go | 53 +- .../telemetry.openstack.org_autoscalings.yaml | 117 ++++ .../telemetry.openstack.org_ceilometers.yaml | 10 + .../telemetry.openstack.org_telemetries.yaml | 108 +++ config/default/manager_default_images.yaml | 8 + config/rbac/role.yaml | 60 ++ .../telemetry_v1beta1_autoscaling.yaml | 14 +- .../samples/telemetry_v1beta1_telemetry.yaml | 3 +- config/webhook/manifests.yaml | 40 ++ controllers/aodh_controller.go | 453 ++++++++++++ controllers/autoscaling_controller.go | 645 ++++++++++++++---- controllers/prometheus_controller.go | 160 +++++ go.mod | 4 +- go.sum | 4 + hack/clean_local_webhook.sh | 2 + hack/configure_local_webhook.sh | 56 ++ main.go | 11 + pkg/autoscaling/aodh_deployment.go | 181 +++++ pkg/autoscaling/const.go | 27 +- pkg/autoscaling/dbsync.go | 87 +++ pkg/autoscaling/prometheus.go | 16 + pkg/autoscaling/volumes.go | 78 +++ .../autoscaling/config/aodh-api-config.json | 23 + .../config/aodh-dbsync-config.json | 11 + .../config/aodh-evaluator-config.json | 17 + .../config/aodh-listener-config.json | 11 + .../config/aodh-notifier-config.json | 11 + templates/autoscaling/config/aodh.conf | 58 ++ templates/autoscaling/config/httpd.conf | 26 + templates/autoscaling/config/prometheus.yaml | 2 + templates/autoscaling/config/wsgi-aodh.conf | 19 + 39 files changed, 2664 insertions(+), 171 deletions(-) create mode 100644 api/v1beta1/autoscaling_webhook.go create mode 100644 controllers/aodh_controller.go create mode 100644 controllers/prometheus_controller.go create mode 100644 pkg/autoscaling/aodh_deployment.go create mode 100644 pkg/autoscaling/dbsync.go create mode 100644 pkg/autoscaling/volumes.go create mode 100644 templates/autoscaling/config/aodh-api-config.json create mode 100644 templates/autoscaling/config/aodh-dbsync-config.json create mode 100644 templates/autoscaling/config/aodh-evaluator-config.json create mode 100644 templates/autoscaling/config/aodh-listener-config.json create mode 100644 templates/autoscaling/config/aodh-notifier-config.json create mode 100644 templates/autoscaling/config/aodh.conf create mode 100644 templates/autoscaling/config/httpd.conf create mode 100644 templates/autoscaling/config/prometheus.yaml create mode 100644 templates/autoscaling/config/wsgi-aodh.conf diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4312239d..29793735 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,6 +41,7 @@ repos: - id: destroyed-symlinks - id: check-yaml args: [-m] + exclude: ^templates/autoscaling/config/prometheus.yaml - id: check-json - id: detect-private-key - id: end-of-file-fixer diff --git a/api/bases/telemetry.openstack.org_autoscalings.yaml b/api/bases/telemetry.openstack.org_autoscalings.yaml index c29786db..763a7817 100644 --- a/api/bases/telemetry.openstack.org_autoscalings.yaml +++ b/api/bases/telemetry.openstack.org_autoscalings.yaml @@ -35,6 +35,104 @@ spec: spec: description: AutoscalingSpec defines the desired state of Autoscaling properties: + aodh: + description: Aodh spec + properties: + apiImage: + type: string + customServiceConfig: + default: '# add your customization here' + description: CustomServiceConfig - customize the service config + using this parameter to change service defaults, or overwrite + rendered information using raw OpenStack config format. The + content gets added to to /etc//.conf.d directory + as custom.conf file. + type: string + databaseInstance: + description: MariaDB instance name Right now required by the maridb-operator + to get the credentials from the instance to create the DB Might + not be required in future + type: string + databaseUser: + default: aodh + description: Database user name Needed to connect to a database + used by aodh + type: string + defaultConfigOverwrite: + additionalProperties: + type: string + description: 'ConfigOverwrite - interface to overwrite default + config files like e.g. logging.conf or policy.json. But can + also be used to add additional files. Those get added to the + service config dir in /etc/ . TODO: -> implement' + type: object + evaluatorImage: + type: string + listenerImage: + type: string + memcachedInstance: + default: memcached + description: Memcached instance name. + type: string + networkAttachmentDefinitions: + description: NetworkAttachmentDefinitions list of network attachment + definitions the service pod gets attached to + items: + type: string + type: array + notifierImage: + type: string + passwordSelector: + default: + aodhService: AodhPassword + database: AodhDatabasePassword + description: PasswordSelectors - Selectors to identify the service + from the Secret + properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service + password from the Secret + type: string + database: + default: AodhDatabasePassword + description: Database - Selector to get the aodh database + user password from the Secret + type: string + service: + default: CeilometerPassword + description: Service - Selector to get the ceilometer service + password from the Secret + type: string + type: object + preserveJobs: + default: false + description: PreserveJobs - do not delete jobs after they finished + e.g. to check logs + type: boolean + rabbitMqClusterName: + default: rabbitmq + description: RabbitMQ instance name Needed to request a transportURL + that is created and used in Aodh + type: string + secret: + description: Secret containing OpenStack password information + for aodh + type: string + serviceUser: + default: aodh + description: ServiceUser - optional username used for this service + to register in keystone + type: string + required: + - apiImage + - databaseInstance + - evaluatorImage + - listenerImage + - memcachedInstance + - notifierImage + - secret + type: object enabled: default: false description: Allows enabling and disabling the autoscaling feature @@ -105,15 +203,34 @@ spec: - type type: object type: array + databaseHostname: + description: DatabaseHostname - Hostname for the database + type: string hash: additionalProperties: type: string description: Map of hashes to track e.g. job status type: object + networks: + description: Networks in addtion to the cluster network, the service + is attached to + items: + type: string + type: array + prometheusHostname: + description: PrometheusHost - Hostname for prometheus used for autoscaling + type: string + prometheusPort: + description: PrometheusPort - Port for prometheus used for autoscaling + format: int32 + type: integer readyCount: description: ReadyCount of autoscaling instances format: int32 type: integer + transportURLSecret: + description: TransportURLSecret - Secret containing RabbitMQ transportURL + type: string type: object type: object served: true diff --git a/api/bases/telemetry.openstack.org_ceilometers.yaml b/api/bases/telemetry.openstack.org_ceilometers.yaml index 42ca0042..5fd3df66 100644 --- a/api/bases/telemetry.openstack.org_ceilometers.yaml +++ b/api/bases/telemetry.openstack.org_ceilometers.yaml @@ -73,6 +73,16 @@ spec: description: PasswordSelectors - Selectors to identify the service from the Secret properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service password + from the Secret + type: string + database: + default: AodhDatabasePassword + description: Database - Selector to get the aodh database user + password from the Secret + type: string service: default: CeilometerPassword description: Service - Selector to get the ceilometer service diff --git a/api/bases/telemetry.openstack.org_telemetries.yaml b/api/bases/telemetry.openstack.org_telemetries.yaml index 5d94ebda..662c8ebf 100644 --- a/api/bases/telemetry.openstack.org_telemetries.yaml +++ b/api/bases/telemetry.openstack.org_telemetries.yaml @@ -39,6 +39,104 @@ spec: description: Autoscaling - Spec definition for the Autoscaling service of this Telemetry deployment properties: + aodh: + description: Aodh spec + properties: + apiImage: + type: string + customServiceConfig: + default: '# add your customization here' + description: CustomServiceConfig - customize the service config + using this parameter to change service defaults, or overwrite + rendered information using raw OpenStack config format. + The content gets added to to /etc//.conf.d + directory as custom.conf file. + type: string + databaseInstance: + description: MariaDB instance name Right now required by the + maridb-operator to get the credentials from the instance + to create the DB Might not be required in future + type: string + databaseUser: + default: aodh + description: Database user name Needed to connect to a database + used by aodh + type: string + defaultConfigOverwrite: + additionalProperties: + type: string + description: 'ConfigOverwrite - interface to overwrite default + config files like e.g. logging.conf or policy.json. But + can also be used to add additional files. Those get added + to the service config dir in /etc/ . TODO: -> implement' + type: object + evaluatorImage: + type: string + listenerImage: + type: string + memcachedInstance: + default: memcached + description: Memcached instance name. + type: string + networkAttachmentDefinitions: + description: NetworkAttachmentDefinitions list of network + attachment definitions the service pod gets attached to + items: + type: string + type: array + notifierImage: + type: string + passwordSelector: + default: + aodhService: AodhPassword + database: AodhDatabasePassword + description: PasswordSelectors - Selectors to identify the + service from the Secret + properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service + password from the Secret + type: string + database: + default: AodhDatabasePassword + description: Database - Selector to get the aodh database + user password from the Secret + type: string + service: + default: CeilometerPassword + description: Service - Selector to get the ceilometer + service password from the Secret + type: string + type: object + preserveJobs: + default: false + description: PreserveJobs - do not delete jobs after they + finished e.g. to check logs + type: boolean + rabbitMqClusterName: + default: rabbitmq + description: RabbitMQ instance name Needed to request a transportURL + that is created and used in Aodh + type: string + secret: + description: Secret containing OpenStack password information + for aodh + type: string + serviceUser: + default: aodh + description: ServiceUser - optional username used for this + service to register in keystone + type: string + required: + - apiImage + - databaseInstance + - evaluatorImage + - listenerImage + - memcachedInstance + - notifierImage + - secret + type: object enabled: default: false description: Allows enabling and disabling the autoscaling feature @@ -105,6 +203,16 @@ spec: description: PasswordSelectors - Selectors to identify the service from the Secret properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service + password from the Secret + type: string + database: + default: AodhDatabasePassword + description: Database - Selector to get the aodh database + user password from the Secret + type: string service: default: CeilometerPassword description: Service - Selector to get the ceilometer service diff --git a/api/v1beta1/autoscaling_types.go b/api/v1beta1/autoscaling_types.go index be7603fc..0b2a187b 100644 --- a/api/v1beta1/autoscaling_types.go +++ b/api/v1beta1/autoscaling_types.go @@ -19,6 +19,21 @@ package v1beta1 import ( condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/openstack-k8s-operators/lib-common/modules/common/util" +) + +const ( + // AodhAPIContainerImage - default fall-back image for Aodh API + AodhAPIContainerImage = "quay.io/podified-antelope-centos9/openstack-aodh-api:current-podified" + // AodhEvaluatorContainerImage - default fall-back image for Aodh Evaluator + AodhEvaluatorContainerImage = "quay.io/podified-antelope-centos9/openstack-aodh-evaluator:current-podified" + // AodhNotifierContainerImage - default fall-back image for Aodh Notifier + AodhNotifierContainerImage = "quay.io/podified-antelope-centos9/openstack-aodh-notifier:current-podified" + // AodhListenerContainerImage - default fall-back image for Aodh Listener + AodhListenerContainerImage = "quay.io/podified-antelope-centos9/openstack-aodh-listener:current-podified" + // DbSyncHash hash + DbSyncHash = "dbsync" ) // Prometheus defines which prometheus to use for Autoscaling @@ -36,11 +51,84 @@ type Prometheus struct { Port int32 `json:"port,omitempty"` } +// Aodh defines the aodh component spec +type Aodh struct { + // RabbitMQ instance name + // Needed to request a transportURL that is created and used in Aodh + // +kubebuilder:default=rabbitmq + RabbitMqClusterName string `json:"rabbitMqClusterName,omitempty"` + + // +kubebuilder:validation:Required + // MariaDB instance name + // Right now required by the maridb-operator to get the credentials from the instance to create the DB + // Might not be required in future + DatabaseInstance string `json:"databaseInstance"` + + // Database user name + // Needed to connect to a database used by aodh + // +kubebuilder:default=aodh + DatabaseUser string `json:"databaseUser,omitempty"` + + // PasswordSelectors - Selectors to identify the service from the Secret + // +kubebuilder:default:={aodhService: AodhPassword, database: AodhDatabasePassword} + PasswordSelectors PasswordsSelector `json:"passwordSelector,omitempty"` + + // ServiceUser - optional username used for this service to register in keystone + // +kubebuilder:validation:Optional + // +kubebuilder:default=aodh + ServiceUser string `json:"serviceUser"` + + // Secret containing OpenStack password information for aodh + // +kubebuilder:validation:Required + Secret string `json:"secret"` + + // CustomServiceConfig - customize the service config using this parameter to change service defaults, + // or overwrite rendered information using raw OpenStack config format. The content gets added to + // to /etc//.conf.d directory as custom.conf file. + // +kubebuilder:default:="# add your customization here" + CustomServiceConfig string `json:"customServiceConfig,omitempty"` + + // ConfigOverwrite - interface to overwrite default config files like e.g. logging.conf or policy.json. + // But can also be used to add additional files. Those get added to the service config dir in /etc/ . + // TODO: -> implement + // +kubebuilder:validation:Optional + DefaultConfigOverwrite map[string]string `json:"defaultConfigOverwrite,omitempty"` + + // NetworkAttachmentDefinitions list of network attachment definitions the service pod gets attached to + // +kubebuilder:validation:Optional + NetworkAttachmentDefinitions []string `json:"networkAttachmentDefinitions,omitempty"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=false + // PreserveJobs - do not delete jobs after they finished e.g. to check logs + PreserveJobs bool `json:"preserveJobs"` + + // +kubebuilder:validation:Required + // +kubebuilder:default=memcached + // Memcached instance name. + MemcachedInstance string `json:"memcachedInstance"` + + // +kubebuilder:validation:Required + APIImage string `json:"apiImage"` + + // +kubebuilder:validation:Required + EvaluatorImage string `json:"evaluatorImage"` + + // +kubebuilder:validation:Required + NotifierImage string `json:"notifierImage"` + + // +kubebuilder:validation:Required + ListenerImage string `json:"listenerImage"` +} + // AutoscalingSpec defines the desired state of Autoscaling type AutoscalingSpec struct { // Specification of which prometheus to use for autoscaling Prometheus Prometheus `json:"prometheus,omitempty"` + // Aodh spec + Aodh Aodh `json:"aodh,omitempty"` + // Allows enabling and disabling the autoscaling feature // +kubebuilder:default=false Enabled bool `json:"enabled,omitempty"` @@ -56,6 +144,21 @@ type AutoscalingStatus struct { // Conditions Conditions condition.Conditions `json:"conditions,omitempty" optional:"true"` + + // Networks in addtion to the cluster network, the service is attached to + Networks []string `json:"networks,omitempty"` + + // TransportURLSecret - Secret containing RabbitMQ transportURL + TransportURLSecret string `json:"transportURLSecret,omitempty"` + + // DatabaseHostname - Hostname for the database + DatabaseHostname string `json:"databaseHostname,omitempty"` + + // PrometheusHost - Hostname for prometheus used for autoscaling + PrometheusHost string `json:"prometheusHostname,omitempty"` + + // PrometheusPort - Port for prometheus used for autoscaling + PrometheusPort int32 `json:"prometheusPort,omitempty"` } //+kubebuilder:object:root=true @@ -102,3 +205,16 @@ func (instance Autoscaling) RbacNamespace() string { func (instance Autoscaling) RbacResourceName() string { return "telemetry-" + instance.Name } + +// SetupDefaultsAutoscaling - initializes any CRD field defaults based on environment variables (the defaulting mechanism itself is implemented via webhooks) +func SetupDefaultsAutoscaling() { + // Acquire environmental defaults and initialize Telemetry defaults with them + autoscalingDefaults := AutoscalingDefaults{ + AodhAPIContainerImageURL: util.GetEnvVar("RELATED_IMAGE_AODH_API_IMAGE_URL_DEFAULT", AodhAPIContainerImage), + AodhEvaluatorContainerImageURL: util.GetEnvVar("RELATED_IMAGE_AODH_EVALUATOR_IMAGE_URL_DEFAULT", AodhEvaluatorContainerImage), + AodhNotifierContainerImageURL: util.GetEnvVar("RELATED_IMAGE_AODH_NOTIFIER_IMAGE_URL_DEFAULT", AodhNotifierContainerImage), + AodhListenerContainerImageURL: util.GetEnvVar("RELATED_IMAGE_AODH_LISTENER_IMAGE_URL_DEFAULT", AodhListenerContainerImage), + } + + SetupAutoscalingDefaults(autoscalingDefaults) +} diff --git a/api/v1beta1/autoscaling_webhook.go b/api/v1beta1/autoscaling_webhook.go new file mode 100644 index 00000000..e5ce5b36 --- /dev/null +++ b/api/v1beta1/autoscaling_webhook.go @@ -0,0 +1,106 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +// AutoscalingDefaults - +type AutoscalingDefaults struct { + AodhAPIContainerImageURL string + AodhEvaluatorContainerImageURL string + AodhNotifierContainerImageURL string + AodhListenerContainerImageURL string +} + +var autoscalingDefaults AutoscalingDefaults + +// log is for logging in this package. +var autoscalinglog = logf.Log.WithName("autoscaling-resource") + +// SetupAutoscalingDefaults - initialize Autoscaling spec defaults for use with either internal or external webhooks +func SetupAutoscalingDefaults(defaults AutoscalingDefaults) { + autoscalingDefaults = defaults + autoscalinglog.Info("Autoscaling defaults initialized", "defaults", defaults) +} + +// SetupWebhookWithManager - setups webhook with the adequate manager +func (r *Autoscaling) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(r). + Complete() +} + +//+kubebuilder:webhook:path=/mutate-telemetry-openstack-org-v1beta1-autoscaling,mutating=true,failurePolicy=fail,sideEffects=None,groups=telemetry.openstack.org,resources=autoscalings,verbs=create;update,versions=v1beta1,name=mautoscaling.kb.io,admissionReviewVersions=v1 + +var _ webhook.Defaulter = &Autoscaling{} + +// Default implements webhook.Defaulter so a webhook will be registered for the type +func (r *Autoscaling) Default() { + autoscalinglog.Info("default", "name", r.Name) + + r.Spec.Default() +} + +// Default - set defaults for this Autoscaling spec +func (spec *AutoscalingSpec) Default() { + if spec.Aodh.APIImage == "" { + spec.Aodh.APIImage = autoscalingDefaults.AodhAPIContainerImageURL + } + if spec.Aodh.EvaluatorImage == "" { + spec.Aodh.EvaluatorImage = autoscalingDefaults.AodhEvaluatorContainerImageURL + } + if spec.Aodh.NotifierImage == "" { + spec.Aodh.NotifierImage = autoscalingDefaults.AodhNotifierContainerImageURL + } + if spec.Aodh.ListenerImage == "" { + spec.Aodh.ListenerImage = autoscalingDefaults.AodhListenerContainerImageURL + } +} + +// TODO(user): change verbs to "verbs=create;update;delete" if you want to enable deletion validation. +//+kubebuilder:webhook:path=/validate-telemetry-openstack-org-v1beta1-autoscaling,mutating=false,failurePolicy=fail,sideEffects=None,groups=telemetry.openstack.org,resources=autoscalings,verbs=create;update,versions=v1beta1,name=vautoscaling.kb.io,admissionReviewVersions=v1 + +var _ webhook.Validator = &Autoscaling{} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type +func (r *Autoscaling) ValidateCreate() error { + autoscalinglog.Info("validate create", "name", r.Name) + + // TODO(user): fill in your validation logic upon object creation. + return nil +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type +func (r *Autoscaling) ValidateUpdate(old runtime.Object) error { + autoscalinglog.Info("validate update", "name", r.Name) + + // TODO(user): fill in your validation logic upon object update. + return nil +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type +func (r *Autoscaling) ValidateDelete() error { + autoscalinglog.Info("validate delete", "name", r.Name) + + // TODO(user): fill in your validation logic upon object deletion. + return nil +} diff --git a/api/v1beta1/telemetry_types.go b/api/v1beta1/telemetry_types.go index 65ad31c0..b58679ab 100644 --- a/api/v1beta1/telemetry_types.go +++ b/api/v1beta1/telemetry_types.go @@ -23,12 +23,24 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/util" ) +// TODO: We might want to split this to aodh and ceilometer and move it to appropriate files + // PasswordsSelector to identify the Service password from the Secret type PasswordsSelector struct { // Service - Selector to get the ceilometer service password from the Secret // +kubebuilder:validation:Optional // +kubebuilder:default:=CeilometerPassword Service string `json:"service"` + + // AodhService - Selector to get the aodh service password from the Secret + // +kubebuilder:validation:Optional + // +kubebuilder:default:=AodhPassword + AodhService string `json:"aodhService"` + + // Database - Selector to get the aodh database user password from the Secret + // +kubebuilder:validation:Optional + // +kubebuilder:default:=AodhDatabasePassword + Database string `json:"database"` } // TelemetrySpec defines the desired state of Telemetry @@ -91,12 +103,20 @@ func init() { func SetupDefaultsTelemetry() { // Acquire environmental defaults and initialize Telemetry defaults with them telemetryDefaults := TelemetryDefaults{ - CentralContainerImageURL: util.GetEnvVar("RELATED_IMAGE_CEILOMETER_CENTRAL_IMAGE_URL_DEFAULT", CeilometerCentralContainerImage), - ComputeContainerImageURL: util.GetEnvVar("RELATED_IMAGE_CEILOMETER_COMPUTE_IMAGE_URL_DEFAULT", CeilometerComputeContainerImage), - IpmiContainerImageURL: util.GetEnvVar("RELATED_IMAGE_CEILOMETER_IPMI_IMAGE_URL_DEFAULT", CeilometerIpmiContainerImage), - NotificationContainerImageURL: util.GetEnvVar("RELATED_IMAGE_CEILOMETER_NOTIFICATION_IMAGE_URL_DEFAULT", CeilometerNotificationContainerImage), - NodeExporterContainerImageURL: util.GetEnvVar("RELATED_IMAGE_TELEMETRY_NODE_EXPORTER_IMAGE_URL_DEFAULT", NodeExporterContainerImage), - SgCoreContainerImageURL: util.GetEnvVar("RELATED_IMAGE_CEILOMETER_SGCORE_IMAGE_URL_DEFAULT", CeilometerSgCoreContainerImage), + CentralContainerImageURL: util.GetEnvVar("RELATED_IMAGE_CEILOMETER_CENTRAL_IMAGE_URL_DEFAULT", CeilometerCentralContainerImage), + ComputeContainerImageURL: util.GetEnvVar("RELATED_IMAGE_CEILOMETER_COMPUTE_IMAGE_URL_DEFAULT", CeilometerComputeContainerImage), + IpmiContainerImageURL: util.GetEnvVar("RELATED_IMAGE_CEILOMETER_IPMI_IMAGE_URL_DEFAULT", CeilometerIpmiContainerImage), + NotificationContainerImageURL: util.GetEnvVar("RELATED_IMAGE_CEILOMETER_NOTIFICATION_IMAGE_URL_DEFAULT", CeilometerNotificationContainerImage), + NodeExporterContainerImageURL: util.GetEnvVar("RELATED_IMAGE_TELEMETRY_NODE_EXPORTER_IMAGE_URL_DEFAULT", NodeExporterContainerImage), + SgCoreContainerImageURL: util.GetEnvVar("RELATED_IMAGE_CEILOMETER_SGCORE_IMAGE_URL_DEFAULT", CeilometerSgCoreContainerImage), + + // Autoscaling + AodhAPIContainerImageURL: util.GetEnvVar("RELATED_IMAGE_AODH_API_IMAGE_URL_DEFAULT", AodhAPIContainerImage), + AodhEvaluatorContainerImageURL: util.GetEnvVar("RELATED_IMAGE_AODH_EVALUATOR_IMAGE_URL_DEFAULT", AodhEvaluatorContainerImage), + AodhNotifierContainerImageURL: util.GetEnvVar("RELATED_IMAGE_AODH_NOTIFIER_IMAGE_URL_DEFAULT", AodhNotifierContainerImage), + AodhListenerContainerImageURL: util.GetEnvVar("RELATED_IMAGE_AODH_LISTENER_IMAGE_URL_DEFAULT", AodhListenerContainerImage), + AodhInitContainerImageURL: util.GetEnvVar("RELATED_IMAGE_AODH_API_IMAGE_URL_DEFAULT", AodhAPIContainerImage), + } SetupTelemetryDefaults(telemetryDefaults) diff --git a/api/v1beta1/telemetry_webhook.go b/api/v1beta1/telemetry_webhook.go index 76c8faee..a92f6bdf 100644 --- a/api/v1beta1/telemetry_webhook.go +++ b/api/v1beta1/telemetry_webhook.go @@ -28,12 +28,17 @@ import ( // TelemetryDefaults - type TelemetryDefaults struct { - CentralContainerImageURL string - ComputeContainerImageURL string - NotificationContainerImageURL string - SgCoreContainerImageURL string - NodeExporterContainerImageURL string - IpmiContainerImageURL string + CentralContainerImageURL string + ComputeContainerImageURL string + NotificationContainerImageURL string + SgCoreContainerImageURL string + NodeExporterContainerImageURL string + IpmiContainerImageURL string + AodhAPIContainerImageURL string + AodhEvaluatorContainerImageURL string + AodhNotifierContainerImageURL string + AodhListenerContainerImageURL string + AodhInitContainerImageURL string } var telemetryDefaults TelemetryDefaults @@ -85,6 +90,19 @@ func (spec *TelemetrySpec) Default() { if spec.Ceilometer.NodeExporterImage == "" { spec.Ceilometer.NodeExporterImage = telemetryDefaults.NodeExporterContainerImageURL } + if spec.Autoscaling.Aodh.APIImage == "" { + spec.Autoscaling.Aodh.APIImage = telemetryDefaults.AodhAPIContainerImageURL + } + if spec.Autoscaling.Aodh.EvaluatorImage == "" { + spec.Autoscaling.Aodh.EvaluatorImage = telemetryDefaults.AodhEvaluatorContainerImageURL + } + if spec.Autoscaling.Aodh.NotifierImage == "" { + spec.Autoscaling.Aodh.NotifierImage = telemetryDefaults.AodhNotifierContainerImageURL + } + if spec.Autoscaling.Aodh.ListenerImage == "" { + spec.Autoscaling.Aodh.ListenerImage = telemetryDefaults.AodhListenerContainerImageURL + } + } // TODO(user): change verbs to "verbs=create;update;delete" if you want to enable deletion validation. diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 2f74d4af..f72a6630 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -26,12 +26,40 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Aodh) DeepCopyInto(out *Aodh) { + *out = *in + out.PasswordSelectors = in.PasswordSelectors + if in.DefaultConfigOverwrite != nil { + in, out := &in.DefaultConfigOverwrite, &out.DefaultConfigOverwrite + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.NetworkAttachmentDefinitions != nil { + in, out := &in.NetworkAttachmentDefinitions, &out.NetworkAttachmentDefinitions + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Aodh. +func (in *Aodh) DeepCopy() *Aodh { + if in == nil { + return nil + } + out := new(Aodh) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Autoscaling) DeepCopyInto(out *Autoscaling) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } @@ -53,6 +81,21 @@ func (in *Autoscaling) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AutoscalingDefaults) DeepCopyInto(out *AutoscalingDefaults) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AutoscalingDefaults. +func (in *AutoscalingDefaults) DeepCopy() *AutoscalingDefaults { + if in == nil { + return nil + } + out := new(AutoscalingDefaults) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AutoscalingList) DeepCopyInto(out *AutoscalingList) { *out = *in @@ -89,6 +132,7 @@ func (in *AutoscalingList) DeepCopyObject() runtime.Object { func (in *AutoscalingSpec) DeepCopyInto(out *AutoscalingSpec) { *out = *in out.Prometheus = in.Prometheus + in.Aodh.DeepCopyInto(&out.Aodh) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AutoscalingSpec. @@ -118,6 +162,11 @@ func (in *AutoscalingStatus) DeepCopyInto(out *AutoscalingStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Networks != nil { + in, out := &in.Networks, &out.Networks + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AutoscalingStatus. @@ -373,7 +422,7 @@ func (in *TelemetryList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TelemetrySpec) DeepCopyInto(out *TelemetrySpec) { *out = *in - out.Autoscaling = in.Autoscaling + in.Autoscaling.DeepCopyInto(&out.Autoscaling) in.Ceilometer.DeepCopyInto(&out.Ceilometer) } diff --git a/config/crd/bases/telemetry.openstack.org_autoscalings.yaml b/config/crd/bases/telemetry.openstack.org_autoscalings.yaml index c29786db..763a7817 100644 --- a/config/crd/bases/telemetry.openstack.org_autoscalings.yaml +++ b/config/crd/bases/telemetry.openstack.org_autoscalings.yaml @@ -35,6 +35,104 @@ spec: spec: description: AutoscalingSpec defines the desired state of Autoscaling properties: + aodh: + description: Aodh spec + properties: + apiImage: + type: string + customServiceConfig: + default: '# add your customization here' + description: CustomServiceConfig - customize the service config + using this parameter to change service defaults, or overwrite + rendered information using raw OpenStack config format. The + content gets added to to /etc//.conf.d directory + as custom.conf file. + type: string + databaseInstance: + description: MariaDB instance name Right now required by the maridb-operator + to get the credentials from the instance to create the DB Might + not be required in future + type: string + databaseUser: + default: aodh + description: Database user name Needed to connect to a database + used by aodh + type: string + defaultConfigOverwrite: + additionalProperties: + type: string + description: 'ConfigOverwrite - interface to overwrite default + config files like e.g. logging.conf or policy.json. But can + also be used to add additional files. Those get added to the + service config dir in /etc/ . TODO: -> implement' + type: object + evaluatorImage: + type: string + listenerImage: + type: string + memcachedInstance: + default: memcached + description: Memcached instance name. + type: string + networkAttachmentDefinitions: + description: NetworkAttachmentDefinitions list of network attachment + definitions the service pod gets attached to + items: + type: string + type: array + notifierImage: + type: string + passwordSelector: + default: + aodhService: AodhPassword + database: AodhDatabasePassword + description: PasswordSelectors - Selectors to identify the service + from the Secret + properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service + password from the Secret + type: string + database: + default: AodhDatabasePassword + description: Database - Selector to get the aodh database + user password from the Secret + type: string + service: + default: CeilometerPassword + description: Service - Selector to get the ceilometer service + password from the Secret + type: string + type: object + preserveJobs: + default: false + description: PreserveJobs - do not delete jobs after they finished + e.g. to check logs + type: boolean + rabbitMqClusterName: + default: rabbitmq + description: RabbitMQ instance name Needed to request a transportURL + that is created and used in Aodh + type: string + secret: + description: Secret containing OpenStack password information + for aodh + type: string + serviceUser: + default: aodh + description: ServiceUser - optional username used for this service + to register in keystone + type: string + required: + - apiImage + - databaseInstance + - evaluatorImage + - listenerImage + - memcachedInstance + - notifierImage + - secret + type: object enabled: default: false description: Allows enabling and disabling the autoscaling feature @@ -105,15 +203,34 @@ spec: - type type: object type: array + databaseHostname: + description: DatabaseHostname - Hostname for the database + type: string hash: additionalProperties: type: string description: Map of hashes to track e.g. job status type: object + networks: + description: Networks in addtion to the cluster network, the service + is attached to + items: + type: string + type: array + prometheusHostname: + description: PrometheusHost - Hostname for prometheus used for autoscaling + type: string + prometheusPort: + description: PrometheusPort - Port for prometheus used for autoscaling + format: int32 + type: integer readyCount: description: ReadyCount of autoscaling instances format: int32 type: integer + transportURLSecret: + description: TransportURLSecret - Secret containing RabbitMQ transportURL + type: string type: object type: object served: true diff --git a/config/crd/bases/telemetry.openstack.org_ceilometers.yaml b/config/crd/bases/telemetry.openstack.org_ceilometers.yaml index 42ca0042..5fd3df66 100644 --- a/config/crd/bases/telemetry.openstack.org_ceilometers.yaml +++ b/config/crd/bases/telemetry.openstack.org_ceilometers.yaml @@ -73,6 +73,16 @@ spec: description: PasswordSelectors - Selectors to identify the service from the Secret properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service password + from the Secret + type: string + database: + default: AodhDatabasePassword + description: Database - Selector to get the aodh database user + password from the Secret + type: string service: default: CeilometerPassword description: Service - Selector to get the ceilometer service diff --git a/config/crd/bases/telemetry.openstack.org_telemetries.yaml b/config/crd/bases/telemetry.openstack.org_telemetries.yaml index 5d94ebda..662c8ebf 100644 --- a/config/crd/bases/telemetry.openstack.org_telemetries.yaml +++ b/config/crd/bases/telemetry.openstack.org_telemetries.yaml @@ -39,6 +39,104 @@ spec: description: Autoscaling - Spec definition for the Autoscaling service of this Telemetry deployment properties: + aodh: + description: Aodh spec + properties: + apiImage: + type: string + customServiceConfig: + default: '# add your customization here' + description: CustomServiceConfig - customize the service config + using this parameter to change service defaults, or overwrite + rendered information using raw OpenStack config format. + The content gets added to to /etc//.conf.d + directory as custom.conf file. + type: string + databaseInstance: + description: MariaDB instance name Right now required by the + maridb-operator to get the credentials from the instance + to create the DB Might not be required in future + type: string + databaseUser: + default: aodh + description: Database user name Needed to connect to a database + used by aodh + type: string + defaultConfigOverwrite: + additionalProperties: + type: string + description: 'ConfigOverwrite - interface to overwrite default + config files like e.g. logging.conf or policy.json. But + can also be used to add additional files. Those get added + to the service config dir in /etc/ . TODO: -> implement' + type: object + evaluatorImage: + type: string + listenerImage: + type: string + memcachedInstance: + default: memcached + description: Memcached instance name. + type: string + networkAttachmentDefinitions: + description: NetworkAttachmentDefinitions list of network + attachment definitions the service pod gets attached to + items: + type: string + type: array + notifierImage: + type: string + passwordSelector: + default: + aodhService: AodhPassword + database: AodhDatabasePassword + description: PasswordSelectors - Selectors to identify the + service from the Secret + properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service + password from the Secret + type: string + database: + default: AodhDatabasePassword + description: Database - Selector to get the aodh database + user password from the Secret + type: string + service: + default: CeilometerPassword + description: Service - Selector to get the ceilometer + service password from the Secret + type: string + type: object + preserveJobs: + default: false + description: PreserveJobs - do not delete jobs after they + finished e.g. to check logs + type: boolean + rabbitMqClusterName: + default: rabbitmq + description: RabbitMQ instance name Needed to request a transportURL + that is created and used in Aodh + type: string + secret: + description: Secret containing OpenStack password information + for aodh + type: string + serviceUser: + default: aodh + description: ServiceUser - optional username used for this + service to register in keystone + type: string + required: + - apiImage + - databaseInstance + - evaluatorImage + - listenerImage + - memcachedInstance + - notifierImage + - secret + type: object enabled: default: false description: Allows enabling and disabling the autoscaling feature @@ -105,6 +203,16 @@ spec: description: PasswordSelectors - Selectors to identify the service from the Secret properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service + password from the Secret + type: string + database: + default: AodhDatabasePassword + description: Database - Selector to get the aodh database + user password from the Secret + type: string service: default: CeilometerPassword description: Service - Selector to get the ceilometer service diff --git a/config/default/manager_default_images.yaml b/config/default/manager_default_images.yaml index fe5933e3..48c42516 100644 --- a/config/default/manager_default_images.yaml +++ b/config/default/manager_default_images.yaml @@ -23,3 +23,11 @@ spec: value: quay.io/infrawatch/sg-core:latest - name: RELATED_IMAGE_TELEMETRY_NODE_EXPORTER_IMAGE_URL_DEFAULT value: quay.io/prometheus/node-exporter:v1.5.0 + - name: RELATED_IMAGE_AODH_API_IMAGE_URL_DEFAULT + value: quay.io/podified-antelope-centos9/openstack-aodh-api:current-podified + - name: RELATED_IMAGE_AODH_EVALUATOR_IMAGE_URL_DEFAULT + value: quay.io/podified-antelope-centos9/openstack-aodh-evaluator:current-podified + - name: RELATED_IMAGE_AODH_NOTIFIER_IMAGE_URL_DEFAULT + value: quay.io/podified-antelope-centos9/openstack-aodh-notifier:current-podified + - name: RELATED_IMAGE_AODH_LISTENER_IMAGE_URL_DEFAULT + value: quay.io/podified-antelope-centos9/openstack-aodh-listener:current-podified diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 6eb262a6..dc2a330c 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -34,6 +34,18 @@ rules: - patch - update - watch +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - "" resources: @@ -90,6 +102,18 @@ rules: - get - list - watch +- apiGroups: + - keystone.openstack.org + resources: + - keystoneendpoints + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - keystone.openstack.org resources: @@ -102,13 +126,37 @@ rules: - patch - update - watch +- apiGroups: + - mariadb.openstack.org + resources: + - mariadbdatabases + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - memcached.openstack.org + resources: + - memcacheds + verbs: + - get + - list + - watch - apiGroups: - monitoring.rhobs resources: - monitoringstacks verbs: + - create + - delete - get - list + - patch + - update - watch - apiGroups: - rabbitmq.openstack.org @@ -142,6 +190,18 @@ rules: - list - update - watch +- apiGroups: + - route.openshift.io + resources: + - routes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - security.openshift.io resourceNames: diff --git a/config/samples/telemetry_v1beta1_autoscaling.yaml b/config/samples/telemetry_v1beta1_autoscaling.yaml index 7ae76c3d..c4988c7f 100644 --- a/config/samples/telemetry_v1beta1_autoscaling.yaml +++ b/config/samples/telemetry_v1beta1_autoscaling.yaml @@ -7,8 +7,18 @@ metadata: app.kubernetes.io/part-of: telemetry-operator app.kubernetes.io/managed-by: kustomize app.kubernetes.io/created-by: telemetry-operator - name: autoscaling-sample + name: autoscaling spec: enabled: true prometheus: - deployPrometheus: true + deployPrometheus: false + aodh: + secret: osp-secret + apiImage: "quay.io/podified-antelope-centos9/openstack-aodh-api:current-podified" + evaluatorImage: "quay.io/podified-antelope-centos9/openstack-aodh-evaluator:current-podified" + notifierImage: "quay.io/podified-antelope-centos9/openstack-aodh-notifier:current-podified" + listenerImage: "quay.io/podified-antelope-centos9/openstack-aodh-listener:current-podified" + passwordSelectors: + databaseUser: aodh + databaseInstance: openstack + memcachedInstance: memcached diff --git a/config/samples/telemetry_v1beta1_telemetry.yaml b/config/samples/telemetry_v1beta1_telemetry.yaml index db6161ef..d2151095 100644 --- a/config/samples/telemetry_v1beta1_telemetry.yaml +++ b/config/samples/telemetry_v1beta1_telemetry.yaml @@ -5,7 +5,6 @@ metadata: namespace: openstack spec: ceilometer: - initImage: quay.io/podified-antelope-centos9/openstack-ceilometer-central:current-podified centralImage: quay.io/podified-antelope-centos9/openstack-ceilometer-central:current-podified notificationImage: quay.io/podified-antelope-centos9/openstack-ceilometer-notification:current-podified sgCoreImage: quay.io/infrawatch/sg-core:latest @@ -15,4 +14,4 @@ spec: autoscaling: enabled: false prometheus: - deployPrometheus: true + deployPrometheus: false diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index 066130e5..ac5b8531 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -5,6 +5,26 @@ metadata: creationTimestamp: null name: mutating-webhook-configuration webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-telemetry-openstack-org-v1beta1-autoscaling + failurePolicy: Fail + name: mautoscaling.kb.io + rules: + - apiGroups: + - telemetry.openstack.org + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - autoscalings + sideEffects: None - admissionReviewVersions: - v1 clientConfig: @@ -52,6 +72,26 @@ metadata: creationTimestamp: null name: validating-webhook-configuration webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-telemetry-openstack-org-v1beta1-autoscaling + failurePolicy: Fail + name: vautoscaling.kb.io + rules: + - apiGroups: + - telemetry.openstack.org + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - autoscalings + sideEffects: None - admissionReviewVersions: - v1 clientConfig: diff --git a/controllers/aodh_controller.go b/controllers/aodh_controller.go new file mode 100644 index 00000000..866b57aa --- /dev/null +++ b/controllers/aodh_controller.go @@ -0,0 +1,453 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + "time" + + k8s_errors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + common "github.com/openstack-k8s-operators/lib-common/modules/common" + condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + deployment "github.com/openstack-k8s-operators/lib-common/modules/common/deployment" + endpoint "github.com/openstack-k8s-operators/lib-common/modules/common/endpoint" + helper "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + job "github.com/openstack-k8s-operators/lib-common/modules/common/job" + secret "github.com/openstack-k8s-operators/lib-common/modules/common/secret" + service "github.com/openstack-k8s-operators/lib-common/modules/common/service" + util "github.com/openstack-k8s-operators/lib-common/modules/common/util" + database "github.com/openstack-k8s-operators/lib-common/modules/database" + + routev1 "github.com/openshift/api/route/v1" + keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" + telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" + autoscaling "github.com/openstack-k8s-operators/telemetry-operator/pkg/autoscaling" +) + +func (r *AutoscalingReconciler) reconcileDisabledAodh( + ctx context.Context, + instance *telemetryv1.Autoscaling, + helper *helper.Helper, +) (ctrl.Result, error) { + r.Log.Info("Reconciling Service Aodh disabled") + serviceLabels := map[string]string{ + common.AppSelector: autoscaling.ServiceName, + } + + // run the Delete to remove all finalizers + ctrlResult, err := r.reconcileDeleteAodh(ctx, instance, helper) + if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + if err != nil { + return ctrl.Result{}, err + } + + // We are disabling autoscaling, not deleting it. We need to + // manually delete all autoscaling resources, + // that were created before + + // Delete deployment + depl, err := autoscaling.AodhDeployment(instance, "", serviceLabels) + if err != nil { + return ctrl.Result{}, err + } + err = r.Client.Delete(ctx, depl) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + + // Delete db + db, err := database.GetDatabaseByName(ctx, helper, autoscaling.ServiceName) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + if !k8s_errors.IsNotFound(err) { + mariadbdatabase := db.GetDatabase() + err = r.Client.Delete(ctx, mariadbdatabase) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + } + + // Delete keystone service + keystoneService, err := keystonev1.GetKeystoneServiceWithName(ctx, helper, autoscaling.ServiceName, instance.Namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + if !k8s_errors.IsNotFound(err) { + err = r.Client.Delete(ctx, keystoneService) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + } + + // Delete keystone endpoint + keystoneEndpoint, err := keystonev1.GetKeystoneEndpointWithName(ctx, helper, autoscaling.ServiceName, instance.Namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + if !k8s_errors.IsNotFound(err) { + err = r.Client.Delete(ctx, keystoneEndpoint) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + } + + // Delete services + svcs, err := service.GetServicesListWithLabel(ctx, helper, instance.Namespace, serviceLabels) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + if !k8s_errors.IsNotFound(err) { + for _, svc := range svcs.Items { + // NOTE: Using r.Client.Delete() to delete a service ends with: + // "k8s.io/api/core/v1".Service does not implement + // client.Object (DeepCopyObject method has pointer receiver) + serviceClient := helper.GetKClient().CoreV1().Services(instance.Namespace) + err = serviceClient.Delete(ctx, svc.ObjectMeta.Name, metav1.DeleteOptions{}) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + } + } + + // Delete public route + route := &routev1.Route{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: instance.Namespace, + Name: autoscaling.ServiceName + "-public", + }, + } + err = r.Client.Delete(ctx, route) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + instance.Status.Conditions = condition.Conditions{} + r.Log.Info(fmt.Sprintf("Reconciled Service Aodh '%s' disable successfully", autoscaling.ServiceName)) + return ctrl.Result{}, nil +} + +func (r *AutoscalingReconciler) reconcileDeleteAodh( + ctx context.Context, + instance *telemetryv1.Autoscaling, + helper *helper.Helper, +) (ctrl.Result, error) { + r.Log.Info("Reconciling Service Aodh delete") + + // remove db finalizer first + db, err := database.GetDatabaseByName(ctx, helper, autoscaling.ServiceName) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + + if !k8s_errors.IsNotFound(err) { + if err := db.DeleteFinalizer(ctx, helper); err != nil { + return ctrl.Result{}, err + } + } + + // Remove the finalizer from our KeystoneService CR + keystoneService, err := keystonev1.GetKeystoneServiceWithName(ctx, helper, autoscaling.ServiceName, instance.Namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + + if err == nil { + if controllerutil.RemoveFinalizer(keystoneService, helper.GetFinalizer()) { + err = r.Update(ctx, keystoneService) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + util.LogForObject(helper, "Removed finalizer from our KeystoneService", instance) + } + } + + // Remove the finalizer from our KeystoneEndpoint CR + keystoneEndpoint, err := keystonev1.GetKeystoneEndpointWithName(ctx, helper, autoscaling.ServiceName, instance.Namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + if err == nil { + if controllerutil.RemoveFinalizer(keystoneEndpoint, helper.GetFinalizer()) { + err = r.Update(ctx, keystoneEndpoint) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + util.LogForObject(helper, "Removed finalizer from our KeystoneEndpoint", instance) + } + } + r.Log.Info(fmt.Sprintf("Reconciled Service Aodh '%s' delete successfully", autoscaling.ServiceName)) + + return ctrl.Result{}, nil +} + +func (r *AutoscalingReconciler) reconcileInitAodh( + ctx context.Context, + instance *telemetryv1.Autoscaling, + helper *helper.Helper, + serviceLabels map[string]string, +) (ctrl.Result, error) { + r.Log.Info("Reconciling Service Aodh init") + _, _, err := secret.GetSecret(ctx, helper, instance.Spec.Aodh.Secret, instance.Namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, fmt.Errorf("OpenStack secret %s not found", instance.Spec.Aodh.Secret) + } + return ctrl.Result{}, err + } + + ksSvcSpec := keystonev1.KeystoneServiceSpec{ + ServiceType: autoscaling.ServiceType, + ServiceName: autoscaling.ServiceName, + ServiceDescription: "Aodh for autoscaling Service", + Enabled: true, + ServiceUser: instance.Spec.Aodh.ServiceUser, + Secret: instance.Spec.Aodh.Secret, + PasswordSelector: instance.Spec.Aodh.PasswordSelectors.AodhService, + } + + ksSvc := keystonev1.NewKeystoneService(ksSvcSpec, instance.Namespace, serviceLabels, 10) + ctrlResult, err := ksSvc.CreateOrPatch(ctx, helper) + if err != nil { + return ctrlResult, err + } + + // mirror the Status, Reason, Severity and Message of the latest keystoneservice condition + // into a local condition with the type condition.KeystoneServiceReadyCondition + c := ksSvc.GetConditions().Mirror(condition.KeystoneServiceReadyCondition) + if c != nil { + instance.Status.Conditions.Set(c) + } + + if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + if instance.Status.Hash == nil { + instance.Status.Hash = map[string]string{} + } + // + // create service DB instance + // + db := database.NewDatabaseWithNamespace( + // instance.Name + // TODO: We might want to change the db name. + // The mariadb-operator is currently implemented + // in a way, that the db name needs to be the + // same as the user + instance.Spec.Aodh.DatabaseUser, + instance.Spec.Aodh.DatabaseUser, + instance.Spec.Aodh.Secret, + map[string]string{ + "dbName": instance.Spec.Aodh.DatabaseInstance, + }, + autoscaling.ServiceName, + instance.Namespace, + ) + // create or patch the DB + ctrlResult, err = db.CreateOrPatchDBByName( + ctx, + helper, + instance.Spec.Aodh.DatabaseInstance, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DBReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DBReadyRunningMessage)) + return ctrlResult, nil + } + // wait for the DB to be setup + ctrlResult, err = db.WaitForDBCreated(ctx, helper) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DBReadyErrorMessage, + err.Error())) + return ctrlResult, err + } + if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DBReadyRunningMessage)) + return ctrlResult, nil + } + // update Status.DatabaseHostname, used to config the service + instance.Status.DatabaseHostname = db.GetDatabaseHostname() + instance.Status.Conditions.MarkTrue(condition.DBReadyCondition, condition.DBReadyMessage) + // create service DB - end + + // + // run Aodh db sync + dbSyncHash := instance.Status.Hash[telemetryv1.DbSyncHash] + jobDef := autoscaling.DbSyncJob(instance, serviceLabels) + + dbSyncjob := job.NewJob( + jobDef, + telemetryv1.DbSyncHash, + instance.Spec.Aodh.PreserveJobs, + time.Duration(5)*time.Second, + dbSyncHash, + ) + ctrlResult, err = dbSyncjob.DoJob( + ctx, + helper, + ) + if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBSyncReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DBSyncReadyRunningMessage)) + return ctrlResult, nil + } + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBSyncReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DBSyncReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + if dbSyncjob.HasChanged() { + instance.Status.Hash[telemetryv1.DbSyncHash] = dbSyncjob.GetHash() + r.Log.Info(fmt.Sprintf("Service '%s' - Job %s hash added - %s", instance.Name, jobDef.Name, instance.Status.Hash[telemetryv1.DbSyncHash])) + } + instance.Status.Conditions.MarkTrue(condition.DBSyncReadyCondition, condition.DBSyncReadyMessage) + + // run Aodh db sync - end + r.Log.Info("Reconciled Service Aodh init successfully") + return ctrl.Result{}, nil +} + +func (r *AutoscalingReconciler) reconcileNormalAodh( + ctx context.Context, + instance *telemetryv1.Autoscaling, + helper *helper.Helper, + inputHash string, +) (ctrl.Result, error) { + r.Log.Info(fmt.Sprintf("Reconciling Service Aodh '%s'", autoscaling.ServiceName)) + serviceLabels := map[string]string{ + common.AppSelector: autoscaling.ServiceName, + } + + deplDef, err := autoscaling.AodhDeployment(instance, inputHash, serviceLabels) + if err != nil { + return ctrl.Result{}, err + } + depl := deployment.NewDeployment( + deplDef, + time.Duration(5)*time.Second, + ) + + ctrlResult, err := depl.CreateOrPatch(ctx, helper) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DeploymentReadyErrorMessage, + err.Error())) + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) + return ctrlResult, nil + } + + err = controllerutil.SetControllerReference(instance, deplDef, r.Scheme) + if err != nil { + return ctrl.Result{}, err + } + + instance.Status.ReadyCount = depl.GetDeployment().Status.ReadyReplicas + if instance.Status.ReadyCount > 0 { + instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) + } + instance.Status.Networks = instance.Spec.Aodh.NetworkAttachmentDefinitions + + ports := map[endpoint.Endpoint]endpoint.Data{} + ports[endpoint.EndpointInternal] = endpoint.Data{ + Port: autoscaling.AodhAPIPort, + } + ports[endpoint.EndpointPublic] = endpoint.Data{ + Port: autoscaling.AodhAPIPort, + } + + apiEndpoints, ctrlResult, err := endpoint.ExposeEndpoints( + ctx, + helper, + autoscaling.ServiceName, + serviceLabels, + ports, + time.Duration(5)*time.Second, + ) + if err != nil { + return ctrlResult, err + } + + // + // create keystone endpoints + // + + ksEndpointSpec := keystonev1.KeystoneEndpointSpec{ + ServiceName: autoscaling.ServiceName, + Endpoints: apiEndpoints, + } + + ksEndptObj := keystonev1.NewKeystoneEndpoint(autoscaling.ServiceName, instance.Namespace, ksEndpointSpec, serviceLabels, time.Duration(10)*time.Second) + ctrlResult, err = ksEndptObj.CreateOrPatch(ctx, helper) + if err != nil { + return ctrlResult, err + } + + c := ksEndptObj.GetConditions().Mirror(condition.KeystoneEndpointReadyCondition) + if c != nil { + instance.Status.Conditions.Set(c) + } + + if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + r.Log.Info("Reconciled Service Aodh successfully") + return ctrl.Result{}, nil +} diff --git a/controllers/autoscaling_controller.go b/controllers/autoscaling_controller.go index 8eb5a0f0..4743f122 100644 --- a/controllers/autoscaling_controller.go +++ b/controllers/autoscaling_controller.go @@ -19,26 +19,41 @@ package controllers import ( "context" "fmt" + "strings" + "time" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + k8s_errors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" logr "github.com/go-logr/logr" common "github.com/openstack-k8s-operators/lib-common/modules/common" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + endpoint "github.com/openstack-k8s-operators/lib-common/modules/common/endpoint" + env "github.com/openstack-k8s-operators/lib-common/modules/common/env" helper "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + labels "github.com/openstack-k8s-operators/lib-common/modules/common/labels" common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac" - service "github.com/openstack-k8s-operators/lib-common/modules/common/service" + secret "github.com/openstack-k8s-operators/lib-common/modules/common/secret" util "github.com/openstack-k8s-operators/lib-common/modules/common/util" - k8s_errors "k8s.io/apimachinery/pkg/api/errors" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" + rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" + mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" autoscaling "github.com/openstack-k8s-operators/telemetry-operator/pkg/autoscaling" obov1 "github.com/rhobs/observability-operator/pkg/apis/monitoring/v1alpha1" @@ -55,8 +70,27 @@ type AutoscalingReconciler struct { // +kubebuilder:rbac:groups=telemetry.openstack.org,resources=autoscalings,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=telemetry.openstack.org,resources=autoscalings/status,verbs=get;update;patch // +kubebuilder:rbac:groups=telemetry.openstack.org,resources=autoscalings/finalizers,verbs=update +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;create;update;patch;delete;watch // +kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=get;list -// +kubebuilder:rbac:groups=monitoring.rhobs,resources=monitoringstacks,verbs=get;list;watch +// +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneservices,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneapis,verbs=get;list;watch; +// +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneendpoints,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups=rabbitmq.openstack.org,resources=transporturls,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=mariadb.openstack.org,resources=mariadbdatabases,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=monitoring.rhobs,resources=monitoringstacks,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=memcached.openstack.org,resources=memcacheds,verbs=get;list;watch; +// service account, role, rolebinding +// +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;create;update +// +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=roles,verbs=get;list;watch;create;update +// +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=rolebindings,verbs=get;list;watch;create;update +// service account permissions that are needed to grant permission to the above +// +kubebuilder:rbac:groups="security.openshift.io",resourceNames=anyuid,resources=securitycontextconstraints,verbs=use // Reconcile reconciles an Autoscaling func (r *AutoscalingReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -89,17 +123,23 @@ func (r *AutoscalingReconciler) Reconcile(ctx context.Context, req ctrl.Request) // Always patch the instance status when exiting this function so we can persist any changes. defer func() { - // update the Ready condition based on the sub conditions - if instance.Status.Conditions.AllSubConditionIsTrue() { + if !instance.Spec.Enabled { instance.Status.Conditions.MarkTrue( - condition.ReadyCondition, condition.ReadyMessage) + condition.ReadyCondition, "Autoscaling disabled", + ) } else { - // something is not ready so reset the Ready condition - instance.Status.Conditions.MarkUnknown( - condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage) - // and recalculate it based on the state of the rest of the conditions - instance.Status.Conditions.Set( - instance.Status.Conditions.Mirror(condition.ReadyCondition)) + // update the Ready condition based on the sub conditions + if instance.Status.Conditions.AllSubConditionIsTrue() { + instance.Status.Conditions.MarkTrue( + condition.ReadyCondition, condition.ReadyMessage) + } else { + // something is not ready so reset the Ready condition + instance.Status.Conditions.MarkUnknown( + condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage) + // and recalculate it based on the state of the rest of the conditions + instance.Status.Conditions.Set( + instance.Status.Conditions.Mirror(condition.ReadyCondition)) + } } err := helper.PatchInstance(ctx, instance) if err != nil { @@ -127,9 +167,21 @@ func (r *AutoscalingReconciler) Reconcile(ctx context.Context, req ctrl.Request) condition.UnknownCondition(condition.RoleBindingReadyCondition, condition.InitReason, condition.RoleBindingReadyInitMessage), + + // Prometheus, Aodh, Heat conditions condition.UnknownCondition("PrometheusReady", condition.InitReason, "PrometheusNotStarted"), + condition.UnknownCondition("HeatReady", condition.InitReason, "HeatNotStarted"), + condition.UnknownCondition(condition.MemcachedReadyCondition, condition.InitReason, condition.MemcachedReadyInitMessage), + + condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage), + condition.UnknownCondition(condition.DBReadyCondition, condition.InitReason, condition.DBReadyInitMessage), + condition.UnknownCondition(condition.DBSyncReadyCondition, condition.InitReason, condition.DBSyncReadyInitMessage), + condition.UnknownCondition(condition.RabbitMqTransportURLReadyCondition, condition.InitReason, condition.RabbitMqTransportURLReadyInitMessage), + + condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), // right now we have no dedicated KeystoneServiceReadyInitMessage - //condition.UnknownCondition(condition.KeystoneServiceReadyCondition, condition.InitReason, ""), + condition.UnknownCondition(condition.KeystoneServiceReadyCondition, condition.InitReason, ""), + condition.UnknownCondition(condition.KeystoneEndpointReadyCondition, condition.InitReason, ""), ) instance.Status.Conditions.Init(&cl) @@ -138,6 +190,9 @@ func (r *AutoscalingReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } + // TODO: Remove this afet Heat integration is added + instance.Status.Conditions.MarkTrue("HeatReady", "Functionality not implemented yet") + if instance.Status.Hash == nil { instance.Status.Hash = map[string]string{} } @@ -162,17 +217,18 @@ func (r *AutoscalingReconciler) reconcileDisabled( helper *helper.Helper, ) (ctrl.Result, error) { r.Log.Info("Reconciling Service disabled") - serviceLabels := map[string]string{ - common.AppSelector: autoscaling.ServiceName, + ctrlResult, err := r.reconcileDisabledPrometheus(ctx, instance, helper) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil } - prom := autoscaling.Prometheus(instance, serviceLabels) - err := r.Client.Delete(ctx, prom) + ctrlResult, err = r.reconcileDisabledAodh(ctx, instance, helper) if err != nil { - if !k8s_errors.IsNotFound(err) { - return ctrl.Result{}, nil - } + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil } - // Set the condition to true, since the service is disabled for _, c := range instance.Status.Conditions { instance.Status.Conditions.MarkTrue(c.Type, "Autoscaling disabled") @@ -188,22 +244,18 @@ func (r *AutoscalingReconciler) reconcileDelete( ) (ctrl.Result, error) { r.Log.Info("Reconciling Service delete") - // Remove the finalizer from our KeystoneService CR - keystoneService, err := keystonev1.GetKeystoneServiceWithName(ctx, helper, autoscaling.ServiceName, instance.Namespace) - if err != nil && !k8s_errors.IsNotFound(err) { - return ctrl.Result{}, err + ctrlResult, err := r.reconcileDeletePrometheus(ctx, instance, helper) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil } - - if err == nil { - if controllerutil.RemoveFinalizer(keystoneService, helper.GetFinalizer()) { - err = r.Update(ctx, keystoneService) - if err != nil && !k8s_errors.IsNotFound(err) { - return ctrl.Result{}, err - } - util.LogForObject(helper, "Removed finalizer from our KeystoneService", instance) - } + ctrlResult, err = r.reconcileDeleteAodh(ctx, instance, helper) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil } - // Service is deleted so remove the finalizer. controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) r.Log.Info(fmt.Sprintf("Reconciled Service '%s' delete successfully", autoscaling.ServiceName)) @@ -218,54 +270,18 @@ func (r *AutoscalingReconciler) reconcileInit( serviceLabels map[string]string, ) (ctrl.Result, error) { r.Log.Info("Reconciling Service init") - // TODO: This code is useles for prometheus, but it might - // be useful for aodh in the future. I'll leave it - // here for now. - - // - // create Keystone service and users - // - /* - _, _, err := secret.GetSecret(ctx, helper, instance.Spec.Secret, instance.Namespace) - if err != nil { - if k8s_errors.IsNotFound(err) { - return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, fmt.Errorf("OpenStack secret %s not found", instance.Spec.Secret) - } - return ctrl.Result{}, err - } - - ksSvcSpec := keystonev1.KeystoneServiceSpec{ - ServiceType: autoscaling.ServiceType, - ServiceName: autoscaling.ServiceName, - ServiceDescription: "Aodh for autoscaling Service", - Enabled: true, - ServiceUser: instance.Spec.ServiceUser, - Secret: instance.Spec.Secret, - PasswordSelector: instance.Spec.PasswordSelectors.Service, - } - - ksSvc := keystonev1.NewKeystoneService(ksSvcSpec, instance.Namespace, serviceLabels, 10) - ctrlResult, err := ksSvc.CreateOrPatch(ctx, helper) - if err != nil { - return ctrlResult, err - } - - // mirror the Status, Reason, Severity and Message of the latest keystoneservice condition - // into a local condition with the type condition.KeystoneServiceReadyCondition - c := ksSvc.GetConditions().Mirror(condition.KeystoneServiceReadyCondition) - if c != nil { - instance.Status.Conditions.Set(c) - } - - if (ctrlResult != ctrl.Result{}) { - return ctrlResult, nil - } - - if instance.Status.Hash == nil { - instance.Status.Hash = map[string]string{} - } - */ - + ctrlResult, err := r.reconcileInitPrometheus(ctx, instance, helper, serviceLabels) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + ctrlResult, err = r.reconcileInitAodh(ctx, instance, helper, serviceLabels) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } r.Log.Info("Reconciled Service init successfully") return ctrl.Result{}, nil @@ -305,95 +321,361 @@ func (r *AutoscalingReconciler) reconcileNormal( common.AppSelector: autoscaling.ServiceName, } - // Handle service init - ctrlResult, err := r.reconcileInit(ctx, instance, helper, serviceLabels) + // + // create RabbitMQ transportURL CR and get the actual URL from the associated secret that is created + // + transportURL, op, err := r.transportURLCreateOrUpdate(instance) + + if err != nil { + r.Log.Info("Error getting transportURL. Setting error condition on status and returning") + instance.Status.Conditions.Set(condition.FalseCondition( + condition.RabbitMqTransportURLReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.RabbitMqTransportURLReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + if op != controllerutil.OperationResultNone { + r.Log.Info(fmt.Sprintf("TransportURL %s successfully reconciled - operation: %s", transportURL.Name, string(op))) + } + + instance.Status.TransportURLSecret = transportURL.Status.SecretName + + if instance.Status.TransportURLSecret == "" { + r.Log.Info(fmt.Sprintf("Waiting for TransportURL %s secret to be created", transportURL.Name)) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.RabbitMqTransportURLReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.RabbitMqTransportURLReadyRunningMessage)) + return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, nil + } + + instance.Status.Conditions.MarkTrue(condition.RabbitMqTransportURLReadyCondition, condition.RabbitMqTransportURLReadyMessage) + // end transportURL + + configMapVars := make(map[string]env.Setter) + + // + // check for required OpenStack secret holding passwords for service/admin user and add hash to the vars map + // + ctrlResult, err := r.getSecret(ctx, helper, instance, instance.Spec.Aodh.Secret, &configMapVars) if err != nil { return ctrlResult, err - } else if (ctrlResult != ctrl.Result{}) { - return ctrlResult, nil + } + // run check OpenStack secret - end + + // + // Check for required memcached used for caching + // + memcached, err := r.getAutoscalingMemcached(ctx, helper, instance) + if err != nil { + if k8s_errors.IsNotFound(err) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.MemcachedReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.MemcachedReadyWaitingMessage)) + return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, fmt.Errorf("memcached %s not found", instance.Spec.Aodh.MemcachedInstance) + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.MemcachedReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.MemcachedReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err } - promResult, err := r.reconcilePrometheus(ctx, instance, helper, serviceLabels) + if !memcached.IsReady() { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.MemcachedReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.MemcachedReadyWaitingMessage)) + return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, fmt.Errorf("memcached %s is not ready", memcached.Name) + } + // Mark the Memcached Service as Ready if we get to this point with no errors + instance.Status.Conditions.MarkTrue( + condition.MemcachedReadyCondition, condition.MemcachedReadyMessage) + // run check memcached - end + + // + // check for required TransportURL secret holding transport URL string + // + ctrlResult, err = r.getSecret(ctx, helper, instance, instance.Status.TransportURLSecret, &configMapVars) if err != nil { - return promResult, err + return ctrlResult, err } + // run check TransportURL secret - end + + // + // create secret required for autoscaling input + // - %-scripts configmap holding scripts to e.g. bootstrap the service + // - %-config configmap holding minimal autoscaling config required to get the service up, user can add additional files to be added to the service + // + err = r.generateServiceConfig(ctx, helper, instance, &configMapVars, memcached) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configMapVars) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } else if hashChanged { + // Hash changed and instance status should be updated (which will be done by main defer func), + // so we need to return and reconcile again + return ctrl.Result{}, nil + } + + instance.Status.Hash[common.InputHashName] = inputHash + // Handle service init + ctrlResult, err = r.reconcileInit(ctx, instance, helper, serviceLabels) + if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + if err != nil { + return ctrlResult, err + } + ctrlResult, err = r.reconcileNormalPrometheus(ctx, instance, helper) + if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + if err != nil { + return ctrlResult, err + } + ctrlResult, err = r.reconcileNormalAodh(ctx, instance, helper, inputHash) + if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + if err != nil { + return ctrlResult, err + } r.Log.Info("Reconciled Service successfully") return ctrl.Result{}, nil } -func (r *AutoscalingReconciler) reconcilePrometheus(ctx context.Context, +// createHashOfInputHashes - creates a hash of hashes which gets added to the resources which requires a restart +// if any of the input resources change, like configs, passwords, ... +// +// returns the hash, whether the hash changed (as a bool) and any error +func (r *AutoscalingReconciler) createHashOfInputHashes( + ctx context.Context, instance *telemetryv1.Autoscaling, - helper *helper.Helper, - serviceLabels map[string]string, -) (ctrl.Result, error) { - prom := autoscaling.Prometheus(instance, serviceLabels) + envVars map[string]env.Setter, +) (string, bool, error) { + var hashMap map[string]string + changed := false + mergedMapVars := env.MergeEnvs([]corev1.EnvVar{}, envVars) + hash, err := util.ObjectHash(mergedMapVars) + if err != nil { + return hash, changed, err + } + if hashMap, changed = util.SetHash(instance.Status.Hash, common.InputHashName, hash); changed { + instance.Status.Hash = hashMap + r.Log.Info(fmt.Sprintf("Input maps hash %s - %s", common.InputHashName, hash)) + } + return hash, changed, nil +} - var promHost string - var promPort int32 +func (r *AutoscalingReconciler) generateServiceConfig( + ctx context.Context, + h *helper.Helper, + instance *telemetryv1.Autoscaling, + envVars *map[string]env.Setter, + mc *memcachedv1.Memcached, +) error { + + cmLabels := labels.GetLabels(instance, labels.GetGroupLabel(autoscaling.ServiceName), map[string]string{}) + customData := map[string]string{common.CustomServiceConfigFileName: instance.Spec.Aodh.CustomServiceConfig} + for key, data := range instance.Spec.Aodh.DefaultConfigOverwrite { + customData[key] = data + } - if instance.Spec.Prometheus.DeployPrometheus { - op, err := controllerutil.CreateOrUpdate(ctx, r.Client, prom, func() error { - err := controllerutil.SetControllerReference(instance, prom, r.Scheme) - return err - }) - if err != nil { - return ctrl.Result{}, err - } - if op != controllerutil.OperationResultNone { - r.Log.Info(fmt.Sprintf("Prometheus %s successfully reconciled - operation: %s", prom.Name, string(op))) - } - promReady := true - for _, c := range prom.Status.Conditions { - if c.Status != "True" { - instance.Status.Conditions.MarkFalse("PrometheusReady", - condition.Reason(c.Reason), - condition.SeverityError, - c.Message) - promReady = false - break - } - } - if len(prom.Status.Conditions) == 0 { - promReady = false - } - if promReady { - instance.Status.Conditions.MarkTrue("PrometheusReady", "Prometheus is ready") - serviceName := prom.Name + "-prometheus" - promSvc, err := service.GetServiceWithName(ctx, helper, serviceName, instance.Namespace) - if err != nil { - return ctrl.Result{}, err - } - // TODO: Remove the nolint after adding aodh and using the variables - promHost = fmt.Sprintf("%s.%s.svc", serviceName, instance.Namespace) //nolint:all - promPort = service.GetServicesPortDetails(promSvc, "web").Port //nolint:all - } - } else { - err := r.Client.Delete(ctx, prom) - if err != nil { - if !k8s_errors.IsNotFound(err) { - return ctrl.Result{}, nil - } - } - promHost = instance.Spec.Prometheus.Host - promPort = instance.Spec.Prometheus.Port - if promHost == "" || promPort == 0 { - instance.Status.Conditions.MarkFalse("PrometheusReady", - condition.ErrorReason, - condition.SeverityError, - "deployPrometheus is false and either port or host isn't set") - } else { - instance.Status.Conditions.MarkTrue("PrometheusReady", "Prometheus is ready") + keystoneAPI, err := keystonev1.GetKeystoneAPI(ctx, h, instance.Namespace, map[string]string{}) + if err != nil { + return err + } + + keystoneInternalURL, err := keystoneAPI.GetEndpoint(endpoint.EndpointInternal) + if err != nil { + return err + } + + ospSecret, _, err := secret.GetSecret(ctx, h, instance.Spec.Aodh.Secret, instance.Namespace) + if err != nil { + return err + } + + transportURLSecret, _, err := secret.GetSecret(ctx, h, instance.Status.TransportURLSecret, instance.Namespace) + if err != nil { + return err + } + + templateParameters := map[string]interface{}{ + "AodhUser": instance.Spec.Aodh.ServiceUser, + "AodhPassword": string(ospSecret.Data[instance.Spec.Aodh.PasswordSelectors.AodhService]), + "KeystoneInternalURL": keystoneInternalURL, + "TransportURL": string(transportURLSecret.Data["transport_url"]), + "PrometheusHost": instance.Status.PrometheusHost, + "PrometheusPort": instance.Status.PrometheusPort, + "MemcachedServers": strings.Join(mc.Status.ServerList, ","), + "MemcachedServersWithInet": strings.Join(mc.Status.ServerListWithInet, ","), + "DatabaseConnection": fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s", + instance.Spec.Aodh.DatabaseUser, + string(ospSecret.Data[instance.Spec.Aodh.PasswordSelectors.Database]), + instance.Status.DatabaseHostname, + autoscaling.DatabaseName), + } + + cms := []util.Template{ + // ScriptsSecret + { + Name: fmt.Sprintf("%s-scripts", autoscaling.ServiceName), + Namespace: instance.Namespace, + Type: util.TemplateTypeScripts, + InstanceType: instance.Kind, + AdditionalTemplate: map[string]string{"common.sh": "/common/common.sh"}, + Labels: cmLabels, + }, + // Secret + { + Name: fmt.Sprintf("%s-config-data", autoscaling.ServiceName), + Namespace: instance.Namespace, + Type: util.TemplateTypeConfig, + InstanceType: instance.Kind, + CustomData: customData, + ConfigOptions: templateParameters, + Labels: cmLabels, + }, + } + return secret.EnsureSecrets(ctx, h, instance, cms, envVars) +} + +// getAutoscalingMemcached - gets the Memcached instance used for aodh cache backend +func (r *AutoscalingReconciler) getAutoscalingMemcached( + ctx context.Context, + h *helper.Helper, + instance *telemetryv1.Autoscaling, +) (*memcachedv1.Memcached, error) { + memcached := &memcachedv1.Memcached{} + err := h.GetClient().Get( + ctx, + types.NamespacedName{ + Name: instance.Spec.Aodh.MemcachedInstance, + Namespace: instance.Namespace, + }, + memcached) + if err != nil { + return nil, err + } + return memcached, err +} + +// getSecret - get the specified secret, and add its hash to envVars +func (r *AutoscalingReconciler) getSecret(ctx context.Context, h *helper.Helper, instance *telemetryv1.Autoscaling, secretName string, envVars *map[string]env.Setter) (ctrl.Result, error) { + secret, hash, err := secret.GetSecret(ctx, h, secretName, instance.Namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.InputReadyWaitingMessage)) + return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, fmt.Errorf("secret %s not found", secretName) } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.InputReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err } - // TODO: Pass the promHost and promPort variables to aodh + // Add a prefix to the var name to avoid accidental collision with other non-secret + // vars. The secret names themselves will be unique. + (*envVars)["secret-"+secret.Name] = env.SetValue(hash) return ctrl.Result{}, nil } +func (r *AutoscalingReconciler) transportURLCreateOrUpdate(instance *telemetryv1.Autoscaling) (*rabbitmqv1.TransportURL, controllerutil.OperationResult, error) { + transportURL := &rabbitmqv1.TransportURL{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-transport", autoscaling.ServiceName), + Namespace: instance.Namespace, + }, + } + op, err := controllerutil.CreateOrUpdate(context.TODO(), r.Client, transportURL, func() error { + transportURL.Spec.RabbitmqClusterName = instance.Spec.Aodh.RabbitMqClusterName + err := controllerutil.SetControllerReference(instance, transportURL, r.Scheme) + return err + }) + return transportURL, op, err +} + // SetupWithManager sets up the controller with the Manager. func (r *AutoscalingReconciler) SetupWithManager(mgr ctrl.Manager) error { + // transportURLSecretFn - Watch for changes made to the secret associated with the RabbitMQ + // TransportURL created and used by Autoscaling CRs. Watch functions return a list of namespace-scoped + // CRs that then get fed to the reconciler. Hence, in this case, we need to know the name of the + // Autoscaling CR associated with the secret we are examining in the function. We could parse the name + // out of the "%s-transport" secret label, which would be faster than getting the list of + // the Autoscaling CRs and trying to match on each one. The downside there, however, is that technically + // someone could randomly label a secret "something-transport" where "something" actually + // matches the name of an existing Autoscaling CR. In that case changes to that secret would trigger + // reconciliation for a Autoscaling CR that does not need it. + // + // TODO: We also need a watch func to monitor for changes to the secret referenced by Autoscaling.Spec.Secret + transportURLSecretFn := func(o client.Object) []reconcile.Request { + result := []reconcile.Request{} + + // get all Autoscaling CRs + autoscalings := &telemetryv1.AutoscalingList{} + listOpts := []client.ListOption{ + client.InNamespace(o.GetNamespace()), + } + if err := r.Client.List(context.Background(), autoscalings, listOpts...); err != nil { + r.Log.Error(err, "Unable to retrieve Autoscaling CRs %v") + return nil + } + + for _, ownerRef := range o.GetOwnerReferences() { + if ownerRef.Kind == "TransportURL" { + for _, cr := range autoscalings.Items { + if ownerRef.Name == fmt.Sprintf("%s-transport", cr.Name) { + // return namespace and Name of CR + name := client.ObjectKey{ + Namespace: o.GetNamespace(), + Name: cr.Name, + } + r.Log.Info(fmt.Sprintf("TransportURL Secret %s belongs to TransportURL belonging to Autoscaling CR %s", o.GetName(), cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + } + } + if len(result) > 0 { + return result + } + return nil + } u := &unstructured.Unstructured{} u.SetGroupVersionKind(schema.GroupVersionKind{ Group: "apiextensions.k8s.io", @@ -403,14 +685,75 @@ func (r *AutoscalingReconciler) SetupWithManager(mgr ctrl.Manager) error { err := r.Client.Get(context.Background(), client.ObjectKey{ Name: "monitoringstacks.monitoring.rhobs", }, u) + + memcachedFn := func(o client.Object) []reconcile.Request { + result := []reconcile.Request{} + + // get all autoscaling CRs + autoscalings := &telemetryv1.AutoscalingList{} + listOpts := []client.ListOption{ + client.InNamespace(o.GetNamespace()), + } + if err := r.Client.List(context.Background(), autoscalings, listOpts...); err != nil { + r.Log.Error(err, "Unable to retrieve Autoscaling CRs %w") + return nil + } + + for _, cr := range autoscalings.Items { + if o.GetName() == cr.Spec.Aodh.MemcachedInstance { + name := client.ObjectKey{ + Namespace: o.GetNamespace(), + Name: cr.Name, + } + r.Log.Info(fmt.Sprintf("Memcached %s is used by Autoscaling CR %s", o.GetName(), cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + if len(result) > 0 { + return result + } + return nil + } if err != nil { return ctrl.NewControllerManagedBy(mgr). For(&telemetryv1.Autoscaling{}). + Owns(&appsv1.Deployment{}). + Owns(&corev1.ConfigMap{}). + Owns(&corev1.Service{}). + Owns(&corev1.Secret{}). + Owns(&corev1.ServiceAccount{}). + Owns(&keystonev1.KeystoneService{}). + Owns(&keystonev1.KeystoneEndpoint{}). + Owns(&mariadbv1.MariaDBDatabase{}). + Owns(&rabbitmqv1.TransportURL{}). + Owns(&rbacv1.Role{}). + Owns(&rbacv1.RoleBinding{}). + // Watch for TransportURL Secrets which belong to any TransportURLs created by Autoscaling CRs + Watches(&source.Kind{Type: &corev1.Secret{}}, + handler.EnqueueRequestsFromMapFunc(transportURLSecretFn)). + Watches(&source.Kind{Type: &memcachedv1.Memcached{}}, + handler.EnqueueRequestsFromMapFunc(memcachedFn)). Complete(r) } - // There is OBO installed and we can own MonitoringStack return ctrl.NewControllerManagedBy(mgr). For(&telemetryv1.Autoscaling{}). + Owns(&appsv1.Deployment{}). + Owns(&corev1.ConfigMap{}). + Owns(&corev1.Service{}). + Owns(&corev1.Secret{}). + Owns(&corev1.ServiceAccount{}). + Owns(&keystonev1.KeystoneService{}). + Owns(&keystonev1.KeystoneEndpoint{}). + Owns(&mariadbv1.MariaDBDatabase{}). + Owns(&rabbitmqv1.TransportURL{}). + Owns(&rbacv1.Role{}). + Owns(&rbacv1.RoleBinding{}). + // There is OBO installed and we can own MonitoringStack Owns(&obov1.MonitoringStack{}). + // Watch for TransportURL Secrets which belong to any TransportURLs created by Autoscaling CRs + Watches(&source.Kind{Type: &corev1.Secret{}}, + handler.EnqueueRequestsFromMapFunc(transportURLSecretFn)). + Watches(&source.Kind{Type: &memcachedv1.Memcached{}}, + handler.EnqueueRequestsFromMapFunc(memcachedFn)). Complete(r) } diff --git a/controllers/prometheus_controller.go b/controllers/prometheus_controller.go new file mode 100644 index 00000000..0f589e27 --- /dev/null +++ b/controllers/prometheus_controller.go @@ -0,0 +1,160 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + "time" + + k8s_errors "k8s.io/apimachinery/pkg/api/errors" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + common "github.com/openstack-k8s-operators/lib-common/modules/common" + condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + helper "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + service "github.com/openstack-k8s-operators/lib-common/modules/common/service" + + telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" + autoscaling "github.com/openstack-k8s-operators/telemetry-operator/pkg/autoscaling" +) + +func (r *AutoscalingReconciler) reconcileDisabledPrometheus( + ctx context.Context, + instance *telemetryv1.Autoscaling, + helper *helper.Helper, +) (ctrl.Result, error) { + r.Log.Info("Reconciling Service Prometheus disabled") + serviceLabels := map[string]string{ + common.AppSelector: autoscaling.ServiceName, + } + + prom := autoscaling.Prometheus(instance, serviceLabels) + err := r.Client.Delete(ctx, prom) + if err != nil { + if !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, nil + } + } + + // Set the condition to true, since the service is disabled + for _, c := range instance.Status.Conditions { + instance.Status.Conditions.MarkTrue(c.Type, "Autoscaling disabled") + } + r.Log.Info(fmt.Sprintf("Reconciled Service '%s' disable successfully", autoscaling.ServiceName)) + return ctrl.Result{}, nil +} + +func (r *AutoscalingReconciler) reconcileDeletePrometheus( + ctx context.Context, + instance *telemetryv1.Autoscaling, + helper *helper.Helper, +) (ctrl.Result, error) { + r.Log.Info("Reconciling Service Prometheus delete") + // TODO: finalizer prometheus + r.Log.Info(fmt.Sprintf("Reconciled Service '%s' delete successfully", autoscaling.ServiceName)) + + return ctrl.Result{}, nil +} + +func (r *AutoscalingReconciler) reconcileInitPrometheus( + ctx context.Context, + instance *telemetryv1.Autoscaling, + helper *helper.Helper, + serviceLabels map[string]string, +) (ctrl.Result, error) { + // TODO: init? + return ctrl.Result{}, nil +} + +func (r *AutoscalingReconciler) reconcileNormalPrometheus( + ctx context.Context, + instance *telemetryv1.Autoscaling, + helper *helper.Helper, +) (ctrl.Result, error) { + serviceLabels := map[string]string{ + common.AppSelector: autoscaling.ServiceName, + } + prom := autoscaling.Prometheus(instance, serviceLabels) + r.Log.Info(fmt.Sprintf("Reconciling Service Aodh '%s'", prom.Name)) + + var promHost string + var promPort int32 + + if instance.Spec.Prometheus.DeployPrometheus { + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, prom, func() error { + err := controllerutil.SetControllerReference(instance, prom, r.Scheme) + return err + }) + if err != nil { + return ctrl.Result{}, err + } + if op != controllerutil.OperationResultNone { + r.Log.Info(fmt.Sprintf("Prometheus %s successfully reconciled - operation: %s", prom.Name, string(op))) + } + promReady := true + for _, c := range prom.Status.Conditions { + if c.Status != "True" { + instance.Status.Conditions.MarkFalse("PrometheusReady", + condition.Reason(c.Reason), + condition.SeverityError, + c.Message) + promReady = false + break + } + } + if len(prom.Status.Conditions) == 0 { + promReady = false + } + if promReady { + instance.Status.Conditions.MarkTrue("PrometheusReady", "Prometheus is ready") + serviceName := prom.Name + "-prometheus" + promSvc, err := service.GetServiceWithName(ctx, helper, serviceName, instance.Namespace) + if err != nil { + return ctrl.Result{}, err + } + promHost = fmt.Sprintf("%s.%s.svc", serviceName, instance.Namespace) + promPort = service.GetServicesPortDetails(promSvc, "web").Port + } else { + return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, fmt.Errorf("Prometheus %s isn't ready", prom.Name) + } + } else { + err := r.Client.Delete(ctx, prom) + if err != nil { + if !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, nil + } + } + promHost = instance.Spec.Prometheus.Host + promPort = instance.Spec.Prometheus.Port + if promHost == "" || promPort == 0 { + instance.Status.Conditions.MarkFalse("PrometheusReady", + condition.ErrorReason, + condition.SeverityError, + "deployPrometheus is false and either port or host isn't set") + } else { + instance.Status.Conditions.MarkTrue("PrometheusReady", "Prometheus is ready") + } + } + + instance.Status.PrometheusHost = promHost + instance.Status.PrometheusPort = promPort + + r.Log.Info(fmt.Sprintf("Reconciled Service Aodh '%s' successfully", prom.Name)) + return ctrl.Result{}, nil +} diff --git a/go.mod b/go.mod index 511bc648..1990eab6 100644 --- a/go.mod +++ b/go.mod @@ -8,9 +8,12 @@ require ( github.com/go-logr/logr v1.2.4 github.com/onsi/ginkgo/v2 v2.12.0 github.com/onsi/gomega v1.27.10 + github.com/openshift/api v3.9.0+incompatible github.com/openstack-k8s-operators/infra-operator/apis v0.1.1-0.20230905074428-c6aefc16dd01 github.com/openstack-k8s-operators/keystone-operator/api v0.1.1-0.20230830083045-d73d07cca617 github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20230824094610-976b18ca2875 + github.com/openstack-k8s-operators/lib-common/modules/database v0.1.0 + github.com/openstack-k8s-operators/mariadb-operator/api v0.1.0 github.com/openstack-k8s-operators/openstack-ansibleee-operator/api v0.1.1-0.20230904091032-d53c9286b6a4 github.com/openstack-k8s-operators/telemetry-operator/api v0.1.1-0.20230904141420-35a75e39495a github.com/rhobs/obo-prometheus-operator/pkg/apis/monitoring v0.64.1-rhobs3 @@ -50,7 +53,6 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/openshift/api v3.9.0+incompatible // indirect github.com/openstack-k8s-operators/lib-common/modules/openstack v0.1.1-0.20230824094610-976b18ca2875 // indirect github.com/openstack-k8s-operators/lib-common/modules/storage v0.1.1-0.20230824094610-976b18ca2875 // indirect github.com/pkg/errors v0.9.1 // indirect diff --git a/go.sum b/go.sum index c538a17d..8c5390ae 100644 --- a/go.sum +++ b/go.sum @@ -150,10 +150,14 @@ github.com/openstack-k8s-operators/keystone-operator/api v0.1.1-0.20230830083045 github.com/openstack-k8s-operators/keystone-operator/api v0.1.1-0.20230830083045-d73d07cca617/go.mod h1:CmUe4tHh990eRUj6Ou8gD9JE0PQ38LGnUu3kaaP8K50= github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20230824094610-976b18ca2875 h1:pj22n6PQy/XAmV5m6XaarMY6X1lvxAh16oVT5ZSVoNI= github.com/openstack-k8s-operators/lib-common/modules/common v0.1.1-0.20230824094610-976b18ca2875/go.mod h1:Mqg9hyHpWPda62750vqmk5TajxP3zbYPDP1rtSH7mg0= +github.com/openstack-k8s-operators/lib-common/modules/database v0.1.0 h1:JWLX0pyQXANEULDbjv4rWcYQ8y4OSqnQl0L6O/gIv7U= +github.com/openstack-k8s-operators/lib-common/modules/database v0.1.0/go.mod h1:bbauLidBocb/iigxC0D4fIbqjkvR80o6fsKpOGyVk00= github.com/openstack-k8s-operators/lib-common/modules/openstack v0.1.1-0.20230824094610-976b18ca2875 h1:aUlwELsLYWQ3FL+/nRG/1uGVNW86c3MhtLrHNVDd57k= github.com/openstack-k8s-operators/lib-common/modules/openstack v0.1.1-0.20230824094610-976b18ca2875/go.mod h1:Vng+vqdTJUuZ+AEzSAaU0I7bn3qwYMMFEUHHhiH0440= github.com/openstack-k8s-operators/lib-common/modules/storage v0.1.1-0.20230824094610-976b18ca2875 h1:lC8Nw4PF2Lcqc7BJAdlBvYPyLqyaKa9R1e15dM9b3BY= github.com/openstack-k8s-operators/lib-common/modules/storage v0.1.1-0.20230824094610-976b18ca2875/go.mod h1:lazDTPD8BYde2yyzZ3HbOfG51Sf87vSr4KXwpF57hDs= +github.com/openstack-k8s-operators/mariadb-operator/api v0.1.0 h1:oM0ZzFHHj+ioCc7NXHIO6+sy7I2yiN29DI9/jh4fe54= +github.com/openstack-k8s-operators/mariadb-operator/api v0.1.0/go.mod h1:m5XuZSa5Zt5uAw3WbJYOIkFAGXy01mybVekcKOq1qHI= github.com/openstack-k8s-operators/openstack-ansibleee-operator/api v0.1.1-0.20230904091032-d53c9286b6a4 h1:lzEcJf9tyzBtVdL889++2NqyqGI34RX87u/fsQWstHw= github.com/openstack-k8s-operators/openstack-ansibleee-operator/api v0.1.1-0.20230904091032-d53c9286b6a4/go.mod h1:gCsHjYsZWdF8DOd4MH++2RZ+tF/VOuhhaVXWB7HrLCg= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/hack/clean_local_webhook.sh b/hack/clean_local_webhook.sh index df4476e2..7aabb936 100755 --- a/hack/clean_local_webhook.sh +++ b/hack/clean_local_webhook.sh @@ -1,6 +1,8 @@ #!/bin/bash set -ex +oc delete validatingwebhookconfiguration/vautoscaling.kb.io --ignore-not-found +oc delete mutatingwebhookconfiguration/mautoscaling.kb.io --ignore-not-found oc delete validatingwebhookconfiguration/vceilometer.kb.io --ignore-not-found oc delete mutatingwebhookconfiguration/mceilometer.kb.io --ignore-not-found oc delete validatingwebhookconfiguration/vtelemetry.kb.io --ignore-not-found diff --git a/hack/configure_local_webhook.sh b/hack/configure_local_webhook.sh index bf030438..5cca419b 100755 --- a/hack/configure_local_webhook.sh +++ b/hack/configure_local_webhook.sh @@ -31,6 +31,62 @@ CA_BUNDLE=`cat ${TMPDIR}/bundle.pem` cat >> ${TMPDIR}/patch_webhook_configurations.yaml <s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined +LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" proxy + +SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded +CustomLog /dev/stdout combined env=!forwarded +CustomLog /dev/stdout proxy env=forwarded + +# XXX: To disable SSL +#Include conf.d/*.conf +# If above include is commented include at least the aodh wsgi file +Include conf.d/00wsgi-aodh.conf diff --git a/templates/autoscaling/config/prometheus.yaml b/templates/autoscaling/config/prometheus.yaml new file mode 100644 index 00000000..fc4c081d --- /dev/null +++ b/templates/autoscaling/config/prometheus.yaml @@ -0,0 +1,2 @@ +host: {{ .PrometheusHost }} +port: {{ .PrometheusPort }} diff --git a/templates/autoscaling/config/wsgi-aodh.conf b/templates/autoscaling/config/wsgi-aodh.conf new file mode 100644 index 00000000..4a73f1bd --- /dev/null +++ b/templates/autoscaling/config/wsgi-aodh.conf @@ -0,0 +1,19 @@ + + + + + Options Indexes FollowSymLinks MultiViews + AllowOverride None + Require all granted + + + ## Logging + ErrorLog "/dev/stdout" + ServerSignature Off + LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\"" logformat + CustomLog "/dev/stdout" logformat + WSGIApplicationGroup %{GLOBAL} + WSGIDaemonProcess aodh group=aodh processes=3 threads=1 user=aodh + WSGIProcessGroup aodh + WSGIScriptAlias / "/var/www/cgi-bin/aodh/aodh-api" +