Introduce ci_dcn_site role

docs/dictionary/en-custom.txt

@@ -21,6 +21,8 @@ authfile
 autoscale
 autostart
 awk
+az
+azs
 backend
 backends
 baremetal
@@ -139,6 +141,7 @@ dnsdata
 dnsmasq
 dockerfile
 dryrun
+dt
 dts
 ecdsa
 edecb
@@ -333,6 +336,7 @@ nodeexporter
 nodenetworkconfigurationpolicy
 nodeps
 nodeset
+nodesets
 nodetemplate
 noop
 nopasswd

playbooks/dcn.yml

@@ -0,0 +1,65 @@
+---
+# Copyright Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Deploy DCN environment
+  hosts: localhost
+  tasks:
+    - name: Load reproducer-variables
+      ansible.builtin.include_vars:
+        file: "~/reproducer-variables.yml"
+
+    - name: Load networking-environment-definition
+      ansible.builtin.include_vars:
+        file: "/etc/ci/env/networking-environment-definition.yml"
+        name: cifmw_networking_env_definition
+
+    - name: Create a network subnet list
+      ansible.builtin.set_fact:
+        _network_ranges: >-
+          {{
+            cifmw_networking_env_definition.networks
+            | dict2items
+            | selectattr('key', 'search', '^ctlplane')
+            | map(attribute='value.network_v4')
+            | list
+          }}
+
+    - name: Get OpenShift access token
+      register: _auth_results
+      community.okd.openshift_auth:
+        host: "{{ cifmw_openshift_api }}"
+        username: "{{ cifmw_openshift_user }}"
+        password: "{{ cifmw_openshift_password }}"
+        validate_certs: false
+
+    - name: Deploy EDPM
+      loop: "{{ groups | dict2items | selectattr('key', 'search', 'compute') | list }}"
+      loop_control:
+        index_var: idx
+        loop_var: itm
+      vars:
+        _az: "az{{ idx }}"
+        _subnet: "subnet{{ idx + 1 }}"
+        _subnet_network_range: "{{ _network_ranges[idx] }}"
+        _group_name: "{{ itm.key }}"
+        _group_hosts: "{{ groups[itm.key] }}"
+        _edpm_hosts: "{{ cifmw_baremetal_hosts | dict2items | selectattr('key', 'in', groups[itm.key]) | items2dict }}"
+        _ceph_bootstrap_node: "{{ (_edpm_hosts | dict2items | first).key if _edpm_hosts | length > 0 else '' }}"
+      when:
+        - _subnet_network_range != ''
+        - _ceph_bootstrap_node != ''
+      ansible.builtin.include_role:
+        name: ci_dcn_site

roles/ci_dcn_site/README.md

@@ -0,0 +1,91 @@
+# ci_dcn_site
+
+Deploys DCN sites for testing. Each DCN site is a new EDPM nodeset
+with a collocated Ceph cluster.
+
+## Privilege escalation
+
+- Applies CRDs in openstack namespace
+- Runs openstack client commands to create aggregates and discover new
+  compute hosts
+
+## Parameters
+
+* `_az`: The name of the availability zone for the AZ, e.g. `az1`
+* `_group_name`: The name of the group of nodes to be deployed, e.g. `dcn1-computes`
+* `_subnet`: The name of the subnet the DCN site will use, e.g. `subnet2`
+* `_subnet_network_range`: The range of the subnet the DCN site will use, e.g. `192.168.133.0/24`
+
+## Examples
+
+To deploy two nodesets named dcn1-computes and dcn2-computes,
+the role may be called like this.
+```yaml
+- name: Deploy
+  include_role: ci_dcn_site
+  with_items: "{{ groups | dict2items | selectattr('key', 'search', 'compute') | list }}"
+  loop_control:
+    index_var: idx
+    loop_var: item
+  vars:
+    _subnet: "subnet{{ idx + 1 }}"
+    _group_name: "{{ item.key }}"
+    _az: "az{{ idx }}"
+    _subnet_network_range: "{{ _network_ranges[idx] }}"
+```
+The above assumes the following values for each iteration:
+```
+_subnet: subnet2 | _group_name: dcn1-computes | _az: az1 | _subnet_network_range: 192.168.133.0/24
+_subnet: subnet3 | _group_name: dcn2-computes | _az: az2 | _subnet_network_range: 192.168.144.0/24
+```
+It relies on the `ci-framework-data/artifacts/zuul_inventory.yml` which the
+ci-framework will populate correctly when the `dt-dcn.yml` scenario is used.
+The variables above can then be built with the following tasks before
+the above is run.
+```yaml
+  - name: Load reproducer-variables
+    ansible.builtin.include_vars:
+      file: "~/reproducer-variables.yml"
+
+  - name: Load networking-environment-definition
+    ansible.builtin.include_vars:
+      file: "/etc/ci/env/networking-environment-definition.yml"
+      name: cifmw_networking_env_definition
+
+  - name: Create a network subnet list
+    ansible.builtin.set_fact:
+      _network_ranges: >-
+        {{
+          cifmw_networking_env_definition.networks
+          | dict2items
+          | selectattr('key', 'search', '^ctlplane')
+          | map(attribute='value.network_v4')
+          | list
+        }}
+```
+
+## Integration with Architecture Repository
+
+The directions in the
+[DCN DT](https://github.com/openstack-k8s-operators/architecture/tree/main/examples/dt/dcn)
+end with deploying the first Availability Zone (AZ) called `az0`.
+Additional AZs may be deployed for testing by calling this role.
+
+The DCN DT contains values yaml files which may be passed to
+kustomize. This role generates additional instances of the same
+type of values files from jinja templates. The templates are populated
+with the values in the environment which are set when the `dt-dcn.yml`
+scenario is used. The role then calls kustomize to apply the CRDs.
+
+The role is executed by the dcn.yml playbook found in the playbooks
+directory. This same playbook is called by the automation structure
+in the DCN DT (`automation/vars/dcn.yaml`) by using a
+`post_stage_run`.
+
+## Maintainers
+
+This role is maintained by the following
+
+- https://github.com/sbekkerm
+- https://github.com/krcmarik
+- https://github.com/fultonj

roles/ci_dcn_site/defaults/main.yml

@@ -0,0 +1,27 @@
+---
+# Copyright Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+ci_dcn_site_arch_repo_path: /home/zuul/src/github.com/openstack-k8s-operators/architecture
+ci_dcn_site_arch_path: "{{ ci_dcn_site_arch_repo_path }}/examples/dt/dcn"
+ci_dcn_site_cifmw_repo_path: /home/zuul/src/github.com/openstack-k8s-operators/ci-framework
+ci_dcn_site_search_storage_network_names:
+  - "storage"
+  - "storagedcn1"
+  - "storagedcn2"
+ci_dcn_site_search_storagemgmt_network_names:
+  - "storagemgmt"
+  - "storagemgmtdcn1"
+  - "storagemgmtdcn2"

roles/ci_dcn_site/meta/main.yml

@@ -0,0 +1,30 @@
+---
+# Copyright Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+galaxy_info:
+  author: CI Framework
+  description: CI Framework Role -- ci_dcn_site
+  company: Red Hat
+  license: Apache-2.0
+  min_ansible_version: "2.14"
+  namespace: cifmw
+  galaxy_tags:
+    - cifmw
+
+# List your role dependencies here, one per line. Be sure to remove the '[]' above,
+# if you add dependencies to this list.
+dependencies: []

roles/ci_dcn_site/tasks/az.yml

@@ -0,0 +1,55 @@
+---
+# Copyright Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Check if AZ exists has hosts
+  register: az_hosts
+  ignore_errors: true
+  kubernetes.core.k8s_exec:
+    api_key: "{{ _auth_results.openshift_auth.api_key }}"
+    namespace: openstack
+    pod: openstackclient
+    command: >-
+      openstack aggregate show {{ _az }} -c hosts -f value
+
+- name: Convert az_hosts string to list and remove extra text
+  ansible.builtin.set_fact:
+    az_hosts_list: >
+      {{ az_hosts.stdout
+         | default([])
+         | from_yaml
+         | map('regex_replace', 'edpm-compute-(.*?)\\..*', 'compute-\\1')
+         | list }}
+
+- name: Create AZ if it does not exist
+  when:
+    - az_hosts.rc == 1
+  kubernetes.core.k8s_exec:
+    api_key: "{{ _auth_results.openshift_auth.api_key }}"
+    namespace: openstack
+    pod: openstackclient
+    command: >-
+      openstack aggregate create {{ _az }} --zone {{ _az }}
+
+- name: Add only the missing edpm hosts to AZ
+  loop: "{{ _edpm_hosts | dict2items }}"
+  when:
+    - item.key not in az_hosts_list
+  kubernetes.core.k8s_exec:
+    api_key: "{{ _auth_results.openshift_auth.api_key }}"
+    namespace: openstack
+    pod: openstackclient
+    command: >-
+      openstack aggregate add host {{ _az }} edpm-{{ item.key }}.ctlplane.example.com

roles/ci_dcn_site/tasks/ceph.yml

@@ -0,0 +1,93 @@
+---
+# Copyright Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Fetch network facts of ceph bootstrap node
+  delegate_to: "{{ _ceph_bootstrap_node }}"
+  run_once: true
+  ansible.builtin.setup:
+    gather_subset:
+      - "!all"
+      - "!min"
+      - network
+
+- name: Update the hosts file on the Ceph bootstrap host
+  become: true
+  vars:
+    ceph_boot_ssh_ip: "{{ ansible_all_ipv4_addresses | ansible.utils.ipaddr(_subnet_network_range) | first }}"
+  delegate_to: "{{ _ceph_bootstrap_node }}"
+  run_once: true
+  ansible.builtin.lineinfile:
+    path: /etc/hosts
+    line: "{{ ceph_boot_ssh_ip }} {{ _ceph_bootstrap_node }}"
+    state: present
+    create: true
+    backup: true
+    insertbefore: EOF
+
+- name: Ensure Ceph bootstrap host can ping itself
+  register: _cmd_result
+  retries: 5
+  delay: 60
+  until: _cmd_result.rc == 0
+  delegate_to: "{{ _ceph_bootstrap_node }}"
+  ansible.builtin.command:
+    cmd: >-
+      ping -c1 "{{ _ceph_bootstrap_node }}"
+
+- name: Create Ceph playbook variables file
+  vars:
+    _content:
+      cifmw_cephadm_cluster: "{{ _az }}"
+      ssh_network_range: "{{ _subnet_network_range }}"
+      cifmw_ceph_target: "{{ _group_name }}"
+      storage_network_range: "{{ _storage_network_range | ansible.utils.ipaddr('network/prefix') }}"
+      storage_mgmt_network_range: "{{ _storage_mgmt_network_range |  ansible.utils.ipaddr('network/prefix') }}"
+      cifmw_ceph_client_service_values_post_ceph_path_dst: "/tmp/edpm_service_values_post_ceph_{{ _az }}.yaml"
+      cifmw_ceph_client_values_post_ceph_path_dst: "{{ ci_dcn_site_arch_repo_path }}/values.yaml"
+      cifmw_ceph_spec_data_devices: >-
+        data_devices:
+          all: true
+      cifmw_ceph_client_vars: "/tmp/ceph_client_{{_az}}.yml"
+  ansible.builtin.copy:
+    mode: "0644"
+    dest: "~/ci-framework-data/parameters/ceph-{{ _az }}.yml"
+    content: "{{ _content | to_nice_yaml }}"
+
+- name: Deploy Ceph
+  cifmw.general.ci_script:
+    output_dir: "/home/zuul/ci-framework-data/artifacts"
+    chdir: "{{ ci_dcn_site_cifmw_repo_path }}"
+    script: >-
+      ansible-playbook
+      -i ~/ci-framework-data/artifacts/zuul_inventory.yml
+      -e @~/ci-framework-data/parameters/reproducer-variables.yml
+      -e @~/ci-framework-data/parameters/ceph-{{ _az }}.yml
+      playbooks/ceph.yml
+
+- name: Load the Ceph cluster variables
+  ansible.builtin.include_vars:
+    file: "/tmp/ceph_client_{{_az}}.yml"
+
+- name: Find all ceph .conf and .keyring files
+  register: _ceph_conf_files
+  ansible.builtin.find:
+    paths: "/tmp"
+    patterns: "ceph*.conf,ceph*.keyring,az*.conf,az*.keyring"
+    recurse: false
+
+- name: Load ceph configuration files
+  ansible.builtin.set_fact:
+    _ceph_files: "{{ _ceph_conf_files.files | map(attribute='path') | list }}"