Merge pull request #76 from abays/kustomize_edpm

Data plane kustomization for SRIOV VA

examples/va/nfv/sriov/README.md

@@ -17,14 +17,14 @@ This is a collection of CR templates that represent a validated Red Hat OpenStac
 
 2. The CRs are applied against an OpenShift cluster in _stages_.  That is, there is an ordering in which each grouping of CRs is fed to the cluster.  It is _not_ a case of simply taking all CRs from all stages and applying them all at once.
 
-3. In stages 1 and 2 [kustomize](https://kustomize.io/) is used to genereate the CRs dynamically. The `*-values.yaml` file(s) must be updated to fit your environment. kustomize version 5 or newer required.
+3. In stages 1 and 2 [kustomize](https://kustomize.io/) is used to generate the control plane CRs dynamically. The `*-values.yaml` file(s) must be updated to fit your environment. kustomize version 5 or newer required.
 
-4. In stage 3 YAML comments are placed throughout the CRs to aid in the process of customizing the CRs.  Fields that _must_ (or most likely need to be) changed are commented with "# CHANGEME" either on the field itself or somewhere nearby.  Other comments are added to explain fields that can be changed and, sometimes, to explain additions that can be made.
+4. In stage 3 [kustomize](https://kustomize.io/) is used to generate the dataplane CRs dynamically. The `edpm/values.yaml` file must be updated to fit your environment. kustomize version 5 or newer required.
 
 ## Stages
 
 All stages must be executed in the order listed below. Everything is required unless otherwise indicated.
 
 1. [Install the OpenStack K8S operators and their dependencies](../../../common/)
 2. [Configuring networking and deploy the OpenStack control plane](control-plane.md)
-3. [Configure and deploy the data plane](edpm)
+3. [Configure and deploy the dataplane](dataplane.md)

examples/va/nfv/sriov/dataplane.md

@@ -0,0 +1,42 @@
+# Configuring networking and deploy the OpenStack control plane
+
+## Assumptions
+
+- The [control plane](control-plane.md) has been created and successfully deployed
+
+## Initialize
+
+Switch to the "openstack" namespace
+```
+oc project openstack
+```
+Change to the nfv/sriov/edpm directory
+```
+cd architecture/examples/va/nfv/sriov/edpm
+```
+Edit the [values.yaml](values.yaml) file to suit 
+your environment.
+```
+vi values.yaml
+```
+Alternatively use your own copies of those files and edit
+[kustomization.yaml](kustomization.yaml) to use those copies.
+```
+resources:
+  - values-ci-framework.yaml
+```
+
+Generate the dataplane CRs.
+```
+kustomize build > dataplane.yaml
+```
+
+## Create CRs
+```
+oc apply -f dataplane.yaml
+```
+
+Wait for dataplane deployment to finish
+```
+oc wait osdpd edpm-deployment --for condition=Ready --timeout=1200s
+```

examples/va/nfv/sriov/edpm/.gitignore

@@ -0,0 +1 @@
+dataplane.yaml

examples/va/nfv/sriov/edpm/kustomization.yaml

@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+components:
+  - ../../../../../va/nfv/sriov/edpm/
+  # - https://github.com/openstack-k8s-operators/architecture/va/nfv/sriov/edpm/?ref=main
+  ## It's possible to replace ../../../va/nfv/sriov/edpm/ with a git checkout URL as per:
+  ## https://github.com/kubernetes-sigs/kustomize/blob/master/examples/remoteBuild.md
+
+resources:
+  - values.yaml

examples/va/nfv/sriov/edpm/openstackdataplanenodeset.yaml → examples/va/nfv/sriov/edpm/values.yaml

@@ -1,89 +1,19 @@
-#
-# CHANGEME:
-#
-# - Change the first interface name specified in "spec.nodeTemplate.ansible.ansibleVars.edpm_network_config_template"
-#   to be the control plane interface on your compute nodes
-#     i.e.
-#     members:
-#     - type: interface
-#       name: enp7s0    <---- change
-# - Change the second interface name specified in "spec.nodeTemplate.ansible.ansibleVars.edpm_network_config_template"
-#   to be the SRIOV interface on your compute nodes
-#     i.e.
-#     - type: sriov_pf
-#       name: eno4      <---- change
-# - Change "spec.nodeTemplate.ansible.ansibleVars.edpm_sshd_allowed_ranges[0]" to
-#   match the CIDR of your control plane network, if necessary
-# - Change "edpm_bootstrap_command" to include subscription-manager and podman login
-#   commands to register the EDPM nodes as in https://access.redhat.com/solutions/253273
-# - Change the "spec.nodes[].ansible.ansibleHost" and "spec.nodes[].networks[0].fixedIP" values
-#   to match the control plane IPs of your nodes, if necessary
-# - Change "spec.nodeTemplate.ansible.ansibleVars.edpm_kernel_args" to be the desired kernel
-#   args for your environment
-# - Change "spec.nodeTemplate.ansible.ansibleVars.edpm_tuned_isolated_cores" to be the desired
-#   CPU core isolation for your environment
-# - Change "spec.nodeTemplate.ansible.ansibleVars.edpm_neutron_sriov_agent_SRIOV_NIC_physical_device_mappings"
-#   to the desired mapping of a network name to your SRIOV interface
-#
-
-apiVersion: dataplane.openstack.org/v1beta1
-kind: OpenStackDataPlaneNodeSet
+# local-config: referenced, but not emitted by kustomize
+apiVersion: v1
+kind: ConfigMap
 metadata:
-  name: openstack-edpm-sriov
-spec:
-  env:
-    - name: ANSIBLE_FORCE_COLOR
-      value: "True"
-  services:
-    - bootstrap
-    - configure-network
-    - validate-network
-    - install-os
-    - configure-os
-    - run-os
-    - reboot-os
-    - libvirt
-    - ovn
-    - neutron-sriov
-    - neutron-metadata
-    - nova-custom-sriov
-  preProvisioned: true
-  nodes:
-    edpm-compute-0:
-      ansible:
-        ansibleHost: 192.168.122.100 # CHANGEME
-      hostName: edpm-compute-0
-      networks:
-      - defaultRoute: true
-        fixedIP: 192.168.122.100 # CHANGEME
-        name: CtlPlane
-        subnetName: subnet1
-      - name: InternalApi
-        subnetName: subnet1
-      - name: Storage
-        subnetName: subnet1
-      - name: Tenant
-        subnetName: subnet1
-    edpm-compute-1:
-      ansible:
-        ansibleHost: 192.168.122.101 # CHANGEME
-      hostName: edpm-compute-1
-      networks:
-      - defaultRoute: true
-        fixedIP: 192.168.122.101 # CHANGEME
-        name: CtlPlane
-        subnetName: subnet1
-      - name: InternalApi
-        subnetName: subnet1
-      - name: Storage
-        subnetName: subnet1
-      - name: Tenant
-        subnetName: subnet1
-  networkAttachments:
-    - ctlplane
-  nodeTemplate:
-    ansibleSSHPrivateKeySecret: dataplane-ansible-ssh-private-key-secret
-    managementNetwork: ctlplane
+  name: edpm-values
+  annotations:
+    config.kubernetes.io/local-config: "true"
+data:
+  ssh_keys:
+    # Authorized keys that will have access to the dataplane computes via SSH
+    authorized: CHANGEME
+    # The private key that will have access to the dataplane computes via SSH
+    private: CHANGEME2
+    # The public key that will have access to the dataplane computes via SSH
+    public: CHANGEME3
+  nodeset:
     ansible:
       ansibleUser: cloud-admin
       ansiblePort: 22
@@ -95,18 +25,15 @@ spec:
         service_net_map:
           nova_api_network: internal_api
           nova_libvirt_network: internal_api
-        # CHANGEME
         timesync_ntp_servers:
           - hostname: clock.redhat.com
         # CPU pinning settings
-        edpm_kernel_args: "default_hugepagesz=1GB hugepagesz=1G hugepages=16 intel_iommu=on iommu=pt isolcpus=4-23,28-47" # CHANGEME
-        edpm_tuned_profile: "cpu-partitioning" 
-        edpm_tuned_isolated_cores: "4-23,28-47" # CHANGEME
+        edpm_kernel_args: "default_hugepagesz=1GB hugepagesz=1G hugepages=16 intel_iommu=on iommu=pt isolcpus=4-23,28-47"
+        edpm_tuned_profile: "cpu-partitioning-powersave" 
+        edpm_tuned_isolated_cores: "4-23,28-47"
         # edpm_network_config
         # These vars are edpm_network_config role vars
         edpm_network_config_hide_sensitive_logs: false
-        # CHANGEME (network_config[0].members[0].name) ---v
-        # CHANGEME (network_config[1].members[0].name) ---v
         edpm_network_config_template: |
           ---
           {% set mtu_list = [ctlplane_mtu] %}
@@ -158,8 +85,70 @@ spec:
         # edpm firewall, change the allowed CIDR if needed
         edpm_sshd_configure_firewall: true
         edpm_sshd_allowed_ranges:
-          - 192.168.122.0/24 # CHANGEME
+          - 192.168.122.0/24
         # SELinux module
         edpm_selinux_mode: enforcing
         # SRIOV settings
-        edpm_neutron_sriov_agent_SRIOV_NIC_physical_device_mappings: 'sriov-phy4:eno4' # CHANGEME
+        edpm_neutron_sriov_agent_SRIOV_NIC_physical_device_mappings: 'sriov-phy4:eno4'
+    nodes:
+      edpm-compute-0:
+        ansible:
+          ansibleHost: 192.168.122.100
+        hostName: edpm-compute-0
+        networks:
+        - defaultRoute: true
+          fixedIP: 192.168.122.100
+          name: CtlPlane
+          subnetName: subnet1
+        - name: InternalApi
+          subnetName: subnet1
+        - name: Storage
+          subnetName: subnet1
+        - name: Tenant
+          subnetName: subnet1
+      edpm-compute-1:
+        ansible:
+          ansibleHost: 192.168.122.101
+        hostName: edpm-compute-1
+        networks:
+        - defaultRoute: true
+          fixedIP: 192.168.122.101
+          name: CtlPlane
+          subnetName: subnet1
+        - name: InternalApi
+          subnetName: subnet1
+        - name: Storage
+          subnetName: subnet1
+        - name: Tenant
+          subnetName: subnet1
+    services:
+    - bootstrap
+    - configure-network
+    - validate-network
+    - install-os
+    - configure-os
+    - run-os
+    - reboot-os
+    - libvirt
+    - ovn
+    - nova-custom-sriov
+    - neutron-sriov
+    - neutron-metadata
+  nova:
+    compute:
+      conf: |
+        [DEFAULT]
+        reserved_host_memory_mb = 4096
+        reserved_huge_pages = node:0,size:4,count:524160
+        reserved_huge_pages = node:1,size:4,count:524160
+        [compute]
+        cpu_shared_set = 0-3,24-27
+        cpu_dedicated_set = 8-23,32-47
+    migration:
+      ssh_keys:
+        private: CHANGEME4
+        public: CHANGEME5
+    pci:
+      conf: |
+        [pci]
+        device_spec = {"vendor_id":"8086", "product_id":"1572", "address": "0000:19:00.3", "physical_network":"sriov-phy4", "trusted":"true"}

lib/dataplane/dataplane-ssh-secret.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+data:
+  authorized_keys: _replaced_
+  ssh-privatekey: _replaced_
+  ssh-publickey: _replaced_
+kind: Secret
+metadata:
+  name: dataplane-ansible-ssh-private-key-secret
+  namespace: openstack
+type: Opaque