Multiple nodesets for ovs-dpdk-sriov scenario

Created a new dt based on ovs-dpdk-sriov va
Depends-on: openstack-k8s-operators/ci-framework#2280

automation/vars/ovs-dpdk-sriov-2nodesets.yaml

@@ -0,0 +1,60 @@
+---
+vas:
+  ovs-dpdk-sriov-2nodesets:
+    stages:
+      - path: examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/nncp
+        wait_conditions:
+          - >-
+            oc -n openstack wait nncp
+            -l osp/nncm-config-type=standard
+            --for jsonpath='{.status.conditions[0].reason}'=SuccessfullyConfigured
+            --timeout=60s
+        values:
+          - name: network-values
+            src_file: ../../../../va/nfv/ovs-dpdk-sriov/nncp/values.yaml
+        build_output: nncp.yaml
+
+      - path: examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets
+        wait_conditions:
+          - >-
+            oc -n openstack wait osctlplane controlplane --for condition=Ready
+            --timeout=1200s
+        values:
+          - name: network-values
+            src_file: ../../../va/nfv/ovs-dpdk-sriov/nncp/values.yaml
+          - name: service-values
+            src_file: ../../../va/nfv/ovs-dpdk-sriov/service-values.yaml
+        build_output: control-plane.yaml
+
+      - path: examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/nodeset
+        values:
+          - name: edpm-nodeset-values
+            src_file: ../../../../../va/nfv/ovs-dpdk-sriov/edpm/nodeset/values.yaml
+        build_output: nodeset.yaml
+
+      - path: examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/nodeset2
+        wait_conditions:
+          - >-
+            oc -n openstack wait
+            osdpns openstack-edpm --for condition=SetupReady
+            --timeout=60m
+          - >-
+            oc -n openstack wait
+            osdpns openstack-edpm-2 --for condition=SetupReady
+            --timeout=60m
+        values:
+          - name: edpm-nodeset2-values
+            src_file: ../../examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/nodeset2/values.yaml
+        build_output: nodeset2.yaml
+
+      - path: examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/deployment
+        wait_conditions:
+          - >-
+            oc -n openstack
+            wait openstackdataplanedeployments.dataplane.openstack.org
+            edpm-deployment
+            --for condition=Ready --timeout=60m
+        values:
+          - name: edpm-deployment-values
+            src_file: ../../../../../va/nfv/ovs-dpdk-sriov/edpm/deployment/values.yaml
+        build_output: deployment.yaml

examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/deployment/.gitignore

@@ -0,0 +1 @@
+dataplane-deployment.yaml

examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/deployment/kustomization.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../../../../va/nfv/ovs-dpdk-sriov/edpm/deployment
+
+patches:
+  - target:
+      kind: OpenStackDataPlaneDeployment
+      name: edpm-deployment
+    patch: |
+      - op: add
+        path: /spec/nodeSets/-
+        value: openstack-edpm-2

examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/nodeset/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../../../../va/nfv/ovs-dpdk-sriov/edpm/nodeset

examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/nodeset2/.gitignore

@@ -0,0 +1 @@
+dataplane-nodeset-2.yaml

examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/nodeset2/kustomization.yaml

@@ -0,0 +1,68 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - values.yaml
+  - ../../../../../va/nfv/ovs-dpdk-sriov/edpm/nodeset
+
+replacements:
+  - source:
+      kind: ConfigMap
+      name: edpm-nodeset2-values
+      fieldPath: data
+    targets:
+      - select:
+          kind: ConfigMap
+          name: edpm-nodeset-values
+        fieldPaths:
+          - data
+        options:
+          create: true
+
+patches:
+  - target:
+      kind: ConfigMap
+      name: sriov-nova
+    patch: |-
+      - op: replace
+        path: /metadata/name
+        value: sriov-nova-2
+  - target:
+      kind: ConfigMap
+      name: ovs-dpdk-sriov-cpu-pinning-nova
+    patch: |-
+      - op: replace
+        path: /metadata/name
+        value: ovs-dpdk-sriov-cpu-pinning-nova-2
+  - target:
+      kind: OpenStackDataPlaneService
+      name: nova-custom-ovsdpdksriov
+    patch: |-
+      - op: replace
+        path: /metadata/name
+        value: nova-custom-ovsdpdksriov-2
+      - op: replace
+        path: /spec/label
+        value: nova-custom-ovsdpdksriov-2
+      - op: replace
+        path: /spec/dataSources/0/configMapRef/name
+        value: ovs-dpdk-sriov-cpu-pinning-nova-2
+      - op: replace
+        path: /spec/dataSources/1/configMapRef/name
+        value: sriov-nova-2
+  - target:
+      kind: OpenStackDataPlaneNodeSet
+      name: openstack-edpm
+    patch: |-
+      - op: replace
+        path: /metadata/name
+        value: openstack-edpm-2
+      - op: replace
+        path: /spec/baremetalSetTemplate/bmhLabelSelector/app
+        value: openstack-2
+      - op: replace
+        path: /spec/services/16
+        value: nova-custom-ovsdpdksriov-2
+      - op: remove
+        path: /spec/services/8

examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/nodeset2/values.yaml

@@ -0,0 +1,230 @@
+---
+# yamllint disable rule:line-length
+
+# local-config: referenced, but not emitted by kustomize
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: edpm-nodeset2-values
+  annotations:
+    config.kubernetes.io/local-config: "true"
+data:
+  root_password: cmVkaGF0Cg==
+  preProvisioned: false
+  baremetalSetTemplate:
+    ctlplaneInterface: eno2  # CHANGEME
+    cloudUserName: cloud-admin
+    provisioningInterface: enp1s0  # CHANGEME
+    bmhLabelSelector:
+      app: openstack-2  # CHANGEME
+    passwordSecret:
+      name: baremetalset-password-secret
+      namespace: openstack
+  ssh_keys:
+    # Authorized keys that will have access to the dataplane computes via SSH
+    authorized: CHANGEME
+    # The private key that will have access to the dataplane computes via SSH
+    private: CHANGEME2
+    # The public key that will have access to the dataplane computes via SSH
+    public: CHANGEME3
+  nodeset:
+    ansible:
+      ansibleUser: cloud-admin
+      ansiblePort: 22
+      ansibleVars:
+        # CHANGEME -- see https://access.redhat.com/solutions/253273
+        # edpm_bootstrap_command: |
+        #       subscription-manager register --username <subscription_manager_username> \
+        #         --password <subscription_manager_password>
+        #       podman login -u <registry_username> -p <registry_password> registry.redhat.io
+        timesync_ntp_servers:
+          - hostname: pool.ntp.org
+        # CPU pinning settings
+        # edpm nfv ovs dpdk config
+        # CHANGEME
+        # yamllint disable-line rule:line-length
+        edpm_kernel_args: "default_hugepagesz=1GB hugepagesz=1G hugepages=64 iommu=pt intel_iommu=on tsx=off isolcpus=2-11,14-23"
+        edpm_tuned_profile: "cpu-partitioning-powersave"
+        edpm_tuned_isolated_cores: "2-11,14-23"
+        edpm_nova_libvirt_qemu_group: "hugetlbfs"
+        edpm_ovs_dpdk_pmd_core_list: "1,13,2,14,3,15"
+        edpm_ovs_dpdk_socket_memory: "4096"
+        edpm_ovs_dpdk_memory_channels: "4"
+        edpm_ovs_dpdk_vhost_postcopy_support: "true"
+        edpm_ovn_bridge_mappings: ['dpdk-mgmt:br-link1', 'dpdk2:br-link2']
+        # edpm nfv sriov config
+        edpm_neutron_sriov_agent_SRIOV_NIC_physical_device_mappings: 'sriov1:eno5,sriov2:eno6'
+        # edpm_network_config
+        # These vars are edpm_network_config role vars
+        edpm_network_config_hide_sensitive_logs: false
+        edpm_network_config_os_net_config_mappings:
+          # Need to provide nic mapping based on system uuid or system product name
+          # here used systetm uuid
+          edpm-compute-0:  # CHANGEME
+            nic1: 6c:fe:54:3f:8a:01  # CHANGEME
+            nic2: 6c:fe:54:3f:8a:02  # CHANGEME
+            nic3: 6c:fe:54:3f:8a:03  # CHANGEME
+            nic4: 6c:fe:54:3f:8a:04  # CHANGEME
+            nic5: 6c:fe:54:3f:8a:05  # CHANGEME
+            nic6: 6c:fe:54:3f:8a:06  # CHANGEME
+            nic7: 6c:fe:54:3f:8a:07  # CHANGEME
+        edpm_network_config_template: |
+          ---
+          {% set mtu_list = [ctlplane_mtu] %}
+          {% for network in nodeset_networks %}
+          {{ mtu_list.append(lookup('vars', networks_lower[network] ~ '_mtu')) }}
+          {%- endfor %}
+          {% set min_viable_mtu = mtu_list | max %}
+          network_config:
+          - type: ovs_bridge
+            name: {{ neutron_physical_bridge_name }}
+            mtu: {{ min_viable_mtu }}
+            use_dhcp: false
+            dns_servers: {{ ctlplane_dns_nameservers }}
+            domain: {{ dns_search_domains }}
+            addresses:
+            - ip_netmask: {{ ctlplane_ip }}/{{ ctlplane_cidr }}
+            routes: {{ ctlplane_host_routes }}
+            members:
+            - type: interface
+              name: nic2
+              mtu: {{ min_viable_mtu }}
+              # force the MAC address of the bridge to this interface
+              primary: true
+          {% for network in nodeset_networks if network not in ['external', 'tenant'] %}
+            - type: vlan
+              mtu: {{ lookup('vars', networks_lower[network] ~ '_mtu') }}
+              vlan_id: {{ lookup('vars', networks_lower[network] ~ '_vlan_id') }}
+              addresses:
+                - ip_netmask: {{ lookup('vars', networks_lower[network] ~ '_ip') }}/{{ lookup('vars', networks_lower[network] ~ '_cidr') }}
+              routes: {{ lookup('vars', networks_lower[network] ~ '_host_routes') }}
+          {% endfor %}
+          - type: ovs_user_bridge
+            name: br-link1
+            use_dhcp: false
+            ovs_extra: "set port br-link1 tag={{ lookup('vars', networks_lower['tenant'] ~ '_vlan_id') }}"
+            addresses:
+              - ip_netmask: {{ lookup('vars', networks_lower['tenant'] ~ '_ip') }}/{{ lookup('vars', networks_lower['tenant'] ~ '_cidr') }}
+            mtu: {{ lookup('vars', networks_lower['tenant'] ~ '_mtu') }}
+            members:
+              - type: ovs_dpdk_bond
+                name: dpdkbond0
+                mtu: 9000
+                rx_queue: 1
+                ovs_options: "bond_mode=balance-tcp lacp=active other_config:lacp-time=fast other-config:lacp-fallback-ab=true other_config:lb-output-action=true"
+                members:
+                - type: ovs_dpdk_port
+                  name: dpdk0
+                  members:
+                  - type: interface
+                    name: nic3
+                - type: ovs_dpdk_port
+                  name: dpdk1
+                  members:
+                  - type: interface
+                    name: nic4
+
+          - type: ovs_user_bridge
+            name: br-link2
+            mtu: 9000
+            use_dhcp: false
+            members:
+              - type: ovs_dpdk_port
+                name: dpdk2
+                mtu: 9000
+                rx_queue: 2
+                members:
+                  - type: interface
+                    name: nic5
+          - type: sriov_pf
+            name: nic6
+            numvfs: 10
+            mtu: 9000
+            use_dhcp: false
+            promisc: true
+          - type: sriov_pf
+            name: nic7
+            numvfs: 10
+            mtu: 9000
+            use_dhcp: false
+            promisc: true
+        # These vars are for the network config templates themselves and are
+        # considered EDPM network defaults.
+        neutron_physical_bridge_name: br-ex
+        neutron_public_interface_name: nic1
+        # edpm_nodes_validation
+        edpm_nodes_validation_validate_controllers_icmp: false
+        edpm_nodes_validation_validate_gateway_icmp: false
+        dns_search_domains: []
+        gather_facts: false
+        # edpm firewall, change the allowed CIDR if needed
+        edpm_sshd_configure_firewall: true
+        edpm_sshd_allowed_ranges:
+          - 192.168.122.0/24
+    networks:
+      - defaultRoute: true
+        name: ctlplane
+        subnetName: subnet1
+      - name: internalapi
+        subnetName: subnet1
+      - name: storage
+        subnetName: subnet1
+      - name: tenant
+        subnetName: subnet1
+    nodes:
+      edpm-compute-0:
+        hostName: edpm-compute-0
+      edpm-compute-1:
+        hostName: edpm-compute-1
+    services:
+      - bootstrap
+      - download-cache
+      - reboot-os
+      - configure-ovs-dpdk
+      - configure-network
+      - validate-network
+      - install-os
+      - configure-os
+      - ssh-known-hosts
+      - run-os
+      - install-certs
+      - ovn
+      - neutron-ovn-igmp
+      - neutron-metadata
+      - neutron-sriov
+      - libvirt
+      - nova-custom-ovsdpdksriov
+      - telemetry
+  nova:
+    compute:
+      conf: |
+        # CHANGEME
+        [DEFAULT]
+        reserved_host_memory_mb = 4096
+        [compute]
+        cpu_shared_set = 0-3,24-27
+        cpu_dedicated_set = 8-23,32-47
+        [neutron]
+        physnets = dpdk1, dpdk2
+        [neutron_physnet_dpdk1]
+        numa_nodes = 0
+        [neutron_physnet_dpdk2]
+        numa_nodes = 0
+        [neutron_tunnel]
+        numa_nodes = 0
+    migration:
+      ssh_keys:
+        private: CHANGEME4
+        public: CHANGEME5
+    pci:
+      # yamllint disable-line rule:line-length
+      conf: |
+        # CHANGEME
+        [pci]
+        device_spec = {"vendor_id":"8086", "product_id":"1572", "address": "0000:19:00.3", "physical_network":"sriov1", "trusted":"true"}
+        device_spec = {"vendor_id":"8086", "product_id":"1572", "address": "0000:20:00.3", "physical_network":"sriov2", "trusted":"true"}
+  neutron:
+    igmp:
+      conf: |
+        [ovs]
+        igmp_snooping_enable = False

examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../../va/nfv/ovs-dpdk-sriov