add upstream dpdk job.

- This change adds playbooks to create dpdk flavors and allocate hugepages. - This change updates ovs to 2.11 and dpdk 18.11 - This change add a new OVS_DPDK_UPDATE_CPU_DEDICATED_SET setting to control auto updating of the cpu dedicated set and replace the legacy use of vcpu_pin_set. OVS_DPDK_UPDATE_CPU_DEDICATED_SET defaults to True for backwards compatiblity. Change-Id: Id99a707a2f7f3f2bf6cb03431829637a641814f9
2019-04-30 20:50:32 +01:00 · 2019-04-30 20:50:32 +01:00 · f645cf3db6
commit f645cf3db6
parent 4526a2470f
5 changed files with 181 additions and 6 deletions
--- a/.zuul.yaml
+++ b/.zuul.yaml
@ -1,7 +1,110 @@
+- nodeset:
+    name: dpdk
+    nodes:
+      - name: controller
+        label: nested-virt-ubuntu-bionic
+    groups:
+      # Node where tests are executed and test results collected
+      - name: tempest
+        nodes:
+          - controller
+      # Nodes running the compute service
+      - name: compute
+        nodes:
+          - controller
+
+- job:
+    name: networking-ovs-dpdk-tempest
+    # this is too high but it will do for now
+    timeout: 10800
+    nodeset: dpdk
+    parent: tempest-all
+    pre-run:
+      - playbooks/dpdk/dpdk.yaml
+    required-projects:
+      - x/networking-ovs-dpdk
+    vars:
+      zuul_copy_output:
+        '/opt/stack/logs/ovs-vswitchd.log': logs
+      tempest_concurrency: 1
+      tempest_test_regex: '^(?!.*(?:tempest\.scenario\.test_network_advanced_server_ops\.TestNetworkAdvancedServerOps\.test_server_connectivity_suspend_resume.*)|(?:tempest\.scenario\.test_network_advanced_server_ops\.TestNetworkAdvancedServerOps\.test_server_connectivity_pause_unpause.*))((?:tempest\.scenario\.test_network_basic_ops.*)|(?:tempest\.scenario\.test_network_advanced_server_ops.*)).*$'
+      tempest_black_regex: 'mtu|keypair|images|negative|disk|quota'
+      devstack_services:
+        dstat: true
+        etcd3: flase
+        peakmem_tracker: true
+        rabbit: true
+        tls-proxy: false
+        s-account: false
+        s-container: false
+        s-object: false
+        s-proxy: false
+        # disable cinder services
+        c-api: false
+        c-bak: false
+        c-sch: false
+        c-vol: false
+        cinder: false
+      devstack_plugins:
+        networking-ovs-dpdk: https://opendev.org/x/networking-ovs-dpdk
+      devstack_localrc:
+        USE_PYTHON3: True
+        LIBVIRT_TYPE: kvm
+        # ovs settings
+        OVS_SOCKET_MEM: 512
+        OVS_DATAPATH_TYPE: netdev
+        OVS_LOG_DIR: /opt/stack/logs
+        OVS_HUGEPAGE_MOUNT_PAGESIZE: 2M
+        OVS_HUGEPAGE_MOUNT: /dev/hugepages
+        OVS_ALLOCATE_HUGEPAGES: False
+        OVS_DPDK_MODE: controller_ovs_dpdk
+        OVS_DPDK_BIND_PORT: False
+        # dpdk settings
+        OVS_DPDK_MEM_SEGMENTS: 10240
+        OVS_DPDK_VHOST_USER_DEBUG: True
+        OVS_DPDK_SERVICE_DEBUG_OUTPUT: True
+        OVS_INTERFACE_DRIVER: vfio-pci
+        # the cpu_dedicated_set will be managed in the job
+        # definition so disable auto updating of it here.
+        OVS_DPDK_UPDATE_CPU_DEDICATED_SET: False
+        OVS_PMD_CORE_MASK: 0x2
+        OVS_CORE_MASK: 0x1
+      devstack_local_conf:
+        post-config:
+          $NOVA_CONF:
+            compute:
+              # NOTE(sean-k-mooney): we have 8 cpus in the gate but
+              # core 1 will be used by ovs-dpdk for the dpdk pmd thread
+              # so do not add it to the cpu_dedicated_set
+              cpu_dedicated_set: 2-7
+              cpu_shared_set: 0
+            filter_scheduler:
+              enabled_filters: 'AvailabilityZoneFilter,ComputeFilter,ComputeCapabilitiesFilter,ImagePropertiesFilter,ServerGroupAntiAffinityFilter,ServerGroupAffinityFilter,SameHostFilter,DifferentHostFilter,NUMATopologyFilter'
+            libvirt:
+              cpu_mode: host-passthrough
+              # This job requires nested virt so we explicitly enable kvm virt_type
+              virt_type: kvm
+        test-config:
+          $TEMPEST_CONFIG:
+            compute:
+              flavor_ref: 420
+              flavor_ref_alt: 840
+            # network tweaks should not be needed but don't hurt.
+            network:
+              ping_timeout: 180
+              connect_timeout: 180
+              ssh_timeout: 300
+            volume-feature-enabled:
+              backup: False
+            compute-feature-enabled:
+              resize: True
+              scheduler_enabled_filters: 'AvailabilityZoneFilter,ComputeFilter,ComputeCapabilitiesFilter,ImagePropertiesFilter,ServerGroupAntiAffinityFilter,ServerGroupAffinityFilter,SameHostFilter,DifferentHostFilter,NUMATopologyFilter'
+
 - project:
    check:
      jobs:
-        - noop
+        - networking-ovs-dpdk-tempest:
+            voting: false
    gate:
      jobs:
        - noop
--- a/devstack/libs/ovs-dpdk
+++ b/devstack/libs/ovs-dpdk
@ -664,7 +664,7 @@ function ovs_dpdk_configure_l3_agent {
    fi
 }

-function set_vcpu_pin_set {
+function set_cpu_dedicated_set {
    OVS_CORE_MASK=$(echo $OVS_CORE_MASK | sed 's/^0x//')
    OVS_PMD_CORE_MASK=$(echo $OVS_PMD_CORE_MASK | sed 's/^0x//')
    BAD_CORES=$((`echo $((16#${OVS_CORE_MASK}))` | `echo $((16#${OVS_PMD_CORE_MASK}))`))
@ -680,7 +680,7 @@ function set_vcpu_pin_set {
    vcpu_pin_set=${vcpu_pin_set::-1}

    if is_service_enabled nova; then
-        iniset $NOVA_CONF DEFAULT vcpu_pin_set $vcpu_pin_set
+        iniset $NOVA_CONF compute cpu_dedicated_set $vcpu_pin_set
    fi
 }

--- a/devstack/plugin.sh
+++ b/devstack/plugin.sh
@ -43,7 +43,9 @@ ERROR_ON_CLONE=False
                    fi
                ;;
                "post-config")
-                    set_vcpu_pin_set
+                    if [ $OVS_DPDK_UPDATE_CPU_DEDICATED_SET == 'True' ]; then
+                        set_cpu_dedicated_set
+                    fi
                    if [ $OVS_DPDK_INSTALL == 'True' ]; then
                        iniset /$Q_PLUGIN_CONF_FILE ovs vhostuser_socket_dir ${OVS_DB_SOCKET_DIR}/${OVS_VHOST_USER_SOCKET_DIR}
                    fi
--- a/devstack/settings
+++ b/devstack/settings
@ -8,7 +8,7 @@ NETWORKING_OVS_DPDK_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd
 #dpdk settings
 OVS_DPDK_MODE=${OVS_DPDK_MODE:-''}
 OVS_DPDK_GIT_REPO=${OVS_DPDK_GIT_REPO:-http://dpdk.org/git/dpdk}
-OVS_DPDK_GIT_TAG=${OVS_DPDK_GIT_TAG:-v17.11}
+OVS_DPDK_GIT_TAG=${OVS_DPDK_GIT_TAG:-v18.11}
 OVS_DPDK_DIR=${OVS_DPDK_DIR:-$DEST/DPDK-$OVS_DPDK_GIT_TAG}
 RTE_TARGET=${RTE_TARGET:-x86_64-native-linuxapp-gcc}
 OVS_DPDK_MEM_SEGMENTS=${OVS_DPDK_MEM_SEGMENTS:-256}
@ -17,6 +17,7 @@ OVS_DPDK_VHOST_USER_DEBUG=${OVS_DPDK_VHOST_USER_DEBUG:-False}
 OVS_DPDK_BUILD_SHARED_LIB=${OVS_DPDK_BUILD_SHARED_LIB:-False}
 OVS_DPDK_CONFIG_RTE_LIBRTE_VHOST_NUMA=${OVS_DPDK_CONFIG_RTE_LIBRTE_VHOST_NUMA:-True}
 OVS_DPDK_USE_QEMU_KVM_WRAPPER=${OVS_DPDK_USE_QEMU_KVM_WRAPPER:-False}
+OVS_DPDK_UPDATE_CPU_DEDICATED_SET=${OVS_DPDK_UPDATE_CPU_DEDICATED_SET:-True}

 # ovs constants
 OVS_DB_CONF_DIR=/etc/openvswitch
@ -27,7 +28,7 @@ OVS_INIT_POLICY=${OVS_INIT_POLICY:-'auto'}
 OVS_VSWITCHD_INIT=${OVS_VSWITCHD_INIT:-'auto'}
 OVS_DIR=${OVS_DIR:-$DEST/ovs}
 OVS_GIT_REPO=${OVS_GIT_REPO:-https://github.com/openvswitch/ovs.git}
-OVS_GIT_TAG=${OVS_GIT_TAG:-'branch-2.9'}
+OVS_GIT_TAG=${OVS_GIT_TAG:-'branch-2.11'}
 OVS_HUGEPAGE_MOUNT=${OVS_HUGEPAGE_MOUNT:-/mnt/huge}
 OVS_HUGEPAGE_MOUNT_PAGESIZE=${OVS_HUGEPAGE_MOUNT_PAGESIZE:-''}
 OVS_NUM_HUGEPAGES=${OVS_NUM_HUGEPAGES:-2048}
--- a/playbooks/dpdk/dpdk.yaml
+++ b/playbooks/dpdk/dpdk.yaml
@ -0,0 +1,69 @@
+- hosts: controller
+  tasks:
+    - name: create local.sh
+      become: yes
+      blockinfile:
+        path: /opt/stack/devstack/local.sh
+        create: True
+        mode: 0777
+        block: |
+          # creating an nfv flavor used in tempest
+          # primary flavor
+          name=nfv-64-1-1
+          openstack flavor create --id 420 --ram 64 --disk 1 --vcpus 1 $name
+          openstack flavor set --property hw:mem_page_size=large 420
+
+          # secondary flavor
+          name=nfv-128-1-2
+          openstack flavor create --id 840 --ram 128 --disk 1 --vcpus 2 $name
+          openstack flavor set --property hw:mem_page_size=2MB \
+          --property hw:cpu_policy=dedicated \
+          --property hw:emulator_threads_policy=share \
+          --property hw:cpu_realtime=yes --property hw:cpu_realtime_mask=^0 \
+          840
+
+- hosts: all
+  tasks:
+    - name: install dpdk deps
+      become: yes
+      package:
+        name: "{{item}}"
+        state: present
+      with_items:
+        - "numactl-devel"
+        - "gcc"
+        - "make"
+        - "python3-devel"
+        - "kernel-headers"
+        - "kernel-devel"
+      when: ansible_os_family == 'RedHat'
+
+
+- hosts: all
+  tasks:
+    - name: install dpdk deps
+      become: yes
+      package:
+        name: "{{item}}"
+        state: present
+      with_items:
+        - "libnuma-dev"
+        - "gcc"
+        - "make"
+        - "python3-dev"
+        - "linux-headers-generic"
+        - "build-essential"
+      when: ansible_os_family == 'Debian'
+
+
+- hosts: controller
+  tasks:
+    - name: allocate hugepages
+      shell: echo 512 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
+      become: yes
+
+- hosts: subnode
+  tasks:
+    - name: allocate hugepages
+      shell: echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
+      become: yes