Native zuulv3 grenade multinode multitenant

Based on the native 'grenade-multinode' job

Change-Id: I4d0a23c371bc42c5bf18e79ea7920bd77b066154

devstack/upgrade/upgrade.sh

@@ -104,6 +104,7 @@ if [[ "$HOST_TOPOLOGY_ROLE" == "primary" ]]; then
         s|%IRONIC_SERVICE_HOST%|$IRONIC_PROVISION_SUBNET_SUBNODE_IP|g;
     " -i $ironic_wsgi_conf
     enable_apache_site ipxe-ironic
+    restart_apache_server
 else
     ensure_started+='ironic-api '
 fi

playbooks/legacy/grenade-dsvm-ironic-multinode-multitenant/run.yaml

@@ -1,173 +0,0 @@
-# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-# NOTE(sambetts) DO NOT UPDATE this job when you update the other jobs with
-# changes related to the current branch. The devstack local config defined in
-# this job is run against the last (old) version of the devstack plugin in the
-# grenade steps.
-# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-
-- hosts: primary
-  name: Autoconverted job legacy-grenade-dsvm-ironic-multinode-multitenant from old
-    job gate-grenade-dsvm-ironic-multinode-multitenant-ubuntu-xenial
-  tasks:
-
-    # NOTE(TheJulia): Python supports recompiling bytecode if a precompiled
-    # (.pyc) file is written to disk. Python will automatically recompile
-    # should that file disappear and attempt to load and use that bytecode.
-    # This can lead to unexpected and undesirable behavior such as python
-    # crashing.
-    #
-    # As this job scenario upgrades across possible structural changes to
-    # python modules, and operates in a mixed environment between releases
-    # it is a good idea to prevent scenarios where newer modules installed
-    # by packages are leveraged by un-upgraded services because their
-    # underlying python packages have been updated during runtime.
-    #
-    # This is unique to Ironic's rolling upgrade grenade job, as Nova is
-    # excluded from being upgraded in the stack, and Ironic is left in
-    # a half-upgraded situation. The net result of which is we have an
-    # unstable Nova installation.
-    # https://bugs.launchpad.net/ironic/+bug/1744139
-    #
-    # TODO(TheJulia): We either need to find a better way to test rolling
-    # upgrades. Something which supports virtualenvs would be ideal, as
-    # well as something that allows us greater upgrade order control as
-    # the Ironic upgrade sequence is problematic and breaks towards the end
-    # of every cycle.
-    - shell:
-        cmd: |
-          echo 'DefaultEnvironment="PYTHONDONTWRITEBYTECODE=1"' >>/etc/systemd/system.conf
-          systemctl daemon-reexec
-      become: yes
-
-    - shell:
-        cmd: |
-          cat << 'EOF' >> ironic-vars-early
-            # Set this early so that we do not have to be as careful with builder ordering in jobs.
-          export GRENADE_PLUGINRC="enable_grenade_plugin ironic https://opendev.org/openstack/ironic"
-
-          EOF
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
-
-    - shell:
-        cmd: |
-
-          # Precreate brbm so that it is created before neutron services are started, as they fail if it
-          # is not present
-          DEBIAN_FRONTEND=noninteractive sudo -E apt-get --option Dpkg::Options::=--force-confold --assume-yes install openvswitch-switch
-          sudo systemctl restart openvswitch-switch
-          sudo ovs-vsctl -- --may-exist add-br brbm
-
-          cat << 'EOF' >> ironic-extra-vars
-            export PROJECTS="openstack/grenade $PROJECTS"
-            export DEVSTACK_GATE_GRENADE=pullup
-            export DEVSTACK_GATE_OS_TEST_TIMEOUT=2600
-            export DEVSTACK_GATE_TEMPEST_BAREMETAL_BUILD_TIMEOUT=1200
-            export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_BUILD_DEPLOY_RAMDISK=False"
-            export DEVSTACK_GATE_TLSPROXY=0
-            export DEVSTACK_GATE_USE_PYTHON3=True
-            export BUILD_TIMEOUT
-
-            export GRENADE_PLUGINRC+=$'\n'"enable_grenade_plugin networking-generic-switch https://opendev.org/openstack/networking-generic-switch"
-
-            export DEVSTACK_GATE_TOPOLOGY="multinode"
-
-            # networking-generic-switch requires sudo to execute ovs-vsctl commands
-            export DEVSTACK_GATE_REMOVE_STACK_SUDO=0
-            export PROJECTS="openstack/networking-generic-switch $PROJECTS"
-            export DEVSTACK_LOCAL_CONFIG+=$'\n'"enable_plugin networking-generic-switch https://opendev.org/openstack/networking-generic-switch"
-            export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_USE_LINK_LOCAL=True"
-            export DEVSTACK_LOCAL_CONFIG+=$'\n'"OVS_BRIDGE_MAPPINGS=mynetwork:brbm,public:br_ironic_vxlan"
-            export DEVSTACK_LOCAL_CONFIG+=$'\n'"OVS_PHYSICAL_BRIDGE=brbm"
-            export DEVSTACK_LOCAL_CONFIG+=$'\n'"PHYSICAL_NETWORK=mynetwork"
-            export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_PROVISION_NETWORK_NAME=ironic-provision"
-            export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_PROVISION_SUBNET_PREFIX=10.0.5.0/24"
-            export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_PROVISION_SUBNET_GATEWAY=10.0.5.1"
-            export DEVSTACK_LOCAL_CONFIG+=$'\n'"Q_PLUGIN=ml2"
-            export DEVSTACK_LOCAL_CONFIG+=$'\n'"PUBLIC_BRIDGE=br_ironic_vxlan"
-            export DEVSTACK_LOCAL_CONFIG+=$'\n'"ENABLE_TENANT_VLANS=True"
-            export DEVSTACK_LOCAL_CONFIG+=$'\n'"Q_ML2_TENANT_NETWORK_TYPE=vlan"
-            export DEVSTACK_LOCAL_CONFIG+=$'\n'"TENANT_VLAN_RANGE=100:150"
-            export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_ENABLED_NETWORK_INTERFACES=flat,neutron"
-            export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_NETWORK_INTERFACE=neutron"
-            export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_DEFAULT_BOOT_OPTION=local"
-            export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_AUTOMATED_CLEAN_ENABLED=False"
-            export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_VM_SPECS_RAM=384"
-            export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_RAMDISK_TYPE=tinyipa"
-            export DEVSTACK_LOCAL_CONFIG+=$'\n'"LIBVIRT_STORAGE_POOL_PATH=/opt/libvirt/images"
-            export DEVSTACK_GATE_TEMPEST_REGEX=test_server_basic_ops
-
-          EOF
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
-
-    - shell:
-        cmd: |
-          cat << 'EOF' >> ironic-vars-early
-            # use tempest plugin
-            export DEVSTACK_LOCAL_CONFIG+=$'\n'"TEMPEST_PLUGINS+=' /opt/stack/new/ironic-tempest-plugin'"
-            export TEMPEST_CONCURRENCY=4
-          EOF
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          export PROJECTS="openstack/ironic $PROJECTS"
-          export PROJECTS="openstack/ironic-lib $PROJECTS"
-          export PROJECTS="openstack/ironic-python-agent $PROJECTS"
-          export PROJECTS="openstack/ironic-python-agent-builder $PROJECTS"
-          export PROJECTS="openstack/ironic-tempest-plugin $PROJECTS"
-          export PROJECTS="openstack/python-ironicclient $PROJECTS"
-          export PROJECTS="openstack/virtualbmc $PROJECTS"
-          export PYTHONUNBUFFERED=true
-          export DEVSTACK_GATE_TEMPEST=1
-          export DEVSTACK_GATE_IRONIC=1
-          export DEVSTACK_GATE_NEUTRON=1
-          export DEVSTACK_GATE_VIRT_DRIVER=ironic
-          export DEVSTACK_GATE_CONFIGDRIVE=1
-          export DEVSTACK_GATE_IRONIC_DRIVER=ipmi
-          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_DEFAULT_DEPLOY_INTERFACE=direct"
-
-          export BRANCH_OVERRIDE="{{ zuul.override_checkout | default('default') }}"
-          if [ "$BRANCH_OVERRIDE" != "default" ] ; then
-              export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
-          fi
-
-          if [[ "$ZUUL_BRANCH" != "stable/ocata" && "$BRANCH_OVERRIDE" != "stable/ocata" ]]; then
-              export DEVSTACK_GATE_TLSPROXY=1
-          fi
-
-          # the direct deploy interface requires Swift temporary URLs
-          export DEVSTACK_LOCAL_CONFIG+=$'\n'"SWIFT_ENABLE_TEMPURLS=True"
-          export DEVSTACK_LOCAL_CONFIG+=$'\n'"SWIFT_TEMPURL_KEY=secretkey"
-
-          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_TEMPEST_WHOLE_DISK_IMAGE=True"
-          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_VM_EPHEMERAL_DISK=0"
-
-          export DEVSTACK_GATE_IRONIC_BUILD_RAMDISK=0
-          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_INSPECTOR_BUILD_RAMDISK=False"
-
-          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_VM_COUNT=7"
-
-          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_REQUIRE_AGENT_TOKEN=False"
-
-          # Ensure the ironic-vars-EARLY file exists
-          touch ironic-vars-early
-          # Pull in the EARLY variables injected by the optional builders
-          source ironic-vars-early
-
-          export DEVSTACK_LOCAL_CONFIG+=$'\n'"enable_plugin ironic https://opendev.org/openstack/ironic"
-
-          # Ensure the ironic-EXTRA-vars file exists
-          touch ironic-extra-vars
-          # Pull in the EXTRA variables injected by the optional builders
-          source ironic-extra-vars
-
-          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
-          ./safe-devstack-vm-gate-wrap.sh
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'

playbooks/legacy/ironic-dsvm-base-multinode/post.yaml

@@ -1,15 +0,0 @@
-- hosts: primary
-  tasks:
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/logs/**
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs

playbooks/legacy/ironic-dsvm-base-multinode/pre.yaml

@@ -1,22 +0,0 @@
-- hosts: primary
-  name: Clone devstack-gate to /opt/git
-  tasks:
-    - name: Ensure legacy workspace directory
-      file:
-        path: '{{ ansible_user_dir }}/workspace'
-        state: directory
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          cat > clonemap.yaml << EOF
-          clonemap:
-            - name: openstack/devstack-gate
-              dest: devstack-gate
-          EOF
-          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
-              https://opendev.org \
-              openstack/devstack-gate
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'

zuul.d/ironic-jobs.yaml

@@ -823,3 +823,157 @@
       tempest_concurrency: 1
       zuul_copy_output:
         '{{ devstack_bases.old }}/ironic-bm-logs': logs
+
+- job:
+    name: ironic-grenade-multinode-multitenant
+    parent: grenade-multinode
+    timeout: 10800
+    irrelevant-files:
+      - ^driver-requirements.txt$
+      - ^.*\.rst$
+      - ^api-ref/.*$
+      - ^doc/.*$
+      - ^install-guide/.*$
+      - ^ironic/locale/.*$
+      - ^ironic/tests/.*$
+      - ^releasenotes/.*$
+      - ^setup.cfg$
+      - ^tools/.*$
+      - ^tox.ini$
+    required-projects:
+      - openstack/grenade
+      - openstack/ironic
+      - openstack/ironic-lib
+      - openstack/ironic-python-agent
+      - openstack/ironic-python-agent-builder
+      - openstack/ironic-tempest-plugin
+      - openstack/networking-generic-switch
+      - openstack/python-ironicclient
+      - openstack/virtualbmc
+    vars:
+      grenade_devstack_localrc:
+        shared:
+          DEFAULT_INSTANCE_TYPE: baremetal
+          ENABLE_TENANT_TUNNELS: False
+          ENABLE_TENANT_VLANS: True
+          FORCE_CONFIG_DRIVE: True
+          HOST_TOPOLOGY: multinode
+          HOST_TOPOLOGY_ROLE: primary
+          BUILD_TIMEOUT: 1200
+          INSTALL_TEMPEST: False # Don't install a tempest package globaly
+          HOST_TOPOLOGY_SUBNODES: "{{ hostvars['compute1']['nodepool']['public_ipv4'] }}"
+          IRONIC_BAREMETAL_BASIC_OPS: True
+          IRONIC_AUTOMATED_CLEAN_ENABLED: False
+          IRONIC_BUILD_DEPLOY_RAMDISK: False
+          IRONIC_CALLBACK_TIMEOUT: 700
+          IRONIC_DEFAULT_DEPLOY_INTERFACE: direct
+          IRONIC_DEFAULT_BOOT_OPTION: netboot
+          IRONIC_DEPLOY_DRIVER: ipmi
+          IRONIC_VM_COUNT: 6
+          IRONIC_VM_EPHEMERAL_DISK: 1
+          IRONIC_VM_SPECS_RAM: 384
+          IRONIC_RAMDISK_TYPE: tinyipa
+          IRONIC_TEMPEST_BUILD_TIMEOUT: 1200
+          IRONIC_TEMPEST_WHOLE_DISK_IMAGE: True
+
+          IRONIC_ENABLED_NETWORK_INTERFACES: flat,neutron
+          IRONIC_NETWORK_INTERFACE: neutron
+          IRONIC_PROVISION_NETWORK_NAME: ironic-provision
+          IRONIC_PROVISION_SUBNET_GATEWAY: 10.0.5.1
+          IRONIC_PROVISION_SUBNET_PREFIX: 10.0.5.0/24
+          IRONIC_USE_LINK_LOCAL: True
+          OVS_BRIDGE_MAPPINGS: 'mynetwork:brbm,public:br-infra'
+          OVS_PHYSICAL_BRIDGE: brbm
+          PHYSICAL_NETWORK: mynetwork
+          PUBLIC_BRIDGE: br-infra
+          Q_AGENT: openvswitch
+          Q_ML2_TENANT_NETWORK_TYPE: vlan
+          Q_PLUGIN: ml2
+          SWIFT_ENABLE_TEMPURLS: True
+          SWIFT_TEMPURL_KEY: secretkey
+          TENANT_VLAN_RANGE: 100:150
+          VIRT_DRIVER: ironic
+          EBTABLES_RACE_FIX: True
+          LIBVIRT_STORAGE_POOL_PATH: /opt/libvirt/images
+          IRONIC_VM_LOG_DIR: '/opt/stack/ironic-bm-logs'
+      grenade_localrc:
+        BASE_RUN_SMOKE: False
+      devstack_plugins:
+        ironic: https://opendev.org/openstack/ironic
+        networking-generic-switch: https://opendev.org/openstack/networking-generic-switch
+      devstack_services:
+        c-api: False
+        c-bak: False
+        c-sch: False
+        c-vol: False
+        cinder: False
+        ir-api: True
+        ir-cond: True
+      tempest_plugins:
+        - ironic-tempest-plugin
+      tempest_test_regex: BaremetalMultitenancy
+      tox_envlist: all
+      tempest_concurrency: 1
+      grenade_tempest_concurrency: 1
+      grenade_test_timeout: 2600
+      zuul_copy_output:
+        '/opt/stack/ironic-bm-logs': logs
+        '{{ devstack_base_dir }}/data/networking-generic-switch/netmiko_session.log': 'logs'
+    group-vars:
+      subnode:
+        devstack_plugins:
+          ironic: https://opendev.org/openstack/ironic
+          networking-generic-switch: https://opendev.org/openstack/networking-generic-switch
+        devstack_services:
+          c-api: False
+          c-bak: False
+          c-sch: False
+          c-vol: False
+          cinder: False
+          g-api: True
+          ir-api: True
+          ir-cond: True
+          n-cpu: True
+          q-agt: True
+          virtualbmc: True
+        grenade_devstack_localrc:
+          shared:
+            DEFAULT_INSTANCE_TYPE: baremetal
+            ENABLE_TENANT_TUNNELS: False
+            ENABLE_TENANT_VLANS: True
+            FORCE_CONFIG_DRIVE: True
+            HOST_TOPOLOGY: multinode
+            HOST_TOPOLOGY_ROLE: subnode
+            IRONIC_AUTOMATED_CLEAN_ENABLED: False
+            IRONIC_BAREMETAL_BASIC_OPS: True
+            IRONIC_BUILD_DEPLOY_RAMDISK: False
+            BUILD_TIMEOUT: 1200
+            IRONIC_CALLBACK_TIMEOUT: 700
+            IRONIC_TEMPEST_BUILD_TIMEOUT: 1200
+            IRONIC_DEFAULT_DEPLOY_INTERFACE: direct
+            IRONIC_DEFAULT_BOOT_OPTION: netboot
+            IRONIC_DEPLOY_DRIVER: ipmi
+            IRONIC_ENABLED_NETWORK_INTERFACES: flat,neutron
+            IRONIC_NETWORK_INTERFACE: neutron
+            IRONIC_PROVISION_NETWORK_NAME: ironic-provision
+            IRONIC_PROVISION_SUBNET_GATEWAY: 10.0.5.1
+            IRONIC_PROVISION_SUBNET_PREFIX: 10.0.5.0/24
+            IRONIC_RAMDISK_TYPE: tinyipa
+            IRONIC_USE_LINK_LOCAL: True
+            IRONIC_VM_COUNT: 6
+            IRONIC_VM_EPHEMERAL_DISK: 0
+            IRONIC_VM_SPECS_RAM: 384
+            IRONIC_VM_NETWORK_BRIDGE: sub1brbm
+            OVS_BRIDGE_MAPPINGS: 'mynetwork:sub1brbm,public:br-infra'
+            OVS_PHYSICAL_BRIDGE: sub1brbm
+            PHYSICAL_NETWORK: mynetwork
+            PUBLIC_BRIDGE: br-infra
+            Q_AGENT: openvswitch
+            Q_ML2_TENANT_NETWORK_TYPE: vlan
+            Q_PLUGIN: ml2
+            SWIFT_ENABLE_TEMPURLS: True
+            SWIFT_TEMPURL_KEY: secretkey
+            VIRT_DRIVER: ironic
+            EBTABLES_RACE_FIX: True
+            LIBVIRT_STORAGE_POOL_PATH: /opt/libvirt/images
+            IRONIC_VM_LOG_DIR: '/opt/stack/ironic-bm-logs'

zuul.d/legacy-ironic-jobs.yaml

@@ -1,46 +0,0 @@
-# TODO(TheJulia): When we migrate to a non-legacy job, we will need to set the BUILD_TIMEOUT
-# and the DEVSTACK_GATE_TEMPEST_BAREMETAL_BUILD_TIMEOUT to 1200 seconds to prevent
-# needless CI job timeouts as the scale of the job is greater than a normal test jobs.
-- job:
-    name: legacy-ironic-dsvm-base-multinode
-    parent: legacy-dsvm-base-multinode
-    irrelevant-files:
-      - ^driver-requirements.txt$
-      - ^.*\.rst$
-      - ^api-ref/.*$
-      - ^doc/.*$
-      - ^install-guide/.*$
-      - ^ironic/locale/.*$
-      - ^ironic/tests/.*$
-      - ^releasenotes/.*$
-      - ^setup.cfg$
-      - ^tools/.*$
-      - ^tox.ini$
-    # NOTE: When adding to 'required-projects' also need to add a corresponding
-    # "export PROJECTS=..." line in all the playbooks/legacy/*/run.yaml files
-    required-projects:
-      - openstack/ironic
-      - openstack/ironic-lib
-      - openstack/ironic-python-agent
-      - openstack/ironic-tempest-plugin
-      - openstack/networking-generic-switch
-      - openstack/python-ironicclient
-      - openstack/virtualbmc
-    pre-run: playbooks/legacy/ironic-dsvm-base-multinode/pre.yaml
-    post-run: playbooks/legacy/ironic-dsvm-base-multinode/post.yaml
-
-
-- job:
-    name: ironic-grenade-dsvm-multinode-multitenant
-    parent: legacy-ironic-dsvm-base-multinode
-    run: playbooks/legacy/grenade-dsvm-ironic-multinode-multitenant/run.yaml
-    timeout: 10800
-    required-projects:
-      - openstack/grenade
-      - openstack/devstack-gate
-      - openstack/ironic
-      - openstack/ironic-lib
-      - openstack/ironic-python-agent
-      - openstack/networking-generic-switch
-      - openstack/python-ironicclient
-      - openstack/virtualbmc

zuul.d/project.yaml

@@ -15,8 +15,7 @@
             voting: false
         - ironic-tempest-functional-python3
         - ironic-grenade
-        # Temporary disable voting because of end of cycle CI instability.
+        - ironic-grenade-multinode-multitenant:
-        - ironic-grenade-dsvm-multinode-multitenant:
             voting: false
         - ironic-tempest-partition-bios-redfish-pxe
         - ironic-tempest-partition-uefi-redfish-vmedia
@@ -53,7 +52,7 @@
         - ironic-tempest-functional-python3
         - ironic-grenade
         # removing from voting due to end of cycle gate instability.
-        # - ironic-grenade-dsvm-multinode-multitenant
+        # - ironic-grenade-multinode-multitenant
         - ironic-tempest-partition-bios-redfish-pxe
         - ironic-tempest-partition-uefi-redfish-vmedia
         - ironic-tempest-wholedisk-bios-snmp-pxe