Port several legacy tempest jobs to Zuul v3

- cinder-tempest-lvm-multibackend is the simple LVM multibackend job
  and replaces legacy-tempest-dsvm-lvm-multibackend.
- remove legacy-tempest-dsvm-zeromq-multibackend, as ZeroMQ
  support was deprecated in Rocky and removed in Stein.
  It could be replaced by inheriting from
  cinder-tempest-lvm-multibackend in older branches
  when this patch is backported;
- replace cinder-tempest-dsvm-lvm-lio-barbican with
  cinder-tempest-dsvm-lvm-lio-barbican provided by
  cinder-tempest-plugins;
- replace legacy-tempest-dsvm-full-devstack-plugin-nfs with
  devstack-plugin-nfs-tempest-full, now provided by
  devstack-plugin-nfs;
- update cinder-plugin-ceph-tempest to also run cinderlib tests
  and make it inherit from devstack-plugin-ceph-tempest-py3.

Change-Id: Icdafaf55494b6dd74ec31c3572d7344fcc028166

.zuul.yaml

@@ -9,7 +9,6 @@
       - release-notes-jobs-python3
     check:
       jobs:
-        - cinder-tempest-dsvm-lvm-lio-barbican
         - cinder-tox-bandit-baseline:
             voting: false
         - openstack-tox-functional-py36:
@@ -48,12 +47,14 @@
               - ^setup.cfg$
               - ^tools/.*$
               - ^tox.ini$
+        - cinder-tempest-plugin-lvm-lio-barbican:
+            irrelevant-files: *functional-irrelevant-files
         - cinder-grenade-dsvm-mn-sub-volbak:
             irrelevant-files: *gate-irrelevant-files
-        - legacy-tempest-dsvm-lvm-multibackend:
+        - cinder-tempest-lvm-multibackend:
             voting: false
             irrelevant-files: *gate-irrelevant-files
-        - legacy-tempest-dsvm-full-devstack-plugin-nfs:
+        - devstack-plugin-nfs-tempest-full:
             voting: false
             irrelevant-files: *gate-irrelevant-files
         - tempest-slow-py3:
@@ -78,8 +79,6 @@
       jobs:
         - tempest-cinder-v2-api:
             irrelevant-files: *gate-irrelevant-files
-        - legacy-tempest-dsvm-zeromq-multibackend:
-            irrelevant-files: *gate-irrelevant-files
         - legacy-tempest-dsvm-multibackend-matrix:
             irrelevant-files: *gate-irrelevant-files
         - cinder-grenade-dsvm-mn-sub-volschbak:
@@ -93,22 +92,6 @@
         - tempest-full-py3-opensuse15:
             irrelevant-files: *gate-irrelevant-files
 
-- job:
-    # Previously named legacy-tempest-dsvm-full-lio
-    name: cinder-tempest-dsvm-lvm-lio-barbican
-    parent: legacy-dsvm-base
-    run: playbooks/legacy/cinder-tempest-dsvm-lvm-lio-barbican/run.yaml
-    post-run: playbooks/legacy/cinder-tempest-dsvm-lvm-lio-barbican/post.yaml
-    timeout: 10800
-    required-projects:
-      - openstack/devstack-gate
-      - openstack/barbican
-      - openstack/cinderlib
-      - openstack/python-barbicanclient
-      - openstack/tempest
-      - openstack/os-brick
-    irrelevant-files: *gate-irrelevant-files
-
 - job:
     # Security testing for known issues
     name: cinder-tox-bandit-baseline
@@ -138,10 +121,16 @@
 - job:
     name: cinder-plugin-ceph-tempest
     parent: devstack-plugin-ceph-tempest-py3
+    roles:
+      - zuul: opendev.org/openstack/cinderlib
+      - zuul: opendev.org/openstack/cinder-tempest-plugin
     run: playbooks/tempest-and-cinderlib-run.yaml
+    post-run: playbooks/post-cinderlib.yaml
     required-projects:
-      - openstack/cinderlib
-      - openstack/tempest
+      - opendev.org/openstack/cinderlib
+    vars:
+      zuul_additional_subunit_dirs:
+        - "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/cinderlib'].src_dir }}"
 
 - job:
     name: cinder-grenade-dsvm-mn-sub-bak
@@ -193,3 +182,23 @@
       - openstack/grenade
       - openstack/devstack-gate
       - openstack/cinder
+
+- job:
+    name: cinder-tempest-lvm-multibackend
+    parent: devstack-tempest
+    description: |
+      Cinder tempest job based on LVM and multiple backends.
+      Former names for this job were:
+        * legacy-tempest-dsvm-lvm-multibackend
+    timeout: 10800
+    required-projects:
+      - opendev.org/openstack/cinder-tempest-plugin
+    vars:
+      tox_envlist: all
+      tempest_test_regex: '(?!.*\[.*\bslow\b.*\])(^tempest\.(api|scenario)|(^cinder_tempest_plugin))'
+      tempest_plugins:
+        - cinder-tempest-plugin
+      devstack_localrc:
+        CINDER_ENABLED_BACKENDS: 'lvm:lvmdriver-1,lvm:lvmdriver-2'
+        CINDER_VOLUME_CLEAR: none
+    irrelevant-files: *gate-irrelevant-files

playbooks/cinderlib-run.yaml

@@ -1,49 +0,0 @@
-# Variables: devstack_base_dir, cinderlib_log_file, cinderlib_ignore_errors
-- hosts: "{{ cinderlib_hosts | default('all') }}"
-  become: True
-  vars:
-    base_dir: "{{ devstack_base_dir | default('/opt/stack/new') }}"
-    default_log_file: "{{ ansible_user_dir }}/zuul-output/logs/cinderlib.txt"
-
-  tasks:
-    - name: Find cinderlib location in required-projects
-      set_fact:
-        cinderlib_location: "{{ ansible_user_dir }}/{{ item.src_dir}}"
-      with_items: "{{ zuul.projects.values() | list }}"
-      when:
-        - cinderlib_location is not defined
-        - item.short_name == 'cinderlib'
-
-    - name: Default cinderlib location to devstack directory
-      set_fact:
-        cinderlib_location: "{{ base_dir }}/cinderlib"
-      when:
-        - cinderlib_location is not defined
-
-    - name: Install cinderlib
-      pip:
-        name: ./
-        editable: yes
-        extra_args: --no-cache-dir
-        chdir: "{{ cinderlib_location }}"
-
-    - name: Locate unit2 binary location
-      shell:
-        cmd: which unit2
-      register: unit2_which
-
-    - name: Add sudoers role for cinderlib unit2
-      copy:
-        dest: /etc/sudoers.d/zuul-sudo-unit2
-        content: "zuul ALL = NOPASSWD:{{ unit2_which.stdout }} discover -v -s cinderlib/tests/functional\n"
-        mode: 0440
-
-    - name: Validate sudoers config after edits
-      command: "/usr/sbin/visudo -c"
-
-    - name: Run cinderlib functional tests
-      shell:
-        cmd: "set -o pipefail && {{ unit2_which.stdout }} discover -v -s cinderlib/tests/functional 2>&1 | tee {{ cinderlib_log_file | default(default_log_file)}}"
-        chdir: "{{ cinderlib_location }}"
-        executable: /bin/bash
-      ignore_errors: "{{ cinderlib_ignore_errors | default(false) | bool}}"

playbooks/legacy/cinder-tempest-dsvm-lvm-lio-barbican/post.yaml

@@ -1,15 +0,0 @@
-- hosts: primary
-  tasks:
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/logs/**
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs

playbooks/legacy/cinder-tempest-dsvm-lvm-lio-barbican/run.yaml

@@ -1,89 +0,0 @@
-- hosts: all
-  name: Test the LVM driver with the LIO iSCSI target and Barbican
-  tasks:
-
-    - name: Ensure legacy workspace directory
-      file:
-        path: '{{ ansible_user_dir }}/workspace'
-        state: directory
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          cat > clonemap.yaml << EOF
-          clonemap:
-            - name: openstack/devstack-gate
-              dest: devstack-gate
-          EOF
-          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
-              https://opendev.org \
-              openstack/devstack-gate
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          cat << 'EOF' >>"/tmp/dg-local.conf"
-          [[local|localrc]]
-          CINDER_ISCSI_HELPER=lioadm
-          CINDER_LVM_TYPE=thin
-          enable_service barbican
-          enable_plugin barbican https://opendev.org/openstack/barbican
-
-          # Test with local file locks
-          CINDER_COORDINATION_URL='file://$state_path'
-
-          EOF
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          export PROJECTS="openstack/barbican $PROJECTS"
-          export PROJECTS="openstack/python-barbicanclient $PROJECTS"
-          export PYTHONUNBUFFERED=true
-          if [ "$ZUUL_BRANCH" == "master" ]; then
-              # Use os-brick head for master, use os-brick from
-              # pip for stable branches.
-              export DEVSTACK_PROJECT_FROM_GIT=os-brick
-          fi
-          export DEVSTACK_GATE_USE_PYTHON3=True
-          export DEVSTACK_GATE_TEMPEST=1
-          export DEVSTACK_GATE_TEMPEST_ALL_PLUGINS=1
-          # As we are setting DEVSTACK_GATE_TEMPEST_ALL_PLUGINS, we need to add
-          # the regex to limit the tests to the same as a tempest -efull would
-          # run.
-          DEVSTACK_GATE_TEMPEST_REGEX='(^tempest\.(api|scenario\.test_encrypted_cinder_volumes|scenario\.test_volume|scenario\.test_shelve_instance)|(^cinder\.tests.tempest))'
-          DEVSTACK_GATE_TEMPEST_REGEX+='(?!.*VolumesSnapshotTestJSON.test_snapshot_backup)'
-          DEVSTACK_GATE_TEMPEST_REGEX+='(?!.*VolumesBackupsAdminTest.test_volume_backup_export_import)'
-          DEVSTACK_GATE_TEMPEST_REGEX+='(?!.*VolumesBackupsAdminTest.test_volume_backup_reset_status)'
-          DEVSTACK_GATE_TEMPEST_REGEX+='(?!.*VolumesBackupsTest.test_backup_create_attached_volume)'
-          DEVSTACK_GATE_TEMPEST_REGEX+='(?!.*VolumesBackupsTest.test_backup_create_and_restore_to_an_existing_volume)'
-          DEVSTACK_GATE_TEMPEST_REGEX+='(?!.*VolumesBackupsTest.test_bootable_volume_backup_and_restore)'
-          DEVSTACK_GATE_TEMPEST_REGEX+='(?!.*VolumesBackupsTest.test_incremental_backup)'
-          DEVSTACK_GATE_TEMPEST_REGEX+='(?!.*VolumesBackupsTest.test_volume_backup_create_get_detailed_list_restore_delete)'
-          DEVSTACK_GATE_TEMPEST_REGEX+='(?!.*VolumesBackupsTest.test_volume_snapshot_backup)'
-          DEVSTACK_GATE_TEMPEST_REGEX+='(?!.*VolumesBackupsV39Test.test_update_backup)'
-          DEVSTACK_GATE_TEMPEST_REGEX+='(?!.*TestVolumeBackupRestore.test_volume_backup_restore)'
-          export DEVSTACK_GATE_TEMPEST_REGEX
-          export BRANCH_OVERRIDE=default
-          if [ "$BRANCH_OVERRIDE" != "default" ] ; then
-              export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
-          fi
-
-          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
-          ./safe-devstack-vm-gate-wrap.sh
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
-
-- import_playbook: ../../cinderlib-run.yaml
-  vars:
-    default_log_file: "{{ ansible_user_dir }}/workspace/logs/cinderlib.txt"

playbooks/post-cinderlib.yaml

@@ -0,0 +1,6 @@
+- hosts: all
+  vars:
+    tox_envlist: functional
+    zuul_work_dir: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/cinderlib'].src_dir }}"
+  roles:
+    - fetch-tox-output

playbooks/tempest-and-cinderlib-run.yaml

@@ -1,4 +1,4 @@
-# Playbook imported from https://opendev.org/openstack/tempest/src/branch/master/playbooks/devstack-tempest.yaml
+# Playbook imported from https://opendev.org/openstack/tempest/src/tag/23.0.0/playbooks/devstack-tempest.yaml
 
 # Changes that run through devstack-tempest are likely to have an impact on
 # the devstack part of the job, so we keep devstack in the main play to
@@ -18,6 +18,11 @@
     - setup-tempest-run-dir
     - setup-tempest-data-dir
     - acl-devstack-files
-    - run-tempest
-
-- import_playbook: cinderlib-run.yaml cinderlib_hosts=tempest
+    - role: run-tempest
+      # ignore the errors, so that run-cinderlib-tests is always executed
+      ignore_errors: yes
+    - role: change-devstack-data-owner
+      devstack_data_subdir_changed: cinder
+      devstack_data_subdir_owner: zuul
+    - role: run-cinderlib-tests
+      cinderlib_base_dir: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/cinderlib'].src_dir }}"