Update cinder-tempest-plugin-lvm-lio (revamp)

The changes should make this job a valid replacement for the legacy
cinder-tempest-dsvm-lvm-lio-barbican job defined inside cinder.git.

- rename as cinder-tempest-plugin-lvm-lio-barbican;
- add it to the gate queue as well, together with
  devstack-plugin-ceph-tempest-py3;
- use the proper functional tox environment provided by cinderlib;
- synchronize the blacklist with the one used by the legacy job,
  and use the same devstack options;
- remove barbicanclient and os-brick from required-projects,
  so that the stable libraries are used, as usual for normal jobs.
  Child jobs can add them if they need to test against
  the master version of those libraries.

And a few minor changes (namespaces for the repositories,
prepare it to be multinode-compatible).

Change-Id: I0ed43c8eda445af8da61d861536da1efdd5a7b1f

.zuul.yaml

@@ -4,35 +4,48 @@
       - tempest-plugin-jobs
     check:
       jobs:
-        - cinder-tempest-plugin-lvm-lio
+        - cinder-tempest-plugin-lvm-lio-barbican
+        - devstack-plugin-ceph-tempest-py3
+    gate:
+      jobs:
+        - cinder-tempest-plugin-lvm-lio-barbican
         - devstack-plugin-ceph-tempest-py3
 
 - job:
-    name: cinder-tempest-plugin-lvm-lio
-    voting: false
+    name: cinder-tempest-plugin-lvm-lio-barbican
     description: |
-      Run Cinder Tempest Plugin tests
+      This jobs configures Cinder with LVM, LIO, barbican and
+      runs tempest tests and cinderlib tests.
     parent: devstack-tempest
+    roles:
+      - zuul: opendev.org/openstack/cinderlib
     required-projects:
-      - openstack/devstack-gate
-      - openstack/barbican
-      - openstack/python-barbicanclient
-      - openstack/tempest
-      - openstack/os-brick
-      - openstack/cinder-tempest-plugin
-      - openstack/cinder
+      - opendev.org/openstack/barbican
+      - opendev.org/openstack/tempest
+      - opendev.org/openstack/cinder-tempest-plugin
+      - opendev.org/openstack/cinder
+      - opendev.org/openstack/cinderlib
+    run: playbooks/tempest-and-cinderlib-run.yaml
+    # Required to collect the tox-based logs of the cinderlib functional tests
+    post-run: playbooks/post-cinderlib.yaml
+    host-vars:
+      controller:
+        devstack_plugins:
+          barbican: https://opendev.org/openstack/barbican
     vars:
-      tempest_test_regex: (tempest\.(api|scenario)|cinder_tempest_plugin)
-      tempest_test_blacklist: '{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tempest_blacklist.txt'
+      tempest_test_regex: '(^tempest\.(api|scenario\.test_encrypted_cinder_volumes|scenario\.test_volume|scenario\.test_shelve_instance)|(^cinder_tempest_plugin))'
+      tempest_test_blacklist: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/cinder-tempest-plugin'].src_dir }}/tempest_blacklist.txt"
       tox_envlist: all
       devstack_localrc:
         CINDER_ISCSI_HELPER: lioadm
-        TEMPEST_PLUGINS: /opt/stack/cinder-tempest-plugin
-        USE_PYTHON3: true
-      devstack_plugins:
-        barbican: https://opendev.org/openstack/barbican
+        CINDER_LVM_TYPE: thin
+        CINDER_COORDINATION_URL: 'file://\$state_path'
       devstack_services:
         barbican: true
+      tempest_plugins:
+        - cinder-tempest-plugin
+      fetch_subunit_output_additional_dirs:
+        - "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/cinderlib'].src_dir }}"
     irrelevant-files:
       - ^.*\.rst$
       - ^doc/.*$

playbooks/post-cinderlib.yaml

@@ -0,0 +1,6 @@
+- hosts: all
+  vars:
+    tox_envlist: functional
+    zuul_work_dir: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/cinderlib'].src_dir }}"
+  roles:
+    - fetch-tox-output

playbooks/tempest-and-cinderlib-run.yaml

@@ -0,0 +1,28 @@
+# Playbook imported from https://opendev.org/openstack/tempest/src/tag/23.0.0/playbooks/devstack-tempest.yaml
+
+# Changes that run through devstack-tempest are likely to have an impact on
+# the devstack part of the job, so we keep devstack in the main play to
+# avoid zuul retrying on legitimate failures.
+- hosts: all
+  roles:
+    - orchestrate-devstack
+
+# We run tests only on one node, regardless how many nodes are in the system
+- hosts: tempest
+  environment:
+    # This enviroment variable is used by the optional tempest-gabbi
+    # job provided by the gabbi-tempest plugin. It can be safely ignored
+    # if that plugin is not being used.
+    GABBI_TEMPEST_PATH: "{{ gabbi_tempest_path | default('') }}"
+  roles:
+    - setup-tempest-run-dir
+    - setup-tempest-data-dir
+    - acl-devstack-files
+    - role: run-tempest
+      # ignore the errors, so that run-cinderlib-tests is always executed
+      ignore_errors: yes
+    - role: change-devstack-data-owner
+      devstack_data_subdir_changed: cinder
+      devstack_data_subdir_owner: zuul
+    - role: run-cinderlib-tests
+      cinderlib_base_dir: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/cinderlib'].src_dir }}"

roles/change-devstack-data-owner/README.rst

@@ -0,0 +1,23 @@
+Change the ownership of a specific devstack data subdirectory
+
+This is needed in order to have cinderlib functional tests,
+which are normally executed by the `zuul` user, run under
+a devstack deployment where the `stack` user is the owner.
+
+**Role Variables**
+
+.. zuul:rolevar:: devstack_data_dir
+   :default: /opt/stack/data
+
+   The devstack data directory.
+
+.. zuul:rolevar:: devstack_data_subdir_changed
+   :default: cinder
+
+   The devstack data subdirectory whose ownership
+   is changed.
+
+.. zuul:rolevar:: devstack_data_subdir_owner
+   :default: zuul
+
+   The new owner of the specified devstack data subdirectory.

roles/change-devstack-data-owner/defaults/main.yaml

@@ -0,0 +1,3 @@
+devstack_data_dir: /opt/stack/data
+devstack_data_subdir_changed: cinder
+devstack_data_subdir_owner: zuul

roles/change-devstack-data-owner/tasks/main.yaml

@@ -0,0 +1,6 @@
+- name: Change the owner of specific devstack data files
+  file:
+    path: "{{ devstack_data_dir }}/{{ devstack_data_subdir_changed }}"
+    owner: "{{ devstack_data_subdir_owner }}"
+    recurse: yes
+  become: yes

tempest_blacklist.txt

@@ -1,10 +1,11 @@
-# List of tests getting skipped
-.*\[.*\bslow\b.*\]
-.*\.*VolumesSnapshotTestJSON\.test_snapshot_backup
-.*\.*VolumesBackupsTest\.test_backup_create_attached_volume
-.*\.*VolumesBackupsTest\.test_backup_create_and_restore_to_an_existing_volume
-.*\.*VolumesBackupsTest\.test_bootable_volume_backup_and_restore
-.*\.*VolumesBackupsTest\.test_incremental_backup
-.*\.*VolumesBackupsTest\.test_volume_backup_create_get_detailed_list_restore_delete
-.*\.*VolumesBackupsTest\.test_volume_snapshot_backup
-.*\.*VolumesBackupsV39Test\.test_update_backup
+.*VolumesSnapshotTestJSON.test_snapshot_backup
+.*VolumesBackupsAdminTest.test_volume_backup_export_import
+.*VolumesBackupsAdminTest.test_volume_backup_reset_status
+.*VolumesBackupsTest.test_backup_create_attached_volume
+.*VolumesBackupsTest.test_backup_create_and_restore_to_an_existing_volume
+.*VolumesBackupsTest.test_bootable_volume_backup_and_restore
+.*VolumesBackupsTest.test_incremental_backup
+.*VolumesBackupsTest.test_volume_backup_create_get_detailed_list_restore_delete
+.*VolumesBackupsTest.test_volume_snapshot_backup
+.*VolumesBackupsV39Test.test_update_backup
+.*TestVolumeBackupRestore.test_volume_backup_restore