Move volume tests to experimental pipeline

The volume tests are unreliable and need to be moved to an experimental
pipeline.

Change-Id: I7414fbdee4da57652bcf10840d2d40e35dc5d1d4

osci.yaml

@@ -8,6 +8,18 @@
           label: bionic-brown
     vars:
       tox_envlist: snap
+
+- job:
+    name: microstack-bionic-lvm
+    parent: openstack-tox-snap-with-sudo
+    timeout: 7200
+    nodeset:
+      nodes:
+        - name: bionic-brown
+          label: bionic-brown
+    vars:
+      tox_envlist: snap-lvm
+
 - job:
     name: microstack-focal
     parent: openstack-tox-snap-with-sudo
@@ -19,9 +31,26 @@
     vars:
       tox_envlist: snap
 
+- job:
+    name: microstack-focal-lvm
+    parent: openstack-tox-snap-with-sudo
+    timeout: 7200
+    nodeset:
+      nodes:
+        - name: focal-brown
+          label: focal-brown
+    vars:
+      tox_envlist: snap-lvm
+
 - project:
     check:
       fail-fast: true
       jobs:
         - microstack-bionic
         - microstack-focal
+
+    experimental:
+      fail-fast: true
+      jobs:
+        - microstack-bionic-lvm
+        - microstack-focal-lvm

tests/framework.py

@@ -121,12 +121,21 @@ class TestHost:
             '--name', 'snap_generated'])
         self.check_call(['microstack-test.tempest-init'])
 
-    def run_verifications(self):
+    def _create_filtered_test_list(self, source):
+        target = '/tmp/snap.microstack-test/tmp/exclude-volume-tests.txt'
+        cmd = ['sudo', 'sh', '-c',
+               f'sed "/.*volume.*/d" {source} > {target}']
+        self.check_call(cmd)
+        return '/tmp/exclude-volume-tests.txt'
+
+    def run_verifications(self, include_volumes=False):
         """Run a set of verification tests on MicroStack from this host."""
+        test_list = '/snap/microstack-test/current/2020.06-test-list.txt'
+        if not include_volumes:
+            test_list = self._create_filtered_test_list(test_list)
         self.check_call([
             'microstack-test.rally', 'verify', 'start',
-            '--load-list',
-            '/snap/microstack-test/current/2020.06-test-list.txt',
+            '--load-list', f'{test_list}',
             '--detailed', '--concurrency', '2'])
         self.check_call([
             'microstack-test.rally', 'verify', 'report',

tests/test_basic.py

@@ -25,7 +25,9 @@ from tests.framework import Framework  # noqa E402
 
 
 class TestBasics(Framework):
+
     snap_try = False
+    use_lvm = False
 
     def test_basics(self):
         """Basic test
@@ -36,12 +38,14 @@ class TestBasics(Framework):
         """
         self._localhost.install_microstack(path='microstack_ussuri_amd64.snap',
                                            snap_try=self.snap_try)
-        self._localhost.init_microstack([
-            '--auto',
-            '--control',
-            '--setup-loop-based-cinder-lvm-backend',
-            '--loop-device-file-size=24'
+        init_args = ['--auto', '--control']
+        if self.use_lvm:
+            init_args.extend([
+                '--setup-loop-based-cinder-lvm-backend',
+                '--loop-device-file-size=24',
             ])
+
+        self._localhost.init_microstack(init_args)
         endpoints = self._localhost.check_output(
             ['/snap/bin/microstack.openstack', 'endpoint', 'list']
         ).decode('utf-8')
@@ -78,7 +82,7 @@ class TestBasics(Framework):
 
         self._localhost.setup_tempest_verifier()
         # Make sure there are no verification failures in the report.
-        failures = self._localhost.run_verifications()
+        failures = self._localhost.run_verifications(self.use_lvm)
         self.assertEqual(failures, 0, 'Verification tests had failure.')
 
         # Try to remove the snap without sudo.
@@ -106,9 +110,12 @@ if __name__ == '__main__':
     parser = argparse.ArgumentParser()
     parser.add_argument("--snap-try", help="Install snap as rw mount from "
                         "squashfs-root directory", action='store_true')
+    parser.add_argument("--use-lvm", help="Init snap with experimental lvm "
+                        "support", action='store_true')
     parser.add_argument('unittest_args', nargs='*')
     args = parser.parse_args()
     TestBasics.snap_try = args.snap_try
+    TestBasics.use_lvm = args.use_lvm
     sys.argv[1:] = args.unittest_args
 
     # Run our tests, ignoring deprecation warnings and warnings about

tox.ini

@@ -30,6 +30,12 @@ commands =
     bash -c "unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY ; {toxinidir}/tests/test_basic.py"
     bash -c "unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY ; {toxinidir}/tests/test_cluster.py"
 
+[testenv:snap-lvm]
+commands =
+    {toxinidir}/tools/lxd_build.sh
+    flake8 {toxinidir}/tests/
+    bash -c "unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY ; {toxinidir}/tests/test_basic.py --use-lvm"
+
 [testenv:snap-try]
 # Testing environment for local debugging. Mounts rw snap from squashfs-root dir.
 commands =