Stop implicit validation_resources provisioning

The provisioning of validation resources is unreadable and hard to
use. Tests have to call an helper before super's resource_setup is
invoked and they will find resources in a dictionary in a class
attribute.

Changing to a model where the test.py base class provides helpers
that takes care of common tasks:
- pulling the right parameters from configuration
- scheduling cleanup

There are two helpers available, one to be used when validation
resources are provisioned for a server created at class setup time;
the second one shall be used with servers provisioned durint tests
or test setup.

The new helper returns the provisioned resources to the test.
Other helpers are affected by this change since they cannot pull
validation resource from class anymore safely, and they have
been updated to accept validation resources as input.

Change-Id: I1106e40c6d7483f66d645f3bb560c6d74a612d0f

tempest/api/compute/admin/test_create_server.py

@@ -17,8 +17,10 @@ import testtools
 
 from tempest.api.compute import base
 from tempest.common.utils.linux import remote_client
+from tempest.common import waiters
 from tempest import config
 from tempest.lib.common.utils import data_utils
+from tempest.lib.common.utils import test_utils
 from tempest.lib import decorators
 
 CONF = config.CONF
@@ -35,12 +37,6 @@ class ServersWithSpecificFlavorTestJSON(base.BaseV2ComputeAdminTest):
         super(ServersWithSpecificFlavorTestJSON, cls).setup_clients()
         cls.client = cls.servers_client
 
-    @classmethod
-    def resource_setup(cls):
-        cls.set_validation_resources()
-
-        super(ServersWithSpecificFlavorTestJSON, cls).resource_setup()
-
     @decorators.idempotent_id('b3c7bcfc-bb5b-4e22-b517-c7f686b802ca')
     @testtools.skipUnless(CONF.validation.run_validation,
                           'Instance validation tests are disabled.')
@@ -67,20 +63,30 @@ class ServersWithSpecificFlavorTestJSON(base.BaseV2ComputeAdminTest):
 
         admin_pass = self.image_ssh_password
 
+        validation_resources = self.get_test_validation_resources(
+            self.os_primary)
         server_no_eph_disk = self.create_test_server(
             validatable=True,
+            validation_resources=validation_resources,
             wait_until='ACTIVE',
             adminPass=admin_pass,
             flavor=flavor_no_eph_disk_id)
 
+        self.addCleanup(waiters.wait_for_server_termination,
+                        self.servers_client, server_no_eph_disk['id'])
+        self.addCleanup(test_utils.call_and_ignore_notfound_exc,
+                        self.servers_client.delete_server,
+                        server_no_eph_disk['id'])
+
         # Get partition number of server without ephemeral disk.
         server_no_eph_disk = self.client.show_server(
             server_no_eph_disk['id'])['server']
         linux_client = remote_client.RemoteClient(
-            self.get_server_ip(server_no_eph_disk),
+            self.get_server_ip(server_no_eph_disk,
+                               validation_resources),
             self.ssh_user,
             admin_pass,
-            self.validation_resources['keypair']['private_key'],
+            validation_resources['keypair']['private_key'],
             server=server_no_eph_disk,
             servers_client=self.client)
         disks_num = len(linux_client.get_disks().split('\n'))
@@ -90,17 +96,25 @@ class ServersWithSpecificFlavorTestJSON(base.BaseV2ComputeAdminTest):
 
         server_with_eph_disk = self.create_test_server(
             validatable=True,
+            validation_resources=validation_resources,
             wait_until='ACTIVE',
             adminPass=admin_pass,
             flavor=flavor_with_eph_disk_id)
 
+        self.addCleanup(waiters.wait_for_server_termination,
+                        self.servers_client, server_with_eph_disk['id'])
+        self.addCleanup(test_utils.call_and_ignore_notfound_exc,
+                        self.servers_client.delete_server,
+                        server_with_eph_disk['id'])
+
         server_with_eph_disk = self.client.show_server(
             server_with_eph_disk['id'])['server']
         linux_client = remote_client.RemoteClient(
-            self.get_server_ip(server_with_eph_disk),
+            self.get_server_ip(server_with_eph_disk,
+                               validation_resources),
             self.ssh_user,
             admin_pass,
-            self.validation_resources['keypair']['private_key'],
+            validation_resources['keypair']['private_key'],
             server=server_with_eph_disk,
             servers_client=self.client)
         disks_num_eph = len(linux_client.get_disks().split('\n'))

tempest/api/compute/base.py

@@ -190,7 +190,7 @@ class BaseV2ComputeTest(api_version_utils.BaseMicroversionTest,
 
     @classmethod
     def create_test_server(cls, validatable=False, volume_backed=False,
-                           **kwargs):
+                           validation_resources=None, **kwargs):
         """Wrapper utility that returns a test server.
 
         This wrapper utility calls the common create test server and
@@ -200,6 +200,10 @@ class BaseV2ComputeTest(api_version_utils.BaseMicroversionTest,
 
         :param validatable: Whether the server will be pingable or sshable.
         :param volume_backed: Whether the instance is volume backed or not.
+        :param validation_resources: Dictionary of validation resources as
+            returned by `get_class_validation_resources`.
+        :param kwargs: Extra arguments are passed down to the
+            `compute.create_test_server` call.
         """
         if 'name' not in kwargs:
             kwargs['name'] = data_utils.rand_name(cls.__name__ + "-server")
@@ -216,7 +220,7 @@ class BaseV2ComputeTest(api_version_utils.BaseMicroversionTest,
         body, servers = compute.create_test_server(
             cls.os_primary,
             validatable,
-            validation_resources=cls.validation_resources,
+            validation_resources=validation_resources,
             tenant_network=tenant_network,
             volume_backed=volume_backed,
             **kwargs)
@@ -326,13 +330,33 @@ class BaseV2ComputeTest(api_version_utils.BaseMicroversionTest,
 
     @classmethod
     def rebuild_server(cls, server_id, validatable=False, **kwargs):
-        # Destroy an existing server and creates a new one
+        """Destroy an existing class level server and creates a new one
+
+        Some test classes use a test server that can be used by multiple
+        tests. This is done to optimise runtime and test load.
+        If something goes wrong with the test server, it can be rebuilt
+        using this helper.
+
+        This helper can also be used for the initial provisioning if no
+        server_id is specified.
+
+        :param server_id: UUID of the server to be rebuilt. If None is
+            specified, a new server is provisioned.
+        :param validatable: whether to the server needs to be
+            validatable. When True, validation resources are acquired via
+            the `get_class_validation_resources` helper.
+        :param kwargs: extra paramaters are passed through to the
+            `create_test_server` call.
+        :return: the UUID of the created server.
+        """
         if server_id:
             cls.delete_server(server_id)
 
         cls.password = data_utils.rand_password()
         server = cls.create_test_server(
             validatable,
+            validation_resources=cls.get_class_validation_resources(
+                cls.os_primary),
             wait_until='ACTIVE',
             adminPass=cls.password,
             **kwargs)
@@ -363,14 +387,23 @@ class BaseV2ComputeTest(api_version_utils.BaseMicroversionTest,
         cls._delete_volume(cls.volumes_client, volume_id)
 
     @classmethod
-    def get_server_ip(cls, server):
+    def get_server_ip(cls, server, validation_resources=None):
         """Get the server fixed or floating IP.
 
         Based on the configuration we're in, return a correct ip
         address for validating that a guest is up.
+
+        :param server: The server dict as returned by the API
+        :param validation_resources: The dict of validation resources
+            provisioned for the server.
         """
         if CONF.validation.connect_method == 'floating':
-            return cls.validation_resources['floating_ip']['ip']
+            if validation_resources:
+                return validation_resources['floating_ip']['ip']
+            else:
+                msg = ('When validation.connect_method equals floating, '
+                       'validation_resources cannot be None')
+                raise exceptions.InvalidParam(invalid_param=msg)
         elif CONF.validation.connect_method == 'fixed':
             addresses = server['addresses'][CONF.validation.network_for_ssh]
             for address in addresses:

tempest/api/compute/servers/test_create_server.py

@@ -42,8 +42,9 @@ class ServersTestJSON(base.BaseV2ComputeTest):
 
     @classmethod
     def resource_setup(cls):
-        cls.set_validation_resources()
         super(ServersTestJSON, cls).resource_setup()
+        validation_resources = cls.get_class_validation_resources(
+            cls.os_primary)
         cls.meta = {'hello': 'world'}
         cls.accessIPv4 = '1.1.1.1'
         cls.accessIPv6 = '0000:0000:0000:0000:0000:babe:220.12.22.2'
@@ -52,6 +53,7 @@ class ServersTestJSON(base.BaseV2ComputeTest):
         disk_config = cls.disk_config
         server_initial = cls.create_test_server(
             validatable=True,
+            validation_resources=validation_resources,
             wait_until='ACTIVE',
             name=cls.name,
             metadata=cls.meta,
@@ -105,11 +107,13 @@ class ServersTestJSON(base.BaseV2ComputeTest):
         # Verify that the number of vcpus reported by the instance matches
         # the amount stated by the flavor
         flavor = self.flavors_client.show_flavor(self.flavor_ref)['flavor']
+        validation_resources = self.get_class_validation_resources(
+            self.os_primary)
         linux_client = remote_client.RemoteClient(
-            self.get_server_ip(self.server),
+            self.get_server_ip(self.server, validation_resources),
             self.ssh_user,
             self.password,
-            self.validation_resources['keypair']['private_key'],
+            validation_resources['keypair']['private_key'],
             server=self.server,
             servers_client=self.client)
         output = linux_client.exec_command('grep -c ^processor /proc/cpuinfo')
@@ -120,11 +124,13 @@ class ServersTestJSON(base.BaseV2ComputeTest):
                           'Instance validation tests are disabled.')
     def test_host_name_is_same_as_server_name(self):
         # Verify the instance host name is the same as the server name
+        validation_resources = self.get_class_validation_resources(
+            self.os_primary)
         linux_client = remote_client.RemoteClient(
-            self.get_server_ip(self.server),
+            self.get_server_ip(self.server, validation_resources),
             self.ssh_user,
             self.password,
-            self.validation_resources['keypair']['private_key'],
+            validation_resources['keypair']['private_key'],
             server=self.server,
             servers_client=self.client)
         hostname = linux_client.exec_command("hostname").rstrip()

tempest/api/compute/servers/test_device_tagging.py

@@ -66,11 +66,6 @@ class DeviceTaggingTest(base.BaseV2ComputeTest):
                                   dhcp=True)
         super(DeviceTaggingTest, cls).setup_credentials()
 
-    @classmethod
-    def resource_setup(cls):
-        cls.set_validation_resources()
-        super(DeviceTaggingTest, cls).resource_setup()
-
     def verify_device_metadata(self, md_json):
         md_dict = json.loads(md_json)
         for d in md_dict['devices']:
@@ -139,9 +134,12 @@ class DeviceTaggingTest(base.BaseV2ComputeTest):
         # Create server
         admin_pass = data_utils.rand_password()
         config_drive_enabled = CONF.compute_feature_enabled.config_drive
+        validation_resources = self.get_test_validation_resources(
+            self.os_primary)
 
         server = self.create_test_server(
             validatable=True,
+            validation_resources=validation_resources,
             config_drive=config_drive_enabled,
             adminPass=admin_pass,
             name=data_utils.rand_name('device-tagging-server'),
@@ -208,10 +206,10 @@ class DeviceTaggingTest(base.BaseV2ComputeTest):
         self.addCleanup(self.delete_server, server['id'])
 
         self.ssh_client = remote_client.RemoteClient(
-            self.get_server_ip(server),
+            self.get_server_ip(server, validation_resources),
             CONF.validation.image_ssh_user,
             admin_pass,
-            self.validation_resources['keypair']['private_key'],
+            validation_resources['keypair']['private_key'],
             server=server,
             servers_client=self.servers_client)
 

tempest/api/compute/servers/test_server_actions.py

@@ -44,8 +44,13 @@ class ServerActionsTestJSON(base.BaseV2ComputeTest):
                                            self.server_id, 'ACTIVE')
         except lib_exc.NotFound:
             # The server was deleted by previous test, create a new one
+            # Use class level validation resources to avoid them being
+            # deleted once a test is over
+            validation_resources = self.get_class_validation_resources(
+                self.os_primary)
             server = self.create_test_server(
                 validatable=True,
+                validation_resources=validation_resources,
                 wait_until='ACTIVE')
             self.__class__.server_id = server['id']
         except Exception:
@@ -69,8 +74,6 @@ class ServerActionsTestJSON(base.BaseV2ComputeTest):
 
     @classmethod
     def resource_setup(cls):
-        cls.set_validation_resources()
-
         super(ServerActionsTestJSON, cls).resource_setup()
         cls.server_id = cls.rebuild_server(None, validatable=True)
 
@@ -80,8 +83,11 @@ class ServerActionsTestJSON(base.BaseV2ComputeTest):
     def test_change_server_password(self):
         # Since this test messes with the password and makes the
         # server unreachable, it should create its own server
+        validation_resources = self.get_test_validation_resources(
+            self.os_primary)
         newserver = self.create_test_server(
             validatable=True,
+            validation_resources=validation_resources,
             wait_until='ACTIVE')
         # The server's password should be set to the provided password
         new_password = 'Newpass1234'
@@ -92,7 +98,7 @@ class ServerActionsTestJSON(base.BaseV2ComputeTest):
             # Verify that the user can authenticate with the new password
             server = self.client.show_server(newserver['id'])['server']
             linux_client = remote_client.RemoteClient(
-                self.get_server_ip(server),
+                self.get_server_ip(server, validation_resources),
                 self.ssh_user,
                 new_password,
                 server=server,
@@ -101,13 +107,15 @@ class ServerActionsTestJSON(base.BaseV2ComputeTest):
 
     def _test_reboot_server(self, reboot_type):
         if CONF.validation.run_validation:
+            validation_resources = self.get_class_validation_resources(
+                self.os_primary)
             # Get the time the server was last rebooted,
             server = self.client.show_server(self.server_id)['server']
             linux_client = remote_client.RemoteClient(
-                self.get_server_ip(server),
+                self.get_server_ip(server, validation_resources),
                 self.ssh_user,
                 self.password,
-                self.validation_resources['keypair']['private_key'],
+                validation_resources['keypair']['private_key'],
                 server=server,
                 servers_client=self.client)
             boot_time = linux_client.get_boot_time()
@@ -122,10 +130,10 @@ class ServerActionsTestJSON(base.BaseV2ComputeTest):
         if CONF.validation.run_validation:
             # Log in and verify the boot time has changed
             linux_client = remote_client.RemoteClient(
-                self.get_server_ip(server),
+                self.get_server_ip(server, validation_resources),
                 self.ssh_user,
                 self.password,
-                self.validation_resources['keypair']['private_key'],
+                validation_resources['keypair']['private_key'],
                 server=server,
                 servers_client=self.client)
             new_boot_time = linux_client.get_boot_time()
@@ -201,6 +209,8 @@ class ServerActionsTestJSON(base.BaseV2ComputeTest):
         self.assertEqual(original_addresses, server['addresses'])
 
         if CONF.validation.run_validation:
+            validation_resources = self.get_class_validation_resources(
+                self.os_primary)
             # Authentication is attempted in the following order of priority:
             # 1.The key passed in, if one was passed in.
             # 2.Any key we can find through an SSH agent (if allowed).
@@ -208,10 +218,10 @@ class ServerActionsTestJSON(base.BaseV2ComputeTest):
             #   ~/.ssh/ (if allowed).
             # 4.Plain username/password auth, if a password was given.
             linux_client = remote_client.RemoteClient(
-                self.get_server_ip(rebuilt_server),
+                self.get_server_ip(rebuilt_server, validation_resources),
                 self.ssh_user,
                 password,
-                self.validation_resources['keypair']['private_key'],
+                validation_resources['keypair']['private_key'],
                 server=rebuilt_server,
                 servers_client=self.client)
             linux_client.validate_authentication()

tempest/api/compute/servers/test_server_personality.py

@@ -20,6 +20,7 @@ from tempest.common.utils.linux import remote_client
 from tempest.common import waiters
 from tempest import config
 from tempest.lib.common.utils import data_utils
+from tempest.lib.common.utils import test_utils
 from tempest.lib import decorators
 from tempest.lib import exceptions as lib_exc
 
@@ -33,11 +34,6 @@ class ServerPersonalityTestJSON(base.BaseV2ComputeTest):
         cls.prepare_instance_network()
         super(ServerPersonalityTestJSON, cls).setup_credentials()
 
-    @classmethod
-    def resource_setup(cls):
-        cls.set_validation_resources()
-        super(ServerPersonalityTestJSON, cls).resource_setup()
-
     @classmethod
     def skip_checks(cls):
         super(ServerPersonalityTestJSON, cls).skip_checks()
@@ -57,16 +53,23 @@ class ServerPersonalityTestJSON(base.BaseV2ComputeTest):
         personality = [{'path': file_path,
                         'contents': base64.encode_as_text(file_contents)}]
         password = data_utils.rand_password()
-        created_server = self.create_test_server(personality=personality,
+        validation_resources = self.get_test_validation_resources(
-                                                 adminPass=password,
+            self.os_primary)
-                                                 wait_until='ACTIVE',
+        created_server = self.create_test_server(
-                                                 validatable=True)
+            personality=personality, adminPass=password, wait_until='ACTIVE',
+            validatable=True,
+            validation_resources=validation_resources)
+        self.addCleanup(waiters.wait_for_server_termination,
+                        self.servers_client, created_server['id'])
+        self.addCleanup(test_utils.call_and_ignore_notfound_exc,
+                        self.servers_client.delete_server,
+                        created_server['id'])
         server = self.client.show_server(created_server['id'])['server']
         if CONF.validation.run_validation:
             linux_client = remote_client.RemoteClient(
-                self.get_server_ip(server),
+                self.get_server_ip(server, validation_resources),
                 self.ssh_user, password,
-                self.validation_resources['keypair']['private_key'],
+                validation_resources['keypair']['private_key'],
                 server=server,
                 servers_client=self.client)
             self.assertEqual(file_contents,
@@ -75,8 +78,16 @@ class ServerPersonalityTestJSON(base.BaseV2ComputeTest):
 
     @decorators.idempotent_id('128966d8-71fc-443c-8cab-08e24114ecc9')
     def test_rebuild_server_with_personality(self):
-        server = self.create_test_server(wait_until='ACTIVE', validatable=True)
+        validation_resources = self.get_test_validation_resources(
+            self.os_primary)
+        server = self.create_test_server(
+            wait_until='ACTIVE', validatable=True,
+            validation_resources=validation_resources)
         server_id = server['id']
+        self.addCleanup(waiters.wait_for_server_termination,
+                        self.servers_client, server_id)
+        self.addCleanup(test_utils.call_and_ignore_notfound_exc,
+                        self.servers_client.delete_server, server_id)
         file_contents = 'Test server rebuild.'
         personality = [{'path': 'rebuild.txt',
                         'contents': base64.encode_as_text(file_contents)}]
@@ -126,16 +137,22 @@ class ServerPersonalityTestJSON(base.BaseV2ComputeTest):
                 'contents': base64.encode_as_text(file_contents + str(i)),
             })
         password = data_utils.rand_password()
-        created_server = self.create_test_server(personality=person,
+        validation_resources = self.get_test_validation_resources(
-                                                 adminPass=password,
+            self.os_primary)
-                                                 wait_until='ACTIVE',
+        created_server = self.create_test_server(
-                                                 validatable=True)
+            personality=person, adminPass=password, wait_until='ACTIVE',
+            validatable=True, validation_resources=validation_resources)
+        self.addCleanup(waiters.wait_for_server_termination,
+                        self.servers_client, created_server['id'])
+        self.addCleanup(test_utils.call_and_ignore_notfound_exc,
+                        self.servers_client.delete_server,
+                        created_server['id'])
         server = self.client.show_server(created_server['id'])['server']
         if CONF.validation.run_validation:
             linux_client = remote_client.RemoteClient(
-                self.get_server_ip(server),
+                self.get_server_ip(server, validation_resources),
                 self.ssh_user, password,
-                self.validation_resources['keypair']['private_key'],
+                validation_resources['keypair']['private_key'],
                 server=server,
                 servers_client=self.client)
             for i in person:

tempest/api/compute/volumes/test_attach_volume.py

@@ -40,35 +40,37 @@ class AttachVolumeTestJSON(base.BaseV2ComputeTest):
 
     @classmethod
     def resource_setup(cls):
-        cls.set_validation_resources()
         super(AttachVolumeTestJSON, cls).resource_setup()
         cls.device = CONF.compute.volume_device_name
 
     def _create_server(self):
         # Start a server and wait for it to become ready
+        validation_resources = self.get_test_validation_resources(
+            self.os_primary)
         server = self.create_test_server(
             validatable=True,
+            validation_resources=validation_resources,
             wait_until='ACTIVE',
             adminPass=self.image_ssh_password)
         self.addCleanup(self.delete_server, server['id'])
         # Record addresses so that we can ssh later
         server['addresses'] = self.servers_client.list_addresses(
             server['id'])['addresses']
-        return server
+        return server, validation_resources
 
     @decorators.idempotent_id('52e9045a-e90d-4c0d-9087-79d657faffff')
     def test_attach_detach_volume(self):
         # Stop and Start a server with an attached volume, ensuring that
         # the volume remains attached.
-        server = self._create_server()
+        server, validation_resources = self._create_server()
 
         # NOTE(andreaf) Create one remote client used throughout the test.
         if CONF.validation.run_validation:
             linux_client = remote_client.RemoteClient(
-                self.get_server_ip(server),
+                self.get_server_ip(server, validation_resources),
                 self.image_ssh_user,
                 self.image_ssh_password,
-                self.validation_resources['keypair']['private_key'],
+                validation_resources['keypair']['private_key'],
                 server=server,
                 servers_client=self.servers_client)
             # NOTE(andreaf) We need to ensure the ssh key has been
@@ -111,7 +113,7 @@ class AttachVolumeTestJSON(base.BaseV2ComputeTest):
     @decorators.idempotent_id('7fa563fe-f0f7-43eb-9e22-a1ece036b513')
     def test_list_get_volume_attachments(self):
         # List volume attachment of the server
-        server = self._create_server()
+        server, _ = self._create_server()
         volume_1st = self.create_volume()
         attachment_1st = self.attach_volume(server, volume_1st,
                                             device=('/dev/%s' % self.device))
@@ -163,15 +165,15 @@ class AttachVolumeShelveTestJSON(AttachVolumeTestJSON):
         if not CONF.compute_feature_enabled.shelve:
             raise cls.skipException('Shelve is not available.')
 
-    def _count_volumes(self, server):
+    def _count_volumes(self, server, validation_resources):
         # Count number of volumes on an instance
         volumes = 0
         if CONF.validation.run_validation:
             linux_client = remote_client.RemoteClient(
-                self.get_server_ip(server),
+                self.get_server_ip(server, validation_resources),
                 self.image_ssh_user,
                 self.image_ssh_password,
-                self.validation_resources['keypair']['private_key'],
+                validation_resources['keypair']['private_key'],
                 server=server,
                 servers_client=self.servers_client)
 
@@ -179,7 +181,7 @@ class AttachVolumeShelveTestJSON(AttachVolumeTestJSON):
             volumes = int(linux_client.exec_command(command).strip())
         return volumes
 
-    def _shelve_server(self, server):
+    def _shelve_server(self, server, validation_resources):
         # NOTE(andreaf) If we are going to shelve a server, we should
         # check first whether the server is ssh-able. Otherwise we
         # won't be able to distinguish failures introduced by shelve
@@ -188,10 +190,10 @@ class AttachVolumeShelveTestJSON(AttachVolumeTestJSON):
         # avoid breaking the VM
         if CONF.validation.run_validation:
             linux_client = remote_client.RemoteClient(
-                self.get_server_ip(server),
+                self.get_server_ip(server, validation_resources),
                 self.image_ssh_user,
                 self.image_ssh_password,
-                self.validation_resources['keypair']['private_key'],
+                validation_resources['keypair']['private_key'],
                 server=server,
                 servers_client=self.servers_client)
             linux_client.validate_authentication()
@@ -199,30 +201,34 @@ class AttachVolumeShelveTestJSON(AttachVolumeTestJSON):
         # If validation went ok, or it was skipped, shelve the server
         compute.shelve_server(self.servers_client, server['id'])
 
-    def _unshelve_server_and_check_volumes(self, server, number_of_volumes):
+    def _unshelve_server_and_check_volumes(self, server,
+                                           validation_resources,
+                                           number_of_volumes):
         # Unshelve the instance and check that there are expected volumes
         self.servers_client.unshelve_server(server['id'])
         waiters.wait_for_server_status(self.servers_client,
                                        server['id'],
                                        'ACTIVE')
         if CONF.validation.run_validation:
-            counted_volumes = self._count_volumes(server)
+            counted_volumes = self._count_volumes(
+                server, validation_resources)
             self.assertEqual(number_of_volumes, counted_volumes)
 
     @decorators.idempotent_id('13a940b6-3474-4c3c-b03f-29b89112bfee')
     def test_attach_volume_shelved_or_offload_server(self):
         # Create server, count number of volumes on it, shelve
         # server and attach pre-created volume to shelved server
-        server = self._create_server()
+        server, validation_resources = self._create_server()
         volume = self.create_volume()
-        num_vol = self._count_volumes(server)
+        num_vol = self._count_volumes(server, validation_resources)
-        self._shelve_server(server)
+        self._shelve_server(server, validation_resources)
         attachment = self.attach_volume(server, volume,
                                         device=('/dev/%s' % self.device),
                                         check_reserved=True)
 
         # Unshelve the instance and check that attached volume exists
-        self._unshelve_server_and_check_volumes(server, num_vol + 1)
+        self._unshelve_server_and_check_volumes(
+            server, validation_resources, num_vol + 1)
 
         # Get volume attachment of the server
         volume_attachment = self.servers_client.show_volume_attachment(
@@ -238,10 +244,10 @@ class AttachVolumeShelveTestJSON(AttachVolumeTestJSON):
     def test_detach_volume_shelved_or_offload_server(self):
         # Count number of volumes on instance, shelve
         # server and attach pre-created volume to shelved server
-        server = self._create_server()
+        server, validation_resources = self._create_server()
         volume = self.create_volume()
-        num_vol = self._count_volumes(server)
+        num_vol = self._count_volumes(server, validation_resources)
-        self._shelve_server(server)
+        self._shelve_server(server, validation_resources)
 
         # Attach and then detach the volume
         self.attach_volume(server, volume, device=('/dev/%s' % self.device),
@@ -252,4 +258,5 @@ class AttachVolumeShelveTestJSON(AttachVolumeTestJSON):
 
         # Unshelve the instance and check that we have the expected number of
         # volume(s)
-        self._unshelve_server_and_check_volumes(server, num_vol)
+        self._unshelve_server_and_check_volumes(
+            server, validation_resources, num_vol)

tempest/common/compute.py

@@ -128,6 +128,8 @@ def create_test_server(clients, validatable=False, validation_resources=None,
                    "this stage.")
             raise ValueError(msg)
 
+        LOG.debug("Provisioning test server with validation resources %s",
+                  validation_resources)
         if 'security_groups' in kwargs:
             kwargs['security_groups'].append(
                 {'name': validation_resources['security_group']['name']})

tempest/common/validation_resources.py

@@ -181,6 +181,9 @@ def create_validation_resources(clients, keypair=False, floating_ip=False,
         floating_ip = resources['floating_ip']['ip']
     """
     # Create and Return the validation resources required to validate a VM
+    msg = ('Requested validation resources keypair %s, floating IP %s, '
+           'security group %s')
+    LOG.debug(msg, keypair, floating_ip, security_group)
     validation_data = {}
     try:
         if keypair:
@@ -429,6 +432,9 @@ class ValidationResourcesFixture(fixtures.Fixture):
         self._validation_resources = None
 
     def _setUp(self):
+        msg = ('Requested setup of ValidationResources keypair %s, floating '
+               'IP %s, security group %s')
+        LOG.debug(msg, self._keypair, self._floating_ip, self._security_group)
         self._validation_resources = create_validation_resources(
             self._clients, keypair=self._keypair,
             floating_ip=self._floating_ip,
@@ -441,9 +447,9 @@ class ValidationResourcesFixture(fixtures.Fixture):
         # cleanup here, so we don't need a try-finally around provisioning
         vr = self._validation_resources
         self.addCleanup(clear_validation_resources, self._clients,
-                        keypair=vr['keypair'],
+                        keypair=vr.get('keypair', None),
-                        floating_ip=vr['floating_ip'],
+                        floating_ip=vr.get('floating_ip', None),
-                        security_group=vr['security_group'],
+                        security_group=vr.get('security_group', None),
                         use_neutron=self._use_neutron)
 
     @property

tempest/test.py

@@ -26,7 +26,7 @@ import testtools
 from tempest import clients
 from tempest.common import credentials_factory as credentials
 from tempest.common import utils
-import tempest.common.validation_resources as vresources
+import tempest.common.validation_resources as vr
 from tempest import config
 from tempest.lib.common import cred_client
 from tempest.lib.common import fixed_network
@@ -105,13 +105,14 @@ class BaseTestCase(testtools.testcase.WithAttributes,
     # a list of roles - the first element of the list being a label, and the
     # rest the actual roles
     credentials = []
-    # Resources required to validate a server using ssh
-    validation_resources = {}
     network_resources = {}
 
     # Stack of resource cleanups
     _class_cleanups = []
 
+    # Resources required to validate a server using ssh
+    _validation_resources = {}
+
     # NOTE(sdague): log_format is defined inline here instead of using the oslo
     # default because going through the config path recouples config to the
     # stress tests too early, and depending on testr order will fail unit tests
@@ -379,29 +380,13 @@ class BaseTestCase(testtools.testcase.WithAttributes,
                             servers.delete_server,
                             cls.shared_server2['id'])
         """
-        if (CONF.validation.ip_version_for_ssh not in (4, 6) and
+        pass
-            CONF.service_available.neutron):
-            msg = "Invalid IP version %s in ip_version_for_ssh. Use 4 or 6"
-            raise lib_exc.InvalidConfiguration(
-                msg % CONF.validation.ip_version_for_ssh)
-        if hasattr(cls, "os_primary"):
-            vr = cls.validation_resources
-            cls.validation_resources = vresources.create_validation_resources(
-                cls.os_primary,
-                use_neutron=CONF.service_available.neutron,
-                ethertype='IPv' + str(CONF.validation.ip_version_for_ssh),
-                floating_network_id=CONF.network.public_network_id,
-                floating_network_name=CONF.network.floating_network_name,
-                **vr)
-        else:
-            LOG.warning("Client manager not found, validation resources not"
-                        " created")
 
     @classmethod
     def resource_cleanup(cls):
         """Class level resource cleanup for test cases.
 
-        Resource cleanup processes the stack or cleanups produced by
+        Resource cleanup processes the stack of cleanups produced by
         `addClassResourceCleanup` and then cleans up validation resources
         if any were provisioned.
 
@@ -440,16 +425,6 @@ class BaseTestCase(testtools.testcase.WithAttributes,
                 fn(*args, **kwargs)
             except Exception:
                 cleanup_errors.append(sys.exc_info())
-        if cls.validation_resources:
-            if hasattr(cls, "os_primary"):
-                vr = cls.validation_resources
-                vresources.clear_validation_resources(
-                    cls.os_primary,
-                    use_neutron=CONF.service_available.neutron, **vr)
-                cls.validation_resources = {}
-            else:
-                LOG.warning("Client manager not found, validation resources "
-                            "not deleted")
         if cleanup_errors:
             raise testtools.MultipleExceptions(*cleanup_errors)
 
@@ -610,45 +585,83 @@ class BaseTestCase(testtools.testcase.WithAttributes,
         if hasattr(cls, '_creds_provider'):
             cls._creds_provider.clear_creds()
 
+    @staticmethod
+    def _validation_resources_params_from_conf():
+        return dict(
+            keypair=(CONF.validation.auth_method.lower() == "keypair"),
+            floating_ip=(CONF.validation.connect_method.lower() == "floating"),
+            security_group=CONF.validation.security_group,
+            security_group_rules=CONF.validation.security_group_rules,
+            use_neutron=CONF.service_available.neutron,
+            ethertype='IPv' + str(CONF.validation.ip_version_for_ssh),
+            floating_network_id=CONF.network.public_network_id,
+            floating_network_name=CONF.network.floating_network_name)
+
     @classmethod
-    def set_validation_resources(cls, keypair=None, floating_ip=None,
+    def get_class_validation_resources(cls, os_clients):
-                                 security_group=None,
+        """Provision validation resources according to configuration
-                                 security_group_rules=None):
-        """Specify which ssh server validation resources should be created.
 
-        Each of the argument must be set to either None, True or False, with
+        This is a wrapper around `create_validation_resources` from
-        None - use default from config (security groups and security group
+        `tempest.common.validation_resources` that passes parameters from
-               rules get created when set to None)
+        Tempest configuration. Only one instance of class level
-        False - Do not create the validation resource
+        validation resources is managed by the helper, so If resources
-        True - create the validation resource
+        were already provisioned before, existing ones will be returned.
 
-        @param keypair
+        Resources are returned as a dictionary. They are also scheduled for
-        @param security_group
+        automatic cleanup during class teardown using
-        @param security_group_rules
+        `addClassResourcesCleanup`.
-        @param floating_ip
+
+        If `CONF.validation.run_validation` is False no resource will be
+        provisioned at all.
+
+        @param os_clients: Clients to be used to provision the resources.
         """
         if not CONF.validation.run_validation:
             return
 
-        if keypair is None:
+        if os_clients in cls._validation_resources:
-            keypair = (CONF.validation.auth_method.lower() == "keypair")
+            return cls._validation_resources[os_clients]
 
-        if floating_ip is None:
+        if (CONF.validation.ip_version_for_ssh not in (4, 6) and
-            floating_ip = (CONF.validation.connect_method.lower() ==
+                CONF.service_available.neutron):
-                           "floating")
+            msg = "Invalid IP version %s in ip_version_for_ssh. Use 4 or 6"
+            raise lib_exc.InvalidConfiguration(
+                msg % CONF.validation.ip_version_for_ssh)
 
-        if security_group is None:
+        resources = vr.create_validation_resources(
-            security_group = CONF.validation.security_group
+            os_clients,
+            **cls._validation_resources_params_from_conf())
 
-        if security_group_rules is None:
+        cls.addClassResourceCleanup(
-            security_group_rules = CONF.validation.security_group_rules
+            vr.clear_validation_resources, os_clients,
+            use_neutron=CONF.service_available.neutron,
+            **resources)
+        cls._validation_resources[os_clients] = resources
+        return resources
 
-        if not cls.validation_resources:
+    def get_test_validation_resources(self, os_clients):
-            cls.validation_resources = {
+        """Returns a dict of validation resources according to configuration
-                'keypair': keypair,
+
-                'security_group': security_group,
+        Initialise a validation resources fixture based on configuration.
-                'security_group_rules': security_group_rules,
+        Start the fixture and returns the validation resources.
-                'floating_ip': floating_ip}
+
+        If `CONF.validation.run_validation` is False no resource will be
+        provisioned at all.
+
+        @param os_clients: Clients to be used to provision the resources.
+        """
+
+        params = {}
+        # Test will try to use the fixture, so for this to be useful
+        # we must return a fixture. If validation is disabled though
+        # we don't need to provision anything, which is the default
+        # behavior for the fixture.
+        if CONF.validation.run_validation:
+            params = self._validation_resources_params_from_conf()
+
+        validation = self.useFixture(
+            vr.ValidationResourcesFixture(os_clients, **params))
+        return validation.resources
 
     @classmethod
     def set_network_resources(cls, network=False, router=False, subnet=False,

tempest/tests/test_test.py

@@ -19,10 +19,15 @@ import mock
 from oslo_config import cfg
 import testtools
 
+from tempest import clients
+from tempest.common import validation_resources as vr
 from tempest import config
+from tempest.lib import exceptions as lib_exc
 from tempest import test
 from tempest.tests import base
 from tempest.tests import fake_config
+from tempest.tests.lib import fake_credentials
+from tempest.tests.lib.services import registry_fixture
 
 
 if sys.version_info >= (2, 7):
@@ -41,6 +46,121 @@ class LoggingTestResult(testtools.TestResult):
         self.log.append((test, err, details))
 
 
+class TestValidationResources(base.TestCase):
+
+    validation_resources_module = 'tempest.common.validation_resources'
+
+    def setUp(self):
+        super(TestValidationResources, self).setUp()
+        self.useFixture(fake_config.ConfigFixture())
+        self.useFixture(registry_fixture.RegistryFixture())
+        self.patchobject(config, 'TempestConfigPrivate',
+                         fake_config.FakePrivate)
+
+        class TestTestClass(test.BaseTestCase):
+            pass
+
+        self.test_test_class = TestTestClass
+
+    def test_validation_resources_no_validation(self):
+        cfg.CONF.set_default('run_validation', False, 'validation')
+        creds = fake_credentials.FakeKeystoneV3Credentials()
+        osclients = clients.Manager(creds)
+        vr = self.test_test_class.get_class_validation_resources(osclients)
+        self.assertIsNone(vr)
+
+    def test_validation_resources_exists(self):
+        cfg.CONF.set_default('run_validation', True, 'validation')
+        creds = fake_credentials.FakeKeystoneV3Credentials()
+        osclients = clients.Manager(creds)
+        expected_vr = 'expected_validation_resources'
+        self.test_test_class._validation_resources[osclients] = expected_vr
+        obtained_vr = self.test_test_class.get_class_validation_resources(
+            osclients)
+        self.assertEqual(expected_vr, obtained_vr)
+
+    @mock.patch(validation_resources_module + '.create_validation_resources',
+                autospec=True)
+    def test_validation_resources_new(self, mock_create_vr):
+        cfg.CONF.set_default('run_validation', True, 'validation')
+        cfg.CONF.set_default('neutron', True, 'service_available')
+        creds = fake_credentials.FakeKeystoneV3Credentials()
+        osclients = clients.Manager(creds)
+        expected_vr = {'expected_validation_resources': None}
+        mock_create_vr.return_value = expected_vr
+        with mock.patch.object(
+                self.test_test_class,
+                'addClassResourceCleanup') as mock_add_class_cleanup:
+            obtained_vr = self.test_test_class.get_class_validation_resources(
+                osclients)
+            self.assertEqual(1, mock_add_class_cleanup.call_count)
+            self.assertEqual(mock.call(vr.clear_validation_resources,
+                                       osclients,
+                                       use_neutron=True,
+                                       **expected_vr),
+                             mock_add_class_cleanup.call_args)
+        self.assertEqual(mock_create_vr.call_count, 1)
+        self.assertIn(osclients, mock_create_vr.call_args_list[0][0])
+        self.assertEqual(expected_vr, obtained_vr)
+        self.assertIn(osclients, self.test_test_class._validation_resources)
+        self.assertEqual(expected_vr,
+                         self.test_test_class._validation_resources[osclients])
+
+    def test_validation_resources_invalid_config(self):
+        invalid_version = 999
+        cfg.CONF.set_default('run_validation', True, 'validation')
+        cfg.CONF.set_default('ip_version_for_ssh', invalid_version,
+                             'validation')
+        cfg.CONF.set_default('neutron', True, 'service_available')
+        creds = fake_credentials.FakeKeystoneV3Credentials()
+        osclients = clients.Manager(creds)
+        with testtools.ExpectedException(
+                lib_exc.InvalidConfiguration,
+                value_re='^.*\n.*' + str(invalid_version)):
+            self.test_test_class.get_class_validation_resources(osclients)
+
+    @mock.patch(validation_resources_module + '.create_validation_resources',
+                autospec=True)
+    def test_validation_resources_invalid_config_nova_net(self,
+                                                          mock_create_vr):
+        invalid_version = 999
+        cfg.CONF.set_default('run_validation', True, 'validation')
+        cfg.CONF.set_default('ip_version_for_ssh', invalid_version,
+                             'validation')
+        cfg.CONF.set_default('neutron', False, 'service_available')
+        creds = fake_credentials.FakeKeystoneV3Credentials()
+        osclients = clients.Manager(creds)
+        expected_vr = {'expected_validation_resources': None}
+        mock_create_vr.return_value = expected_vr
+        obtained_vr = self.test_test_class.get_class_validation_resources(
+            osclients)
+        self.assertEqual(mock_create_vr.call_count, 1)
+        self.assertIn(osclients, mock_create_vr.call_args_list[0][0])
+        self.assertEqual(expected_vr, obtained_vr)
+        self.assertIn(osclients, self.test_test_class._validation_resources)
+        self.assertEqual(expected_vr,
+                         self.test_test_class._validation_resources[osclients])
+
+    @mock.patch(validation_resources_module + '.create_validation_resources',
+                autospec=True)
+    @mock.patch(validation_resources_module + '.clear_validation_resources',
+                autospec=True)
+    def test_validation_resources_fixture(self, mock_clean_vr, mock_create_vr):
+
+        class TestWithRun(self.test_test_class):
+
+            def runTest(self):
+                pass
+
+        cfg.CONF.set_default('run_validation', True, 'validation')
+        test_case = TestWithRun()
+        creds = fake_credentials.FakeKeystoneV3Credentials()
+        osclients = clients.Manager(creds)
+        test_case.get_test_validation_resources(osclients)
+        self.assertEqual(1, mock_create_vr.call_count)
+        self.assertEqual(0, mock_clean_vr.call_count)
+
+
 class TestTempestBaseTestClass(base.TestCase):
 
     def setUp(self):
@@ -56,26 +176,16 @@ class TestTempestBaseTestClass(base.TestCase):
 
         self.parent_test = ParentTest
 
-    @mock.patch(
+    def test_resource_cleanup(self):
-        'tempest.common.validation_resources.clear_validation_resources',
-        autospec=True)
-    def test_resource_cleanup(self, mock_vr):
         cfg.CONF.set_default('neutron', False, 'service_available')
         exp_args = (1, 2,)
         exp_kwargs = {'a': 1, 'b': 2}
-        exp_vr = {'keypair': 'kp1', 'floating_ip': 'fip2'}
         mock1 = mock.Mock()
         mock2 = mock.Mock()
         exp_functions = [mock1, mock2]
 
         class TestWithCleanups(self.parent_test):
 
-            # set fake validation resources
-            validation_resources = exp_vr
-
-            # set fake clients
-            os_primary = 'os_primary'
-
             @classmethod
             def resource_setup(cls):
                 for fn in exp_functions:
@@ -92,34 +202,22 @@ class TestTempestBaseTestClass(base.TestCase):
         # All stacked resource cleanups invoked
         mock1.assert_called_once_with(*exp_args, **exp_kwargs)
         mock2.assert_called_once_with(*exp_args, **exp_kwargs)
-        self.assertEqual(1, mock_vr.call_count)
         # Cleanup stack is empty
         self.assertEqual(0, len(test_cleanups._class_cleanups))
-        # Assert vrs are cleaned up
-        self.assertIn(mock.call(TestWithCleanups.os_primary, use_neutron=False,
-                                **exp_vr), mock_vr.call_args_list)
 
-    @mock.patch(
+    def test_resource_cleanup_failures(self):
-        'tempest.common.validation_resources.clear_validation_resources',
-        autospec=True)
-    def test_resource_cleanup_failures(self, mock_vr):
         cfg.CONF.set_default('neutron', False, 'service_available')
         exp_args = (1, 2,)
         exp_kwargs = {'a': 1, 'b': 2}
-        exp_vr = {'keypair': 'kp1', 'floating_ip': 'fip2'}
         mock1 = mock.Mock()
         mock1.side_effect = Exception('mock1 resource cleanup failure')
         mock2 = mock.Mock()
-        exp_functions = [mock1, mock2]
+        mock3 = mock.Mock()
+        mock3.side_effect = Exception('mock3 resource cleanup failure')
+        exp_functions = [mock1, mock2, mock3]
 
         class TestWithFailingCleanups(self.parent_test):
 
-            # set fake validation resources
-            validation_resources = exp_vr
-
-            # set fake clients
-            os_primary = 'os_primary'
-
             @classmethod
             def resource_setup(cls):
                 for fn in exp_functions:
@@ -137,19 +235,15 @@ class TestTempestBaseTestClass(base.TestCase):
         # Type, Exception, traceback [1] -> MultipleException
         found_exc = log[0][1][1]
         self.assertTrue(isinstance(found_exc, testtools.MultipleExceptions))
-        self.assertEqual(1, len(found_exc.args))
+        self.assertEqual(2, len(found_exc.args))
         # Each arg is exc_info - match messages and order
-        self.assertIn('mock1 resource', str(found_exc.args[0][1]))
+        self.assertIn('mock3 resource', str(found_exc.args[0][1]))
+        self.assertIn('mock1 resource', str(found_exc.args[1][1]))
         # All stacked resource cleanups invoked
         mock1.assert_called_once_with(*exp_args, **exp_kwargs)
         mock2.assert_called_once_with(*exp_args, **exp_kwargs)
-        self.assertEqual(1, mock_vr.call_count)
         # Cleanup stack is empty
         self.assertEqual(0, len(test_cleanups._class_cleanups))
-        # Assert fake vr are cleaned up
-        self.assertIn(mock.call(TestWithFailingCleanups.os_primary,
-                                use_neutron=False, **exp_vr),
-                      mock_vr.call_args_list)
 
     def test_super_resource_cleanup_not_invoked(self):