Update Nova server APIs an

- ensure there quota limits before creating a new server
- ensure a server is in desired state (ACTIVE or SHUTOFF)

Change-Id: Id91123fd4ca4114d92bc7b836257e64505b10404

tobiko/openstack/keystone/__init__.py

@@ -16,6 +16,7 @@ from __future__ import absolute_import
 from tobiko.openstack.keystone import _client
 from tobiko.openstack.keystone import _clouds_file
 from tobiko.openstack.keystone import _credentials
+from tobiko.openstack.keystone import _resource
 from tobiko.openstack.keystone import _services
 from tobiko.openstack.keystone import _session
 
@@ -47,6 +48,13 @@ InvalidKeystoneCredentials = _credentials.InvalidKeystoneCredentials
 DEFAULT_KEYSTONE_CREDENTIALS_FIXTURES = \
     _credentials.DEFAULT_KEYSTONE_CREDENTIALS_FIXTURES
 
+get_keystone_resource_id = _resource.get_keystone_resource_id
+get_project_id = _resource.get_project_id
+get_user_id = _resource.get_user_id
+KeystoneResourceType = _resource.KeystoneResourceType
+ProjectType = _resource.ProjectType
+UserType = _resource.UserType
+
 has_service = _services.has_service
 is_service_missing = _services.is_service_missing
 skip_if_missing_service = _services.skip_if_missing_service

tobiko/openstack/keystone/_client.py

@@ -13,6 +13,8 @@
 #    under the License.
 from __future__ import absolute_import
 
+import typing
+
 from keystoneclient import base
 from keystoneclient import client as keystoneclient
 from keystoneclient.v2_0 import client as v2_client
@@ -36,13 +38,16 @@ class KeystoneClientManager(_client.OpenstackClientManager):
 
 
 CLIENTS = KeystoneClientManager()
-
-
 CLIENT_CLASSES = (v2_client.Client, v3_client.Client)
+KeystoneClient = typing.Union[v2_client.Client, v3_client.Client]
+KeystoneClientType = typing.Union[KeystoneClient,
+                                  KeystoneClientFixture,
+                                  typing.Type[KeystoneClientFixture],
+                                  None]
 
 
-def keystone_client(obj):
-    if not obj:
+def keystone_client(obj: KeystoneClientType) -> KeystoneClient:
+    if obj is None:
         return get_keystone_client()
 
     if isinstance(obj, CLIENT_CLASSES):
@@ -50,14 +55,13 @@ def keystone_client(obj):
 
     fixture = tobiko.setup_fixture(obj)
     if isinstance(fixture, KeystoneClientFixture):
-        return fixture.client
+        return tobiko.setup_fixture(obj).client
 
-    message = "Object {!r} is not a KeystoneClientFixture".format(obj)
-    raise TypeError(message)
+    raise TypeError(f"Object {obj} is not a KeystoneClientFixture")
 
 
 def get_keystone_client(session=None, shared=True, init_client=None,
-                        manager=None):
+                        manager=None) -> KeystoneClient:
     manager = manager or CLIENTS
     client = manager.get_client(session=session, shared=shared,
                                 init_client=init_client)

tobiko/openstack/keystone/_resource.py

@@ -0,0 +1,60 @@
+# Copyright 2021 Red Hat
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+from __future__ import absolute_import
+
+import typing
+
+from keystoneclient import base
+from keystoneclient.v2_0 import tenants as tenants_v2
+from keystoneclient.v2_0 import users as users_v2
+from keystoneclient.v3 import projects as projects_v3
+from keystoneclient.v3 import users as users_v3
+
+
+KeystoneResourceType = typing.Union[str, base.Resource]
+ProjectType = typing.Union[str, tenants_v2.Tenant, projects_v3.Project]
+UserType = typing.Union[str, users_v2.User, users_v3.User]
+
+
+def get_project_id(
+        project: typing.Optional[ProjectType] = None,
+        session=None) -> str:
+    if project is not None:
+        return get_keystone_resource_id(project)
+    if session is not None:
+        return session.auth.auth_ref.project_id
+    raise ValueError("'project' and 'session' can't be None ata the same "
+                     "time.")
+
+
+def get_user_id(
+        user: typing.Optional[UserType] = None,
+        session=None) -> str:
+    if user is not None:
+        return get_keystone_resource_id(user)
+    if session is not None:
+        return session.auth.auth_ref.user_id
+    raise ValueError("'project' and 'session' can't be None ata the same "
+                     "time.")
+
+
+def get_keystone_resource_id(resource: KeystoneResourceType) -> str:
+    if isinstance(resource, str):
+        return resource
+
+    if isinstance(resource, base.Resource):
+        return resource.id
+
+    raise TypeError(f"Object {resource} is not a valid Keystone resource "
+                    "type")

tobiko/openstack/nova/__init__.py

@@ -16,11 +16,13 @@ from __future__ import absolute_import
 from tobiko.openstack.nova import _client
 from tobiko.openstack.nova import _cloud_init
 from tobiko.openstack.nova import _hypervisor
+from tobiko.openstack.nova import _quota_set
 from tobiko.openstack.nova import _server
 from tobiko.openstack.nova import _service
 
-
 CLIENT_CLASSES = _client.CLIENT_CLASSES
+NovaClient = _client.NovaClient
+NovaClientType = _client.NovaClientType
 get_console_output = _client.get_console_output
 get_nova_client = _client.get_nova_client
 get_server = _client.get_server
@@ -38,8 +40,10 @@ WaitForServerStatusError = _client.WaitForServerStatusError
 WaitForServerStatusTimeout = _client.WaitForServerStatusTimeout
 shutoff_server = _client.shutoff_server
 activate_server = _client.activate_server
+ensure_server_status = _client.ensure_server_status
 migrate_server = _client.migrate_server
 confirm_resize = _client.confirm_resize
+NovaServer = _client.NovaServer
 
 WaitForCloudInitTimeoutError = _cloud_init.WaitForCloudInitTimeoutError
 cloud_config = _cloud_init.cloud_config
@@ -54,6 +58,10 @@ get_different_host_hypervisors = _hypervisor.get_different_host_hypervisors
 get_server_hypervisor = _hypervisor.get_server_hypervisor
 get_servers_hypervisors = _hypervisor.get_servers_hypervisors
 
+get_nova_quota_set = _quota_set.get_nova_quota_set
+ensure_nova_quota_limits = _quota_set.ensure_nova_quota_limits
+set_nova_quota_set = _quota_set.set_nova_quota_set
+
 find_server_ip_address = _server.find_server_ip_address
 HasServerMixin = _server.HasServerMixin
 list_server_ip_addresses = _server.list_server_ip_addresses

tobiko/openstack/nova/_client.py

@@ -13,37 +13,45 @@
 #    under the License.
 from __future__ import absolute_import
 
-import time
+import typing
 
-from novaclient import client as novaclient
-from novaclient.v2 import client as client_v2
+import novaclient
+import novaclient.v2.client
 from oslo_log import log
 
 import tobiko
 from tobiko.openstack import _client
 
 
-CLIENT_CLASSES = (client_v2.Client,)
 LOG = log.getLogger(__name__)
 
+CLIENT_CLASSES = (novaclient.v2.client.Client,)
+NovaClient = typing.Union[novaclient.v2.client.Client]
+NovaServer = typing.Union[novaclient.v2.servers.Server]
+
 
 class NovaClientFixture(_client.OpenstackClientFixture):
 
-    def init_client(self, session):
-        return novaclient.Client('2', session=session)
+    def init_client(self, session) -> NovaClient:
+        return novaclient.client.Client('2', session=session)
 
 
 class NovaClientManager(_client.OpenstackClientManager):
 
-    def create_client(self, session):
+    def create_client(self, session) -> NovaClientFixture:
         return NovaClientFixture(session=session)
 
 
 CLIENTS = NovaClientManager()
 
+NovaClientType = typing.Union[NovaClient,
+                              NovaClientFixture,
+                              typing.Type[NovaClientFixture],
+                              None]
 
-def nova_client(obj):
-    if not obj:
+
+def nova_client(obj: NovaClientType) -> NovaClient:
+    if obj is None:
         return get_nova_client()
 
     if isinstance(obj, CLIENT_CLASSES):
@@ -51,14 +59,15 @@ def nova_client(obj):
 
     fixture = tobiko.setup_fixture(obj)
     if isinstance(fixture, NovaClientFixture):
+        assert fixture.client is not None
         return fixture.client
 
-    message = "Object {!r} is not a NovaClientFixture".format(obj)
+    message = f"Object '{obj}' is not a NovaClientFixture"
     raise TypeError(message)
 
 
 def get_nova_client(session=None, shared=True, init_client=None,
-                    manager=None):
+                    manager=None) -> NovaClient:
     manager = manager or CLIENTS
     client = manager.get_client(session=session, shared=shared,
                                 init_client=init_client)
@@ -66,13 +75,13 @@ def get_nova_client(session=None, shared=True, init_client=None,
     return client.client
 
 
-def list_hypervisors(client=None, detailed=True, **params):
+def list_hypervisors(client: NovaClientType = None, detailed=True, **params):
     client = nova_client(client)
     hypervisors = client.hypervisors.list(detailed=detailed)
     return tobiko.select(hypervisors).with_attributes(**params)
 
 
-def find_hypervisor(client=None, unique=False, **params):
+def find_hypervisor(client: NovaClientType = None, unique=False, **params):
     hypervisors = list_hypervisors(client=client, **params)
     if unique:
         return hypervisors.unique
@@ -80,13 +89,14 @@ def find_hypervisor(client=None, unique=False, **params):
         return hypervisors.first
 
 
-def list_servers(client=None, **params):
-    client = nova_client(client)
-    servers = client.servers.list()
+def list_servers(client: NovaClientType = None, **params) -> \
+        tobiko.Selection[NovaServer]:
+    servers = nova_client(client).servers.list()
     return tobiko.select(servers).with_attributes(**params)
 
 
-def find_server(client=None, unique=False, **params):
+def find_server(client: NovaClientType = None, unique=False, **params) -> \
+        NovaServer:
     servers = list_servers(client=client, **params)
     if unique:
         return servers.unique
@@ -94,13 +104,13 @@ def find_server(client=None, unique=False, **params):
         return servers.first
 
 
-def list_services(client=None, **params) -> tobiko.Selection:
+def list_services(client: NovaClientType = None, **params) -> tobiko.Selection:
     client = nova_client(client)
     services = client.services.list()
     return tobiko.select(services).with_attributes(**params)
 
 
-def find_service(client=None, unique=False, **params):
+def find_service(client: NovaClientType = None, unique=False, **params):
     services = list_services(client=client, **params)
     if unique:
         return services.unique
@@ -108,29 +118,42 @@ def find_service(client=None, unique=False, **params):
         return services.first
 
 
-def get_server_id(server):
-    if isinstance(server, str):
-        return server
-    else:
-        return server.id
+ServerType = typing.Union[str, NovaServer]
 
 
-def get_server(server, client=None, **params):
-    server_id = get_server_id(server)
+def get_server_id(server: typing.Optional[ServerType] = None,
+                  server_id: typing.Optional[str] = None) -> str:
+    if server_id is None:
+        if isinstance(server, str):
+            server_id = server
+        else:
+            assert server is not None
+            server_id = server.id
+    return server_id
+
+
+def get_server(server: typing.Optional[ServerType] = None,
+               server_id: typing.Optional[str] = None,
+               client: NovaClientType = None, **params) -> NovaServer:
+    server_id = get_server_id(server=server, server_id=server_id)
     return nova_client(client).servers.get(server_id, **params)
 
 
-def migrate_server(server, client=None, **params):
+def migrate_server(server: typing.Optional[ServerType] = None,
+                   server_id: typing.Optional[str] = None,
+                   client: NovaClientType = None, **params):
     # pylint: disable=protected-access
-    server_id = get_server_id(server)
+    server_id = get_server_id(server=server, server_id=server_id)
     LOG.debug(f"Start server migration (server_id='{server_id}', "
               f"info={params})")
     return nova_client(client).servers._action('migrate', server_id,
                                                info=params)
 
 
-def confirm_resize(server, client=None, **params):
-    server_id = get_server_id(server)
+def confirm_resize(server: typing.Optional[ServerType] = None,
+                   server_id: typing.Optional[str] = None,
+                   client: NovaClientType = None, **params):
+    server_id = get_server_id(server=server, server_id=server_id)
     LOG.debug(f"Confirm server resize (server_id='{server_id}', "
               f"info={params})")
     return nova_client(client).servers.confirm_resize(server_id, **params)
@@ -139,46 +162,67 @@ def confirm_resize(server, client=None, **params):
 MAX_SERVER_CONSOLE_OUTPUT_LENGTH = 1024 * 256
 
 
-def get_console_output(server, timeout=None, interval=1., length=None,
-                       client=None):
-    client = nova_client(client)
-    start_time = time.time()
+def get_console_output(server: typing.Optional[ServerType] = None,
+                       server_id: typing.Optional[str] = None,
+                       timeout: tobiko.Seconds = None,
+                       interval: tobiko.Seconds = None,
+                       length: typing.Optional[int] = None,
+                       client: NovaClientType = None) -> \
+        typing.Optional[str]:
     if length is not None:
         length = min(length, MAX_SERVER_CONSOLE_OUTPUT_LENGTH)
     else:
         length = MAX_SERVER_CONSOLE_OUTPUT_LENGTH
-    while True:
+
+    server_id = get_server_id(server=server, server_id=server_id)
+
+    for attempt in tobiko.retry(timeout=timeout,
+                                interval=interval,
+                                default_timeout=60.,
+                                default_interval=5.):
         try:
-            output = client.servers.get_console_output(server=server,
-                                                       length=length)
-        except TypeError:
-            # For some reason it could happen resulting body cannot be
-            # translated to json object and it is converted to None
-            # on such case get_console_output would raise a TypeError
-            return None
+            output = nova_client(client).servers.get_console_output(
+                server=server_id, length=length)
+        except (TypeError, novaclient.exceptions.NotFound):
+            # Only active servers have console output
+            server = get_server(server_id=server_id)
+            if server.status != 'ACTIVE':
+                LOG.debug(f"Server '{server_id}' has no console output "
+                          f"(status = '{server.status}').")
+                break
+            else:
+                # For some reason it could happen resulting body cannot be
+                # translated to json object and it is converted to None
+                # on such case get_console_output would raise a TypeError
+                LOG.exception(f"Error getting server '{server_id}' console "
+                              "output")
+        else:
+            if output:
+                LOG.debug(f"got server '{server_id}' console output "
+                          f"(length = {len(output)}).")
+                return output
 
-        if timeout is None or output:
+        try:
+            attempt.check_limits()
+        except tobiko.RetryLimitError:
+            LOG.info(f"No console output produced by server '{server_id}') "
+                     f" after {attempt.elapsed_time} seconds")
             break
+        else:
+            LOG.debug(f"Waiting for server '{server_id}' console output...")
 
-        if time.time() - start_time > timeout:
-            LOG.warning("No console output produced by server (%r) after "
-                        "%r seconds", server, timeout)
-            break
-
-        LOG.debug('Waiting for server (%r) console output...', server)
-        time.sleep(interval)
-
-    return output
+    return None
 
 
 class HasNovaClientMixin(object):
 
-    nova_client = None
+    nova_client: NovaClientType = None
 
-    def get_server(self, server, **params):
+    def get_server(self, server: ServerType, **params) -> NovaServer:
         return get_server(server=server, client=self.nova_client, **params)
 
-    def get_server_console_output(self, server, **params):
+    def get_server_console_output(self, server: ServerType, **params) -> \
+            typing.Optional[str]:
         return get_console_output(server=server, client=self.nova_client,
                                   **params)
 
@@ -193,76 +237,113 @@ class WaitForServerStatusTimeout(WaitForServerStatusError):
                "{server_status} to {status} status after {timeout} seconds")
 
 
-NOVA_SERVER_TRANSIENT_STATUS = {
-    'ACTIVE': ('BUILD', 'SHUTOFF'),
-    'SHUTOFF': ('ACTIVE'),
-    'VERIFY_RESIZE': ('RESIZE'),
+NOVA_SERVER_TRANSIENT_STATUS: typing.Dict[str, typing.List[str]] = {
+    'ACTIVE': ['BUILD', 'SHUTOFF'],
+    'SHUTOFF': ['ACTIVE'],
+    'VERIFY_RESIZE': ['RESIZE'],
 }
 
 
-def wait_for_server_status(server, status, client=None, timeout=None,
-                           sleep_time=None, transient_status=None):
-    if timeout is None:
-        timeout = 300.
-    if sleep_time is None:
-        sleep_time = 5.
-    start_time = time.time()
+def wait_for_server_status(
+        server: ServerType,
+        status: str,
+        client: NovaClientType = None,
+        timeout: tobiko.Seconds = None,
+        sleep_time: tobiko.Seconds = None,
+        transient_status: typing.Optional[typing.List[str]] = None) -> \
+            NovaServer:
     if transient_status is None:
-        transient_status = NOVA_SERVER_TRANSIENT_STATUS.get(status) or tuple()
-    while True:
-        server = get_server(server=server, client=client)
-        if server.status == status:
+        transient_status = NOVA_SERVER_TRANSIENT_STATUS.get(status) or []
+    server_id = get_server_id(server)
+    for attempt in tobiko.retry(timeout=timeout,
+                                interval=sleep_time,
+                                default_timeout=300.,
+                                default_interval=5.):
+        _server = get_server(server_id=server_id, client=client)
+        if _server.status == status:
             break
 
-        if server.status not in transient_status:
-            raise WaitForServerStatusError(server_id=server.id,
-                                           server_status=server.status,
+        if _server.status not in transient_status:
+            raise WaitForServerStatusError(server_id=server_id,
+                                           server_status=_server.status,
                                            status=status)
-
-        if time.time() - start_time >= timeout:
-            raise WaitForServerStatusTimeout(server_id=server.id,
-                                             server_status=server.status,
+        try:
+            attempt.check_time_left()
+        except tobiko.RetryTimeLimitError as ex:
+            raise WaitForServerStatusTimeout(server_id=server_id,
+                                             server_status=_server.status,
                                              status=status,
-                                             timeout=timeout)
+                                             timeout=timeout) from ex
 
         progress = getattr(server, 'progress', None)
-        LOG.debug(f"Waiting for server {server.id} status to get from "
-                  f"{server.status} to {status} "
+        LOG.debug(f"Waiting for server {server_id} status to get from "
+                  f"{_server.status} to {status} "
                   f"(progress={progress}%)")
-        time.sleep(sleep_time)
-    return server
+
+    return _server
 
 
-def shutoff_server(server, client=None, timeout=None, sleep_time=None):
+def shutoff_server(server: ServerType = None,
+                   client: NovaClientType = None,
+                   timeout: tobiko.Seconds = None,
+                   sleep_time: tobiko.Seconds = None) -> NovaServer:
     client = nova_client(client)
     server = get_server(server=server, client=client)
     if server.status == 'SHUTOFF':
         return server
 
+    LOG.info(f"stop server '{server.id}' (status='{server.status}').")
     client.servers.stop(server.id)
-    return wait_for_server_status(server=server.id, status='SHUTOFF',
-                                  client=client, timeout=timeout,
+    return wait_for_server_status(server=server.id,
+                                  status='SHUTOFF',
+                                  client=client,
+                                  timeout=timeout,
                                   sleep_time=sleep_time)
 
 
-def activate_server(server, client=None, timeout=None, sleep_time=None):
+def activate_server(server: ServerType,
+                    client: NovaClientType = None,
+                    timeout: tobiko.Seconds = None,
+                    sleep_time: tobiko.Seconds = None) -> NovaServer:
     client = nova_client(client)
     server = get_server(server=server, client=client)
     if server.status == 'ACTIVE':
         return server
 
     if server.status == 'SHUTOFF':
+        LOG.info(f"Start server '{server.id}' (status='{server.status}').")
         client.servers.start(server.id)
     elif server.status == 'RESIZE':
-        wait_for_server_status(server=server.id, status='VERIFY_RESIZE',
-                               client=client, timeout=timeout,
-                               sleep_time=sleep_time)
+        server = wait_for_server_status(
+            server=server.id, status='VERIFY_RESIZE', client=client,
+            timeout=timeout, sleep_time=sleep_time)
+        LOG.info(f"Confirm resize of server '{server.id}' "
+                 f"(status='{server.status}').")
         client.servers.confirm_resize(server)
     elif server.status == 'VERIFY_RESIZE':
+        LOG.info(f"Confirm resize of server '{server.id}' "
+                 f"(status='{server.status}').")
         client.servers.confirm_resize(server)
     else:
+        LOG.warning(f"Try activating server '{server.id}' by rebooting "
+                    f"it  (status='{server.status}').")
         client.servers.reboot(server.id, reboot_type='HARD')
 
     return wait_for_server_status(server=server.id, status='ACTIVE',
                                   client=client, timeout=timeout,
                                   sleep_time=sleep_time)
+
+
+def ensure_server_status(server: ServerType,
+                         status: str,
+                         client: NovaClientType = None,
+                         timeout: tobiko.Seconds = None,
+                         sleep_time: tobiko.Seconds = None) -> NovaServer:
+    if status == 'ACTIVE':
+        return activate_server(server=server, client=client, timeout=timeout,
+                               sleep_time=sleep_time)
+    elif status == 'SHUTOFF':
+        return shutoff_server(server=server, client=client, timeout=timeout,
+                              sleep_time=sleep_time)
+    else:
+        raise ValueError(f"Unsupported server status: '{status}'")

tobiko/openstack/nova/_quota_set.py

@@ -0,0 +1,87 @@
+# Copyright 2021 Red Hat
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+from __future__ import absolute_import
+
+from oslo_log import log
+
+from tobiko.openstack import keystone
+from tobiko.openstack.nova import _client
+
+
+LOG = log.getLogger(__name__)
+
+
+def get_nova_quota_set(project: keystone.ProjectType = None,
+                       user: keystone.UserType = None,
+                       client: _client.NovaClientType = None,
+                       **params):
+    client = _client.nova_client(client)
+    project_id = keystone.get_project_id(project=project,
+                                         session=client.client.session)
+    user_id = user and keystone.get_user_id(user=user) or None
+    return client.quotas.get(project_id, user_id=user_id, **params)
+
+
+def set_nova_quota_set(project: keystone.ProjectType = None,
+                       user: keystone.UserType = None,
+                       client: _client.NovaClientType = None,
+                       **params):
+    client = _client.nova_client(client)
+    project_id = keystone.get_project_id(project=project,
+                                         session=client.client.session)
+    user_id = user and keystone.get_user_id(user=user) or None
+    return client.quotas.update(project_id, user_id=user_id, **params)
+
+
+def ensure_nova_quota_limits(project: keystone.ProjectType = None,
+                             user: keystone.UserType = None,
+                             client: _client.NovaClientType = None,
+                             **required):
+    client = _client.nova_client(client)
+    project = keystone.get_project_id(project=project,
+                                      session=client.client.session)
+    user = user and keystone.get_user_id(user=user) or None
+    if user:
+        # Must increase project limits before user ones
+        ensure_nova_quota_limits(project=project, client=client,
+                                 **required)
+
+    quota_set = get_nova_quota_set(project=project, user=user,
+                                   client=client, detail=True)
+    actual_limits = {}
+    increment_limits = {}
+    for name, needed in required.items():
+        quota = getattr(quota_set, name)
+        limit: int = quota['limit']
+        if limit > 0:
+            in_use: int = max(0, quota['in_use']) + max(0, quota['reserved'])
+            if in_use + needed >= limit:
+                actual_limits[name] = limit
+                increment_limits[name] = max(10, limit * 2)
+
+    if increment_limits:
+        LOG.info(f"Increment Nova quota limits (project={project}, "
+                 f"user={user}): {actual_limits} -> {increment_limits}...")
+        set_nova_quota_set(project=project, user=user, client=client,
+                           **increment_limits)
+        quota_set = get_nova_quota_set(project=project, user=user,
+                                       client=client, detail=True)
+        new_limits = {
+            name: getattr(quota_set, name)['limit']
+            for name in increment_limits.keys()}
+
+        LOG.info(f"Nova quota limit increased (project={project}, "
+                 f"user={user}): {actual_limits} -> {new_limits}...")
+
+    return quota_set

tobiko/openstack/stacks/_nova.py

@@ -27,6 +27,7 @@ import tobiko
 from tobiko import config
 from tobiko.openstack import glance
 from tobiko.openstack import heat
+from tobiko.openstack import keystone
 from tobiko.openstack import neutron
 from tobiko.openstack import nova
 from tobiko.openstack.stacks import _hot
@@ -96,6 +97,10 @@ class ServerStackFixture(heat.HeatStackFixture, abc.ABC):
     #: stack with the internal where the server port is created
     network_stack = tobiko.required_setup_fixture(_neutron.NetworkStackFixture)
 
+    def create_stack(self, retry=None):
+        self.ensure_quota_limits()
+        super(ServerStackFixture, self).create_stack(retry=retry)
+
     @property
     def image_fixture(self) -> glance.GlanceImageFixture:
         """Glance image used to create a Nova server instance"""
@@ -320,22 +325,50 @@ class ServerStackFixture(heat.HeatStackFixture, abc.ABC):
                       'maxsize': self.swap_maxsize})
         return cloud_config
 
-    def ensure_server_status(self, status):
-        tobiko.setup_fixture(self)
-        try:
-            server = nova.wait_for_server_status(self.server_id, status)
-        except nova.WaitForServerStatusError:
-            server = nova.get_server(self.server_id)
-            LOG.debug(f"Server {server.id} status is {server.status} instead "
-                      f"of {status}", exc_info=1)
-        if server.status == status:
-            return server
-        elif status == "ACTIVE":
-            tobiko.reset_fixture(self)
-            return nova.wait_for_server_status(self.server_id, 'ACTIVE')
-        else:
-            tobiko.skip_test(f"{type(self).__name__}.ensure_server_status "
-                             "method not implemented")
+    def ensure_server_status(
+            self, status: str,
+            retry_count: typing.Optional[int] = None,
+            retry_timeout: tobiko.Seconds = None,
+            retry_interval: tobiko.Seconds = None):
+        self.ssh_client.close()
+        for attempt in tobiko.retry(count=retry_count,
+                                    timeout=retry_timeout,
+                                    interval=retry_interval,
+                                    default_count=3,
+                                    default_timeout=900.,
+                                    default_interval=5.):
+            tobiko.setup_fixture(self)
+            server_id = self.server_id
+            try:
+                server = nova.ensure_server_status(
+                    server=server_id,
+                    status=status,
+                    timeout=attempt.time_left)
+            except nova.WaitForServerStatusError:
+                attempt.check_limits()
+                LOG.warning(
+                    f"Unable to change server '{server_id}' status to "
+                    f"'{status}'",
+                    exc_info=1)
+                tobiko.cleanup_fixture(self)
+            else:
+                assert server.status == status
+                break
+
+        return server
+
+    def ensure_quota_limits(self):
+        """Ensures Nova quota limits before creating a new server
+        """
+        project = keystone.get_project_id(
+            session=self.client.http_client.session)
+        user = keystone.get_user_id(
+            session=self.client.http_client.session)
+        nova.ensure_nova_quota_limits(
+            project=project,
+            user=user,
+            instances=1,
+            cores=self.flavor_stack.vcpus or 1)
 
 
 class ExternalServerStackFixture(ServerStackFixture, abc.ABC):

tobiko/tests/functional/openstack/test_nova.py

@@ -103,21 +103,32 @@ class ClientTest(testtools.TestCase):
         self.assertEqual(server_id, server.id)
         self.assertEqual('ACTIVE', server.status)
 
-    def test_shutof_and_activate_server(self):
-        server_id = self.useFixture(stacks.CirrosServerStackFixture(
-            stack_name=self.id())).server_id
 
-        server = nova.wait_for_server_status(server=server_id, status='ACTIVE')
-        self.assertEqual(server_id, server.id)
+class ServerActionsStack(stacks.CirrosServerStackFixture):
+    pass
+
+
+class ServerActionsTest(testtools.TestCase):
+
+    stack = tobiko.required_setup_fixture(ServerActionsStack)
+
+    def test_activate_server(self, initial_status='SHUTOFF'):
+        self.stack.ensure_server_status(initial_status)
+        server = nova.activate_server(self.stack.server_id)
         self.assertEqual('ACTIVE', server.status)
+        ping.assert_reachable_hosts([self.stack.ip_address])
 
-        server = nova.shutoff_server(server=server_id)
-        self.assertEqual(server_id, server.id)
+    def test_activate_server_when_shutoff(self):
+        self.test_activate_server(initial_status='SHUTOFF')
+
+    def test_shutoff_server(self, initial_status='ACTIVE'):
+        self.stack.ensure_server_status(initial_status)
+        server = nova.shutoff_server(self.stack.server_id)
         self.assertEqual('SHUTOFF', server.status)
+        ping.assert_unreachable_hosts([self.stack.ip_address])
 
-        server = nova.activate_server(server=server_id)
-        self.assertEqual(server_id, server.id)
-        self.assertEqual('ACTIVE', server.status)
+    def test_shutoff_server_when_shutoff(self):
+        self.test_shutoff_server(initial_status='SHUTOFF')
 
 
 @keystone.skip_unless_has_keystone_credentials()
@@ -194,8 +205,8 @@ class MigrateServerTest(testtools.TestCase):
                 target_hypervisor = hypervisor.hypervisor_hostname
                 break
         else:
-            self.skip("Cannot find a valid hypervisor host to migrate server "
-                      "to")
+            self.skipTest("Cannot find a valid hypervisor host to migrate "
+                          "server to")
 
         server = self.migrate_server(server=server, host=target_hypervisor)