From 5d04e37ab05c214fbfa36d1d299b2cb8d0c0025c Mon Sep 17 00:00:00 2001 From: lkuchlan Date: Tue, 1 Aug 2023 14:08:10 +0300 Subject: [PATCH] [Manila] pre-upgrade resource creation and post-procedure validation There's a need to create a Manila share resource before the upgrade process and then validate the share after the process is complete. To achieve this objective, it is essential to incorporate support for the Manila client, a task that is accomplished through this commit. Change-Id: I0d520e40a1a491ee864e707d63413d10035b99ca --- lower-constraints.txt | 1 + .../add-manila-support-d2dd3aab05b75439.yaml | 4 + requirements.txt | 1 + roles/tobiko-cleanup/tasks/main.yaml | 8 ++ .../test-workflow-check-resources-manila.yaml | 8 ++ ...test-workflow-create-resources-manila.yaml | 8 ++ tobiko/config.py | 6 + tobiko/openstack/manila/__init__.py | 44 ++++++++ tobiko/openstack/manila/_client.py | 104 ++++++++++++++++++ tobiko/openstack/manila/_constants.py | 24 ++++ tobiko/openstack/manila/_exceptions.py | 24 ++++ tobiko/openstack/manila/_waiters.py | 93 ++++++++++++++++ tobiko/openstack/manila/config.py | 36 ++++++ tobiko/tests/scenario/manila/__init__.py | 0 tobiko/tests/scenario/manila/test_manila.py | 58 ++++++++++ tox.ini | 9 ++ upper-constraints.txt | 2 +- 17 files changed, 429 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/add-manila-support-d2dd3aab05b75439.yaml create mode 100644 roles/tobiko-run/vars/test-workflow-check-resources-manila.yaml create mode 100644 roles/tobiko-run/vars/test-workflow-create-resources-manila.yaml create mode 100644 tobiko/openstack/manila/__init__.py create mode 100644 tobiko/openstack/manila/_client.py create mode 100644 tobiko/openstack/manila/_constants.py create mode 100644 tobiko/openstack/manila/_exceptions.py create mode 100644 tobiko/openstack/manila/_waiters.py create mode 100644 tobiko/openstack/manila/config.py create mode 100644 tobiko/tests/scenario/manila/__init__.py create mode 100644 tobiko/tests/scenario/manila/test_manila.py diff --git a/lower-constraints.txt b/lower-constraints.txt index 69368ac35..50ec14b6b 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -25,6 +25,7 @@ python-designateclient==4.4.0 python-glanceclient==3.2.2 python-heatclient==2.3.0 python-ironicclient==4.6.1 +python-manilaclient==4.5.1 python-neutronclient==7.2.1 python-novaclient==17.2.1 python-octaviaclient==2.2.0 diff --git a/releasenotes/notes/add-manila-support-d2dd3aab05b75439.yaml b/releasenotes/notes/add-manila-support-d2dd3aab05b75439.yaml new file mode 100644 index 000000000..94a773cf4 --- /dev/null +++ b/releasenotes/notes/add-manila-support-d2dd3aab05b75439.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Added Manila support and testing. Now Manila tests can be run on tobiko as well. diff --git a/requirements.txt b/requirements.txt index 3615d6ff9..8e09e8fb2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,6 +22,7 @@ python-designateclient>=4.4.0 # Apache-2.0 python-glanceclient>=3.2.2 # Apache-2.0 python-heatclient>=2.3.0 # Apache-2.0 python-ironicclient>=4.6.1 # Apache-2.0 +python-manilaclient>=4.5.1 # Apache-2.0 python-neutronclient>=7.2.1 # Apache-2.0 python-novaclient>=17.2.1 # Apache-2.0 python-octaviaclient>=2.2.0 # Apache-2.0 diff --git a/roles/tobiko-cleanup/tasks/main.yaml b/roles/tobiko-cleanup/tasks/main.yaml index fa5ade32c..6c92064ed 100644 --- a/roles/tobiko-cleanup/tasks/main.yaml +++ b/roles/tobiko-cleanup/tasks/main.yaml @@ -38,3 +38,11 @@ grep "^tobiko\." | \ xargs -r {{ openstack_cmd }} image delete ignore_errors: yes + +- name: "cleanup Manila shares created by Tobiko tests" + shell: | + source {{ stackrc_file }} + openstack share list -f value -c 'Name' | \ + grep "^tobiko" | \ + xargs -r openstack share delete --force + ignore_errors: yes diff --git a/roles/tobiko-run/vars/test-workflow-check-resources-manila.yaml b/roles/tobiko-run/vars/test-workflow-check-resources-manila.yaml new file mode 100644 index 000000000..10a206d13 --- /dev/null +++ b/roles/tobiko-run/vars/test-workflow-check-resources-manila.yaml @@ -0,0 +1,8 @@ +--- + +test_workflow_steps: + - tox_description: 'check manila resources' + tox_envlist: manila + tox_step_name: check_manila_resources + tox_environment: + TOBIKO_PREVENT_CREATE: yes diff --git a/roles/tobiko-run/vars/test-workflow-create-resources-manila.yaml b/roles/tobiko-run/vars/test-workflow-create-resources-manila.yaml new file mode 100644 index 000000000..1c69ca813 --- /dev/null +++ b/roles/tobiko-run/vars/test-workflow-create-resources-manila.yaml @@ -0,0 +1,8 @@ +--- + +test_workflow_steps: + - tox_description: 'create manila resources' + tox_envlist: manila + tox_step_name: create_manila_resources + tox_environment: + TOBIKO_PREVENT_CREATE: no diff --git a/tobiko/config.py b/tobiko/config.py index 3e3dbc9e9..54aa86d66 100644 --- a/tobiko/config.py +++ b/tobiko/config.py @@ -30,6 +30,7 @@ LOG = log.getLogger(__name__) CONFIG_MODULES = ['tobiko.common._case', 'tobiko.openstack.glance.config', + 'tobiko.openstack.manila.config', 'tobiko.openstack.keystone.config', 'tobiko.openstack.neutron.config', 'tobiko.openstack.nova.config', @@ -418,3 +419,8 @@ def is_prevent_create() -> bool: def skip_if_prevent_create(reason='TOBIKO_PREVENT_CREATE is True'): return tobiko.skip_if(reason=reason, predicate=is_prevent_create) + + +def skip_unless_prevent_create(reason='TOBIKO_PREVENT_CREATE is False'): + return tobiko.skip_unless(reason=reason, + predicate=is_prevent_create) diff --git a/tobiko/openstack/manila/__init__.py b/tobiko/openstack/manila/__init__.py new file mode 100644 index 000000000..85d0788c3 --- /dev/null +++ b/tobiko/openstack/manila/__init__.py @@ -0,0 +1,44 @@ +# Copyright 2023 Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import absolute_import + +from tobiko.openstack.manila import _client +from tobiko.openstack.manila import _constants +from tobiko.openstack.manila import _exceptions +from tobiko.openstack.manila import _waiters + +manila_client = _client.manila_client +get_manila_client = _client.get_manila_client +ManilaClientFixture = _client.ManilaClientFixture +create_share = _client.create_share +get_share = _client.get_share +get_shares_by_name = _client.get_shares_by_name +delete_share = _client.delete_share +extend_share = _client.extend_share +list_shares = _client.list_shares + +# Waiters +wait_for_share_status = _waiters.wait_for_share_status +wait_for_resource_deletion = _waiters.wait_for_resource_deletion + +# Exceptions +ShareNotFound = _exceptions.ShareNotFound +ShareReleaseFailed = _exceptions.ShareReleaseFailed + +# Constants +RESOURCE_STATUS = _constants.RESOURCE_STATUS +STATUS_AVAILABLE = _constants.STATUS_AVAILABLE +STATUS_ERROR = _constants.STATUS_ERROR +STATUS_ERROR_DELETING = _constants.STATUS_ERROR_DELETING +SHARE_NAME = _constants.SHARE_NAME diff --git a/tobiko/openstack/manila/_client.py b/tobiko/openstack/manila/_client.py new file mode 100644 index 000000000..7d062c677 --- /dev/null +++ b/tobiko/openstack/manila/_client.py @@ -0,0 +1,104 @@ +# Copyright 2023 Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import absolute_import + +from manilaclient.v2 import client as manilaclient +from manilaclient import exceptions +from oslo_log import log + +import tobiko +from tobiko import config +from tobiko.openstack import keystone +from tobiko.openstack import _client +from tobiko.openstack.manila import _exceptions + +LOG = log.getLogger(__name__) +CONF = config.CONF + + +class ManilaClientFixture(_client.OpenstackClientFixture): + + def init_client(self, session): + return manilaclient.Client(session=session) + + +class ManilaClientManager(_client.OpenstackClientManager): + + def create_client(self, session): + return ManilaClientFixture(session=session) + + +CLIENTS = ManilaClientManager() + + +@keystone.skip_if_missing_service(name='manila') +def manila_client(obj=None): + obj = obj or default_manila_client() + if tobiko.is_fixture(obj): + obj = tobiko.setup_fixture(obj).client + return tobiko.check_valid_type(obj, manilaclient.Client) + + +def default_manila_client(): + return get_manila_client() + + +def get_manila_client(session=None, shared=True, init_client=None, + manager=None): + manager = manager or CLIENTS + fixture = manager.get_client(session=session, shared=shared, + init_client=init_client) + return manila_client(fixture) + + +def create_share(share_protocol=None, size=None, client=None, **kwargs): + share_protocol = share_protocol or CONF.tobiko.manila.share_protocol + share_size = size or CONF.tobiko.manila.size + return manila_client(client).shares.create( + share_proto=share_protocol, size=share_size, return_raw=True, + **kwargs) + + +def list_shares(client=None, **kwargs): + return manila_client(client).shares.list(return_raw=True, **kwargs) + + +def delete_share(share_id, client=None, **kwargs): + try: + manila_client(client).shares.delete(share_id, **kwargs) + except exceptions.NotFound: + LOG.debug(f'Share {share_id} was not found') + return False + else: + LOG.debug(f'Share {share_id} was deleted successfully') + return True + + +def extend_share(share_id, new_size, client=None): + return manila_client(client).shares.extend(share_id, new_size) + + +def get_share(share_id, client=None): + try: + return manila_client(client).shares.get(share_id, return_raw=True) + except exceptions.NotFound as ex: + raise _exceptions.ShareNotFound(id=share_id) from ex + + +def get_shares_by_name(share_name, client=None): + share_list = list_shares(client=client) + shares = [ + s for s in share_list if s['name'] == share_name + ] + return shares diff --git a/tobiko/openstack/manila/_constants.py b/tobiko/openstack/manila/_constants.py new file mode 100644 index 000000000..e826053e6 --- /dev/null +++ b/tobiko/openstack/manila/_constants.py @@ -0,0 +1,24 @@ +# Copyright 2023 Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Manila attributes +RESOURCE_STATUS = 'status' + +# Shares +STATUS_AVAILABLE = 'available' +STATUS_ERROR = 'error' +STATUS_ERROR_DELETING = 'error_deleting' + +# Manila resource names +SHARE_NAME = "tobiko-manila-share" diff --git a/tobiko/openstack/manila/_exceptions.py b/tobiko/openstack/manila/_exceptions.py new file mode 100644 index 000000000..75082686c --- /dev/null +++ b/tobiko/openstack/manila/_exceptions.py @@ -0,0 +1,24 @@ +# Copyright 2023 Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import absolute_import + +import tobiko + + +class ShareNotFound(tobiko.ObjectNotFound): + message = "No such manila share {id!r}" + + +class ShareReleaseFailed(tobiko.ObjectNotFound): + message = "Share has 'error_deleting' status and can not be deleted" diff --git a/tobiko/openstack/manila/_waiters.py b/tobiko/openstack/manila/_waiters.py new file mode 100644 index 000000000..0a0129b19 --- /dev/null +++ b/tobiko/openstack/manila/_waiters.py @@ -0,0 +1,93 @@ +# Copyright 2023 Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import absolute_import + +import typing +import time + +from oslo_log import log +from manilaclient import exceptions + +import tobiko +from tobiko.openstack.manila import _client +from tobiko.openstack.manila import _constants +from tobiko.openstack.manila import _exceptions + +LOG = log.getLogger(__name__) + + +def wait_for_status(object_id: str, + status_key: str = _constants.RESOURCE_STATUS, + status: str = _constants.STATUS_AVAILABLE, + get_client: typing.Callable = None, + interval: tobiko.Seconds = None, + timeout: tobiko.Seconds = None, + **kwargs): + """Waits for an object to reach a specific status. + + :param status_key: The key of the status field in the response. + Ex. status + :param status: The status to wait for. Ex. "ACTIVE" + :param get_client: The tobiko client get method. + Ex. _client.get_zone + :param object_id: The id of the object to query. + :param interval: How often to check the status, in seconds. + :param timeout: The maximum time, in seconds, to check the status. + :raises TimeoutException: The object did not achieve the status or ERROR in + the check_timeout period. + :raises UnexpectedStatusException: The request returned an unexpected + response code. + """ + + get_client = get_client or _client.get_share + + for attempt in tobiko.retry(timeout=timeout, + interval=interval, + default_timeout=300., + default_interval=5.): + response = get_client(object_id, **kwargs) + if response[status_key] == status: + return response + + attempt.check_limits() + + LOG.debug(f"Waiting for {get_client.__name__} {status_key} to get " + f"from '{response[status_key]}' to '{status}'...") + + +def wait_for_share_status(share_id): + wait_for_status(object_id=share_id) + + +def _is_share_deleted(share_id): + try: + res = _client.get_share(share_id) + except _exceptions.ShareNotFound: + return True + if res.get(_constants.RESOURCE_STATUS) in [ + _constants.STATUS_ERROR, _constants.STATUS_ERROR_DELETING]: + # Share has "error_deleting" status and can not be deleted. + raise _exceptions.ShareReleaseFailed(id=share_id) + return False + + +def wait_for_resource_deletion(share_id, build_interval=1, build_timeout=60): + """Waits for a resource to be deleted.""" + start_time = int(time.time()) + while True: + if _is_share_deleted(share_id): + return + if int(time.time()) - start_time >= build_timeout: + raise exceptions.TimeoutException + time.sleep(build_interval) diff --git a/tobiko/openstack/manila/config.py b/tobiko/openstack/manila/config.py new file mode 100644 index 000000000..a34b68421 --- /dev/null +++ b/tobiko/openstack/manila/config.py @@ -0,0 +1,36 @@ +# Copyright 2023 Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import absolute_import + +import itertools + +from oslo_config import cfg + +GROUP_NAME = 'manila' +OPTIONS = [ + cfg.StrOpt('share_protocol', + default='nfs', + help="Share protocol"), + cfg.IntOpt('size', + default=1, + help="Default size in GB for shares created by share tests."), +] + + +def register_tobiko_options(conf): + conf.register_opts(group=cfg.OptGroup(GROUP_NAME), opts=OPTIONS) + + +def list_options(): + return [(GROUP_NAME, itertools.chain(OPTIONS))] diff --git a/tobiko/tests/scenario/manila/__init__.py b/tobiko/tests/scenario/manila/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tobiko/tests/scenario/manila/test_manila.py b/tobiko/tests/scenario/manila/test_manila.py new file mode 100644 index 000000000..7567f0adf --- /dev/null +++ b/tobiko/tests/scenario/manila/test_manila.py @@ -0,0 +1,58 @@ +# Copyright (c) 2023 Red Hat, Inc. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import absolute_import + +import testtools +from oslo_log import log + +from tobiko import config +from tobiko.openstack import keystone +from tobiko.openstack import manila + +LOG = log.getLogger(__name__) +CONF = config.CONF + + +@keystone.skip_if_missing_service(name='manila') +class ManilaApiTestCase(testtools.TestCase): + """Manila scenario tests. + + Create a manila share. + Check it reaches status 'available'. + After upgrade/disruptions/etc, check the share is still valid and it can be + extended. + """ + @classmethod + def setUpClass(cls): + if config.get_bool_env('TOBIKO_PREVENT_CREATE'): + LOG.debug('skipping creation of manila resources') + cls.share = manila.get_shares_by_name(manila.SHARE_NAME)[0] + else: + cls.share = manila.create_share(name=manila.SHARE_NAME) + + manila.wait_for_share_status(cls.share['id']) + + @config.skip_if_prevent_create() + def test_1_create_share(self): + self.assertEqual(manila.SHARE_NAME, self.share['name']) + + @config.skip_unless_prevent_create() + def test_2_extend_share(self): + share_id = self.share['id'] + manila.extend_share(share_id, new_size=CONF.tobiko.manila.size + 1) + manila.wait_for_share_status(share_id) + share_size = manila.get_share(share_id)['size'] + self.assertEqual(CONF.tobiko.manila.size + 1, share_size) diff --git a/tox.ini b/tox.ini index 188ae7569..f4332a52f 100644 --- a/tox.ini +++ b/tox.ini @@ -172,6 +172,15 @@ setenv = OS_TEST_PATH = {toxinidir}/tobiko/tests/functional +[testenv:manila] +basepython = {[integration]basepython} +envdir = {[integration]envdir} +passenv = {[integration]passenv} +setenv = + {[integration]setenv} + OS_TEST_PATH = {toxinidir}/tobiko/tests/scenario/manila + + [testenv:scenario] basepython = {[integration]basepython} diff --git a/upper-constraints.txt b/upper-constraints.txt index 55fe06a8f..04e2e250b 100644 --- a/upper-constraints.txt +++ b/upper-constraints.txt @@ -359,7 +359,7 @@ python-keystoneclient===4.4.0 python-ldap===3.4.0 python-linstor===1.13.0 python-magnumclient===3.6.0 -python-manilaclient===3.3.1 +python-manilaclient===4.5.1 python-masakariclient===7.2.0 python-memcached===1.59 python-mistralclient===4.4.0