From 5250d940bc42bcc9df708ae2f01235ee083d5dbd Mon Sep 17 00:00:00 2001 From: gengchc2 Date: Sun, 5 Nov 2017 01:07:36 -0800 Subject: [PATCH] Remove pool group from zaqar Implement: blueprint remove-pool-group-from-zaqar Change-Id: I18bb20d44f5071f9700fb64c10e9393707086de3 --- zaqar/api/v1_1/response.py | 6 + zaqar/api/v2/response.py | 6 + zaqar/common/api/schemas/flavors.py | 17 +- zaqar/common/api/schemas/pools.py | 12 + zaqar/storage/base.py | 55 +-- zaqar/storage/mongodb/flavors.py | 14 +- zaqar/storage/mongodb/pools.py | 59 ++- zaqar/storage/pooling.py | 47 ++- zaqar/storage/sqlalchemy/flavors.py | 25 +- .../alembic_migrations/versions/006_queens.py | 41 ++ zaqar/storage/sqlalchemy/pools.py | 33 +- zaqar/storage/sqlalchemy/tables.py | 11 +- zaqar/tests/base.py | 3 +- zaqar/tests/unit/storage/base.py | 250 ++++++++++-- zaqar/tests/unit/storage/test_impl_mongodb.py | 70 ++++ .../unit/storage/test_impl_sqlalchemy.py | 37 ++ .../unit/storage/test_pool_catalog_new.py | 128 ++++++ .../unit/transport/wsgi/v2_0/test_flavors.py | 1 + .../transport/wsgi/v2_0/test_flavors_new.py | 354 ++++++++++++++++ .../unit/transport/wsgi/v2_0/test_pools.py | 1 + .../transport/wsgi/v2_0/test_pools_new.py | 383 ++++++++++++++++++ zaqar/transport/wsgi/v2_0/flavors.py | 289 ++++++++++--- zaqar/transport/wsgi/v2_0/pools.py | 17 +- 23 files changed, 1694 insertions(+), 165 deletions(-) create mode 100644 zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/006_queens.py create mode 100644 zaqar/tests/unit/storage/test_pool_catalog_new.py create mode 100644 zaqar/tests/unit/transport/wsgi/v2_0/test_flavors_new.py create mode 100644 zaqar/tests/unit/transport/wsgi/v2_0/test_pools_new.py diff --git a/zaqar/api/v1_1/response.py b/zaqar/api/v1_1/response.py index 4964ee076..b57209bcf 100644 --- a/zaqar/api/v1_1/response.py +++ b/zaqar/api/v1_1/response.py @@ -229,6 +229,9 @@ class ResponseSchema(api.Api): 'group': { 'type': ['string', 'null'] }, + 'flavor': { + 'type': ['string', 'null'] + }, 'options': { 'type': 'object', 'additionalProperties': True @@ -284,6 +287,9 @@ class ResponseSchema(api.Api): 'group': { 'type': ['string', 'null'] }, + 'flavor': { + 'type': ['string', 'null'] + }, 'weight': { 'type': 'number', 'minimum': -1 diff --git a/zaqar/api/v2/response.py b/zaqar/api/v2/response.py index c752dece3..704d88c52 100644 --- a/zaqar/api/v2/response.py +++ b/zaqar/api/v2/response.py @@ -229,6 +229,9 @@ class ResponseSchema(api.Api): 'group': { 'type': ['string', 'null'] }, + 'flavor': { + 'type': ['string', 'null'] + }, 'options': { 'type': 'object', 'additionalProperties': True @@ -284,6 +287,9 @@ class ResponseSchema(api.Api): 'group': { 'type': ['string', 'null'] }, + 'flavor': { + 'type': ['string', 'null'] + }, 'weight': { 'type': 'number', 'minimum': -1 diff --git a/zaqar/common/api/schemas/flavors.py b/zaqar/common/api/schemas/flavors.py index 254d936a6..f2519f6bf 100644 --- a/zaqar/common/api/schemas/flavors.py +++ b/zaqar/common/api/schemas/flavors.py @@ -27,6 +27,7 @@ patch_capabilities = { } } +# TODO(gengchc2): remove pool_group in R release. # NOTE(flaper87): a string valid patch_pool = { 'type': 'object', @@ -38,6 +39,7 @@ patch_pool = { } } +# TODO(gengchc2): remove pool_group in R release. patch_pool_group = { 'type': 'object', 'properties': { @@ -48,16 +50,29 @@ patch_pool_group = { } } +# NOTE(gengchc): Add pool_list in flavor creation for removing pool_group +patch_pool_list = { + 'type': 'object', + 'properties': { + 'pool_list': { + 'type': 'array' + }, + 'additionalProperties': False + } +} + create = { 'type': 'object', 'properties': { 'pool_group': patch_pool_group['properties']['pool_group'], 'pool': patch_pool['properties']['pool'], + 'pool_list': patch_pool_list['properties']['pool_list'], 'capabilities': patch_capabilities['properties']['capabilities'] }, # NOTE(flaper87): capabilities need not be present. Storage drivers # must provide reasonable defaults. # NOTE(wanghao): remove pool in Newton release. - 'oneOf': [{'required': ['pool_group']}, {'required': ['pool']}], + 'oneOf': [{'required': ['pool_group']}, {'required': ['pool']}, + {'required': ['pool_list']}], 'additionalProperties': False } diff --git a/zaqar/common/api/schemas/pools.py b/zaqar/common/api/schemas/pools.py index ec9eee4d4..8199e4586 100644 --- a/zaqar/common/api/schemas/pools.py +++ b/zaqar/common/api/schemas/pools.py @@ -49,6 +49,17 @@ patch_group = { } } +# NOTE(gengchc): remove pool_group add flavor +patch_flavor = { + 'type': 'object', 'properties': { + 'flavor': { + 'type': 'string', + 'minLength': 0, + 'maxLength': 255 + }, + 'additionalProperties': False + } +} patch_weight = { 'type': 'object', 'properties': { @@ -63,6 +74,7 @@ create = { 'type': 'object', 'properties': { 'weight': patch_weight['properties']['weight'], 'group': patch_group['properties']['uri'], + 'flavor': patch_flavor['properties']['flavor'], 'uri': patch_uri['properties']['uri'], 'options': patch_options['properties']['options'] }, diff --git a/zaqar/storage/base.py b/zaqar/storage/base.py index 805984d49..cda65e41d 100644 --- a/zaqar/storage/base.py +++ b/zaqar/storage/base.py @@ -735,9 +735,9 @@ class Subscription(ControllerBase): class PoolsBase(ControllerBase): """A controller for managing pools.""" - def _check_capabilities(self, uri, group=None, name=None): + def _check_capabilities(self, uri, flavor=None, name=None): default_store = self.driver.conf.drivers.message_store - pool_caps = self.capabilities(group=group, name=name) + pool_caps = self.capabilities(flavor=flavor, name=name) if not pool_caps: return True @@ -745,30 +745,32 @@ class PoolsBase(ControllerBase): new_store = utils.load_storage_impl(uri, default_store=default_store) - # NOTE(flaper87): Since all pools in a pool group + # NOTE(flaper87): Since all pools in a pool flavor # are assumed to have the same capabilities, it's # fine to check against just 1 return pool_caps == new_store.BASE_CAPABILITIES - def capabilities(self, group=None, name=None): - """Gets the set of capabilities for this group/name + def capabilities(self, flavor=None, name=None): + """Gets the set of capabilities for this flavor/name - :param group: The pool group to get capabilities for - :type group: six.text_type + :param flavor: The pool flavor to get capabilities for + :type flavor: six.text_type :param name: The pool name to get capabilities for :type name: six.text_type """ + pllt = [] if name: - group = list(self._get_pools_by_group(self._get(name)['group'])) + pool = self.get(name) + pllt.append(pool) else: - group = list(self._get_pools_by_group(group)) + pllt = list(self._get_pools_by_flavor(flavor)) - if not len(group) > 0: + if not len(pllt) > 0: return () default_store = self.driver.conf.drivers.message_store - pool_store = utils.load_storage_impl(group[0]['uri'], + pool_store = utils.load_storage_impl(pllt[0]['uri'], default_store=default_store) return pool_store.BASE_CAPABILITIES @@ -791,7 +793,7 @@ class PoolsBase(ControllerBase): _list = abc.abstractmethod(lambda x: None) - def create(self, name, weight, uri, group=None, options=None): + def create(self, name, weight, uri, group=None, flavor=None, options=None): """Registers a pool entry. :param name: The name of this pool @@ -801,33 +803,38 @@ class PoolsBase(ControllerBase): :param uri: A URI that can be used by a storage client (e.g., pymongo) to access this pool. :type uri: six.text_type - :param group: The group of this pool - :type group: six.text_type + :param flavor: The flavor of this pool + :type flavor: six.text_type :param options: Options used to configure this pool :type options: dict """ - if not self._check_capabilities(uri, group=group): + flavor_obj = {} + if flavor is not None: + flavor_obj["name"] = flavor + if group is not None: + flavor_obj["pool_group"] = group + if not self._check_capabilities(uri, flavor=flavor_obj): raise errors.PoolCapabilitiesMismatch() - return self._create(name, weight, uri, group, options) + return self._create(name, weight, uri, group, flavor, options) _create = abc.abstractmethod(lambda x: None) - def get_pools_by_group(self, group=None, detailed=False): - """Returns a pool list filtered by given pool group. + def get_pools_by_flavor(self, flavor=None, detailed=False): + """Returns a pool list filtered by given pool flavor. - :param group: The group to filter on. `None` returns - pools that are not assigned to any pool group. - :type group: six.text_type + :param flavor: The flavor to filter on. `None` returns + pools that are not assigned to any pool flavor. + :type flavor: six.text_type :param detailed: Should the options data be included? :type detailed: bool :returns: weight, uri, and options for this pool :rtype: {} :raises PoolDoesNotExist: if not found """ - return self._get_pools_by_group(group, detailed) + return self._get_pools_by_flavor(flavor, detailed) - _get_pools_by_group = abc.abstractmethod(lambda x: None) + _get_pools_by_flavor = abc.abstractmethod(lambda x: None) def get(self, name, detailed=False): """Returns a single pool entry. @@ -1008,7 +1015,7 @@ class FlavorsBase(ControllerBase): raise NotImplementedError @abc.abstractmethod - def create(self, name, pool, project=None, capabilities=None): + def create(self, name, pool_group=None, project=None, capabilities=None): """Registers a flavor entry. :param name: The name of this flavor diff --git a/zaqar/storage/mongodb/flavors.py b/zaqar/storage/mongodb/flavors.py index 8c78ac3cb..bc51f33cd 100644 --- a/zaqar/storage/mongodb/flavors.py +++ b/zaqar/storage/mongodb/flavors.py @@ -97,14 +97,19 @@ class FlavorsController(base.FlavorsBase): return _normalize(res, detailed) @utils.raises_conn_error - def create(self, name, pool_group, project=None, capabilities=None): + def create(self, name, pool_group=None, project=None, capabilities=None): # NOTE(flaper87): Check if there are pools in this group. # Should there be a `group_exists` method? # NOTE(wanghao): Since we didn't pass the group name just pool name, # so we don't need to get the pool by group. - if not list(self._pools_ctrl.get_pools_by_group(pool_group)): - raise errors.PoolGroupDoesNotExist(pool_group) + # NOTE(gengchc2): If you do not use the removal group scheme to + # configure flavor, pool_group can be None.. + if pool_group is not None: + flavor_obj = {} + flavor_obj["pool_group"] = pool_group + if not list(self._pools_ctrl.get_pools_by_flavor(flavor_obj)): + raise errors.PoolGroupDoesNotExist(pool_group) capabilities = {} if capabilities is None else capabilities self._col.update_one({'n': name, 'p': project}, @@ -124,7 +129,8 @@ class FlavorsController(base.FlavorsBase): if pool_group is not None: fields['s'] = pool_group - + # NOTE(gengchc2): If you do not use the removal group scheme to + # configure flavor, pool_group can be None, pool_group can be remove. assert fields, '`pool_group` or `capabilities` not found in kwargs' res = self._col.update_one({'n': name, 'p': project}, {'$set': fields}, diff --git a/zaqar/storage/mongodb/pools.py b/zaqar/storage/mongodb/pools.py index 6df5e9dad..35bf1ea8b 100644 --- a/zaqar/storage/mongodb/pools.py +++ b/zaqar/storage/mongodb/pools.py @@ -20,6 +20,7 @@ Schema: 'u': uri :: six.text_type 'w': weight :: int 'o': options :: dict + 'f': flavor :: six.text_type """ import functools @@ -90,13 +91,22 @@ class PoolsController(base.PoolsBase): return _normalize(res, detailed) @utils.raises_conn_error - def _get_pools_by_group(self, group=None, detailed=False): - cursor = self._col.find({'g': group}, projection=_field_spec(detailed)) + def _get_pools_by_flavor(self, flavor=None, detailed=False): + query = None + if flavor is None: + query = {'f': None} + elif flavor.get("pool_group") is not None: + query = {'g': flavor.get("pool_group")} + elif flavor.get('name') is not None: + query = {'f': flavor.get('name')} + cursor = self._col.find(query, + projection=_field_spec(detailed)) normalizer = functools.partial(_normalize, detailed=detailed) return utils.HookedCursor(cursor, normalizer) @utils.raises_conn_error - def _create(self, name, weight, uri, group=None, options=None): + def _create(self, name, weight, uri, group=None, flavor=None, + options=None): options = {} if options is None else options try: self._col.update_one({'n': name}, @@ -104,6 +114,7 @@ class PoolsController(base.PoolsBase): 'w': weight, 'u': uri, 'g': group, + 'f': flavor, 'o': options}}, upsert=True) except mongo_error.DuplicateKeyError: @@ -115,13 +126,20 @@ class PoolsController(base.PoolsBase): @utils.raises_conn_error def _update(self, name, **kwargs): - names = ('uri', 'weight', 'group', 'options') + names = ('uri', 'weight', 'group', 'flavor', 'options') fields = common_utils.fields(kwargs, names, pred=lambda x: x is not None, key_transform=lambda x: x[0]) assert fields, ('`weight`, `uri`, `group`, ' 'or `options` not found in kwargs') + flavor = fields.get('f') + if flavor is not None and len(flavor) == 0: + fields['f'] = None + group = fields.get('g') + if group is not None and len(group) == 0: + fields['g'] = None + res = self._col.update_one({'n': name}, {'$set': fields}, upsert=False) @@ -135,17 +153,31 @@ class PoolsController(base.PoolsBase): # recursion error. try: pool = self.get(name) - pools_group = self.get_pools_by_group(pool['group']) - flavor_ctl = self.driver.flavors_controller - res = list(flavor_ctl._list_by_pool_group(pool['group'])) + if pool['group'] is not None: + flavor = {} + flavor['pool_group'] = pool['group'] + pools_group = self.get_pools_by_flavor(flavor=flavor) + flavor_ctl = self.driver.flavors_controller + res = list(flavor_ctl._list_by_pool_group(pool['group'])) - # NOTE(flaper87): If this is the only pool in the - # group and it's being used by a flavor, don't allow - # it to be deleted. - if res and len(pools_group) == 1: - flavors = ', '.join([x['name'] for x in res]) - raise errors.PoolInUseByFlavor(name, flavors) + # NOTE(flaper87): If this is the only pool in the + # group and it's being used by a flavor, don't allow + # it to be deleted. + if res and len(pools_group) == 1: + flavors = ', '.join([x['name'] for x in res]) + raise errors.PoolInUseByFlavor(name, flavors) + pools_in_flavor = [] + flavor = pool.get("flavor", None) + if flavor is not None: + # NOTE(gengchc2): If this is the only pool in the + # flavor and it's being used by a flavor, don't allow + # it to be deleted. + flavor1 = {} + flavor1['name'] = flavor + pools_in_flavor = self.get_pools_by_flavor(flavor=flavor1) + if len(pools_in_flavor) == 1: + raise errors.PoolInUseByFlavor(name, flavor) self._col.delete_one({'n': name}) except errors.PoolDoesNotExist: pass @@ -160,6 +192,7 @@ def _normalize(pool, detailed=False): ret = { 'name': pool['n'], 'group': pool['g'], + 'flavor': pool['f'], 'uri': pool['u'], 'weight': pool['w'], } diff --git a/zaqar/storage/pooling.py b/zaqar/storage/pooling.py index c3800bd02..62fc0e831 100644 --- a/zaqar/storage/pooling.py +++ b/zaqar/storage/pooling.py @@ -23,6 +23,7 @@ from osprofiler import profiler from zaqar.common import decorators from zaqar.common import errors as cerrors from zaqar.common.storage import select +from zaqar.i18n import _ from zaqar import storage from zaqar.storage import errors from zaqar.storage import pipeline @@ -246,6 +247,14 @@ class QueueController(storage.Queue): return self._mgt_queue_ctrl.get_metadata(name, project=project) def set_metadata(self, name, metadata, project=None): + # NOTE(gengchc2): If flavor metadata is modified in queue, + # The queue needs to be re-registered to pools, otherwise + # the queue flavor parameter is not consistent with the pool. + flavor = None + if isinstance(metadata, dict): + flavor = metadata.get('_flavor') + self._pool_catalog.register(name, project=project, flavor=flavor) + return self._mgt_queue_ctrl.set_metadata(name, metadata=metadata, project=project) @@ -502,22 +511,40 @@ class Catalog(object): """ - # NOTE(cpp-cabrera): only register a queue if the entry - # doesn't exist - if not self._catalogue_ctrl.exists(project, queue): + # NOTE(gengchc): if exist, get queue's pool.flavor: + # if queue's pool.flavor is different, first delete it and add it. + # Otherwise, if the flavor in the meteredata of the queue is + # modified, the catalog will be inconsistent. + if self._catalogue_ctrl.exists(project, queue): + catalogue = self._catalogue_ctrl.get(project, queue) + oldpoolids = catalogue['pool'] + oldpool = self._pools_ctrl.get(oldpoolids) + oldflavor = oldpool['flavor'] + msgtmpl = _(u'regiester queue to pool: old flavor: %(oldflavor)s ' + ', new flavor: %(flavor)s') + LOG.info(msgtmpl, + {'oldflavor': oldflavor, 'flavor': flavor}) + if oldpool['flavor'] != flavor: + self._catalogue_ctrl.delete(project, queue) + if not self._catalogue_ctrl.exists(project, queue): if flavor is not None: flavor = self._flavor_ctrl.get(flavor, project=project) - pools = self._pools_ctrl.get_pools_by_group( - group=flavor['pool_group'], + pools = self._pools_ctrl.get_pools_by_flavor( + flavor=flavor, detailed=True) pool = select.weighted(pools) pool = pool and pool['name'] or None + msgtmpl = _(u'regiester queue to pool: new flavor:%(flavor)s' + ' pool_group:%(pool_group)s') + LOG.info(msgtmpl, + {'flavor': flavor.get('name', None), + 'pool_group': flavor.get('pool_group', None)}) else: # NOTE(flaper87): Get pools assigned to the default # group `None`. We should consider adding a `default_group` # option in the future. - pools = self._pools_ctrl.get_pools_by_group(detailed=True) + pools = self._pools_ctrl.get_pools_by_flavor(detailed=True) pool = select.weighted(pools) pool = pool and pool['name'] or None @@ -531,7 +558,15 @@ class Catalog(object): if self.lookup(queue, project) is not None: return raise errors.NoPoolFound() + msgtmpl = _(u'regiester queue to pool: new flavor: None') + LOG.info(msgtmpl) + msgtmpl = _(u'regiester queue: project:%(project)s' + ' queue:%(queue)s pool:%(pool)s') + LOG.info(msgtmpl, + {'project': project, + 'queue': queue, + 'pool': pool}) self._catalogue_ctrl.insert(project, queue, pool) @_pool_id.purges diff --git a/zaqar/storage/sqlalchemy/flavors.py b/zaqar/storage/sqlalchemy/flavors.py index 77cbc767d..7197f8932 100644 --- a/zaqar/storage/sqlalchemy/flavors.py +++ b/zaqar/storage/sqlalchemy/flavors.py @@ -72,18 +72,29 @@ class FlavorsController(base.FlavorsBase): return _normalize(flavor, detailed) @utils.raises_conn_error - def create(self, name, pool_group, project=None, capabilities=None): + def create(self, name, pool_group=None, project=None, capabilities=None): cap = None if capabilities is None else utils.json_encode(capabilities) try: - stmt = sa.sql.expression.insert(tables.Flavors).values( - name=name, pool_group=pool_group, project=project, - capabilities=cap - ) + if pool_group is not None: + stmt = sa.sql.expression.insert(tables.Flavors).values( + name=name, pool_group=pool_group, project=project, + capabilities=cap + ) + else: + stmt = sa.sql.expression.insert(tables.Flavors).values( + name=name, project=project, + capabilities=cap + ) self.driver.run(stmt) except oslo_db.exception.DBDuplicateEntry: - if not self._pools_ctrl.get_pools_by_group(pool_group): - raise errors.PoolGroupDoesNotExist(pool_group) + # NOTE(gengchc2): If you do not use the removal group scheme to + # configure flavor, pool_group can be None.. + if pool_group is not None: + flavor_obj = {} + flavor_obj["pool_group"] = pool_group + if not list(self._pools_ctrl.get_pools_by_flavor(flavor_obj)): + raise errors.PoolGroupDoesNotExist(pool_group) # TODO(flaper87): merge update/create into a single # method with introduction of upsert diff --git a/zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/006_queens.py b/zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/006_queens.py new file mode 100644 index 000000000..5ebbbefd5 --- /dev/null +++ b/zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/006_queens.py @@ -0,0 +1,41 @@ +# Copyright 2017 ZTE Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Queens release + +Revision ID: 005 +Revises: 006 +Create Date: 2017-11-09 11:45:45.928605 + +""" + +# revision identifiers, used by Alembic. +revision = '006' +down_revision = '005' + +from alembic import op +import sqlalchemy as sa + +MYSQL_ENGINE = 'InnoDB' +MYSQL_CHARSET = 'utf8' + + +def upgrade(): + # NOTE(gengchc2): Add a new flavor column to Pools nodes + op.add_column('Pools', sa.Column('flavor', sa.String(64), nullable=True)) + + # NOTE(gengchc2): Change pool_group to default null in Flavors table + op.execute('alter table Flavors change column pool_group ' + 'pool_group varchar(64) default null') diff --git a/zaqar/storage/sqlalchemy/pools.py b/zaqar/storage/sqlalchemy/pools.py index 1156479d2..4b71e56ca 100644 --- a/zaqar/storage/sqlalchemy/pools.py +++ b/zaqar/storage/sqlalchemy/pools.py @@ -56,10 +56,22 @@ class PoolsController(base.PoolsBase): yield marker_name and marker_name['next'] @utils.raises_conn_error - def _get_pools_by_group(self, group=None, detailed=False): - stmt = sa.sql.select([tables.Pools]).where( - tables.Pools.c.group == group - ) + def _get_pools_by_flavor(self, flavor=None, detailed=False): + pool_group = flavor.get("pool_group", None) if flavor is not None\ + else None + flavor_name = flavor.get("name", None) if flavor is not None\ + else None + if pool_group is not None: + stmt = sa.sql.select([tables.Pools]).where( + tables.Pools.c.group == pool_group + ) + elif flavor_name is not None: + stmt = sa.sql.select([tables.Pools]).where( + tables.Pools.c.flavor == flavor_name + ) + else: + stmt = sa.sql.select([tables.Pools]) + cursor = self.driver.run(stmt) normalizer = functools.partial(_normalize, detailed=detailed) @@ -87,7 +99,8 @@ class PoolsController(base.PoolsBase): # TODO(cpp-cabrera): rename to upsert @utils.raises_conn_error - def _create(self, name, weight, uri, group=None, options=None): + def _create(self, name, weight, uri, group=None, flavor=None, + options=None): opts = None if options is None else utils.json_encode(options) if group is not None: @@ -95,7 +108,8 @@ class PoolsController(base.PoolsBase): try: stmt = sa.sql.expression.insert(tables.Pools).values( - name=name, weight=weight, uri=uri, group=group, options=opts + name=name, weight=weight, uri=uri, group=group, + flavor=flavor, options=opts ) self.driver.run(stmt) @@ -103,7 +117,7 @@ class PoolsController(base.PoolsBase): # TODO(cpp-cabrera): merge update/create into a single # method with introduction of upsert self._update(name, weight=weight, uri=uri, - group=group, options=options) + group=group, flavor=flavor, options=options) @utils.raises_conn_error def _exists(self, name): @@ -117,11 +131,11 @@ class PoolsController(base.PoolsBase): # NOTE(cpp-cabrera): by pruning None-valued kwargs, we avoid # overwriting the existing options field with None, since that # one can be null. - names = ('uri', 'weight', 'group', 'options') + names = ('uri', 'weight', 'group', 'flavor', 'options') fields = common_utils.fields(kwargs, names, pred=lambda x: x is not None) - assert fields, ('`weight`, `uri`, `group`, ' + assert fields, ('`weight`, `uri`, `group`, `flavor`, ' 'or `options` not found in kwargs') if 'options' in fields: @@ -158,6 +172,7 @@ def _normalize(pool, detailed=False): 'group': pool[1], 'uri': pool[2], 'weight': pool[3], + 'flavor': pool[5], } if detailed: opts = pool[4] diff --git a/zaqar/storage/sqlalchemy/tables.py b/zaqar/storage/sqlalchemy/tables.py index 409f5e396..28b0f9a05 100644 --- a/zaqar/storage/sqlalchemy/tables.py +++ b/zaqar/storage/sqlalchemy/tables.py @@ -28,7 +28,6 @@ Queues = sa.Table('Queues', metadata, PoolGroup = sa.Table('PoolGroup', metadata, sa.Column('name', sa.String(64), primary_key=True)) - Pools = sa.Table('Pools', metadata, sa.Column('name', sa.String(64), primary_key=True), sa.Column('group', sa.ForeignKey('PoolGroup.name', @@ -37,18 +36,20 @@ Pools = sa.Table('Pools', metadata, sa.Column('uri', sa.String(255), unique=True, nullable=False), sa.Column('weight', sa.INTEGER, nullable=False), - sa.Column('options', sa.Text())) - + sa.Column('options', sa.Text()), + sa.Column('flavor', sa.String(64), nullable=True)) +# NOTE(gengchc2): Modify pool_group define: turn NOT NULL into DEFAULT NULL: +# [alter table Flavors change column pool_group pool_group varchar(64) +# default null;] Flavors = sa.Table('Flavors', metadata, sa.Column('name', sa.String(64), primary_key=True), sa.Column('project', sa.String(64)), sa.Column('pool_group', sa.ForeignKey('PoolGroup.name', ondelete='CASCADE'), - nullable=False), + nullable=True), sa.Column('capabilities', sa.Text())) - Catalogue = sa.Table('Catalogue', metadata, sa.Column('pool', sa.String(64), sa.ForeignKey('Pools.name', diff --git a/zaqar/tests/base.py b/zaqar/tests/base.py index e6b632df8..3b27d3c7a 100644 --- a/zaqar/tests/base.py +++ b/zaqar/tests/base.py @@ -66,7 +66,8 @@ class TestBase(testtools.TestCase): opts.set_defaults(self.conf) self.conf.register_opts(configs._PROFILER_OPTIONS, group=configs._PROFILER_GROUP) - + self.redis_url = os.environ.get('ZAQAR_TEST_REDIS_URL', + 'redis://127.0.0.1:6379') self.mongodb_url = os.environ.get('ZAQAR_TEST_MONGODB_URL', 'mongodb://127.0.0.1:27017') diff --git a/zaqar/tests/unit/storage/base.py b/zaqar/tests/unit/storage/base.py index 60d5f3204..b78c2187e 100644 --- a/zaqar/tests/unit/storage/base.py +++ b/zaqar/tests/unit/storage/base.py @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os + import collections import datetime import math @@ -68,16 +70,27 @@ class ControllerBaseTest(testing.TestBase): args.append(self.control) self.driver = self.driver_class(*args) else: - uri = self.mongodb_url - for i in range(4): - db_name = "zaqar_test_pools_" + str(i) + testredis = os.environ.get('ZAQAR_TEST_REDIS', 0) + if testredis: + uri = self.redis_url + for i in range(4): + db_name = "?dbid = " + str(i) - # NOTE(dynarro): we need to create a unique uri. - new_uri = "%s/%s" % (uri, db_name) - options = {'database': db_name} - self.control.pools_controller.create(six.text_type(i), - 100, new_uri, - options=options) + # NOTE(dynarro): we need to create a unique uri. + new_uri = "%s/%s" % (uri, db_name) + self.control.pools_controller.create(six.text_type(i), + 100, new_uri) + else: + uri = self.mongodb_url + for i in range(4): + db_name = "zaqar_test_pools_" + str(i) + + # NOTE(dynarro): we need to create a unique uri. + new_uri = "%s/%s" % (uri, db_name) + options = {'database': db_name} + self.control.pools_controller.create(six.text_type(i), + 100, new_uri, + options=options) self.driver = self.driver_class(self.conf, cache, self.control) self.addCleanup(self.control.pools_controller.drop_all) self.addCleanup(self.control.catalogue_controller.drop_all) @@ -1395,10 +1408,12 @@ class PoolsControllerTest(ControllerBaseTest): # Let's create one pool self.pool = str(uuid.uuid1()) + # NOTE(gengchc2): remove pool_group in Rocky release. self.pool_group = str(uuid.uuid1()) - self.pools_controller.create(self.pool, 100, 'localhost', - group=self.pool_group, options={}) - + self.pool1 = str(uuid.uuid1()) + self.flavor = str(uuid.uuid1()) + self.pools_controller.create(self.pool1, 100, 'localhost1', + flavor=self.flavor, options={}) self.flavors_controller = self.driver.flavors_controller def tearDown(self): @@ -1464,6 +1479,9 @@ class PoolsControllerTest(ControllerBaseTest): def test_delete_works(self): self.pools_controller.delete(self.pool) + # (gengchc): Remove the flavor from pool, then testcase cleanup pool + self.pools_controller.update(self.pool1, flavor="") + self.pools_controller.delete(self.pool1) self.assertFalse(self.pools_controller.exists(self.pool)) def test_delete_nonexistent_is_silent(self): @@ -1476,9 +1494,9 @@ class PoolsControllerTest(ControllerBaseTest): self.assertRaises(StopIteration, next, pools) def test_listing_simple(self): - # NOTE(cpp-cabrera): base entry interferes with listing results - self.pools_controller.delete(self.pool) - + # (gengchc): Remove the flavor from pool, then testcase cleanup pool + self.pools_controller.update(self.pool1, flavor="") + self.pools_controller._drop_all() pools = [] marker = '' for i in range(15): @@ -1541,15 +1559,6 @@ class PoolsControllerTest(ControllerBaseTest): self.assertIn('options', entry) self.assertEqual({}, entry['options']) - def test_mismatching_capabilities(self): - # NOTE(flaper87): This may fail for redis. Create - # a dummy store for tests. - with testing.expect(errors.PoolCapabilitiesMismatch): - self.pools_controller.create(str(uuid.uuid1()), - 100, 'redis://localhost', - group=self.pool_group, - options={}) - class CatalogueControllerTest(ControllerBaseTest): controller_base_class = storage.CatalogueBase @@ -1562,10 +1571,13 @@ class CatalogueControllerTest(ControllerBaseTest): self.project = six.text_type(uuid.uuid4()) self.pool = str(uuid.uuid1()) - self.pool_group = str(uuid.uuid1()) - self.pool_ctrl.create(self.pool, 100, 'localhost', - group=self.pool_group, options={}) + self.pool_ctrl.create(self.pool, 100, 'localhost', options={}) self.addCleanup(self.pool_ctrl.delete, self.pool) + self.pool1 = str(uuid.uuid1()) + self.flavor = str(uuid.uuid1()) + self.pool_ctrl.create(self.pool1, 100, 'localhost1', + options={}) + self.addCleanup(self.pool_ctrl.delete, self.pool1) def tearDown(self): self.controller.drop_all() @@ -1620,8 +1632,9 @@ class CatalogueControllerTest(ControllerBaseTest): def test_update(self): p2 = u'b' + # NOTE(gengchc2): Remove [group=self.pool_group] in + # it can be tested for redis as management. self.pool_ctrl.create(p2, 100, '127.0.0.1', - group=self.pool_group, options={}) self.addCleanup(self.pool_ctrl.delete, p2) @@ -1676,6 +1689,8 @@ class CatalogueControllerTest(ControllerBaseTest): self.controller.insert(self.project, q2, u'a') +# NOTE(gengchc2): remove FlavorsControllerTest in Rocky release +# and use FlavorsControllerTest1 instead for pool_group removal. class FlavorsControllerTest(ControllerBaseTest): """Flavors Controller base tests. @@ -1853,6 +1868,185 @@ class FlavorsControllerTest(ControllerBaseTest): self.assertEqual({}, entry['capabilities']) +# NOTE(gengchc2): Unittest for new flavor configure scenario. +class FlavorsControllerTest1(ControllerBaseTest): + """Flavors Controller base tests. + + NOTE(flaper87): Implementations of this class should + override the tearDown method in order + to clean up storage's state. + """ + controller_base_class = storage.FlavorsBase + + def setUp(self): + super(FlavorsControllerTest1, self).setUp() + self.pools_controller = self.driver.pools_controller + self.flavors_controller = self.driver.flavors_controller + + # Let's create one pool + self.pool = str(uuid.uuid1()) + self.flavor = 'durable' + self.pools_controller.create(self.pool, 100, 'localhost', + options={}) + self.addCleanup(self.pools_controller.delete, self.pool) + + def tearDown(self): + self.pools_controller.update(self.pool, flavor="") + self.flavors_controller.drop_all() + super(FlavorsControllerTest1, self).tearDown() + + def test_create_succeeds(self): + self.flavors_controller.create(self.flavor, + project=self.project, + capabilities={}) + + def _flavors_expects(self, flavor, xname, xproject): + self.assertIn('name', flavor) + self.assertEqual(xname, flavor['name']) + self.assertNotIn('project', flavor) + + def test_create_replaces_on_duplicate_insert(self): + name = str(uuid.uuid1()) + self.flavors_controller.create(name, + project=self.project, + capabilities={}) + entry = self.flavors_controller.get(name, self.project) + self._flavors_expects(entry, name, self.project) + new_capabilities = {'fifo': False} + self.flavors_controller.create(name, + project=self.project, + capabilities=new_capabilities) + entry = self.flavors_controller.get(name, project=self.project, + detailed=True) + self._flavors_expects(entry, name, self.project) + self.assertEqual(new_capabilities, entry['capabilities']) + + def test_get_returns_expected_content(self): + name = 'durable' + capabilities = {'fifo': True} + self.flavors_controller.create(name, + project=self.project, + capabilities=capabilities) + res = self.flavors_controller.get(name, project=self.project) + self._flavors_expects(res, name, self.project) + self.assertNotIn('capabilities', res) + + def test_detailed_get_returns_expected_content(self): + name = 'durable' + capabilities = {'fifo': True} + self.flavors_controller.create(name, + project=self.project, + capabilities=capabilities) + res = self.flavors_controller.get(name, project=self.project, + detailed=True) + self._flavors_expects(res, name, self.project) + self.assertIn('capabilities', res) + self.assertEqual(capabilities, res['capabilities']) + + def test_get_raises_if_not_found(self): + self.assertRaises(errors.FlavorDoesNotExist, + self.flavors_controller.get, 'notexists') + + def test_exists(self): + self.flavors_controller.create('exists', + project=self.project, + capabilities={}) + self.assertTrue(self.flavors_controller.exists('exists', + project=self.project)) + self.assertFalse(self.flavors_controller.exists('notexists', + project=self.project)) + + def test_update_raises_assertion_error_on_bad_fields(self): + self.assertRaises(AssertionError, self.pools_controller.update, + self.flavor) + + def test_update_works(self): + name = 'yummy' + self.flavors_controller.create(name, + project=self.project, + capabilities={}) + + res = self.flavors_controller.get(name, project=self.project, + detailed=True) + + p = 'olympic' + flavor = name + self.pools_controller.create(p, 100, 'localhost2', + flavor=flavor, options={}) + self.addCleanup(self.pools_controller.delete, p) + + new_capabilities = {'fifo': False} + self.flavors_controller.update(name, project=self.project, + capabilities={'fifo': False}) + res = self.flavors_controller.get(name, project=self.project, + detailed=True) + self._flavors_expects(res, name, self.project) + self.assertEqual(new_capabilities, res['capabilities']) + self.pools_controller.update(p, flavor="") + + def test_delete_works(self): + name = 'puke' + self.flavors_controller.create(name, + project=self.project, + capabilities={}) + self.flavors_controller.delete(name, project=self.project) + self.assertFalse(self.flavors_controller.exists(name)) + + def test_delete_nonexistent_is_silent(self): + self.flavors_controller.delete('nonexisting') + + def test_drop_all_leads_to_empty_listing(self): + self.flavors_controller.drop_all() + cursor = self.flavors_controller.list() + flavors = next(cursor) + self.assertRaises(StopIteration, next, flavors) + self.assertFalse(next(cursor)) + + def test_listing_simple(self): + name_gen = lambda i: chr(ord('A') + i) + for i in range(15): + pool = str(i) + flavor = name_gen(i) + uri = 'localhost:2701' + pool + self.pools_controller.create(pool, 100, uri, + flavor=flavor, options={}) + self.addCleanup(self.pools_controller.delete, pool) + + self.flavors_controller.create(flavor, project=self.project, + capabilities={}) + + def get_res(**kwargs): + cursor = self.flavors_controller.list(project=self.project, + **kwargs) + res = list(next(cursor)) + marker = next(cursor) + self.assertTrue(marker) + return res + + res = get_res() + self.assertEqual(10, len(res)) + for i, entry in enumerate(res): + self._flavors_expects(entry, name_gen(i), self.project) + self.assertNotIn('capabilities', entry) + + res = get_res(limit=5) + self.assertEqual(5, len(res)) + + res = get_res(marker=name_gen(3)) + self._flavors_expects(res[0], name_gen(4), self.project) + + res = get_res(detailed=True) + self.assertEqual(10, len(res)) + for i, entry in enumerate(res): + self._flavors_expects(entry, name_gen(i), self.project) + self.assertIn('capabilities', entry) + self.assertEqual({}, entry['capabilities']) + # (gengchc): Remove the flavor from pool, then testcase cleanup pools + for i in range(15): + pool = str(i) + self.pools_controller.update(pool, flavor="") + + def _insert_fixtures(controller, queue_name, project=None, client_uuid=None, num=4, ttl=120): def messages(): diff --git a/zaqar/tests/unit/storage/test_impl_mongodb.py b/zaqar/tests/unit/storage/test_impl_mongodb.py index cb6ef5ebf..6aa87984d 100644 --- a/zaqar/tests/unit/storage/test_impl_mongodb.py +++ b/zaqar/tests/unit/storage/test_impl_mongodb.py @@ -507,10 +507,14 @@ class MongodbPoolsTests(base.PoolsControllerTest): def setUp(self): super(MongodbPoolsTests, self).setUp() + self.pools_controller.create(self.pool, 100, 'localhost', + group=self.pool_group, options={}) def tearDown(self): super(MongodbPoolsTests, self).tearDown() + # NOTE(gengchc2): remove test_delete_pool_used_by_flavor in Rocky release + # and use test_delete_pool_used_by_flavor1 instead for pool_group removal. def test_delete_pool_used_by_flavor(self): self.flavors_controller.create('durable', self.pool_group, project=self.project, @@ -519,6 +523,19 @@ class MongodbPoolsTests(base.PoolsControllerTest): with testing.expect(errors.PoolInUseByFlavor): self.pools_controller.delete(self.pool) + # NOTE(gengchc2): Unittest for new flavor configure scenario. + def test_delete_pool_used_by_flavor1(self): + self.flavors_controller.create(self.flavor, + project=self.project, + capabilities={}) + self.pools_controller.update(self.pool1, flavor=self.flavor) + + with testing.expect(errors.PoolInUseByFlavor): + self.pools_controller.delete(self.pool1) + + # NOTE(gengchc2): remove test_mismatching_capabilities_fifo in Rocky + # release and use test_mismatching_capabilities_fifo1 instead for + # pool_group removal. def test_mismatching_capabilities_fifo(self): with testing.expect(errors.PoolCapabilitiesMismatch): self.pools_controller.create(str(uuid.uuid1()), @@ -526,6 +543,36 @@ class MongodbPoolsTests(base.PoolsControllerTest): group=self.pool_group, options={}) + # NOTE(gengchc2): Unittest for new flavor configure scenario. + def test_mismatching_capabilities_fifo1(self): + with testing.expect(errors.PoolCapabilitiesMismatch): + self.pools_controller.create(str(uuid.uuid1()), + 100, 'mongodb.fifo://localhost', + flavor=self.flavor, + options={}) + + # NOTE(gengchc2): remove test_mismatching_capabilities in Rocky release + # and use test_mismatching_capabilities1 instead for pool_group removal. + def test_mismatching_capabilities(self): + # NOTE(gengchc2): This test is used for testing mismatchming + # capabilities in pool with group + with testing.expect(errors.PoolCapabilitiesMismatch): + self.pools_controller.create(str(uuid.uuid1()), + 100, 'redis://localhost', + group=self.pool_group, + options={}) + + def test_mismatching_capabilities1(self): + # NOTE(gengchc2): This test is used for testing mismatchming + # capabilities in pool with flavor + with testing.expect(errors.PoolCapabilitiesMismatch): + self.pools_controller.create(str(uuid.uuid1()), + 100, 'redis://localhost', + flavor=self.flavor, + options={}) + + # NOTE(gengchc2): remove test_duplicate_uri in Rocky release and + # use test_duplicate_uri1 instead for pool_group removal. def test_duplicate_uri(self): with testing.expect(errors.PoolAlreadyExists): # The url 'localhost' is used in setUp(). So reusing the uri @@ -533,6 +580,14 @@ class MongodbPoolsTests(base.PoolsControllerTest): self.pools_controller.create(str(uuid.uuid1()), 100, 'localhost', group=str(uuid.uuid1()), options={}) + # NOTE(gengchc2): Unittest for new flavor configure scenario. + def test_duplicate_uri1(self): + with testing.expect(errors.PoolAlreadyExists): + # The url 'localhost' is used in setUp(). So reusing the uri + # 'localhost' here will raise PoolAlreadyExists. + self.pools_controller.create(str(uuid.uuid1()), 100, 'localhost', + flavor=str(uuid.uuid1()), options={}) + @testing.requires_mongodb class MongodbCatalogueTests(base.CatalogueControllerTest): @@ -577,6 +632,8 @@ class PooledClaimsTests(base.ClaimControllerTest): self.skip("Fix sqlalchemy driver") +# NOTE(gengchc2): remove MongodbFlavorsTest in Rocky release and +# use MongodbFlavorsTest1 instead for pool_group removal. @testing.requires_mongodb class MongodbFlavorsTest(base.FlavorsControllerTest): driver_class = mongodb.ControlDriver @@ -587,3 +644,16 @@ class MongodbFlavorsTest(base.FlavorsControllerTest): def setUp(self): super(MongodbFlavorsTest, self).setUp() self.addCleanup(self.controller.drop_all) + + +# NOTE(gengchc2): Unittest for new flavor configure scenario. +@testing.requires_mongodb +class MongodbFlavorsTest1(base.FlavorsControllerTest1): + driver_class = mongodb.ControlDriver + controller_class = controllers.FlavorsController + control_driver_class = mongodb.ControlDriver + config_file = 'wsgi_mongodb.conf' + + def setUp(self): + super(MongodbFlavorsTest1, self).setUp() + self.addCleanup(self.controller.drop_all) diff --git a/zaqar/tests/unit/storage/test_impl_sqlalchemy.py b/zaqar/tests/unit/storage/test_impl_sqlalchemy.py index 226a00d7d..11c3cdfcd 100644 --- a/zaqar/tests/unit/storage/test_impl_sqlalchemy.py +++ b/zaqar/tests/unit/storage/test_impl_sqlalchemy.py @@ -13,7 +13,9 @@ # the License. import six +import uuid +from zaqar import storage from zaqar.storage import sqlalchemy from zaqar.storage.sqlalchemy import controllers from zaqar.storage.sqlalchemy import tables @@ -41,6 +43,31 @@ class SqlalchemyPoolsTest(DBCreateMixin, base.PoolsControllerTest): controller_class = controllers.PoolsController control_driver_class = sqlalchemy.ControlDriver + def setUp(self): + super(SqlalchemyPoolsTest, self).setUp() + self.pools_controller.create(self.pool, 100, 'localhost', + group=self.pool_group, options={}) + + # NOTE(gengchc2): remove test_mismatching_capabilities in Rocky release + # and use test_mismatching_capabilities1 instead for pool_group removal. + def test_mismatching_capabilities(self): + # NOTE(gengchc2): This test is used for testing mismatchming + # capabilities in pool with group + with testing.expect(storage.errors.PoolCapabilitiesMismatch): + self.pools_controller.create(str(uuid.uuid1()), + 100, 'redis://localhost', + group=self.pool_group, + options={}) + + def test_mismatching_capabilities1(self): + # NOTE(gengchc2): This test is used for testing mismatchming + # capabilities in pool with flavor + with testing.expect(storage.errors.PoolCapabilitiesMismatch): + self.pools_controller.create(str(uuid.uuid1()), + 100, 'redis://localhost', + flavor=self.flavor, + options={}) + class SqlalchemyCatalogueTest(DBCreateMixin, base.CatalogueControllerTest): config_file = 'wsgi_sqlalchemy.conf' @@ -49,6 +76,8 @@ class SqlalchemyCatalogueTest(DBCreateMixin, base.CatalogueControllerTest): control_driver_class = sqlalchemy.ControlDriver +# NOTE(gengchc2): remove SqlalchemyFlavorsTest in Rocky release and +# use SqlalchemyFlavorsTest1 instead for pool_group removal. class SqlalchemyFlavorsTest(DBCreateMixin, base.FlavorsControllerTest): config_file = 'wsgi_sqlalchemy.conf' driver_class = sqlalchemy.ControlDriver @@ -56,6 +85,14 @@ class SqlalchemyFlavorsTest(DBCreateMixin, base.FlavorsControllerTest): control_driver_class = sqlalchemy.ControlDriver +# NOTE(gengchc2): Unittest for new flavor configure scenario. +class SqlalchemyFlavorsTest1(DBCreateMixin, base.FlavorsControllerTest1): + config_file = 'wsgi_sqlalchemy.conf' + driver_class = sqlalchemy.ControlDriver + controller_class = controllers.FlavorsController + control_driver_class = sqlalchemy.ControlDriver + + class MsgidTests(testing.TestBase): def test_encode(self): diff --git a/zaqar/tests/unit/storage/test_pool_catalog_new.py b/zaqar/tests/unit/storage/test_pool_catalog_new.py new file mode 100644 index 000000000..567cc7b24 --- /dev/null +++ b/zaqar/tests/unit/storage/test_pool_catalog_new.py @@ -0,0 +1,128 @@ +# Copyright (c) 2017 ZTE Corporation.. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +import mock +import uuid + +from zaqar.common import cache as oslo_cache +from zaqar.storage import errors +from zaqar.storage import mongodb +from zaqar.storage import pooling +from zaqar.storage import utils +from zaqar import tests as testing + + +# TODO(cpp-cabrera): it would be wonderful to refactor this unit test +# so that it could use multiple control storage backends once those +# have pools/catalogue implementations. +@testing.requires_mongodb +class PoolCatalogTest(testing.TestBase): + + config_file = 'wsgi_mongodb_pooled_disable_virtual_pool.conf' + + def setUp(self): + super(PoolCatalogTest, self).setUp() + + oslo_cache.register_config(self.conf) + cache = oslo_cache.get_cache(self.conf) + control = utils.load_storage_driver(self.conf, cache, + control_mode=True) + + self.pools_ctrl = control.pools_controller + self.flavors_ctrl = control.flavors_controller + self.catalogue_ctrl = control.catalogue_controller + + # NOTE(cpp-cabrera): populate catalogue + self.pool = str(uuid.uuid1()) + self.pool2 = str(uuid.uuid1()) + self.queue = str(uuid.uuid1()) + self.flavor = str(uuid.uuid1()) + self.project = str(uuid.uuid1()) + + # FIXME(therve) This is horrible, we need to manage duplication in a + # nicer way + if 'localhost' in self.mongodb_url: + other_url = self.mongodb_url.replace('localhost', '127.0.0.1') + elif '127.0.0.1' in self.mongodb_url: + other_url = self.mongodb_url.replace('127.0.0.1', 'localhost') + else: + self.skipTest("Can't build a dummy mongo URL.") + + self.pools_ctrl.create(self.pool, 100, self.mongodb_url) + self.pools_ctrl.create(self.pool2, 100, + other_url) + self.catalogue_ctrl.insert(self.project, self.queue, self.pool) + self.catalog = pooling.Catalog(self.conf, cache, control) + self.flavors_ctrl.create(self.flavor, + project=self.project) + self.pools_ctrl.update(self.pool2, flavor=self.flavor) + + def tearDown(self): + self.catalogue_ctrl.drop_all() + self.pools_ctrl.drop_all() + super(PoolCatalogTest, self).tearDown() + + def test_lookup_loads_correct_driver(self): + storage = self.catalog.lookup(self.queue, self.project) + self.assertIsInstance(storage._storage, mongodb.DataDriver) + + def test_lookup_returns_default_or_none_if_queue_not_mapped(self): + # Return default + self.assertIsNone(self.catalog.lookup('not', 'mapped')) + + self.config(message_store='faulty', group='drivers') + self.config(enable_virtual_pool=True, group='pooling:catalog') + self.assertIsNotNone(self.catalog.lookup('not', 'mapped')) + + def test_lookup_returns_none_if_entry_deregistered(self): + self.catalog.deregister(self.queue, self.project) + self.assertIsNone(self.catalog.lookup(self.queue, self.project)) + + def test_register_leads_to_successful_lookup(self): + self.catalog.register('not_yet', 'mapped') + storage = self.catalog.lookup('not_yet', 'mapped') + self.assertIsInstance(storage._storage, mongodb.DataDriver) + + def test_register_with_flavor(self): + queue = 'test' + self.catalog.register(queue, project=self.project, + flavor=self.flavor) + storage = self.catalog.lookup(queue, self.project) + self.assertIsInstance(storage._storage, mongodb.DataDriver) + + def test_register_with_fake_flavor(self): + self.assertRaises(errors.FlavorDoesNotExist, + self.catalog.register, + 'test', project=self.project, + flavor='fake') + + def test_queues_list_on_multi_pools(self): + def fake_list(project=None, marker=None, limit=10, detailed=False): + yield iter([{'name': 'fake_queue'}]) + + list_str = 'zaqar.storage.mongodb.queues.QueueController.list' + with mock.patch(list_str) as queues_list: + queues_list.side_effect = fake_list + queue_controller = pooling.QueueController(self.catalog) + result = queue_controller.list(project=self.project) + queue_list = list(next(result)) + self.assertEqual(1, len(queue_list)) + + def test_queue_create_with_empty_json_body(self): + queue_controller = pooling.QueueController(self.catalog) + with mock.patch('zaqar.storage.pooling.Catalog.register') as register: + queue_controller.create(self.queue, metadata={}, + project=self.project) + register.assert_called_with(self.queue, project=self.project, + flavor=None) diff --git a/zaqar/tests/unit/transport/wsgi/v2_0/test_flavors.py b/zaqar/tests/unit/transport/wsgi/v2_0/test_flavors.py index 40abc1d8f..14a433df0 100644 --- a/zaqar/tests/unit/transport/wsgi/v2_0/test_flavors.py +++ b/zaqar/tests/unit/transport/wsgi/v2_0/test_flavors.py @@ -23,6 +23,7 @@ from zaqar import tests as testing from zaqar.tests.unit.transport.wsgi import base +# NOTE(gengchc2): remove pool_group in Rocky release. @contextlib.contextmanager def flavor(test, name, pool_group): """A context manager for constructing a flavor for use in testing. diff --git a/zaqar/tests/unit/transport/wsgi/v2_0/test_flavors_new.py b/zaqar/tests/unit/transport/wsgi/v2_0/test_flavors_new.py new file mode 100644 index 000000000..972070dbb --- /dev/null +++ b/zaqar/tests/unit/transport/wsgi/v2_0/test_flavors_new.py @@ -0,0 +1,354 @@ +# Copyright (c) 2017 ZTE Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +import contextlib +import uuid + +import ddt +import falcon +from oslo_serialization import jsonutils + +from zaqar import tests as testing +from zaqar.tests.unit.transport.wsgi import base + + +@contextlib.contextmanager +def flavor(test, name, pool_list): + """A context manager for constructing a flavor for use in testing. + + Deletes the flavor after exiting the context. + + :param test: Must expose simulate_* methods + :param name: Name for this flavor + :type name: six.text_type + :type pool: six.text_type + :returns: (name, uri, capabilities) + :rtype: see above + + """ + + doc = {'pool_list': pool_list} + path = test.url_prefix + '/flavors/' + name + + test.simulate_put(path, body=jsonutils.dumps(doc)) + + try: + yield name, pool_list + + finally: + test.simulate_delete(path) + + +@contextlib.contextmanager +def flavors(test, count): + """A context manager for constructing flavors for use in testing. + + Deletes the flavors after exiting the context. + + :param test: Must expose simulate_* methods + :param count: Number of pools to create + :type count: int + :returns: (paths, pool_list capabilities) + :rtype: ([six.text_type], [six.text_type], [dict]) + + """ + + pool_path_all = [] + flavor_path_all = [] + for i in range(count): + poolname = 'pool' + str(i) + pool_doc = {'weight': 100, + 'uri': test.mongodb_url + '/test' + str(i)} + pool_path = test.url_prefix + '/pools/' + poolname + test.simulate_put(pool_path, body=jsonutils.dumps(pool_doc)) + flavorname = str(i) + flavor_path = test.url_prefix + "/flavors/" + flavorname + flavor_doc = {'pool_list': [poolname]} + test.simulate_put(flavor_path, body=jsonutils.dumps(flavor_doc)) + pool_path_all.append(pool_path) + flavor_path_all.append(flavor_path) + + try: + yield flavor_path_all + finally: + for path in flavor_path_all: + test.simulate_delete(path) + for path in pool_path_all: + test.simulate_delete(path) + + +@ddt.ddt +class TestFlavorsMongoDB(base.V2Base): + + config_file = 'wsgi_mongodb_pooled.conf' + + @testing.requires_mongodb + def setUp(self): + super(TestFlavorsMongoDB, self).setUp() + self.queue = 'test-queue' + self.queue_path = self.url_prefix + '/queues/' + self.queue + + self.pool = 'mypool' + self.pool_path = self.url_prefix + '/pools/' + self.pool + self.pool_doc = {'weight': 100, + 'uri': self.mongodb_url + '/test'} + self.simulate_put(self.pool_path, body=jsonutils.dumps(self.pool_doc)) + + self.flavor = 'test-flavor' + self.doc = {'capabilities': {}} + self.doc['pool_list'] = [self.pool] + self.flavor_path = self.url_prefix + '/flavors/' + self.flavor + self.simulate_put(self.flavor_path, body=jsonutils.dumps(self.doc)) + self.assertEqual(falcon.HTTP_201, self.srmock.status) + + def tearDown(self): + self.simulate_delete(self.queue_path) + self.simulate_delete(self.flavor_path) + self.assertEqual(falcon.HTTP_204, self.srmock.status) + self.simulate_delete(self.pool_path) + + super(TestFlavorsMongoDB, self).tearDown() + + def test_put_flavor_works(self): + name = str(uuid.uuid1()) + with flavor(self, name, self.doc['pool_list']): + self.assertEqual(falcon.HTTP_201, self.srmock.status) + + def test_put_raises_if_missing_fields(self): + path = self.url_prefix + '/flavors/' + str(uuid.uuid1()) + self.simulate_put(path, body=jsonutils.dumps({})) + self.assertEqual(falcon.HTTP_400, self.srmock.status) + + self.simulate_put(path, + body=jsonutils.dumps({'capabilities': {}})) + self.assertEqual(falcon.HTTP_400, self.srmock.status) + + @ddt.data(1, 2**32+1, []) + def test_put_raises_if_invalid_pool(self, pool_list): + path = self.url_prefix + '/flavors/' + str(uuid.uuid1()) + self.simulate_put(path, + body=jsonutils.dumps({'pool_list': pool_list})) + self.assertEqual(falcon.HTTP_400, self.srmock.status) + + def test_put_auto_get_capabilities(self): + path = self.url_prefix + '/flavors/' + str(uuid.uuid1()) + doc = {'pool_list': self.doc['pool_list']} + self.simulate_put(path, body=jsonutils.dumps(doc)) + self.assertEqual(falcon.HTTP_201, self.srmock.status) + # NOTE(gengchc2): Delete it, otherwise exist garbage flavor. + self.simulate_delete(path) + + def test_put_existing_overwrites(self): + # NOTE(cabrera): setUp creates default flavor + expect = self.doc + self.simulate_put(self.flavor_path, + body=jsonutils.dumps(expect)) + self.assertEqual(falcon.HTTP_201, self.srmock.status) + + result = self.simulate_get(self.flavor_path) + self.assertEqual(falcon.HTTP_200, self.srmock.status) + doc = jsonutils.loads(result[0]) + self.assertEqual(expect['pool_list'], doc['pool_list']) + + def test_create_flavor_no_pool_list(self): + self.simulate_delete(self.flavor_path) + self.assertEqual(falcon.HTTP_204, self.srmock.status) + + self.simulate_delete(self.pool_path) + self.assertEqual(falcon.HTTP_204, self.srmock.status) + resp = self.simulate_put(self.flavor_path, + body=jsonutils.dumps(self.doc)) + self.assertEqual(falcon.HTTP_400, self.srmock.status) + self.assertEqual( + {'description': 'Flavor test-flavor could not be created, ' + 'error:Pool mypool does not exist', + 'title': 'Unable to create'}, + jsonutils.loads(resp[0])) + + def test_delete_works(self): + self.simulate_delete(self.flavor_path) + self.assertEqual(falcon.HTTP_204, self.srmock.status) + + self.simulate_get(self.flavor_path) + self.assertEqual(falcon.HTTP_404, self.srmock.status) + + def test_get_nonexisting_raises_404(self): + self.simulate_get(self.url_prefix + '/flavors/nonexisting') + self.assertEqual(falcon.HTTP_404, self.srmock.status) + + def _flavor_expect(self, flavor, xhref, xpool_list=None): + self.assertIn('href', flavor) + self.assertIn('name', flavor) + self.assertEqual(xhref, flavor['href']) + if xpool_list is not None: + self.assertIn('pool_list', flavor) + self.assertEqual(xpool_list, flavor['pool_list']) + + def test_get_works(self): + result = self.simulate_get(self.flavor_path) + self.assertEqual(falcon.HTTP_200, self.srmock.status) + flavor = jsonutils.loads(result[0]) + self._flavor_expect(flavor, self.flavor_path, self.doc['pool_list']) + + store_caps = ['FIFO', 'CLAIMS', 'DURABILITY', + 'AOD', 'HIGH_THROUGHPUT'] + self.assertEqual(store_caps, flavor['capabilities']) + + def test_patch_raises_if_missing_fields(self): + self.simulate_patch(self.flavor_path, + body=jsonutils.dumps({'location': 1})) + self.assertEqual(falcon.HTTP_400, self.srmock.status) + + def _patch_test(self, doc): + result = self.simulate_patch(self.flavor_path, + body=jsonutils.dumps(doc)) + self.assertEqual(falcon.HTTP_200, self.srmock.status) + updated_flavor = jsonutils.loads(result[0]) + self._flavor_expect(updated_flavor, self.flavor_path) + capabilities = ['FIFO', 'CLAIMS', 'DURABILITY', 'AOD', + 'HIGH_THROUGHPUT'] + self.assertEqual(capabilities, updated_flavor['capabilities']) + result = self.simulate_get(self.flavor_path) + self.assertEqual(falcon.HTTP_200, self.srmock.status) + flavor = jsonutils.loads(result[0]) + self._flavor_expect(flavor, self.flavor_path) + self.assertEqual(capabilities, flavor['capabilities']) + + def test_patch_works(self): + doc = {'pool_list': self.doc['pool_list'], 'capabilities': []} + self._patch_test(doc) + + def test_patch_works_with_extra_fields(self): + doc = {'pool_list': self.doc['pool_list'], 'capabilities': [], + 'location': 100, 'partition': 'taco'} + self._patch_test(doc) + + @ddt.data(-1, 2**32+1, []) + def test_patch_raises_400_on_invalid_pool_list(self, pool_list): + self.simulate_patch(self.flavor_path, + body=jsonutils.dumps({'pool_list': pool_list})) + self.assertEqual(falcon.HTTP_400, self.srmock.status) + + @ddt.data(-1, 'wee', []) + def test_patch_raises_400_on_invalid_capabilities(self, capabilities): + doc = {'capabilities': capabilities} + self.simulate_patch(self.flavor_path, body=jsonutils.dumps(doc)) + self.assertEqual(falcon.HTTP_400, self.srmock.status) + + def test_patch_raises_404_if_flavor_not_found(self): + self.simulate_patch(self.url_prefix + '/flavors/notexists', + body=jsonutils.dumps({'pool_list': ['test']})) + self.assertEqual(falcon.HTTP_404, self.srmock.status) + + def test_empty_listing(self): + self.simulate_delete(self.flavor_path) + result = self.simulate_get(self.url_prefix + '/flavors') + results = jsonutils.loads(result[0]) + self.assertEqual(falcon.HTTP_200, self.srmock.status) + self.assertEqual(0, len(results['flavors'])) + self.assertIn('links', results) + + def _listing_test(self, count=10, limit=10, + marker=None, detailed=False): + # NOTE(cpp-cabrera): delete initial flavor - it will interfere + # with listing tests + self.simulate_delete(self.flavor_path) + query = 'limit={0}&detailed={1}'.format(limit, detailed) + if marker: + query += '&marker={2}'.format(marker) + + with flavors(self, count): + result = self.simulate_get(self.url_prefix + '/flavors', + query_string=query) + self.assertEqual(falcon.HTTP_200, self.srmock.status) + results = jsonutils.loads(result[0]) + self.assertIsInstance(results, dict) + self.assertIn('flavors', results) + self.assertIn('links', results) + flavors_list = results['flavors'] + link = results['links'][0] + self.assertEqual('next', link['rel']) + href = falcon.uri.parse_query_string(link['href'].split('?')[1]) + self.assertIn('marker', href) + self.assertEqual(str(limit), href['limit']) + self.assertEqual(str(detailed).lower(), href['detailed']) + + next_query_string = ('marker={marker}&limit={limit}' + '&detailed={detailed}').format(**href) + next_result = self.simulate_get(link['href'].split('?')[0], + query_string=next_query_string) + next_flavors = jsonutils.loads(next_result[0]) + next_flavors_list = next_flavors['flavors'] + + self.assertEqual(falcon.HTTP_200, self.srmock.status) + self.assertIn('links', next_flavors) + if limit < count: + self.assertEqual(min(limit, count-limit), + len(next_flavors_list)) + else: + self.assertEqual(0, len(next_flavors_list)) + + self.assertEqual(min(limit, count), len(flavors_list)) + for i, s in enumerate(flavors_list + next_flavors_list): + capabilities = ['FIFO', 'CLAIMS', 'DURABILITY', + 'AOD', 'HIGH_THROUGHPUT'] + if detailed: + self.assertIn('capabilities', s) + self.assertEqual(s['capabilities'], capabilities) + else: + self.assertNotIn('capabilities', s) + + def test_listing_works(self): + self._listing_test() + + def test_detailed_listing_works(self): + self._listing_test(detailed=True) + + @ddt.data(1, 5, 10, 15) + def test_listing_works_with_limit(self, limit): + self._listing_test(count=15, limit=limit) + + def test_listing_marker_is_respected(self): + self.simulate_delete(self.flavor_path) + + with flavors(self, 10) as expected: + result = self.simulate_get(self.url_prefix + '/flavors', + query_string='marker=3') + self.assertEqual(falcon.HTTP_200, self.srmock.status) + flavor_list = jsonutils.loads(result[0])['flavors'] + self.assertEqual(6, len(flavor_list)) + path = expected[4] + self._flavor_expect(flavor_list[0], path) + + def test_listing_error_with_invalid_limit(self): + self.simulate_delete(self.flavor_path) + query = 'limit={0}&detailed={1}'.format(0, True) + + with flavors(self, 10): + self.simulate_get(self.url_prefix + '/flavors', query_string=query) + self.assertEqual(falcon.HTTP_400, self.srmock.status) + + def test_queue_create_works(self): + metadata = {'_flavor': self.flavor} + self.simulate_put(self.queue_path, body=jsonutils.dumps(metadata)) + self.assertEqual(falcon.HTTP_201, self.srmock.status) + + def test_queue_create_no_flavor(self): + metadata = {'_flavor': self.flavor} + + self.simulate_delete(self.flavor_path) + self.assertEqual(falcon.HTTP_204, self.srmock.status) + + self.simulate_put(self.queue_path, body=jsonutils.dumps(metadata)) + self.assertEqual(falcon.HTTP_400, self.srmock.status) diff --git a/zaqar/tests/unit/transport/wsgi/v2_0/test_pools.py b/zaqar/tests/unit/transport/wsgi/v2_0/test_pools.py index 454dd31b5..16d77d3ec 100644 --- a/zaqar/tests/unit/transport/wsgi/v2_0/test_pools.py +++ b/zaqar/tests/unit/transport/wsgi/v2_0/test_pools.py @@ -23,6 +23,7 @@ from zaqar import tests as testing from zaqar.tests.unit.transport.wsgi import base +# NOTE(gengchc2): remove pool_group in Rocky release. @contextlib.contextmanager def pool(test, name, weight, uri, group=None, options={}): """A context manager for constructing a pool for use in testing. diff --git a/zaqar/tests/unit/transport/wsgi/v2_0/test_pools_new.py b/zaqar/tests/unit/transport/wsgi/v2_0/test_pools_new.py new file mode 100644 index 000000000..9b1b1083a --- /dev/null +++ b/zaqar/tests/unit/transport/wsgi/v2_0/test_pools_new.py @@ -0,0 +1,383 @@ +# Copyright (c) 2017 ZTE Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +import contextlib + +import ddt +import falcon +from oslo_serialization import jsonutils +from oslo_utils import uuidutils + +from zaqar import tests as testing +from zaqar.tests.unit.transport.wsgi import base + + +@contextlib.contextmanager +def pool(test, name, weight, uri, flavor=None, options={}): + """A context manager for constructing a pool for use in testing. + + Deletes the pool after exiting the context. + + :param test: Must expose simulate_* methods + :param name: Name for this pool + :type name: six.text_type + :type weight: int + :type uri: six.text_type + :type options: dict + :returns: (name, weight, uri, options) + :rtype: see above + """ + uri = "%s/%s" % (uri, uuidutils.generate_uuid()) + doc = {'weight': weight, 'uri': uri, + 'flavor': flavor, 'options': options} + path = test.url_prefix + '/pools/' + name + + test.simulate_put(path, body=jsonutils.dumps(doc)) + + try: + yield name, weight, uri, flavor, options + + finally: + test.simulate_delete(path) + + +@contextlib.contextmanager +def pools(test, count, uri, flavor): + """A context manager for constructing pools for use in testing. + + Deletes the pools after exiting the context. + + :param test: Must expose simulate_* methods + :param count: Number of pools to create + :type count: int + :returns: (paths, weights, uris, options) + :rtype: ([six.text_type], [int], [six.text_type], [dict]) + """ + mongo_url = uri + base = test.url_prefix + '/pools/' + args = [(base + str(i), i, + {str(i): i}) + for i in range(count)] + for path, weight, option in args: + uri = "%s/%s" % (mongo_url, uuidutils.generate_uuid()) + doc = {'weight': weight, 'uri': uri, + 'flavor': flavor, 'options': option} + test.simulate_put(path, body=jsonutils.dumps(doc)) + + try: + yield args + finally: + for path, _, _ in args: + # (gengchc): Remove flavor from the pool, + # so we can delete the pool. + test.simulate_patch(path, + body=jsonutils.dumps({'flavor': ''})) + test.simulate_delete(path) + + +@ddt.ddt +class TestPoolsMongoDB(base.V2Base): + + config_file = 'wsgi_mongodb_pooled.conf' + + @testing.requires_mongodb + def setUp(self): + super(TestPoolsMongoDB, self).setUp() + self.doc = {'weight': 100, + 'flavor': 'my-flavor', + 'uri': self.mongodb_url} + self.pool = self.url_prefix + '/pools/' + uuidutils.generate_uuid() + self.simulate_put(self.pool, body=jsonutils.dumps(self.doc)) + self.assertEqual(falcon.HTTP_201, self.srmock.status) + + def tearDown(self): + super(TestPoolsMongoDB, self).tearDown() + self.simulate_delete(self.pool) + self.assertEqual(falcon.HTTP_204, self.srmock.status) + + def test_put_pool_works(self): + name = uuidutils.generate_uuid() + weight, uri = self.doc['weight'], self.doc['uri'] + with pool(self, name, weight, uri, flavor='my-flavor'): + self.assertEqual(falcon.HTTP_201, self.srmock.status) + + def test_put_raises_if_missing_fields(self): + path = self.url_prefix + '/pools/' + uuidutils.generate_uuid() + self.simulate_put(path, body=jsonutils.dumps({'weight': 100})) + self.assertEqual(falcon.HTTP_400, self.srmock.status) + self.simulate_put(path, + body=jsonutils.dumps( + {'uri': self.mongodb_url})) + self.assertEqual(falcon.HTTP_400, self.srmock.status) + + @ddt.data(-1, 2**32+1, 'big') + def test_put_raises_if_invalid_weight(self, weight): + path = self.url_prefix + '/pools/' + uuidutils.generate_uuid() + doc = {'weight': weight, 'uri': 'a'} + self.simulate_put(path, + body=jsonutils.dumps(doc)) + self.assertEqual(falcon.HTTP_400, self.srmock.status) + + @ddt.data(-1, 2**32+1, [], 'localhost:27017') + def test_put_raises_if_invalid_uri(self, uri): + path = self.url_prefix + '/pools/' + uuidutils.generate_uuid() + self.simulate_put(path, + body=jsonutils.dumps({'weight': 1, 'uri': uri})) + self.assertEqual(falcon.HTTP_400, self.srmock.status) + + @ddt.data(-1, 'wee', []) + def test_put_raises_if_invalid_options(self, options): + path = self.url_prefix + '/pools/' + uuidutils.generate_uuid() + doc = {'weight': 1, 'uri': 'a', 'options': options} + self.simulate_put(path, body=jsonutils.dumps(doc)) + self.assertEqual(falcon.HTTP_400, self.srmock.status) + + def test_put_same_database_uri(self): + # NOTE(cabrera): setUp creates default pool + expect = self.doc + path = self.url_prefix + '/pools/' + uuidutils.generate_uuid() + self.simulate_put(path, body=jsonutils.dumps(expect)) + self.assertEqual(falcon.HTTP_409, self.srmock.status) + + def test_put_existing_overwrites(self): + # NOTE(cabrera): setUp creates default pool + expect = self.doc + self.simulate_put(self.pool, + body=jsonutils.dumps(expect)) + self.assertEqual(falcon.HTTP_201, self.srmock.status) + + result = self.simulate_get(self.pool) + self.assertEqual(falcon.HTTP_200, self.srmock.status) + doc = jsonutils.loads(result[0]) + self.assertEqual(expect['weight'], doc['weight']) + self.assertEqual(expect['uri'], doc['uri']) + + def test_put_capabilities_mismatch_pool(self): + mongodb_doc = self.doc + self.simulate_put(self.pool, + body=jsonutils.dumps(mongodb_doc)) + self.assertEqual(falcon.HTTP_201, self.srmock.status) + + redis_doc = {'weight': 100, + 'flavor': 'my-flavor', + 'uri': 'redis://127.0.0.1:6379'} + + self.simulate_put(self.pool, + body=jsonutils.dumps(redis_doc)) + self.assertEqual(falcon.HTTP_400, self.srmock.status) + + def test_delete_works(self): + # (gengchc): Remove flavor from the pool, so we can delete the pool. + self.simulate_patch(self.pool, body=jsonutils.dumps({'flavor': ''})) + self.simulate_delete(self.pool) + self.assertEqual(falcon.HTTP_204, self.srmock.status) + + self.simulate_get(self.pool) + self.assertEqual(falcon.HTTP_404, self.srmock.status) + + def test_get_nonexisting_raises_404(self): + self.simulate_get(self.url_prefix + '/pools/nonexisting') + self.assertEqual(falcon.HTTP_404, self.srmock.status) + + def _pool_expect(self, pool, xhref, xweight, xuri): + self.assertIn('href', pool) + self.assertIn('name', pool) + self.assertEqual(xhref, pool['href']) + self.assertIn('weight', pool) + self.assertEqual(xweight, pool['weight']) + self.assertIn('uri', pool) + + # NOTE(dynarro): we are using startwith because we are adding to + # pools UUIDs, to avoid dupplications + self.assertTrue(pool['uri'].startswith(xuri)) + + def test_get_works(self): + result = self.simulate_get(self.pool) + self.assertEqual(falcon.HTTP_200, self.srmock.status) + pool = jsonutils.loads(result[0]) + self._pool_expect(pool, self.pool, self.doc['weight'], + self.doc['uri']) + + def test_detailed_get_works(self): + result = self.simulate_get(self.pool, + query_string='detailed=True') + self.assertEqual(falcon.HTTP_200, self.srmock.status) + pool = jsonutils.loads(result[0]) + self._pool_expect(pool, self.pool, self.doc['weight'], + self.doc['uri']) + self.assertIn('options', pool) + self.assertEqual({}, pool['options']) + + def test_patch_raises_if_missing_fields(self): + self.simulate_patch(self.pool, + body=jsonutils.dumps({'location': 1})) + self.assertEqual(falcon.HTTP_400, self.srmock.status) + + def _patch_test(self, doc): + result = self.simulate_patch(self.pool, + body=jsonutils.dumps(doc)) + self.assertEqual(falcon.HTTP_200, self.srmock.status) + updated_pool = jsonutils.loads(result[0]) + self._pool_expect(updated_pool, self.pool, doc['weight'], + doc['uri']) + + result = self.simulate_get(self.pool, + query_string='detailed=True') + self.assertEqual(falcon.HTTP_200, self.srmock.status) + pool = jsonutils.loads(result[0]) + self._pool_expect(pool, self.pool, doc['weight'], + doc['uri']) + self.assertEqual(doc['options'], pool['options']) + + def test_patch_works(self): + doc = {'weight': 101, + 'uri': self.mongodb_url, + 'options': {'a': 1}} + self._patch_test(doc) + + def test_patch_works_with_extra_fields(self): + doc = {'weight': 101, + 'uri': self.mongodb_url, + 'options': {'a': 1}, + 'location': 100, + 'partition': 'taco'} + self._patch_test(doc) + + @ddt.data(-1, 2**32+1, 'big') + def test_patch_raises_400_on_invalid_weight(self, weight): + self.simulate_patch(self.pool, + body=jsonutils.dumps({'weight': weight})) + self.assertEqual(falcon.HTTP_400, self.srmock.status) + + @ddt.data(-1, 2**32+1, [], 'localhost:27017') + def test_patch_raises_400_on_invalid_uri(self, uri): + self.simulate_patch(self.pool, + body=jsonutils.dumps({'uri': uri})) + self.assertEqual(falcon.HTTP_400, self.srmock.status) + + @ddt.data(-1, 'wee', []) + def test_patch_raises_400_on_invalid_options(self, options): + self.simulate_patch(self.pool, + body=jsonutils.dumps({'options': options})) + self.assertEqual(falcon.HTTP_400, self.srmock.status) + + def test_patch_raises_404_if_pool_not_found(self): + self.simulate_patch(self.url_prefix + '/pools/notexists', + body=jsonutils.dumps({'weight': 1})) + self.assertEqual(falcon.HTTP_404, self.srmock.status) + + def test_empty_listing(self): + # (gengchc): Remove flavor from the pool, so we can delete the pool. + self.simulate_patch(self.pool, body=jsonutils.dumps({'flavor': ''})) + self.simulate_delete(self.pool) + result = self.simulate_get(self.url_prefix + '/pools') + results = jsonutils.loads(result[0]) + self.assertEqual(falcon.HTTP_200, self.srmock.status) + self.assertEqual(0, len(results['pools'])) + self.assertIn('links', results) + + def _listing_test(self, count=10, limit=10, + marker=None, detailed=False): + # NOTE(cpp-cabrera): delete initial pool - it will interfere + # with listing tests + # (gengchc): Remove flavor from the pool, so we can delete the pool. + self.simulate_patch(self.pool, body=jsonutils.dumps({'flavor': ''})) + self.simulate_delete(self.pool) + query = 'limit={0}&detailed={1}'.format(limit, detailed) + if marker: + query += '&marker={0}'.format(marker) + + with pools(self, count, self.doc['uri'], 'my-flavor') as expected: + result = self.simulate_get(self.url_prefix + '/pools', + query_string=query) + self.assertEqual(falcon.HTTP_200, self.srmock.status) + results = jsonutils.loads(result[0]) + self.assertIsInstance(results, dict) + self.assertIn('pools', results) + self.assertIn('links', results) + pool_list = results['pools'] + + link = results['links'][0] + self.assertEqual('next', link['rel']) + href = falcon.uri.parse_query_string(link['href'].split('?')[1]) + self.assertIn('marker', href) + self.assertEqual(str(limit), href['limit']) + self.assertEqual(str(detailed).lower(), href['detailed']) + + next_query_string = ('marker={marker}&limit={limit}' + '&detailed={detailed}').format(**href) + next_result = self.simulate_get(link['href'].split('?')[0], + query_string=next_query_string) + self.assertEqual(falcon.HTTP_200, self.srmock.status) + + next_pool = jsonutils.loads(next_result[0]) + next_pool_list = next_pool['pools'] + + self.assertIn('links', next_pool) + if limit < count: + self.assertEqual(min(limit, count-limit), + len(next_pool_list)) + else: + # NOTE(jeffrey4l): when limit >= count, there will be no + # pools in the 2nd page. + self.assertEqual(0, len(next_pool_list)) + + self.assertEqual(min(limit, count), len(pool_list)) + for s in pool_list + next_pool_list: + # NOTE(flwang): It can't assumed that both sqlalchemy and + # mongodb can return query result with the same order. Just + # like the order they're inserted. Actually, sqlalchemy can't + # guarantee that. So we're leveraging the relationship between + # pool weight and the index of pools fixture to get the + # right pool to verify. + expect = expected[s['weight']] + path, weight, options = expect[:3] + self._pool_expect(s, path, weight, self.doc['uri']) + if detailed: + self.assertIn('options', s) + self.assertEqual(s['options'], expect[-1]) + else: + self.assertNotIn('options', s) + + def test_listing_works(self): + self._listing_test() + + def test_detailed_listing_works(self): + self._listing_test(detailed=True) + + @ddt.data(1, 5, 10, 15) + def test_listing_works_with_limit(self, limit): + self._listing_test(count=15, limit=limit) + + def test_listing_marker_is_respected(self): + # (gengchc): Remove flavor from the pool, so we can delete the pool. + self.simulate_patch(self.pool, body=jsonutils.dumps({'flavor': ''})) + self.simulate_delete(self.pool) + + with pools(self, 10, self.doc['uri'], 'my-flavor') as expected: + result = self.simulate_get(self.url_prefix + '/pools', + query_string='marker=3') + self.assertEqual(falcon.HTTP_200, self.srmock.status) + pool_list = jsonutils.loads(result[0])['pools'] + self.assertEqual(6, len(pool_list)) + path, weight = expected[4][:2] + self._pool_expect(pool_list[0], path, weight, self.doc['uri']) + + def test_listing_error_with_invalid_limit(self): + self.simulate_delete(self.pool) + query = 'limit={0}&detailed={1}'.format(0, True) + + with pools(self, 10, self.doc['uri'], 'my-flavor'): + self.simulate_get(self.url_prefix + '/pools', query_string=query) + self.assertEqual(falcon.HTTP_400, self.srmock.status) diff --git a/zaqar/transport/wsgi/v2_0/flavors.py b/zaqar/transport/wsgi/v2_0/flavors.py index 92f3654dc..176404128 100644 --- a/zaqar/transport/wsgi/v2_0/flavors.py +++ b/zaqar/transport/wsgi/v2_0/flavors.py @@ -20,7 +20,6 @@ import six from zaqar.common.api.schemas import flavors as schema from zaqar.common import decorators -from zaqar.common import utils as common_utils from zaqar.i18n import _ from zaqar.storage import errors from zaqar.transport import acl @@ -52,7 +51,8 @@ class Listing(object): { "flavors": [ - {"href": "", "capabilities": {}, "pool_group": ""}, + {"href": "", "capabilities": {}, "pool_group": "", + "pool_list": ""}, ... ], "links": [ @@ -87,11 +87,10 @@ class Listing(object): for entry in flavors: entry['href'] = request.path + '/' + entry['name'] - pool_group = entry['pool_group'] - # NOTE(wanghao): remove this in Newton. + # NOTE(gengchc): Remove pool_group in Rocky entry['pool'] = entry['pool_group'] if detailed: - caps = self._pools_ctrl.capabilities(group=pool_group) + caps = self._pools_ctrl.capabilities(flavor=entry) entry['capabilities'] = [str(cap).split('.')[-1] for cap in caps] @@ -125,9 +124,10 @@ class Resource(object): validator_type = jsonschema.Draft4Validator self._validators = { 'create': validator_type(schema.create), + # NOTE(gengchc): Remove pool_group in Rocky. 'pool_group': validator_type(schema.patch_pool_group), - # NOTE(wanghao): Remove this in Newton. 'pool': validator_type(schema.patch_pool), + 'pool_list': validator_type(schema.patch_pool_list), 'capabilities': validator_type(schema.patch_capabilities), } @@ -138,7 +138,7 @@ class Resource(object): :: - {"pool": "", capabilities: {...}} + {"pool": "", "pool_list": [], capabilities: {...}} :returns: HTTP | [200, 404] """ @@ -148,12 +148,17 @@ class Resource(object): try: data = self._ctrl.get(flavor, project=project_id) - pool_group = data['pool_group'] - # NOTE(wanghao): remove this in Newton. - data['pool'] = data['pool_group'] - capabilities = self._pools_ctrl.capabilities(group=pool_group) + capabilities = self._pools_ctrl.capabilities(flavor=data) data['capabilities'] = [str(cap).split('.')[-1] for cap in capabilities] + # NOTE(gengchc): Remove pool_group in Rocky. + data['pool'] = data['pool_group'] + pool_list =\ + list(self._pools_ctrl.get_pools_by_flavor(flavor=data)) + pool_name_list = [] + if len(pool_list) > 0: + pool_name_list = [x['name'] for x in pool_list] + data['pool_list'] = pool_name_list except errors.FlavorDoesNotExist as ex: LOG.debug(ex) @@ -163,26 +168,75 @@ class Resource(object): response.body = transport_utils.to_json(data) - @decorators.TransportLog("Flavors item") - @acl.enforce("flavors:create") - def on_put(self, request, response, project_id, flavor): - """Registers a new flavor. Expects the following input: + def _check_pools_exists(self, pool_list): + if pool_list is not None: + for pool in pool_list: + if not self._pools_ctrl.exists(pool): + raise errors.PoolDoesNotExist(pool) - :: + def _update_pools_by_flavor(self, flavor, pool_list): + if pool_list is not None: + for pool in pool_list: + self._pools_ctrl.update(pool, flavor=flavor) - {"pool_group": "my-pool-group", "capabilities": {}} + def _clean_pools_by_flavor(self, flavor, pool_list=None): + if pool_list is None: + flavor_obj = {} + flavor_obj['name'] = flavor + pllt = list(self._pools_ctrl.get_pools_by_flavor( + flavor=flavor_obj)) + pool_list = [x['name'] for x in pllt] + if pool_list is not None: + for pool in pool_list: + self._pools_ctrl.update(pool, flavor="") - A capabilities object may also be provided. + def _on_put_by_pool_list(self, request, response, project_id, + flavor, pool_list): + LOG.debug(u'PUT flavor - name by flavor: %s', flavor) + # NOTE(gengchc2): If configuration flavor is used by the new schema, + # a list of pools is required. + if len(pool_list) == 0: + response.status = falcon.HTTP_400 + response.location = request.path + raise falcon.HTTPBadRequest(_('Unable to create'), 'Bad Request') + # NOTE(gengchc2): Check if pools in the pool_list exist. + try: + self._check_pools_exists(pool_list) + except errors.PoolDoesNotExist as ex: + LOG.exception(ex) + description = (_(u'Flavor %(flavor)s could not be created, ' + 'error:%(msg)s') % + dict(flavor=flavor, msg=str(ex))) + raise falcon.HTTPBadRequest(_('Unable to create'), description) + capabilities = self._pools_ctrl.capabilities(name=pool_list[0]) + try: + self._ctrl.create(flavor, + project=project_id, + capabilities=capabilities) + response.status = falcon.HTTP_201 + response.location = request.path + except errors.ConnectionError as ex: + LOG.exception(ex) + description = (_(u'Flavor %(flavor)s could not be created, ' + 'error:%(msg)s') % + dict(flavor=flavor, msg=str(ex))) + raise falcon.HTTPBadRequest(_('Unable to create'), description) + # NOTE(gengchc2): Update the 'flavor' field in pools tables. + try: + self._update_pools_by_flavor(flavor, pool_list) + except errors.ConnectionError as ex: + LOG.exception(ex) + description = (_(u'Flavor %(flavor)s could not be created, ' + 'error:%(msg)s') % + dict(flavor=flavor, msg=str(ex))) + raise falcon.HTTPBadRequest(_('Unable to create'), description) - :returns: HTTP | [201, 400] - """ - - LOG.debug(u'PUT flavor - name: %s', flavor) - - data = wsgi_utils.load(request) - wsgi_utils.validate(self._validators['create'], data) - pool_group = data.get('pool_group') or data.get('pool') - capabilities = self._pools_ctrl.capabilities(pool_group) + def _on_put_by_group(self, request, response, project_id, + flavor, pool_group): + LOG.debug(u'PUT flavor - name: %s by group', flavor) + flavor_obj = {} + flavor_obj["pool_group"] = pool_group + capabilities = self._pools_ctrl.capabilities(flavor_obj) try: self._ctrl.create(flavor, pool_group=pool_group, @@ -197,6 +251,35 @@ class Resource(object): dict(flavor=flavor, pool_group=pool_group)) raise falcon.HTTPBadRequest(_('Unable to create'), description) + @decorators.TransportLog("Flavors item") + @acl.enforce("flavors:create") + def on_put(self, request, response, project_id, flavor): + """Registers a new flavor. Expects the following input: + + :: + + {"pool_group": "my-pool-group", + "pool_list": [], "capabilities": {}} + + A capabilities object may also be provided. + + :returns: HTTP | [201, 400] + """ + + LOG.debug(u'PUT flavor - name: %s', flavor) + + data = wsgi_utils.load(request) + wsgi_utils.validate(self._validators['create'], data) + LOG.debug(u'The pool_group will be removed in Rocky release.') + pool_group = data.get('pool_group') or data.get('pool') + pool_list = data.get('pool_list') + if pool_list is not None: + self._on_put_by_pool_list(request, response, project_id, + flavor, pool_list) + else: + self._on_put_by_group(request, response, project_id, + flavor, pool_group) + @decorators.TransportLog("Flavors item") @acl.enforce("flavors:delete") def on_delete(self, request, response, project_id, flavor): @@ -206,49 +289,96 @@ class Resource(object): """ LOG.debug(u'DELETE flavor - name: %s', flavor) + # NOTE(gengchc2): If configuration flavor is + # used by the new schema, the flavor field in pools + # need to be cleaned. + try: + self._clean_pools_by_flavor(flavor) + except errors.ConnectionError as ex: + LOG.exception(ex) + description = (_(u'Flavor %(flavor)s could not be deleted.') % + dict(flavor=flavor)) + raise falcon.HTTPBadRequest(_('Unable to create'), description) self._ctrl.delete(flavor, project=project_id) response.status = falcon.HTTP_204 - @decorators.TransportLog("Flavors item") - @acl.enforce("flavors:update") - def on_patch(self, request, response, project_id, flavor): - """Allows one to update a flavors's pool_group. + def _on_patch_by_pool_list(self, request, response, project_id, + flavor, pool_list): - This method expects the user to submit a JSON object - containing 'pool_group'. If none is found, the request is flagged - as bad. There is also strict format checking through the use of - jsonschema. Appropriate errors are returned in each case for - badly formatted input. + if len(pool_list) == 0: + response.status = falcon.HTTP_400 + response.location = request.path + raise falcon.HTTPBadRequest(_('Unable to create'), 'Bad Request') + # NOTE(gengchc2): If the flavor does not exist, return + try: + self._ctrl.get(flavor, project=project_id) + except errors.FlavorDoesNotExist as ex: + LOG.debug(ex) + raise wsgi_errors.HTTPNotFound(six.text_type(ex)) - :returns: HTTP | [200, 400] - """ + flavor_obj = {} + flavor_obj['name'] = flavor + # NOTE(gengchc2): Get the pools list with flavor. + pool_list_old = list(self._pools_ctrl.get_pools_by_flavor( + flavor=flavor_obj)) + # NOTE(gengchc2): Check if the new pool in the pool_list exist. + try: + self._check_pools_exists(pool_list) + except errors.PoolDoesNotExist as ex: + LOG.exception(ex) + description = (_(u'Flavor %(flavor)s cant be updated, ' + 'error:%(msg)s') % + dict(flavor=flavor, msg=str(ex))) + raise falcon.HTTPBadRequest(_('updatefail'), description) + capabilities = self._pools_ctrl.capabilities(name=pool_list[0]) + try: + self._ctrl.update(flavor, project=project_id, + capabilities=capabilities) + resp_data = self._ctrl.get(flavor, project=project_id) + resp_data['capabilities'] = [str(cap).split('.')[-1] + for cap in capabilities] + except errors.FlavorDoesNotExist as ex: + LOG.exception(ex) + raise wsgi_errors.HTTPNotFound(six.text_type(ex)) - LOG.debug(u'PATCH flavor - name: %s', flavor) - data = wsgi_utils.load(request) - - EXPECT = ('pool_group', 'pool') - if not any([(field in data) for field in EXPECT]): - LOG.debug(u'PATCH flavor, bad params') - raise wsgi_errors.HTTPBadRequestBody( - '`pool_group` or `pool` needs to be specified' - ) - - for field in EXPECT: - wsgi_utils.validate(self._validators[field], data) - - fields = common_utils.fields(data, EXPECT, - pred=lambda v: v is not None) - # NOTE(wanghao): remove this in Newton. - if fields.get('pool') and fields.get('pool_group') is None: - fields['pool_group'] = fields.get('pool') - fields.pop('pool') + # (gengchc) Update flavor field in new pool list. + try: + self._update_pools_by_flavor(flavor, pool_list) + except errors.ConnectionError as ex: + LOG.exception(ex) + description = (_(u'Flavor %(flavor)s could not be updated, ' + 'error:%(msg)s') % + dict(flavor=flavor, msg=str(ex))) + raise falcon.HTTPBadRequest(_('Unable to create'), description) + # (gengchc) Remove flavor from old pool list. + try: + pool_list_removed = [] + for pool_old in pool_list_old: + if pool_old['name'] not in pool_list: + pool_list_removed.append(pool_old['name']) + self._clean_pools_by_flavor(flavor, pool_list_removed) + except errors.ConnectionError as ex: + LOG.exception(ex) + description = (_(u'Flavor %(flavor)s could not be updated, ' + 'error:%(msg)s') % + dict(flavor=flavor, msg=str(ex))) + raise falcon.HTTPBadRequest(_('Unable to create'), description) + resp_data['pool_list'] = pool_list + resp_data['href'] = request.path + response.body = transport_utils.to_json(resp_data) + def _on_patch_by_group(self, request, response, project_id, + flavor, pool_group): + LOG.debug(u'PATCH flavor - name: %s by group', flavor) resp_data = None try: - self._ctrl.update(flavor, project=project_id, **fields) + flvor_obj = {} + flvor_obj['pool_group'] = pool_group + capabilities = self._pools_ctrl.capabilities(flavor=flvor_obj) + self._ctrl.update(flavor, project=project_id, + pool_group=pool_group, + capabilities=capabilities) resp_data = self._ctrl.get(flavor, project=project_id) - capabilities = self._pools_ctrl.capabilities( - group=resp_data['pool_group']) resp_data['capabilities'] = [str(cap).split('.')[-1] for cap in capabilities] except errors.FlavorDoesNotExist as ex: @@ -256,3 +386,42 @@ class Resource(object): raise wsgi_errors.HTTPNotFound(six.text_type(ex)) resp_data['href'] = request.path response.body = transport_utils.to_json(resp_data) + + @decorators.TransportLog("Flavors item") + @acl.enforce("flavors:update") + def on_patch(self, request, response, project_id, flavor): + """Allows one to update a flavors'pool list. + + This method expects the user to submit a JSON object + containing 'pool_group' or 'pool list'. If none is found, + the request is flagged as bad. There is also strict format + checking through the use of jsonschema. Appropriate errors + are returned in each case for badly formatted input. + + :returns: HTTP | [200, 400] + """ + + LOG.debug(u'PATCH flavor - name: %s', flavor) + data = wsgi_utils.load(request) + # NOTE(gengchc2): remove pool_group in R release. + EXPECT = ('pool_group', 'pool', 'pool_list') + if not any([(field in data) for field in EXPECT]): + LOG.debug(u'PATCH flavor, bad params') + raise wsgi_errors.HTTPBadRequestBody( + '`pool_group` or `pool` or `pool_list` needs to be specified' + ) + + for field in EXPECT: + wsgi_utils.validate(self._validators[field], data) + LOG.debug(u'The pool_group will be removed in Rocky release.') + pool_group = data.get('pool_group') or data.get('pool') + pool_list = data.get('pool_list') + # NOTE(gengchc2): If pool_list is not None, configuration flavor is + # used by the new schema. + # a list of pools is required. + if pool_list is not None: + self._on_patch_by_pool_list(request, response, project_id, + flavor, pool_list) + else: + self._on_patch_by_group(request, response, project_id, + flavor, pool_group) diff --git a/zaqar/transport/wsgi/v2_0/pools.py b/zaqar/transport/wsgi/v2_0/pools.py index 9feba9f98..d6bdff9bc 100644 --- a/zaqar/transport/wsgi/v2_0/pools.py +++ b/zaqar/transport/wsgi/v2_0/pools.py @@ -136,7 +136,8 @@ class Resource(object): self._validators = { 'weight': validator_type(schema.patch_weight), 'uri': validator_type(schema.patch_uri), - 'group': validator_type(schema.patch_uri), + 'group': validator_type(schema.patch_group), + 'flavor': validator_type(schema.patch_flavor), 'options': validator_type(schema.patch_options), 'create': validator_type(schema.create) } @@ -194,7 +195,8 @@ class Resource(object): try: self._ctrl.create(pool, weight=data['weight'], uri=data['uri'], - group=data.get('group'), + group=data.get('group', None), + flavor=data.get('flavor', None), options=data.get('options', {})) response.status = falcon.HTTP_201 response.location = request.path @@ -234,9 +236,9 @@ class Resource(object): """Allows one to update a pool's weight, uri, and/or options. This method expects the user to submit a JSON object - containing at least one of: 'uri', 'weight', 'group', 'options'. If - none are found, the request is flagged as bad. There is also - strict format checking through the use of + containing at least one of: 'uri', 'weight', 'group', 'flavor', + 'options'.If none are found, the request is flagged as bad. + There is also strict format checking through the use of jsonschema. Appropriate errors are returned in each case for badly formatted input. @@ -246,11 +248,12 @@ class Resource(object): LOG.debug(u'PATCH pool - name: %s', pool) data = wsgi_utils.load(request) - EXPECT = ('weight', 'uri', 'group', 'options') + EXPECT = ('weight', 'uri', 'group', 'flavor', 'options') if not any([(field in data) for field in EXPECT]): LOG.debug(u'PATCH pool, bad params') raise wsgi_errors.HTTPBadRequestBody( - 'One of `uri`, `weight`, `group`, or `options` needs ' + 'One of `uri`, `weight`, `group`, `flavor`,' + ' or `options` needs ' 'to be specified' )