openstack
/
zaqar
Code Issues Proposed changes

Remove pool group from zaqar 

Implement: blueprint remove-pool-group-from-zaqar

Change-Id: I18bb20d44f5071f9700fb64c10e9393707086de3
This commit is contained in:
gengchc2 2017-11-05 01:07:36 -08:00
parent 39d5b2bc53
commit 5250d940bc
23 changed files with 1694 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
zaqar/api/v1_1/response.py
View File

@ -229,6 +229,9 @@ class ResponseSchema(api.Api):
'group': {
'type': ['string', 'null']
},
'flavor': {
'type': ['string', 'null']
},
'options': {
'type': 'object',
'additionalProperties': True
@ -284,6 +287,9 @@ class ResponseSchema(api.Api):
'group': {
'type': ['string', 'null']
},
'flavor': {
'type': ['string', 'null']
},
'weight': {
'type': 'number',
'minimum': -1

6
zaqar/api/v2/response.py
View File

@ -229,6 +229,9 @@ class ResponseSchema(api.Api):
'group': {
'type': ['string', 'null']
},
'flavor': {
'type': ['string', 'null']
},
'options': {
'type': 'object',
'additionalProperties': True
@ -284,6 +287,9 @@ class ResponseSchema(api.Api):
'group': {
'type': ['string', 'null']
},
'flavor': {
'type': ['string', 'null']
},
'weight': {
'type': 'number',
'minimum': -1

17
zaqar/common/api/schemas/flavors.py
View File

@ -27,6 +27,7 @@ patch_capabilities = {
}
}
# TODO(gengchc2): remove pool_group in R release.
# NOTE(flaper87): a string valid
patch_pool = {
'type': 'object',
@ -38,6 +39,7 @@ patch_pool = {
}
}
# TODO(gengchc2): remove pool_group in R release.
patch_pool_group = {
'type': 'object',
'properties': {
@ -48,16 +50,29 @@ patch_pool_group = {
}
}
# NOTE(gengchc): Add pool_list in flavor creation for removing pool_group
patch_pool_list = {
'type': 'object',
'properties': {
'pool_list': {
'type': 'array'
},
'additionalProperties': False
}
}
create = {
'type': 'object',
'properties': {
'pool_group': patch_pool_group['properties']['pool_group'],
'pool': patch_pool['properties']['pool'],
'pool_list': patch_pool_list['properties']['pool_list'],
'capabilities': patch_capabilities['properties']['capabilities']
},
# NOTE(flaper87): capabilities need not be present. Storage drivers
# must provide reasonable defaults.
# NOTE(wanghao): remove pool in Newton release.
'oneOf': [{'required': ['pool_group']}, {'required': ['pool']}],
'oneOf': [{'required': ['pool_group']}, {'required': ['pool']},
{'required': ['pool_list']}],
'additionalProperties': False
}

12
zaqar/common/api/schemas/pools.py
View File

@ -49,6 +49,17 @@ patch_group = {
}
}
# NOTE(gengchc): remove pool_group add flavor
patch_flavor = {
'type': 'object', 'properties': {
'flavor': {
'type': 'string',
'minLength': 0,
'maxLength': 255
},
'additionalProperties': False
}
}
patch_weight = {
'type': 'object', 'properties': {
@ -63,6 +74,7 @@ create = {
'type': 'object', 'properties': {
'weight': patch_weight['properties']['weight'],
'group': patch_group['properties']['uri'],
'flavor': patch_flavor['properties']['flavor'],
'uri': patch_uri['properties']['uri'],
'options': patch_options['properties']['options']
},

55
zaqar/storage/base.py
View File

@ -735,9 +735,9 @@ class Subscription(ControllerBase):
class PoolsBase(ControllerBase):
"""A controller for managing pools."""
def _check_capabilities(self, uri, group=None, name=None):
def _check_capabilities(self, uri, flavor=None, name=None):
default_store = self.driver.conf.drivers.message_store
pool_caps = self.capabilities(group=group, name=name)
pool_caps = self.capabilities(flavor=flavor, name=name)
if not pool_caps:
return True
@ -745,30 +745,32 @@ class PoolsBase(ControllerBase):
new_store = utils.load_storage_impl(uri,
default_store=default_store)
# NOTE(flaper87): Since all pools in a pool group
# NOTE(flaper87): Since all pools in a pool flavor
# are assumed to have the same capabilities, it's
# fine to check against just 1
return pool_caps == new_store.BASE_CAPABILITIES
def capabilities(self, group=None, name=None):
"""Gets the set of capabilities for this group/name
def capabilities(self, flavor=None, name=None):
"""Gets the set of capabilities for this flavor/name
:param group: The pool group to get capabilities for
:type group: six.text_type
:param flavor: The pool flavor to get capabilities for
:type flavor: six.text_type
:param name: The pool name to get capabilities for
:type name: six.text_type
"""
pllt = []
if name:
group = list(self._get_pools_by_group(self._get(name)['group']))
pool = self.get(name)
pllt.append(pool)
else:
group = list(self._get_pools_by_group(group))
pllt = list(self._get_pools_by_flavor(flavor))
if not len(group) > 0:
if not len(pllt) > 0:
return ()
default_store = self.driver.conf.drivers.message_store
pool_store = utils.load_storage_impl(group[0]['uri'],
pool_store = utils.load_storage_impl(pllt[0]['uri'],
default_store=default_store)
return pool_store.BASE_CAPABILITIES
@ -791,7 +793,7 @@ class PoolsBase(ControllerBase):
_list = abc.abstractmethod(lambda x: None)
def create(self, name, weight, uri, group=None, options=None):
def create(self, name, weight, uri, group=None, flavor=None, options=None):
"""Registers a pool entry.
:param name: The name of this pool
@ -801,33 +803,38 @@ class PoolsBase(ControllerBase):
:param uri: A URI that can be used by a storage client
(e.g., pymongo) to access this pool.
:type uri: six.text_type
:param group: The group of this pool
:type group: six.text_type
:param flavor: The flavor of this pool
:type flavor: six.text_type
:param options: Options used to configure this pool
:type options: dict
"""
if not self._check_capabilities(uri, group=group):
flavor_obj = {}
if flavor is not None:
flavor_obj["name"] = flavor
if group is not None:
flavor_obj["pool_group"] = group
if not self._check_capabilities(uri, flavor=flavor_obj):
raise errors.PoolCapabilitiesMismatch()
return self._create(name, weight, uri, group, options)
return self._create(name, weight, uri, group, flavor, options)
_create = abc.abstractmethod(lambda x: None)
def get_pools_by_group(self, group=None, detailed=False):
"""Returns a pool list filtered by given pool group.
def get_pools_by_flavor(self, flavor=None, detailed=False):
"""Returns a pool list filtered by given pool flavor.
:param group: The group to filter on. `None` returns
pools that are not assigned to any pool group.
:type group: six.text_type
:param flavor: The flavor to filter on. `None` returns
pools that are not assigned to any pool flavor.
:type flavor: six.text_type
:param detailed: Should the options data be included?
:type detailed: bool
:returns: weight, uri, and options for this pool
:rtype: {}
:raises PoolDoesNotExist: if not found
"""
return self._get_pools_by_group(group, detailed)
return self._get_pools_by_flavor(flavor, detailed)
_get_pools_by_group = abc.abstractmethod(lambda x: None)
_get_pools_by_flavor = abc.abstractmethod(lambda x: None)
def get(self, name, detailed=False):
"""Returns a single pool entry.
@ -1008,7 +1015,7 @@ class FlavorsBase(ControllerBase):
raise NotImplementedError
@abc.abstractmethod
def create(self, name, pool, project=None, capabilities=None):
def create(self, name, pool_group=None, project=None, capabilities=None):
"""Registers a flavor entry.
:param name: The name of this flavor

14
zaqar/storage/mongodb/flavors.py
View File

@ -97,14 +97,19 @@ class FlavorsController(base.FlavorsBase):
return _normalize(res, detailed)
@utils.raises_conn_error
def create(self, name, pool_group, project=None, capabilities=None):
def create(self, name, pool_group=None, project=None, capabilities=None):
# NOTE(flaper87): Check if there are pools in this group.
# Should there be a `group_exists` method?
# NOTE(wanghao): Since we didn't pass the group name just pool name,
# so we don't need to get the pool by group.
if not list(self._pools_ctrl.get_pools_by_group(pool_group)):
raise errors.PoolGroupDoesNotExist(pool_group)
# NOTE(gengchc2): If you do not use the removal group scheme to
# configure flavor, pool_group can be None..
if pool_group is not None:
flavor_obj = {}
flavor_obj["pool_group"] = pool_group
if not list(self._pools_ctrl.get_pools_by_flavor(flavor_obj)):
raise errors.PoolGroupDoesNotExist(pool_group)
capabilities = {} if capabilities is None else capabilities
self._col.update_one({'n': name, 'p': project},
@ -124,7 +129,8 @@ class FlavorsController(base.FlavorsBase):
if pool_group is not None:
fields['s'] = pool_group
# NOTE(gengchc2): If you do not use the removal group scheme to
# configure flavor, pool_group can be None, pool_group can be remove.
assert fields, '`pool_group` or `capabilities` not found in kwargs'
res = self._col.update_one({'n': name, 'p': project},
{'$set': fields},

59
zaqar/storage/mongodb/pools.py
View File

@ -20,6 +20,7 @@ Schema:
'u': uri :: six.text_type
'w': weight :: int
'o': options :: dict
'f': flavor :: six.text_type
"""
import functools
@ -90,13 +91,22 @@ class PoolsController(base.PoolsBase):
return _normalize(res, detailed)
@utils.raises_conn_error
def _get_pools_by_group(self, group=None, detailed=False):
cursor = self._col.find({'g': group}, projection=_field_spec(detailed))
def _get_pools_by_flavor(self, flavor=None, detailed=False):
query = None
if flavor is None:
query = {'f': None}
elif flavor.get("pool_group") is not None:
query = {'g': flavor.get("pool_group")}
elif flavor.get('name') is not None:
query = {'f': flavor.get('name')}
cursor = self._col.find(query,
projection=_field_spec(detailed))
normalizer = functools.partial(_normalize, detailed=detailed)
return utils.HookedCursor(cursor, normalizer)
@utils.raises_conn_error
def _create(self, name, weight, uri, group=None, options=None):
def _create(self, name, weight, uri, group=None, flavor=None,
options=None):
options = {} if options is None else options
try:
self._col.update_one({'n': name},
@ -104,6 +114,7 @@ class PoolsController(base.PoolsBase):
'w': weight,
'u': uri,
'g': group,
'f': flavor,
'o': options}},
upsert=True)
except mongo_error.DuplicateKeyError:
@ -115,13 +126,20 @@ class PoolsController(base.PoolsBase):
@utils.raises_conn_error
def _update(self, name, **kwargs):
names = ('uri', 'weight', 'group', 'options')
names = ('uri', 'weight', 'group', 'flavor', 'options')
fields = common_utils.fields(kwargs, names,
pred=lambda x: x is not None,
key_transform=lambda x: x[0])
assert fields, ('`weight`, `uri`, `group`, '
'or `options` not found in kwargs')
flavor = fields.get('f')
if flavor is not None and len(flavor) == 0:
fields['f'] = None
group = fields.get('g')
if group is not None and len(group) == 0:
fields['g'] = None
res = self._col.update_one({'n': name},
{'$set': fields},
upsert=False)
@ -135,17 +153,31 @@ class PoolsController(base.PoolsBase):
# recursion error.
try:
pool = self.get(name)
pools_group = self.get_pools_by_group(pool['group'])
flavor_ctl = self.driver.flavors_controller
res = list(flavor_ctl._list_by_pool_group(pool['group']))
if pool['group'] is not None:
flavor = {}
flavor['pool_group'] = pool['group']
pools_group = self.get_pools_by_flavor(flavor=flavor)
flavor_ctl = self.driver.flavors_controller
res = list(flavor_ctl._list_by_pool_group(pool['group']))
# NOTE(flaper87): If this is the only pool in the
# group and it's being used by a flavor, don't allow
# it to be deleted.
if res and len(pools_group) == 1:
flavors = ', '.join([x['name'] for x in res])
raise errors.PoolInUseByFlavor(name, flavors)
# NOTE(flaper87): If this is the only pool in the
# group and it's being used by a flavor, don't allow
# it to be deleted.
if res and len(pools_group) == 1:
flavors = ', '.join([x['name'] for x in res])
raise errors.PoolInUseByFlavor(name, flavors)
pools_in_flavor = []
flavor = pool.get("flavor", None)
if flavor is not None:
# NOTE(gengchc2): If this is the only pool in the
# flavor and it's being used by a flavor, don't allow
# it to be deleted.
flavor1 = {}
flavor1['name'] = flavor
pools_in_flavor = self.get_pools_by_flavor(flavor=flavor1)
if len(pools_in_flavor) == 1:
raise errors.PoolInUseByFlavor(name, flavor)
self._col.delete_one({'n': name})
except errors.PoolDoesNotExist:
pass
@ -160,6 +192,7 @@ def _normalize(pool, detailed=False):
ret = {
'name': pool['n'],
'group': pool['g'],
'flavor': pool['f'],
'uri': pool['u'],
'weight': pool['w'],
}

47
zaqar/storage/pooling.py
View File

@ -23,6 +23,7 @@ from osprofiler import profiler
from zaqar.common import decorators
from zaqar.common import errors as cerrors
from zaqar.common.storage import select
from zaqar.i18n import _
from zaqar import storage
from zaqar.storage import errors
from zaqar.storage import pipeline
@ -246,6 +247,14 @@ class QueueController(storage.Queue):
return self._mgt_queue_ctrl.get_metadata(name, project=project)
def set_metadata(self, name, metadata, project=None):
# NOTE(gengchc2): If flavor metadata is modified in queue,
# The queue needs to be re-registered to pools, otherwise
# the queue flavor parameter is not consistent with the pool.
flavor = None
if isinstance(metadata, dict):
flavor = metadata.get('_flavor')
self._pool_catalog.register(name, project=project, flavor=flavor)
return self._mgt_queue_ctrl.set_metadata(name, metadata=metadata,
project=project)
@ -502,22 +511,40 @@ class Catalog(object):
"""
# NOTE(cpp-cabrera): only register a queue if the entry
# doesn't exist
if not self._catalogue_ctrl.exists(project, queue):
# NOTE(gengchc): if exist, get queue's pool.flavor:
# if queue's pool.flavor is different, first delete it and add it.
# Otherwise, if the flavor in the meteredata of the queue is
# modified, the catalog will be inconsistent.
if self._catalogue_ctrl.exists(project, queue):
catalogue = self._catalogue_ctrl.get(project, queue)
oldpoolids = catalogue['pool']
oldpool = self._pools_ctrl.get(oldpoolids)
oldflavor = oldpool['flavor']
msgtmpl = _(u'regiester queue to pool: old flavor: %(oldflavor)s '
', new flavor: %(flavor)s')
LOG.info(msgtmpl,
{'oldflavor': oldflavor, 'flavor': flavor})
if oldpool['flavor'] != flavor:
self._catalogue_ctrl.delete(project, queue)
if not self._catalogue_ctrl.exists(project, queue):
if flavor is not None:
flavor = self._flavor_ctrl.get(flavor, project=project)
pools = self._pools_ctrl.get_pools_by_group(
group=flavor['pool_group'],
pools = self._pools_ctrl.get_pools_by_flavor(
flavor=flavor,
detailed=True)
pool = select.weighted(pools)
pool = pool and pool['name'] or None
msgtmpl = _(u'regiester queue to pool: new flavor:%(flavor)s'
' pool_group:%(pool_group)s')
LOG.info(msgtmpl,
{'flavor': flavor.get('name', None),
'pool_group': flavor.get('pool_group', None)})
else:
# NOTE(flaper87): Get pools assigned to the default
# group `None`. We should consider adding a `default_group`
# option in the future.
pools = self._pools_ctrl.get_pools_by_group(detailed=True)
pools = self._pools_ctrl.get_pools_by_flavor(detailed=True)
pool = select.weighted(pools)
pool = pool and pool['name'] or None
@ -531,7 +558,15 @@ class Catalog(object):
if self.lookup(queue, project) is not None:
return
raise errors.NoPoolFound()
msgtmpl = _(u'regiester queue to pool: new flavor: None')
LOG.info(msgtmpl)
msgtmpl = _(u'regiester queue: project:%(project)s'
' queue:%(queue)s pool:%(pool)s')
LOG.info(msgtmpl,
{'project': project,
'queue': queue,
'pool': pool})
self._catalogue_ctrl.insert(project, queue, pool)
@_pool_id.purges

25
zaqar/storage/sqlalchemy/flavors.py
View File

@ -72,18 +72,29 @@ class FlavorsController(base.FlavorsBase):
return _normalize(flavor, detailed)
@utils.raises_conn_error
def create(self, name, pool_group, project=None, capabilities=None):
def create(self, name, pool_group=None, project=None, capabilities=None):
cap = None if capabilities is None else utils.json_encode(capabilities)
try:
stmt = sa.sql.expression.insert(tables.Flavors).values(
name=name, pool_group=pool_group, project=project,
capabilities=cap
)
if pool_group is not None:
stmt = sa.sql.expression.insert(tables.Flavors).values(
name=name, pool_group=pool_group, project=project,
capabilities=cap
)
else:
stmt = sa.sql.expression.insert(tables.Flavors).values(
name=name, project=project,
capabilities=cap
)
self.driver.run(stmt)
except oslo_db.exception.DBDuplicateEntry:
if not self._pools_ctrl.get_pools_by_group(pool_group):
raise errors.PoolGroupDoesNotExist(pool_group)
# NOTE(gengchc2): If you do not use the removal group scheme to
# configure flavor, pool_group can be None..
if pool_group is not None:
flavor_obj = {}
flavor_obj["pool_group"] = pool_group
if not list(self._pools_ctrl.get_pools_by_flavor(flavor_obj)):
raise errors.PoolGroupDoesNotExist(pool_group)
# TODO(flaper87): merge update/create into a single
# method with introduction of upsert

41
zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/006_queens.py Normal file
View File

@ -0,0 +1,41 @@
# Copyright 2017 ZTE Corporation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Queens release
Revision ID: 005
Revises: 006
Create Date: 2017-11-09 11:45:45.928605
"""
# revision identifiers, used by Alembic.
revision = '006'
down_revision = '005'
from alembic import op
import sqlalchemy as sa
MYSQL_ENGINE = 'InnoDB'
MYSQL_CHARSET = 'utf8'
def upgrade():
# NOTE(gengchc2): Add a new flavor column to Pools nodes
op.add_column('Pools', sa.Column('flavor', sa.String(64), nullable=True))
# NOTE(gengchc2): Change pool_group to default null in Flavors table
op.execute('alter table Flavors change column pool_group '
'pool_group varchar(64) default null')

33
zaqar/storage/sqlalchemy/pools.py
View File

@ -56,10 +56,22 @@ class PoolsController(base.PoolsBase):
yield marker_name and marker_name['next']
@utils.raises_conn_error
def _get_pools_by_group(self, group=None, detailed=False):
stmt = sa.sql.select([tables.Pools]).where(
tables.Pools.c.group == group
)
def _get_pools_by_flavor(self, flavor=None, detailed=False):
pool_group = flavor.get("pool_group", None) if flavor is not None\
else None
flavor_name = flavor.get("name", None) if flavor is not None\
else None
if pool_group is not None:
stmt = sa.sql.select([tables.Pools]).where(
tables.Pools.c.group == pool_group
)
elif flavor_name is not None:
stmt = sa.sql.select([tables.Pools]).where(
tables.Pools.c.flavor == flavor_name
)
else:
stmt = sa.sql.select([tables.Pools])
cursor = self.driver.run(stmt)
normalizer = functools.partial(_normalize, detailed=detailed)
@ -87,7 +99,8 @@ class PoolsController(base.PoolsBase):
# TODO(cpp-cabrera): rename to upsert
@utils.raises_conn_error
def _create(self, name, weight, uri, group=None, options=None):
def _create(self, name, weight, uri, group=None, flavor=None,
options=None):
opts = None if options is None else utils.json_encode(options)
if group is not None:
@ -95,7 +108,8 @@ class PoolsController(base.PoolsBase):
try:
stmt = sa.sql.expression.insert(tables.Pools).values(
name=name, weight=weight, uri=uri, group=group, options=opts
name=name, weight=weight, uri=uri, group=group,
flavor=flavor, options=opts
)
self.driver.run(stmt)
@ -103,7 +117,7 @@ class PoolsController(base.PoolsBase):
# TODO(cpp-cabrera): merge update/create into a single
# method with introduction of upsert
self._update(name, weight=weight, uri=uri,
group=group, options=options)
group=group, flavor=flavor, options=options)
@utils.raises_conn_error
def _exists(self, name):
@ -117,11 +131,11 @@ class PoolsController(base.PoolsBase):
# NOTE(cpp-cabrera): by pruning None-valued kwargs, we avoid
# overwriting the existing options field with None, since that
# one can be null.
names = ('uri', 'weight', 'group', 'options')
names = ('uri', 'weight', 'group', 'flavor', 'options')
fields = common_utils.fields(kwargs, names,
pred=lambda x: x is not None)
assert fields, ('`weight`, `uri`, `group`, '
assert fields, ('`weight`, `uri`, `group`, `flavor`, '
'or `options` not found in kwargs')
if 'options' in fields:
@ -158,6 +172,7 @@ def _normalize(pool, detailed=False):
'group': pool[1],
'uri': pool[2],
'weight': pool[3],
'flavor': pool[5],
}
if detailed:
opts = pool[4]

11
zaqar/storage/sqlalchemy/tables.py
View File

@ -28,7 +28,6 @@ Queues = sa.Table('Queues', metadata,
PoolGroup = sa.Table('PoolGroup', metadata,
sa.Column('name', sa.String(64), primary_key=True))
Pools = sa.Table('Pools', metadata,
sa.Column('name', sa.String(64), primary_key=True),
sa.Column('group', sa.ForeignKey('PoolGroup.name',
@ -37,18 +36,20 @@ Pools = sa.Table('Pools', metadata,
sa.Column('uri', sa.String(255),
unique=True, nullable=False),
sa.Column('weight', sa.INTEGER, nullable=False),
sa.Column('options', sa.Text()))
sa.Column('options', sa.Text()),
sa.Column('flavor', sa.String(64), nullable=True))
# NOTE(gengchc2): Modify pool_group define: turn NOT NULL into DEFAULT NULL:
# [alter table Flavors change column pool_group pool_group varchar(64)
# default null;]
Flavors = sa.Table('Flavors', metadata,
sa.Column('name', sa.String(64), primary_key=True),
sa.Column('project', sa.String(64)),
sa.Column('pool_group', sa.ForeignKey('PoolGroup.name',
ondelete='CASCADE'),
nullable=False),
nullable=True),
sa.Column('capabilities', sa.Text()))
Catalogue = sa.Table('Catalogue', metadata,
sa.Column('pool', sa.String(64),
sa.ForeignKey('Pools.name',

3
zaqar/tests/base.py
View File

@ -66,7 +66,8 @@ class TestBase(testtools.TestCase):
opts.set_defaults(self.conf)
self.conf.register_opts(configs._PROFILER_OPTIONS,
group=configs._PROFILER_GROUP)
self.redis_url = os.environ.get('ZAQAR_TEST_REDIS_URL',
'redis://127.0.0.1:6379')
self.mongodb_url = os.environ.get('ZAQAR_TEST_MONGODB_URL',
'mongodb://127.0.0.1:27017')

250
zaqar/tests/unit/storage/base.py
View File

@ -14,6 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import collections
import datetime
import math
@ -68,16 +70,27 @@ class ControllerBaseTest(testing.TestBase):
args.append(self.control)
self.driver = self.driver_class(*args)
else:
uri = self.mongodb_url
for i in range(4):
db_name = "zaqar_test_pools_" + str(i)
testredis = os.environ.get('ZAQAR_TEST_REDIS', 0)
if testredis:
uri = self.redis_url
for i in range(4):
db_name = "?dbid = " + str(i)
# NOTE(dynarro): we need to create a unique uri.
new_uri = "%s/%s" % (uri, db_name)
options = {'database': db_name}
self.control.pools_controller.create(six.text_type(i),
100, new_uri,
options=options)
# NOTE(dynarro): we need to create a unique uri.
new_uri = "%s/%s" % (uri, db_name)
self.control.pools_controller.create(six.text_type(i),
100, new_uri)
else:
uri = self.mongodb_url
for i in range(4):
db_name = "zaqar_test_pools_" + str(i)
# NOTE(dynarro): we need to create a unique uri.
new_uri = "%s/%s" % (uri, db_name)
options = {'database': db_name}
self.control.pools_controller.create(six.text_type(i),
100, new_uri,
options=options)
self.driver = self.driver_class(self.conf, cache, self.control)
self.addCleanup(self.control.pools_controller.drop_all)
self.addCleanup(self.control.catalogue_controller.drop_all)
@ -1395,10 +1408,12 @@ class PoolsControllerTest(ControllerBaseTest):
# Let's create one pool
self.pool = str(uuid.uuid1())
# NOTE(gengchc2): remove pool_group in Rocky release.
self.pool_group = str(uuid.uuid1())
self.pools_controller.create(self.pool, 100, 'localhost',
group=self.pool_group, options={})
self.pool1 = str(uuid.uuid1())
self.flavor = str(uuid.uuid1())
self.pools_controller.create(self.pool1, 100, 'localhost1',
flavor=self.flavor, options={})
self.flavors_controller = self.driver.flavors_controller
def tearDown(self):
@ -1464,6 +1479,9 @@ class PoolsControllerTest(ControllerBaseTest):
def test_delete_works(self):
self.pools_controller.delete(self.pool)
# (gengchc): Remove the flavor from pool, then testcase cleanup pool
self.pools_controller.update(self.pool1, flavor="")
self.pools_controller.delete(self.pool1)
self.assertFalse(self.pools_controller.exists(self.pool))
def test_delete_nonexistent_is_silent(self):
@ -1476,9 +1494,9 @@ class PoolsControllerTest(ControllerBaseTest):
self.assertRaises(StopIteration, next, pools)
def test_listing_simple(self):
# NOTE(cpp-cabrera): base entry interferes with listing results
self.pools_controller.delete(self.pool)
# (gengchc): Remove the flavor from pool, then testcase cleanup pool
self.pools_controller.update(self.pool1, flavor="")
self.pools_controller._drop_all()
pools = []
marker = ''
for i in range(15):
@ -1541,15 +1559,6 @@ class PoolsControllerTest(ControllerBaseTest):
self.assertIn('options', entry)
self.assertEqual({}, entry['options'])
def test_mismatching_capabilities(self):
# NOTE(flaper87): This may fail for redis. Create
# a dummy store for tests.
with testing.expect(errors.PoolCapabilitiesMismatch):
self.pools_controller.create(str(uuid.uuid1()),
100, 'redis://localhost',
group=self.pool_group,
options={})
class CatalogueControllerTest(ControllerBaseTest):
controller_base_class = storage.CatalogueBase
@ -1562,10 +1571,13 @@ class CatalogueControllerTest(ControllerBaseTest):
self.project = six.text_type(uuid.uuid4())
self.pool = str(uuid.uuid1())
self.pool_group = str(uuid.uuid1())
self.pool_ctrl.create(self.pool, 100, 'localhost',
group=self.pool_group, options={})
self.pool_ctrl.create(self.pool, 100, 'localhost', options={})
self.addCleanup(self.pool_ctrl.delete, self.pool)
self.pool1 = str(uuid.uuid1())
self.flavor = str(uuid.uuid1())
self.pool_ctrl.create(self.pool1, 100, 'localhost1',
options={})
self.addCleanup(self.pool_ctrl.delete, self.pool1)
def tearDown(self):
self.controller.drop_all()
@ -1620,8 +1632,9 @@ class CatalogueControllerTest(ControllerBaseTest):
def test_update(self):
p2 = u'b'
# NOTE(gengchc2): Remove [group=self.pool_group] in
# it can be tested for redis as management.
self.pool_ctrl.create(p2, 100, '127.0.0.1',
group=self.pool_group,
options={})
self.addCleanup(self.pool_ctrl.delete, p2)
@ -1676,6 +1689,8 @@ class CatalogueControllerTest(ControllerBaseTest):
self.controller.insert(self.project, q2, u'a')
# NOTE(gengchc2): remove FlavorsControllerTest in Rocky release
# and use FlavorsControllerTest1 instead for pool_group removal.
class FlavorsControllerTest(ControllerBaseTest):
"""Flavors Controller base tests.
@ -1853,6 +1868,185 @@ class FlavorsControllerTest(ControllerBaseTest):
self.assertEqual({}, entry['capabilities'])
# NOTE(gengchc2): Unittest for new flavor configure scenario.
class FlavorsControllerTest1(ControllerBaseTest):
"""Flavors Controller base tests.
NOTE(flaper87): Implementations of this class should
override the tearDown method in order
to clean up storage's state.
"""
controller_base_class = storage.FlavorsBase
def setUp(self):
super(FlavorsControllerTest1, self).setUp()
self.pools_controller = self.driver.pools_controller
self.flavors_controller = self.driver.flavors_controller
# Let's create one pool
self.pool = str(uuid.uuid1())
self.flavor = 'durable'
self.pools_controller.create(self.pool, 100, 'localhost',
options={})
self.addCleanup(self.pools_controller.delete, self.pool)
def tearDown(self):
self.pools_controller.update(self.pool, flavor="")
self.flavors_controller.drop_all()
super(FlavorsControllerTest1, self).tearDown()
def test_create_succeeds(self):
self.flavors_controller.create(self.flavor,
project=self.project,
capabilities={})
def _flavors_expects(self, flavor, xname, xproject):
self.assertIn('name', flavor)
self.assertEqual(xname, flavor['name'])
self.assertNotIn('project', flavor)
def test_create_replaces_on_duplicate_insert(self):
name = str(uuid.uuid1())
self.flavors_controller.create(name,
project=self.project,
capabilities={})
entry = self.flavors_controller.get(name, self.project)
self._flavors_expects(entry, name, self.project)
new_capabilities = {'fifo': False}
self.flavors_controller.create(name,
project=self.project,
capabilities=new_capabilities)
entry = self.flavors_controller.get(name, project=self.project,
detailed=True)
self._flavors_expects(entry, name, self.project)
self.assertEqual(new_capabilities, entry['capabilities'])
def test_get_returns_expected_content(self):
name = 'durable'
capabilities = {'fifo': True}
self.flavors_controller.create(name,
project=self.project,
capabilities=capabilities)
res = self.flavors_controller.get(name, project=self.project)
self._flavors_expects(res, name, self.project)
self.assertNotIn('capabilities', res)
def test_detailed_get_returns_expected_content(self):
name = 'durable'
capabilities = {'fifo': True}
self.flavors_controller.create(name,
project=self.project,
capabilities=capabilities)
res = self.flavors_controller.get(name, project=self.project,
detailed=True)
self._flavors_expects(res, name, self.project)
self.assertIn('capabilities', res)
self.assertEqual(capabilities, res['capabilities'])
def test_get_raises_if_not_found(self):
self.assertRaises(errors.FlavorDoesNotExist,
self.flavors_controller.get, 'notexists')
def test_exists(self):
self.flavors_controller.create('exists',
project=self.project,
capabilities={})
self.assertTrue(self.flavors_controller.exists('exists',
project=self.project))
self.assertFalse(self.flavors_controller.exists('notexists',
project=self.project))
def test_update_raises_assertion_error_on_bad_fields(self):
self.assertRaises(AssertionError, self.pools_controller.update,
self.flavor)
def test_update_works(self):
name = 'yummy'
self.flavors_controller.create(name,
project=self.project,
capabilities={})
res = self.flavors_controller.get(name, project=self.project,
detailed=True)
p = 'olympic'
flavor = name
self.pools_controller.create(p, 100, 'localhost2',
flavor=flavor, options={})
self.addCleanup(self.pools_controller.delete, p)
new_capabilities = {'fifo': False}
self.flavors_controller.update(name, project=self.project,
capabilities={'fifo': False})
res = self.flavors_controller.get(name, project=self.project,
detailed=True)
self._flavors_expects(res, name, self.project)
self.assertEqual(new_capabilities, res['capabilities'])
self.pools_controller.update(p, flavor="")
def test_delete_works(self):
name = 'puke'
self.flavors_controller.create(name,
project=self.project,
capabilities={})
self.flavors_controller.delete(name, project=self.project)
self.assertFalse(self.flavors_controller.exists(name))
def test_delete_nonexistent_is_silent(self):
self.flavors_controller.delete('nonexisting')
def test_drop_all_leads_to_empty_listing(self):
self.flavors_controller.drop_all()
cursor = self.flavors_controller.list()
flavors = next(cursor)
self.assertRaises(StopIteration, next, flavors)
self.assertFalse(next(cursor))
def test_listing_simple(self):
name_gen = lambda i: chr(ord('A') + i)
for i in range(15):
pool = str(i)
flavor = name_gen(i)
uri = 'localhost:2701' + pool
self.pools_controller.create(pool, 100, uri,
flavor=flavor, options={})
self.addCleanup(self.pools_controller.delete, pool)
self.flavors_controller.create(flavor, project=self.project,
capabilities={})
def get_res(**kwargs):
cursor = self.flavors_controller.list(project=self.project,
**kwargs)
res = list(next(cursor))
marker = next(cursor)
self.assertTrue(marker)
return res
res = get_res()
self.assertEqual(10, len(res))
for i, entry in enumerate(res):
self._flavors_expects(entry, name_gen(i), self.project)
self.assertNotIn('capabilities', entry)
res = get_res(limit=5)
self.assertEqual(5, len(res))
res = get_res(marker=name_gen(3))
self._flavors_expects(res[0], name_gen(4), self.project)
res = get_res(detailed=True)
self.assertEqual(10, len(res))
for i, entry in enumerate(res):
self._flavors_expects(entry, name_gen(i), self.project)
self.assertIn('capabilities', entry)
self.assertEqual({}, entry['capabilities'])
# (gengchc): Remove the flavor from pool, then testcase cleanup pools
for i in range(15):
pool = str(i)
self.pools_controller.update(pool, flavor="")
def _insert_fixtures(controller, queue_name, project=None,
client_uuid=None, num=4, ttl=120):
def messages():

70
zaqar/tests/unit/storage/test_impl_mongodb.py
View File

@ -507,10 +507,14 @@ class MongodbPoolsTests(base.PoolsControllerTest):
def setUp(self):
super(MongodbPoolsTests, self).setUp()
self.pools_controller.create(self.pool, 100, 'localhost',
group=self.pool_group, options={})
def tearDown(self):
super(MongodbPoolsTests, self).tearDown()
# NOTE(gengchc2): remove test_delete_pool_used_by_flavor in Rocky release
# and use test_delete_pool_used_by_flavor1 instead for pool_group removal.
def test_delete_pool_used_by_flavor(self):
self.flavors_controller.create('durable', self.pool_group,
project=self.project,
@ -519,6 +523,19 @@ class MongodbPoolsTests(base.PoolsControllerTest):
with testing.expect(errors.PoolInUseByFlavor):
self.pools_controller.delete(self.pool)
# NOTE(gengchc2): Unittest for new flavor configure scenario.
def test_delete_pool_used_by_flavor1(self):
self.flavors_controller.create(self.flavor,
project=self.project,
capabilities={})
self.pools_controller.update(self.pool1, flavor=self.flavor)
with testing.expect(errors.PoolInUseByFlavor):
self.pools_controller.delete(self.pool1)
# NOTE(gengchc2): remove test_mismatching_capabilities_fifo in Rocky
# release and use test_mismatching_capabilities_fifo1 instead for
# pool_group removal.
def test_mismatching_capabilities_fifo(self):
with testing.expect(errors.PoolCapabilitiesMismatch):
self.pools_controller.create(str(uuid.uuid1()),
@ -526,6 +543,36 @@ class MongodbPoolsTests(base.PoolsControllerTest):
group=self.pool_group,
options={})
# NOTE(gengchc2): Unittest for new flavor configure scenario.
def test_mismatching_capabilities_fifo1(self):
with testing.expect(errors.PoolCapabilitiesMismatch):
self.pools_controller.create(str(uuid.uuid1()),
100, 'mongodb.fifo://localhost',
flavor=self.flavor,
options={})
# NOTE(gengchc2): remove test_mismatching_capabilities in Rocky release
# and use test_mismatching_capabilities1 instead for pool_group removal.
def test_mismatching_capabilities(self):
# NOTE(gengchc2): This test is used for testing mismatchming
# capabilities in pool with group
with testing.expect(errors.PoolCapabilitiesMismatch):
self.pools_controller.create(str(uuid.uuid1()),
100, 'redis://localhost',
group=self.pool_group,
options={})
def test_mismatching_capabilities1(self):
# NOTE(gengchc2): This test is used for testing mismatchming
# capabilities in pool with flavor
with testing.expect(errors.PoolCapabilitiesMismatch):
self.pools_controller.create(str(uuid.uuid1()),
100, 'redis://localhost',
flavor=self.flavor,
options={})
# NOTE(gengchc2): remove test_duplicate_uri in Rocky release and
# use test_duplicate_uri1 instead for pool_group removal.
def test_duplicate_uri(self):
with testing.expect(errors.PoolAlreadyExists):
# The url 'localhost' is used in setUp(). So reusing the uri
@ -533,6 +580,14 @@ class MongodbPoolsTests(base.PoolsControllerTest):
self.pools_controller.create(str(uuid.uuid1()), 100, 'localhost',
group=str(uuid.uuid1()), options={})
# NOTE(gengchc2): Unittest for new flavor configure scenario.
def test_duplicate_uri1(self):
with testing.expect(errors.PoolAlreadyExists):
# The url 'localhost' is used in setUp(). So reusing the uri
# 'localhost' here will raise PoolAlreadyExists.
self.pools_controller.create(str(uuid.uuid1()), 100, 'localhost',
flavor=str(uuid.uuid1()), options={})
@testing.requires_mongodb
class MongodbCatalogueTests(base.CatalogueControllerTest):
@ -577,6 +632,8 @@ class PooledClaimsTests(base.ClaimControllerTest):
self.skip("Fix sqlalchemy driver")
# NOTE(gengchc2): remove MongodbFlavorsTest in Rocky release and
# use MongodbFlavorsTest1 instead for pool_group removal.
@testing.requires_mongodb
class MongodbFlavorsTest(base.FlavorsControllerTest):
driver_class = mongodb.ControlDriver
@ -587,3 +644,16 @@ class MongodbFlavorsTest(base.FlavorsControllerTest):
def setUp(self):
super(MongodbFlavorsTest, self).setUp()
self.addCleanup(self.controller.drop_all)
# NOTE(gengchc2): Unittest for new flavor configure scenario.
@testing.requires_mongodb
class MongodbFlavorsTest1(base.FlavorsControllerTest1):
driver_class = mongodb.ControlDriver
controller_class = controllers.FlavorsController
control_driver_class = mongodb.ControlDriver
config_file = 'wsgi_mongodb.conf'
def setUp(self):
super(MongodbFlavorsTest1, self).setUp()
self.addCleanup(self.controller.drop_all)

37
zaqar/tests/unit/storage/test_impl_sqlalchemy.py
View File

@ -13,7 +13,9 @@
# the License.
import six
import uuid
from zaqar import storage
from zaqar.storage import sqlalchemy
from zaqar.storage.sqlalchemy import controllers
from zaqar.storage.sqlalchemy import tables
@ -41,6 +43,31 @@ class SqlalchemyPoolsTest(DBCreateMixin, base.PoolsControllerTest):
controller_class = controllers.PoolsController
control_driver_class = sqlalchemy.ControlDriver
def setUp(self):
super(SqlalchemyPoolsTest, self).setUp()
self.pools_controller.create(self.pool, 100, 'localhost',
group=self.pool_group, options={})
# NOTE(gengchc2): remove test_mismatching_capabilities in Rocky release
# and use test_mismatching_capabilities1 instead for pool_group removal.
def test_mismatching_capabilities(self):
# NOTE(gengchc2): This test is used for testing mismatchming
# capabilities in pool with group
with testing.expect(storage.errors.PoolCapabilitiesMismatch):
self.pools_controller.create(str(uuid.uuid1()),
100, 'redis://localhost',
group=self.pool_group,
options={})
def test_mismatching_capabilities1(self):
# NOTE(gengchc2): This test is used for testing mismatchming
# capabilities in pool with flavor
with testing.expect(storage.errors.PoolCapabilitiesMismatch):
self.pools_controller.create(str(uuid.uuid1()),
100, 'redis://localhost',
flavor=self.flavor,
options={})
class SqlalchemyCatalogueTest(DBCreateMixin, base.CatalogueControllerTest):
config_file = 'wsgi_sqlalchemy.conf'
@ -49,6 +76,8 @@ class SqlalchemyCatalogueTest(DBCreateMixin, base.CatalogueControllerTest):
control_driver_class = sqlalchemy.ControlDriver
# NOTE(gengchc2): remove SqlalchemyFlavorsTest in Rocky release and
# use SqlalchemyFlavorsTest1 instead for pool_group removal.
class SqlalchemyFlavorsTest(DBCreateMixin, base.FlavorsControllerTest):
config_file = 'wsgi_sqlalchemy.conf'
driver_class = sqlalchemy.ControlDriver
@ -56,6 +85,14 @@ class SqlalchemyFlavorsTest(DBCreateMixin, base.FlavorsControllerTest):
control_driver_class = sqlalchemy.ControlDriver
# NOTE(gengchc2): Unittest for new flavor configure scenario.
class SqlalchemyFlavorsTest1(DBCreateMixin, base.FlavorsControllerTest1):
config_file = 'wsgi_sqlalchemy.conf'
driver_class = sqlalchemy.ControlDriver
controller_class = controllers.FlavorsController
control_driver_class = sqlalchemy.ControlDriver
class MsgidTests(testing.TestBase):
def test_encode(self):

128
zaqar/tests/unit/storage/test_pool_catalog_new.py Normal file
View File

@ -0,0 +1,128 @@
# Copyright (c) 2017 ZTE Corporation..
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
import mock
import uuid
from zaqar.common import cache as oslo_cache
from zaqar.storage import errors
from zaqar.storage import mongodb
from zaqar.storage import pooling
from zaqar.storage import utils
from zaqar import tests as testing
# TODO(cpp-cabrera): it would be wonderful to refactor this unit test
# so that it could use multiple control storage backends once those
# have pools/catalogue implementations.
@testing.requires_mongodb
class PoolCatalogTest(testing.TestBase):
config_file = 'wsgi_mongodb_pooled_disable_virtual_pool.conf'
def setUp(self):
super(PoolCatalogTest, self).setUp()
oslo_cache.register_config(self.conf)
cache = oslo_cache.get_cache(self.conf)
control = utils.load_storage_driver(self.conf, cache,
control_mode=True)
self.pools_ctrl = control.pools_controller
self.flavors_ctrl = control.flavors_controller
self.catalogue_ctrl = control.catalogue_controller
# NOTE(cpp-cabrera): populate catalogue
self.pool = str(uuid.uuid1())
self.pool2 = str(uuid.uuid1())
self.queue = str(uuid.uuid1())
self.flavor = str(uuid.uuid1())
self.project = str(uuid.uuid1())
# FIXME(therve) This is horrible, we need to manage duplication in a
# nicer way
if 'localhost' in self.mongodb_url:
other_url = self.mongodb_url.replace('localhost', '127.0.0.1')
elif '127.0.0.1' in self.mongodb_url:
other_url = self.mongodb_url.replace('127.0.0.1', 'localhost')
else:
self.skipTest("Can't build a dummy mongo URL.")
self.pools_ctrl.create(self.pool, 100, self.mongodb_url)
self.pools_ctrl.create(self.pool2, 100,
other_url)
self.catalogue_ctrl.insert(self.project, self.queue, self.pool)
self.catalog = pooling.Catalog(self.conf, cache, control)
self.flavors_ctrl.create(self.flavor,
project=self.project)
self.pools_ctrl.update(self.pool2, flavor=self.flavor)
def tearDown(self):
self.catalogue_ctrl.drop_all()
self.pools_ctrl.drop_all()
super(PoolCatalogTest, self).tearDown()
def test_lookup_loads_correct_driver(self):
storage = self.catalog.lookup(self.queue, self.project)
self.assertIsInstance(storage._storage, mongodb.DataDriver)
def test_lookup_returns_default_or_none_if_queue_not_mapped(self):
# Return default
self.assertIsNone(self.catalog.lookup('not', 'mapped'))
self.config(message_store='faulty', group='drivers')
self.config(enable_virtual_pool=True, group='pooling:catalog')
self.assertIsNotNone(self.catalog.lookup('not', 'mapped'))
def test_lookup_returns_none_if_entry_deregistered(self):
self.catalog.deregister(self.queue, self.project)
self.assertIsNone(self.catalog.lookup(self.queue, self.project))
def test_register_leads_to_successful_lookup(self):
self.catalog.register('not_yet', 'mapped')
storage = self.catalog.lookup('not_yet', 'mapped')
self.assertIsInstance(storage._storage, mongodb.DataDriver)
def test_register_with_flavor(self):
queue = 'test'
self.catalog.register(queue, project=self.project,
flavor=self.flavor)
storage = self.catalog.lookup(queue, self.project)
self.assertIsInstance(storage._storage, mongodb.DataDriver)
def test_register_with_fake_flavor(self):
self.assertRaises(errors.FlavorDoesNotExist,
self.catalog.register,
'test', project=self.project,
flavor='fake')
def test_queues_list_on_multi_pools(self):
def fake_list(project=None, marker=None, limit=10, detailed=False):
yield iter([{'name': 'fake_queue'}])
list_str = 'zaqar.storage.mongodb.queues.QueueController.list'
with mock.patch(list_str) as queues_list:
queues_list.side_effect = fake_list
queue_controller = pooling.QueueController(self.catalog)
result = queue_controller.list(project=self.project)
queue_list = list(next(result))
self.assertEqual(1, len(queue_list))
def test_queue_create_with_empty_json_body(self):
queue_controller = pooling.QueueController(self.catalog)
with mock.patch('zaqar.storage.pooling.Catalog.register') as register:
queue_controller.create(self.queue, metadata={},
project=self.project)
register.assert_called_with(self.queue, project=self.project,
flavor=None)

1
zaqar/tests/unit/transport/wsgi/v2_0/test_flavors.py
View File

@ -23,6 +23,7 @@ from zaqar import tests as testing
from zaqar.tests.unit.transport.wsgi import base
# NOTE(gengchc2): remove pool_group in Rocky release.
@contextlib.contextmanager
def flavor(test, name, pool_group):
"""A context manager for constructing a flavor for use in testing.

354
zaqar/tests/unit/transport/wsgi/v2_0/test_flavors_new.py Normal file
View File

@ -0,0 +1,354 @@
# Copyright (c) 2017 ZTE Corporation.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
import contextlib
import uuid
import ddt
import falcon
from oslo_serialization import jsonutils
from zaqar import tests as testing
from zaqar.tests.unit.transport.wsgi import base
@contextlib.contextmanager
def flavor(test, name, pool_list):
"""A context manager for constructing a flavor for use in testing.
Deletes the flavor after exiting the context.
:param test: Must expose simulate_* methods
:param name: Name for this flavor
:type name: six.text_type
:type pool: six.text_type
:returns: (name, uri, capabilities)
:rtype: see above
"""
doc = {'pool_list': pool_list}
path = test.url_prefix + '/flavors/' + name
test.simulate_put(path, body=jsonutils.dumps(doc))
try:
yield name, pool_list
finally:
test.simulate_delete(path)
@contextlib.contextmanager
def flavors(test, count):
"""A context manager for constructing flavors for use in testing.
Deletes the flavors after exiting the context.
:param test: Must expose simulate_* methods
:param count: Number of pools to create
:type count: int
:returns: (paths, pool_list capabilities)
:rtype: ([six.text_type], [six.text_type], [dict])
"""
pool_path_all = []
flavor_path_all = []
for i in range(count):
poolname = 'pool' + str(i)
pool_doc = {'weight': 100,
'uri': test.mongodb_url + '/test' + str(i)}
pool_path = test.url_prefix + '/pools/' + poolname
test.simulate_put(pool_path, body=jsonutils.dumps(pool_doc))
flavorname = str(i)
flavor_path = test.url_prefix + "/flavors/" + flavorname
flavor_doc = {'pool_list': [poolname]}
test.simulate_put(flavor_path, body=jsonutils.dumps(flavor_doc))
pool_path_all.append(pool_path)
flavor_path_all.append(flavor_path)
try:
yield flavor_path_all
finally:
for path in flavor_path_all:
test.simulate_delete(path)
for path in pool_path_all:
test.simulate_delete(path)
@ddt.ddt
class TestFlavorsMongoDB(base.V2Base):
config_file = 'wsgi_mongodb_pooled.conf'
@testing.requires_mongodb
def setUp(self):
super(TestFlavorsMongoDB, self).setUp()
self.queue = 'test-queue'
self.queue_path = self.url_prefix + '/queues/' + self.queue
self.pool = 'mypool'
self.pool_path = self.url_prefix + '/pools/' + self.pool
self.pool_doc = {'weight': 100,
'uri': self.mongodb_url + '/test'}
self.simulate_put(self.pool_path, body=jsonutils.dumps(self.pool_doc))
self.flavor = 'test-flavor'
self.doc = {'capabilities': {}}
self.doc['pool_list'] = [self.pool]
self.flavor_path = self.url_prefix + '/flavors/' + self.flavor
self.simulate_put(self.flavor_path, body=jsonutils.dumps(self.doc))
self.assertEqual(falcon.HTTP_201, self.srmock.status)
def tearDown(self):
self.simulate_delete(self.queue_path)
self.simulate_delete(self.flavor_path)
self.assertEqual(falcon.HTTP_204, self.srmock.status)
self.simulate_delete(self.pool_path)
super(TestFlavorsMongoDB, self).tearDown()
def test_put_flavor_works(self):
name = str(uuid.uuid1())
with flavor(self, name, self.doc['pool_list']):
self.assertEqual(falcon.HTTP_201, self.srmock.status)
def test_put_raises_if_missing_fields(self):
path = self.url_prefix + '/flavors/' + str(uuid.uuid1())
self.simulate_put(path, body=jsonutils.dumps({}))
self.assertEqual(falcon.HTTP_400, self.srmock.status)
self.simulate_put(path,
body=jsonutils.dumps({'capabilities': {}}))
self.assertEqual(falcon.HTTP_400, self.srmock.status)
@ddt.data(1, 2**32+1, [])
def test_put_raises_if_invalid_pool(self, pool_list):
path = self.url_prefix + '/flavors/' + str(uuid.uuid1())
self.simulate_put(path,
body=jsonutils.dumps({'pool_list': pool_list}))
self.assertEqual(falcon.HTTP_400, self.srmock.status)
def test_put_auto_get_capabilities(self):
path = self.url_prefix + '/flavors/' + str(uuid.uuid1())
doc = {'pool_list': self.doc['pool_list']}
self.simulate_put(path, body=jsonutils.dumps(doc))
self.assertEqual(falcon.HTTP_201, self.srmock.status)
# NOTE(gengchc2): Delete it, otherwise exist garbage flavor.
self.simulate_delete(path)
def test_put_existing_overwrites(self):
# NOTE(cabrera): setUp creates default flavor
expect = self.doc
self.simulate_put(self.flavor_path,
body=jsonutils.dumps(expect))
self.assertEqual(falcon.HTTP_201, self.srmock.status)
result = self.simulate_get(self.flavor_path)
self.assertEqual(falcon.HTTP_200, self.srmock.status)
doc = jsonutils.loads(result[0])
self.assertEqual(expect['pool_list'], doc['pool_list'])
def test_create_flavor_no_pool_list(self):
self.simulate_delete(self.flavor_path)
self.assertEqual(falcon.HTTP_204, self.srmock.status)
self.simulate_delete(self.pool_path)
self.assertEqual(falcon.HTTP_204, self.srmock.status)
resp = self.simulate_put(self.flavor_path,
body=jsonutils.dumps(self.doc))
self.assertEqual(falcon.HTTP_400, self.srmock.status)
self.assertEqual(
{'description': 'Flavor test-flavor could not be created, '
'error:Pool mypool does not exist',
'title': 'Unable to create'},
jsonutils.loads(resp[0]))
def test_delete_works(self):
self.simulate_delete(self.flavor_path)
self.assertEqual(falcon.HTTP_204, self.srmock.status)
self.simulate_get(self.flavor_path)
self.assertEqual(falcon.HTTP_404, self.srmock.status)
def test_get_nonexisting_raises_404(self):
self.simulate_get(self.url_prefix + '/flavors/nonexisting')
self.assertEqual(falcon.HTTP_404, self.srmock.status)
def _flavor_expect(self, flavor, xhref, xpool_list=None):
self.assertIn('href', flavor)
self.assertIn('name', flavor)
self.assertEqual(xhref, flavor['href'])
if xpool_list is not None:
self.assertIn('pool_list', flavor)
self.assertEqual(xpool_list, flavor['pool_list'])
def test_get_works(self):
result = self.simulate_get(self.flavor_path)
self.assertEqual(falcon.HTTP_200, self.srmock.status)
flavor = jsonutils.loads(result[0])
self._flavor_expect(flavor, self.flavor_path, self.doc['pool_list'])
store_caps = ['FIFO', 'CLAIMS', 'DURABILITY',
'AOD', 'HIGH_THROUGHPUT']
self.assertEqual(store_caps, flavor['capabilities'])
def test_patch_raises_if_missing_fields(self):
self.simulate_patch(self.flavor_path,
body=jsonutils.dumps({'location': 1}))
self.assertEqual(falcon.HTTP_400, self.srmock.status)
def _patch_test(self, doc):
result = self.simulate_patch(self.flavor_path,
body=jsonutils.dumps(doc))
self.assertEqual(falcon.HTTP_200, self.srmock.status)
updated_flavor = jsonutils.loads(result[0])
self._flavor_expect(updated_flavor, self.flavor_path)
capabilities = ['FIFO', 'CLAIMS', 'DURABILITY', 'AOD',
'HIGH_THROUGHPUT']
self.assertEqual(capabilities, updated_flavor['capabilities'])
result = self.simulate_get(self.flavor_path)
self.assertEqual(falcon.HTTP_200, self.srmock.status)
flavor = jsonutils.loads(result[0])
self._flavor_expect(flavor, self.flavor_path)
self.assertEqual(capabilities, flavor['capabilities'])
def test_patch_works(self):
doc = {'pool_list': self.doc['pool_list'], 'capabilities': []}
self._patch_test(doc)
def test_patch_works_with_extra_fields(self):
doc = {'pool_list': self.doc['pool_list'], 'capabilities': [],
'location': 100, 'partition': 'taco'}
self._patch_test(doc)
@ddt.data(-1, 2**32+1, [])
def test_patch_raises_400_on_invalid_pool_list(self, pool_list):
self.simulate_patch(self.flavor_path,
body=jsonutils.dumps({'pool_list': pool_list}))
self.assertEqual(falcon.HTTP_400, self.srmock.status)
@ddt.data(-1, 'wee', [])
def test_patch_raises_400_on_invalid_capabilities(self, capabilities):
doc = {'capabilities': capabilities}
self.simulate_patch(self.flavor_path, body=jsonutils.dumps(doc))
self.assertEqual(falcon.HTTP_400, self.srmock.status)
def test_patch_raises_404_if_flavor_not_found(self):
self.simulate_patch(self.url_prefix + '/flavors/notexists',
body=jsonutils.dumps({'pool_list': ['test']}))
self.assertEqual(falcon.HTTP_404, self.srmock.status)
def test_empty_listing(self):
self.simulate_delete(self.flavor_path)
result = self.simulate_get(self.url_prefix + '/flavors')
results = jsonutils.loads(result[0])
self.assertEqual(falcon.HTTP_200, self.srmock.status)
self.assertEqual(0, len(results['flavors']))
self.assertIn('links', results)
def _listing_test(self, count=10, limit=10,
marker=None, detailed=False):
# NOTE(cpp-cabrera): delete initial flavor - it will interfere
# with listing tests
self.simulate_delete(self.flavor_path)
query = 'limit={0}&detailed={1}'.format(limit, detailed)
if marker:
query += '&marker={2}'.format(marker)
with flavors(self, count):
result = self.simulate_get(self.url_prefix + '/flavors',
query_string=query)
self.assertEqual(falcon.HTTP_200, self.srmock.status)
results = jsonutils.loads(result[0])
self.assertIsInstance(results, dict)
self.assertIn('flavors', results)
self.assertIn('links', results)
flavors_list = results['flavors']
link = results['links'][0]
self.assertEqual('next', link['rel'])
href = falcon.uri.parse_query_string(link['href'].split('?')[1])
self.assertIn('marker', href)
self.assertEqual(str(limit), href['limit'])
self.assertEqual(str(detailed).lower(), href['detailed'])
next_query_string = ('marker={marker}&limit={limit}'
'&detailed={detailed}').format(**href)
next_result = self.simulate_get(link['href'].split('?')[0],
query_string=next_query_string)
next_flavors = jsonutils.loads(next_result[0])
next_flavors_list = next_flavors['flavors']
self.assertEqual(falcon.HTTP_200, self.srmock.status)
self.assertIn('links', next_flavors)
if limit < count:
self.assertEqual(min(limit, count-limit),
len(next_flavors_list))
else:
self.assertEqual(0, len(next_flavors_list))
self.assertEqual(min(limit, count), len(flavors_list))
for i, s in enumerate(flavors_list + next_flavors_list):
capabilities = ['FIFO', 'CLAIMS', 'DURABILITY',
'AOD', 'HIGH_THROUGHPUT']
if detailed:
self.assertIn('capabilities', s)
self.assertEqual(s['capabilities'], capabilities)
else:
self.assertNotIn('capabilities', s)
def test_listing_works(self):
self._listing_test()
def test_detailed_listing_works(self):
self._listing_test(detailed=True)
@ddt.data(1, 5, 10, 15)
def test_listing_works_with_limit(self, limit):
self._listing_test(count=15, limit=limit)
def test_listing_marker_is_respected(self):
self.simulate_delete(self.flavor_path)
with flavors(self, 10) as expected:
result = self.simulate_get(self.url_prefix + '/flavors',
query_string='marker=3')
self.assertEqual(falcon.HTTP_200, self.srmock.status)
flavor_list = jsonutils.loads(result[0])['flavors']
self.assertEqual(6, len(flavor_list))
path = expected[4]
self._flavor_expect(flavor_list[0], path)
def test_listing_error_with_invalid_limit(self):
self.simulate_delete(self.flavor_path)
query = 'limit={0}&detailed={1}'.format(0, True)
with flavors(self, 10):
self.simulate_get(self.url_prefix + '/flavors', query_string=query)
self.assertEqual(falcon.HTTP_400, self.srmock.status)
def test_queue_create_works(self):
metadata = {'_flavor': self.flavor}
self.simulate_put(self.queue_path, body=jsonutils.dumps(metadata))
self.assertEqual(falcon.HTTP_201, self.srmock.status)
def test_queue_create_no_flavor(self):
metadata = {'_flavor': self.flavor}
self.simulate_delete(self.flavor_path)
self.assertEqual(falcon.HTTP_204, self.srmock.status)
self.simulate_put(self.queue_path, body=jsonutils.dumps(metadata))
self.assertEqual(falcon.HTTP_400, self.srmock.status)

1
zaqar/tests/unit/transport/wsgi/v2_0/test_pools.py
View File

@ -23,6 +23,7 @@ from zaqar import tests as testing
from zaqar.tests.unit.transport.wsgi import base
# NOTE(gengchc2): remove pool_group in Rocky release.
@contextlib.contextmanager
def pool(test, name, weight, uri, group=None, options={}):
"""A context manager for constructing a pool for use in testing.

383
zaqar/tests/unit/transport/wsgi/v2_0/test_pools_new.py Normal file
View File

@ -0,0 +1,383 @@
# Copyright (c) 2017 ZTE Corporation.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
import contextlib
import ddt
import falcon
from oslo_serialization import jsonutils
from oslo_utils import uuidutils
from zaqar import tests as testing
from zaqar.tests.unit.transport.wsgi import base
@contextlib.contextmanager
def pool(test, name, weight, uri, flavor=None, options={}):
"""A context manager for constructing a pool for use in testing.
Deletes the pool after exiting the context.
:param test: Must expose simulate_* methods
:param name: Name for this pool
:type name: six.text_type
:type weight: int
:type uri: six.text_type
:type options: dict
:returns: (name, weight, uri, options)
:rtype: see above
"""
uri = "%s/%s" % (uri, uuidutils.generate_uuid())
doc = {'weight': weight, 'uri': uri,
'flavor': flavor, 'options': options}
path = test.url_prefix + '/pools/' + name
test.simulate_put(path, body=jsonutils.dumps(doc))
try:
yield name, weight, uri, flavor, options
finally:
test.simulate_delete(path)
@contextlib.contextmanager
def pools(test, count, uri, flavor):
"""A context manager for constructing pools for use in testing.
Deletes the pools after exiting the context.
:param test: Must expose simulate_* methods
:param count: Number of pools to create
:type count: int
:returns: (paths, weights, uris, options)
:rtype: ([six.text_type], [int], [six.text_type], [dict])
"""
mongo_url = uri
base = test.url_prefix + '/pools/'
args = [(base + str(i), i,
{str(i): i})
for i in range(count)]
for path, weight, option in args:
uri = "%s/%s" % (mongo_url, uuidutils.generate_uuid())
doc = {'weight': weight, 'uri': uri,
'flavor': flavor, 'options': option}
test.simulate_put(path, body=jsonutils.dumps(doc))
try:
yield args
finally:
for path, _, _ in args:
# (gengchc): Remove flavor from the pool,
# so we can delete the pool.
test.simulate_patch(path,
body=jsonutils.dumps({'flavor': ''}))
test.simulate_delete(path)
@ddt.ddt
class TestPoolsMongoDB(base.V2Base):
config_file = 'wsgi_mongodb_pooled.conf'
@testing.requires_mongodb
def setUp(self):
super(TestPoolsMongoDB, self).setUp()
self.doc = {'weight': 100,
'flavor': 'my-flavor',
'uri': self.mongodb_url}
self.pool = self.url_prefix + '/pools/' + uuidutils.generate_uuid()
self.simulate_put(self.pool, body=jsonutils.dumps(self.doc))
self.assertEqual(falcon.HTTP_201, self.srmock.status)
def tearDown(self):
super(TestPoolsMongoDB, self).tearDown()
self.simulate_delete(self.pool)
self.assertEqual(falcon.HTTP_204, self.srmock.status)
def test_put_pool_works(self):
name = uuidutils.generate_uuid()
weight, uri = self.doc['weight'], self.doc['uri']
with pool(self, name, weight, uri, flavor='my-flavor'):
self.assertEqual(falcon.HTTP_201, self.srmock.status)
def test_put_raises_if_missing_fields(self):
path = self.url_prefix + '/pools/' + uuidutils.generate_uuid()
self.simulate_put(path, body=jsonutils.dumps({'weight': 100}))
self.assertEqual(falcon.HTTP_400, self.srmock.status)
self.simulate_put(path,
body=jsonutils.dumps(
{'uri': self.mongodb_url}))
self.assertEqual(falcon.HTTP_400, self.srmock.status)
@ddt.data(-1, 2**32+1, 'big')
def test_put_raises_if_invalid_weight(self, weight):
path = self.url_prefix + '/pools/' + uuidutils.generate_uuid()
doc = {'weight': weight, 'uri': 'a'}
self.simulate_put(path,
body=jsonutils.dumps(doc))
self.assertEqual(falcon.HTTP_400, self.srmock.status)
@ddt.data(-1, 2**32+1, [], 'localhost:27017')
def test_put_raises_if_invalid_uri(self, uri):
path = self.url_prefix + '/pools/' + uuidutils.generate_uuid()
self.simulate_put(path,
body=jsonutils.dumps({'weight': 1, 'uri': uri}))
self.assertEqual(falcon.HTTP_400, self.srmock.status)
@ddt.data(-1, 'wee', [])
def test_put_raises_if_invalid_options(self, options):
path = self.url_prefix + '/pools/' + uuidutils.generate_uuid()
doc = {'weight': 1, 'uri': 'a', 'options': options}
self.simulate_put(path, body=jsonutils.dumps(doc))
self.assertEqual(falcon.HTTP_400, self.srmock.status)
def test_put_same_database_uri(self):
# NOTE(cabrera): setUp creates default pool
expect = self.doc
path = self.url_prefix + '/pools/' + uuidutils.generate_uuid()
self.simulate_put(path, body=jsonutils.dumps(expect))
self.assertEqual(falcon.HTTP_409, self.srmock.status)
def test_put_existing_overwrites(self):
# NOTE(cabrera): setUp creates default pool
expect = self.doc
self.simulate_put(self.pool,
body=jsonutils.dumps(expect))
self.assertEqual(falcon.HTTP_201, self.srmock.status)
result = self.simulate_get(self.pool)
self.assertEqual(falcon.HTTP_200, self.srmock.status)
doc = jsonutils.loads(result[0])
self.assertEqual(expect['weight'], doc['weight'])
self.assertEqual(expect['uri'], doc['uri'])
def test_put_capabilities_mismatch_pool(self):
mongodb_doc = self.doc
self.simulate_put(self.pool,
body=jsonutils.dumps(mongodb_doc))
self.assertEqual(falcon.HTTP_201, self.srmock.status)
redis_doc = {'weight': 100,
'flavor': 'my-flavor',
'uri': 'redis://127.0.0.1:6379'}
self.simulate_put(self.pool,
body=jsonutils.dumps(redis_doc))
self.assertEqual(falcon.HTTP_400, self.srmock.status)
def test_delete_works(self):
# (gengchc): Remove flavor from the pool, so we can delete the pool.
self.simulate_patch(self.pool, body=jsonutils.dumps({'flavor': ''}))
self.simulate_delete(self.pool)
self.assertEqual(falcon.HTTP_204, self.srmock.status)
self.simulate_get(self.pool)
self.assertEqual(falcon.HTTP_404, self.srmock.status)
def test_get_nonexisting_raises_404(self):
self.simulate_get(self.url_prefix + '/pools/nonexisting')
self.assertEqual(falcon.HTTP_404, self.srmock.status)
def _pool_expect(self, pool, xhref, xweight, xuri):
self.assertIn('href', pool)
self.assertIn('name', pool)
self.assertEqual(xhref, pool['href'])
self.assertIn('weight', pool)
self.assertEqual(xweight, pool['weight'])
self.assertIn('uri', pool)
# NOTE(dynarro): we are using startwith because we are adding to
# pools UUIDs, to avoid dupplications
self.assertTrue(pool['uri'].startswith(xuri))
def test_get_works(self):
result = self.simulate_get(self.pool)
self.assertEqual(falcon.HTTP_200, self.srmock.status)
pool = jsonutils.loads(result[0])
self._pool_expect(pool, self.pool, self.doc['weight'],
self.doc['uri'])
def test_detailed_get_works(self):
result = self.simulate_get(self.pool,
query_string='detailed=True')
self.assertEqual(falcon.HTTP_200, self.srmock.status)
pool = jsonutils.loads(result[0])
self._pool_expect(pool, self.pool, self.doc['weight'],
self.doc['uri'])
self.assertIn('options', pool)
self.assertEqual({}, pool['options'])
def test_patch_raises_if_missing_fields(self):
self.simulate_patch(self.pool,
body=jsonutils.dumps({'location': 1}))
self.assertEqual(falcon.HTTP_400, self.srmock.status)
def _patch_test(self, doc):
result = self.simulate_patch(self.pool,
body=jsonutils.dumps(doc))
self.assertEqual(falcon.HTTP_200, self.srmock.status)
updated_pool = jsonutils.loads(result[0])
self._pool_expect(updated_pool, self.pool, doc['weight'],
doc['uri'])
result = self.simulate_get(self.pool,
query_string='detailed=True')
self.assertEqual(falcon.HTTP_200, self.srmock.status)
pool = jsonutils.loads(result[0])
self._pool_expect(pool, self.pool, doc['weight'],
doc['uri'])
self.assertEqual(doc['options'], pool['options'])
def test_patch_works(self):
doc = {'weight': 101,
'uri': self.mongodb_url,
'options': {'a': 1}}
self._patch_test(doc)
def test_patch_works_with_extra_fields(self):
doc = {'weight': 101,
'uri': self.mongodb_url,
'options': {'a': 1},
'location': 100,
'partition': 'taco'}
self._patch_test(doc)
@ddt.data(-1, 2**32+1, 'big')
def test_patch_raises_400_on_invalid_weight(self, weight):
self.simulate_patch(self.pool,
body=jsonutils.dumps({'weight': weight}))
self.assertEqual(falcon.HTTP_400, self.srmock.status)
@ddt.data(-1, 2**32+1, [], 'localhost:27017')
def test_patch_raises_400_on_invalid_uri(self, uri):
self.simulate_patch(self.pool,
body=jsonutils.dumps({'uri': uri}))
self.assertEqual(falcon.HTTP_400, self.srmock.status)
@ddt.data(-1, 'wee', [])
def test_patch_raises_400_on_invalid_options(self, options):
self.simulate_patch(self.pool,
body=jsonutils.dumps({'options': options}))
self.assertEqual(falcon.HTTP_400, self.srmock.status)
def test_patch_raises_404_if_pool_not_found(self):
self.simulate_patch(self.url_prefix + '/pools/notexists',
body=jsonutils.dumps({'weight': 1}))
self.assertEqual(falcon.HTTP_404, self.srmock.status)
def test_empty_listing(self):
# (gengchc): Remove flavor from the pool, so we can delete the pool.
self.simulate_patch(self.pool, body=jsonutils.dumps({'flavor': ''}))
self.simulate_delete(self.pool)
result = self.simulate_get(self.url_prefix + '/pools')
results = jsonutils.loads(result[0])
self.assertEqual(falcon.HTTP_200, self.srmock.status)
self.assertEqual(0, len(results['pools']))
self.assertIn('links', results)
def _listing_test(self, count=10, limit=10,
marker=None, detailed=False):
# NOTE(cpp-cabrera): delete initial pool - it will interfere
# with listing tests
# (gengchc): Remove flavor from the pool, so we can delete the pool.
self.simulate_patch(self.pool, body=jsonutils.dumps({'flavor': ''}))
self.simulate_delete(self.pool)
query = 'limit={0}&detailed={1}'.format(limit, detailed)
if marker:
query += '&marker={0}'.format(marker)
with pools(self, count, self.doc['uri'], 'my-flavor') as expected:
result = self.simulate_get(self.url_prefix + '/pools',
query_string=query)
self.assertEqual(falcon.HTTP_200, self.srmock.status)
results = jsonutils.loads(result[0])
self.assertIsInstance(results, dict)
self.assertIn('pools', results)
self.assertIn('links', results)
pool_list = results['pools']
link = results['links'][0]
self.assertEqual('next', link['rel'])
href = falcon.uri.parse_query_string(link['href'].split('?')[1])
self.assertIn('marker', href)
self.assertEqual(str(limit), href['limit'])
self.assertEqual(str(detailed).lower(), href['detailed'])
next_query_string = ('marker={marker}&limit={limit}'
'&detailed={detailed}').format(**href)
next_result = self.simulate_get(link['href'].split('?')[0],
query_string=next_query_string)
self.assertEqual(falcon.HTTP_200, self.srmock.status)
next_pool = jsonutils.loads(next_result[0])
next_pool_list = next_pool['pools']
self.assertIn('links', next_pool)
if limit < count:
self.assertEqual(min(limit, count-limit),
len(next_pool_list))
else:
# NOTE(jeffrey4l): when limit >= count, there will be no
# pools in the 2nd page.
self.assertEqual(0, len(next_pool_list))
self.assertEqual(min(limit, count), len(pool_list))
for s in pool_list + next_pool_list:
# NOTE(flwang): It can't assumed that both sqlalchemy and
# mongodb can return query result with the same order. Just
# like the order they're inserted. Actually, sqlalchemy can't
# guarantee that. So we're leveraging the relationship between
# pool weight and the index of pools fixture to get the
# right pool to verify.
expect = expected[s['weight']]
path, weight, options = expect[:3]
self._pool_expect(s, path, weight, self.doc['uri'])
if detailed:
self.assertIn('options', s)
self.assertEqual(s['options'], expect[-1])
else:
self.assertNotIn('options', s)
def test_listing_works(self):
self._listing_test()
def test_detailed_listing_works(self):
self._listing_test(detailed=True)
@ddt.data(1, 5, 10, 15)
def test_listing_works_with_limit(self, limit):
self._listing_test(count=15, limit=limit)
def test_listing_marker_is_respected(self):
# (gengchc): Remove flavor from the pool, so we can delete the pool.
self.simulate_patch(self.pool, body=jsonutils.dumps({'flavor': ''}))
self.simulate_delete(self.pool)
with pools(self, 10, self.doc['uri'], 'my-flavor') as expected:
result = self.simulate_get(self.url_prefix + '/pools',
query_string='marker=3')
self.assertEqual(falcon.HTTP_200, self.srmock.status)
pool_list = jsonutils.loads(result[0])['pools']
self.assertEqual(6, len(pool_list))
path, weight = expected[4][:2]
self._pool_expect(pool_list[0], path, weight, self.doc['uri'])
def test_listing_error_with_invalid_limit(self):
self.simulate_delete(self.pool)
query = 'limit={0}&detailed={1}'.format(0, True)
with pools(self, 10, self.doc['uri'], 'my-flavor'):
self.simulate_get(self.url_prefix + '/pools', query_string=query)
self.assertEqual(falcon.HTTP_400, self.srmock.status)

289
zaqar/transport/wsgi/v2_0/flavors.py
View File

@ -20,7 +20,6 @@ import six
from zaqar.common.api.schemas import flavors as schema
from zaqar.common import decorators
from zaqar.common import utils as common_utils
from zaqar.i18n import _
from zaqar.storage import errors
from zaqar.transport import acl
@ -52,7 +51,8 @@ class Listing(object):
{
"flavors": [
{"href": "", "capabilities": {}, "pool_group": ""},
{"href": "", "capabilities": {}, "pool_group": "",
"pool_list": ""},
...
],
"links": [
@ -87,11 +87,10 @@ class Listing(object):
for entry in flavors:
entry['href'] = request.path + '/' + entry['name']
pool_group = entry['pool_group']
# NOTE(wanghao): remove this in Newton.
# NOTE(gengchc): Remove pool_group in Rocky
entry['pool'] = entry['pool_group']
if detailed:
caps = self._pools_ctrl.capabilities(group=pool_group)
caps = self._pools_ctrl.capabilities(flavor=entry)
entry['capabilities'] = [str(cap).split('.')[-1]
for cap in caps]
@ -125,9 +124,10 @@ class Resource(object):
validator_type = jsonschema.Draft4Validator
self._validators = {
'create': validator_type(schema.create),
# NOTE(gengchc): Remove pool_group in Rocky.
'pool_group': validator_type(schema.patch_pool_group),
# NOTE(wanghao): Remove this in Newton.
'pool': validator_type(schema.patch_pool),
'pool_list': validator_type(schema.patch_pool_list),
'capabilities': validator_type(schema.patch_capabilities),
}
@ -138,7 +138,7 @@ class Resource(object):
::
{"pool": "", capabilities: {...}}
{"pool": "", "pool_list": [], capabilities: {...}}
:returns: HTTP | [200, 404]
"""
@ -148,12 +148,17 @@ class Resource(object):
try:
data = self._ctrl.get(flavor, project=project_id)
pool_group = data['pool_group']
# NOTE(wanghao): remove this in Newton.
data['pool'] = data['pool_group']
capabilities = self._pools_ctrl.capabilities(group=pool_group)
capabilities = self._pools_ctrl.capabilities(flavor=data)
data['capabilities'] = [str(cap).split('.')[-1]
for cap in capabilities]
# NOTE(gengchc): Remove pool_group in Rocky.
data['pool'] = data['pool_group']
pool_list =\
list(self._pools_ctrl.get_pools_by_flavor(flavor=data))
pool_name_list = []
if len(pool_list) > 0:
pool_name_list = [x['name'] for x in pool_list]
data['pool_list'] = pool_name_list
except errors.FlavorDoesNotExist as ex:
LOG.debug(ex)
@ -163,26 +168,75 @@ class Resource(object):
response.body = transport_utils.to_json(data)
@decorators.TransportLog("Flavors item")
@acl.enforce("flavors:create")
def on_put(self, request, response, project_id, flavor):
"""Registers a new flavor. Expects the following input:
def _check_pools_exists(self, pool_list):
if pool_list is not None:
for pool in pool_list:
if not self._pools_ctrl.exists(pool):
raise errors.PoolDoesNotExist(pool)
::
def _update_pools_by_flavor(self, flavor, pool_list):
if pool_list is not None:
for pool in pool_list:
self._pools_ctrl.update(pool, flavor=flavor)
{"pool_group": "my-pool-group", "capabilities": {}}
def _clean_pools_by_flavor(self, flavor, pool_list=None):
if pool_list is None:
flavor_obj = {}
flavor_obj['name'] = flavor
pllt = list(self._pools_ctrl.get_pools_by_flavor(
flavor=flavor_obj))
pool_list = [x['name'] for x in pllt]
if pool_list is not None:
for pool in pool_list:
self._pools_ctrl.update(pool, flavor="")
A capabilities object may also be provided.
def _on_put_by_pool_list(self, request, response, project_id,
flavor, pool_list):
LOG.debug(u'PUT flavor - name by flavor: %s', flavor)
# NOTE(gengchc2): If configuration flavor is used by the new schema,
# a list of pools is required.
if len(pool_list) == 0:
response.status = falcon.HTTP_400
response.location = request.path
raise falcon.HTTPBadRequest(_('Unable to create'), 'Bad Request')
# NOTE(gengchc2): Check if pools in the pool_list exist.
try:
self._check_pools_exists(pool_list)
except errors.PoolDoesNotExist as ex:
LOG.exception(ex)
description = (_(u'Flavor %(flavor)s could not be created, '
'error:%(msg)s') %
dict(flavor=flavor, msg=str(ex)))
raise falcon.HTTPBadRequest(_('Unable to create'), description)
capabilities = self._pools_ctrl.capabilities(name=pool_list[0])
try:
self._ctrl.create(flavor,
project=project_id,
capabilities=capabilities)
response.status = falcon.HTTP_201
response.location = request.path
except errors.ConnectionError as ex:
LOG.exception(ex)
description = (_(u'Flavor %(flavor)s could not be created, '
'error:%(msg)s') %
dict(flavor=flavor, msg=str(ex)))
raise falcon.HTTPBadRequest(_('Unable to create'), description)
# NOTE(gengchc2): Update the 'flavor' field in pools tables.
try:
self._update_pools_by_flavor(flavor, pool_list)
except errors.ConnectionError as ex:
LOG.exception(ex)
description = (_(u'Flavor %(flavor)s could not be created, '
'error:%(msg)s') %
dict(flavor=flavor, msg=str(ex)))
raise falcon.HTTPBadRequest(_('Unable to create'), description)
:returns: HTTP | [201, 400]
"""
LOG.debug(u'PUT flavor - name: %s', flavor)
data = wsgi_utils.load(request)
wsgi_utils.validate(self._validators['create'], data)
pool_group = data.get('pool_group') or data.get('pool')
capabilities = self._pools_ctrl.capabilities(pool_group)
def _on_put_by_group(self, request, response, project_id,
flavor, pool_group):
LOG.debug(u'PUT flavor - name: %s by group', flavor)
flavor_obj = {}
flavor_obj["pool_group"] = pool_group
capabilities = self._pools_ctrl.capabilities(flavor_obj)
try:
self._ctrl.create(flavor,
pool_group=pool_group,
@ -197,6 +251,35 @@ class Resource(object):
dict(flavor=flavor, pool_group=pool_group))
raise falcon.HTTPBadRequest(_('Unable to create'), description)
@decorators.TransportLog("Flavors item")
@acl.enforce("flavors:create")
def on_put(self, request, response, project_id, flavor):
"""Registers a new flavor. Expects the following input:
::
{"pool_group": "my-pool-group",
"pool_list": [], "capabilities": {}}
A capabilities object may also be provided.
:returns: HTTP | [201, 400]
"""
LOG.debug(u'PUT flavor - name: %s', flavor)
data = wsgi_utils.load(request)
wsgi_utils.validate(self._validators['create'], data)
LOG.debug(u'The pool_group will be removed in Rocky release.')
pool_group = data.get('pool_group') or data.get('pool')
pool_list = data.get('pool_list')
if pool_list is not None:
self._on_put_by_pool_list(request, response, project_id,
flavor, pool_list)
else:
self._on_put_by_group(request, response, project_id,
flavor, pool_group)
@decorators.TransportLog("Flavors item")
@acl.enforce("flavors:delete")
def on_delete(self, request, response, project_id, flavor):
@ -206,49 +289,96 @@ class Resource(object):
"""
LOG.debug(u'DELETE flavor - name: %s', flavor)
# NOTE(gengchc2): If configuration flavor is
# used by the new schema, the flavor field in pools
# need to be cleaned.
try:
self._clean_pools_by_flavor(flavor)
except errors.ConnectionError as ex:
LOG.exception(ex)
description = (_(u'Flavor %(flavor)s could not be deleted.') %
dict(flavor=flavor))
raise falcon.HTTPBadRequest(_('Unable to create'), description)
self._ctrl.delete(flavor, project=project_id)
response.status = falcon.HTTP_204
@decorators.TransportLog("Flavors item")
@acl.enforce("flavors:update")
def on_patch(self, request, response, project_id, flavor):
"""Allows one to update a flavors's pool_group.
def _on_patch_by_pool_list(self, request, response, project_id,
flavor, pool_list):
This method expects the user to submit a JSON object
containing 'pool_group'. If none is found, the request is flagged
as bad. There is also strict format checking through the use of
jsonschema. Appropriate errors are returned in each case for
badly formatted input.
if len(pool_list) == 0:
response.status = falcon.HTTP_400
response.location = request.path
raise falcon.HTTPBadRequest(_('Unable to create'), 'Bad Request')
# NOTE(gengchc2): If the flavor does not exist, return
try:
self._ctrl.get(flavor, project=project_id)
except errors.FlavorDoesNotExist as ex:
LOG.debug(ex)
raise wsgi_errors.HTTPNotFound(six.text_type(ex))
:returns: HTTP | [200, 400]
"""
flavor_obj = {}
flavor_obj['name'] = flavor
# NOTE(gengchc2): Get the pools list with flavor.
pool_list_old = list(self._pools_ctrl.get_pools_by_flavor(
flavor=flavor_obj))
# NOTE(gengchc2): Check if the new pool in the pool_list exist.
try:
self._check_pools_exists(pool_list)
except errors.PoolDoesNotExist as ex:
LOG.exception(ex)
description = (_(u'Flavor %(flavor)s cant be updated, '
'error:%(msg)s') %
dict(flavor=flavor, msg=str(ex)))
raise falcon.HTTPBadRequest(_('updatefail'), description)
capabilities = self._pools_ctrl.capabilities(name=pool_list[0])
try:
self._ctrl.update(flavor, project=project_id,
capabilities=capabilities)
resp_data = self._ctrl.get(flavor, project=project_id)
resp_data['capabilities'] = [str(cap).split('.')[-1]
for cap in capabilities]
except errors.FlavorDoesNotExist as ex:
LOG.exception(ex)
raise wsgi_errors.HTTPNotFound(six.text_type(ex))
LOG.debug(u'PATCH flavor - name: %s', flavor)
data = wsgi_utils.load(request)
EXPECT = ('pool_group', 'pool')
if not any([(field in data) for field in EXPECT]):
LOG.debug(u'PATCH flavor, bad params')
raise wsgi_errors.HTTPBadRequestBody(
'`pool_group` or `pool` needs to be specified'
)
for field in EXPECT:
wsgi_utils.validate(self._validators[field], data)
fields = common_utils.fields(data, EXPECT,
pred=lambda v: v is not None)
# NOTE(wanghao): remove this in Newton.
if fields.get('pool') and fields.get('pool_group') is None:
fields['pool_group'] = fields.get('pool')
fields.pop('pool')
# (gengchc) Update flavor field in new pool list.
try:
self._update_pools_by_flavor(flavor, pool_list)
except errors.ConnectionError as ex:
LOG.exception(ex)
description = (_(u'Flavor %(flavor)s could not be updated, '
'error:%(msg)s') %
dict(flavor=flavor, msg=str(ex)))
raise falcon.HTTPBadRequest(_('Unable to create'), description)
# (gengchc) Remove flavor from old pool list.
try:
pool_list_removed = []
for pool_old in pool_list_old:
if pool_old['name'] not in pool_list:
pool_list_removed.append(pool_old['name'])
self._clean_pools_by_flavor(flavor, pool_list_removed)
except errors.ConnectionError as ex:
LOG.exception(ex)
description = (_(u'Flavor %(flavor)s could not be updated, '
'error:%(msg)s') %
dict(flavor=flavor, msg=str(ex)))
raise falcon.HTTPBadRequest(_('Unable to create'), description)
resp_data['pool_list'] = pool_list
resp_data['href'] = request.path
response.body = transport_utils.to_json(resp_data)
def _on_patch_by_group(self, request, response, project_id,
flavor, pool_group):
LOG.debug(u'PATCH flavor - name: %s by group', flavor)
resp_data = None
try:
self._ctrl.update(flavor, project=project_id, **fields)
flvor_obj = {}
flvor_obj['pool_group'] = pool_group
capabilities = self._pools_ctrl.capabilities(flavor=flvor_obj)
self._ctrl.update(flavor, project=project_id,
pool_group=pool_group,
capabilities=capabilities)
resp_data = self._ctrl.get(flavor, project=project_id)
capabilities = self._pools_ctrl.capabilities(
group=resp_data['pool_group'])
resp_data['capabilities'] = [str(cap).split('.')[-1]
for cap in capabilities]
except errors.FlavorDoesNotExist as ex:
@ -256,3 +386,42 @@ class Resource(object):
raise wsgi_errors.HTTPNotFound(six.text_type(ex))
resp_data['href'] = request.path
response.body = transport_utils.to_json(resp_data)
@decorators.TransportLog("Flavors item")
@acl.enforce("flavors:update")
def on_patch(self, request, response, project_id, flavor):
"""Allows one to update a flavors'pool list.
This method expects the user to submit a JSON object
containing 'pool_group' or 'pool list'. If none is found,
the request is flagged as bad. There is also strict format
checking through the use of jsonschema. Appropriate errors
are returned in each case for badly formatted input.
:returns: HTTP | [200, 400]
"""
LOG.debug(u'PATCH flavor - name: %s', flavor)
data = wsgi_utils.load(request)
# NOTE(gengchc2): remove pool_group in R release.
EXPECT = ('pool_group', 'pool', 'pool_list')
if not any([(field in data) for field in EXPECT]):
LOG.debug(u'PATCH flavor, bad params')
raise wsgi_errors.HTTPBadRequestBody(
'`pool_group` or `pool` or `pool_list` needs to be specified'
)
for field in EXPECT:
wsgi_utils.validate(self._validators[field], data)
LOG.debug(u'The pool_group will be removed in Rocky release.')
pool_group = data.get('pool_group') or data.get('pool')
pool_list = data.get('pool_list')
# NOTE(gengchc2): If pool_list is not None, configuration flavor is
# used by the new schema.
# a list of pools is required.
if pool_list is not None:
self._on_patch_by_pool_list(request, response, project_id,
flavor, pool_list)
else:
self._on_patch_by_group(request, response, project_id,
flavor, pool_group)

17
zaqar/transport/wsgi/v2_0/pools.py
View File

@ -136,7 +136,8 @@ class Resource(object):
self._validators = {
'weight': validator_type(schema.patch_weight),
'uri': validator_type(schema.patch_uri),
'group': validator_type(schema.patch_uri),
'group': validator_type(schema.patch_group),
'flavor': validator_type(schema.patch_flavor),
'options': validator_type(schema.patch_options),
'create': validator_type(schema.create)
}
@ -194,7 +195,8 @@ class Resource(object):
try:
self._ctrl.create(pool, weight=data['weight'],
uri=data['uri'],
group=data.get('group'),
group=data.get('group', None),
flavor=data.get('flavor', None),
options=data.get('options', {}))
response.status = falcon.HTTP_201
response.location = request.path
@ -234,9 +236,9 @@ class Resource(object):
"""Allows one to update a pool's weight, uri, and/or options.
This method expects the user to submit a JSON object
containing at least one of: 'uri', 'weight', 'group', 'options'. If
none are found, the request is flagged as bad. There is also
strict format checking through the use of
containing at least one of: 'uri', 'weight', 'group', 'flavor',
'options'.If none are found, the request is flagged as bad.
There is also strict format checking through the use of
jsonschema. Appropriate errors are returned in each case for
badly formatted input.
@ -246,11 +248,12 @@ class Resource(object):
LOG.debug(u'PATCH pool - name: %s', pool)
data = wsgi_utils.load(request)
EXPECT = ('weight', 'uri', 'group', 'options')
EXPECT = ('weight', 'uri', 'group', 'flavor', 'options')
if not any([(field in data) for field in EXPECT]):
LOG.debug(u'PATCH pool, bad params')
raise wsgi_errors.HTTPBadRequestBody(
'One of `uri`, `weight`, `group`, or `options` needs '
'One of `uri`, `weight`, `group`, `flavor`,'
' or `options` needs '
'to be specified'
)