diff --git a/placement/attribute_cache.py b/placement/attribute_cache.py new file mode 100644 index 000000000..e4f735211 --- /dev/null +++ b/placement/attribute_cache.py @@ -0,0 +1,163 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy as sa + +from placement.db.sqlalchemy import models +from placement import db_api +from placement import exception + +_RC_TBL = models.ResourceClass.__table__ +_TRAIT_TBL = models.Trait.__table__ + + +class _AttributeCache(object): + """A cache of integer and string lookup values for string-based attributes. + + Subclasses must define `_table` and `_not_found` members describing the + database table which is the authoritative source of data and the exception + raised if data for an attribute is not found, respectively. + + The cache is required to be correct for the extent of any individual API + request and be used only for those entities where any change to the + underlying data is only making that change and will have no subsequent + queries into the cache. For example, when we add a new resource class we + do not then list all the resource classes from within the same session. + + Despite that requirement, any time an entity associated with a cache is + created, updated, or deleted `clear()` should be called on the cache. + """ + _table = None + _not_found = None + + def __init__(self, ctx): + """Initialize the cache of resource class identifiers. + + :param ctx: `placement.context.RequestContext` from which we can grab a + `SQLAlchemy.Connection` object to use for any DB lookups. + """ + # Prevent this class being created directly, relevent during + # development. + assert self._table is not None, "_table must be defined" + assert self._not_found is not None, "_not_found must be defined" + self._ctx = ctx + self.clear() + + def clear(self): + self._id_cache = {} + self._str_cache = {} + self._all_cache = {} + + def id_from_string(self, attr_str): + """Given a string representation of an attribute -- e.g. "DISK_GB" + or "CUSTOM_IRON_SILVER" -- return the integer code for the attribute + by doing a DB lookup into the appropriate table; however, the results + of these DB lookups are cached since the lookups are so frequent. + + :param attr_str: The string representation of the attribute to look up + a numeric identifier for. + :returns Integer identifier for the attribute. + :raises An instance of the subclass' _not_found exception if attribute + cannot be found in the DB. + """ + attr_id = self._id_cache.get(attr_str) + if attr_id is not None: + return attr_id + + # Otherwise, check the database table + self._refresh_from_db(self._ctx) + if attr_str in self._id_cache: + return self._id_cache[attr_str] + raise self._not_found(name=attr_str) + + def all_from_string(self, attr_str): + """Given a string representation of an attribute -- e.g. "DISK_GB" + or "CUSTOM_IRON_SILVER" -- return all the attribute info. + + :param attr_str: The string representation of the attribute for which + to look up the object. + :returns: dict representing the attribute fields, if the attribute was + found in the appropriate database table. + :raises An instance of the subclass' _not_found exception if attr_str + cannot be found in the DB. + """ + attr_id_str = self._all_cache.get(attr_str) + if attr_id_str is not None: + return attr_id_str + + # Otherwise, check the database table + self._refresh_from_db(self._ctx) + if attr_str in self._all_cache: + return self._all_cache[attr_str] + raise self._not_found(name=attr_str) + + def string_from_id(self, attr_id): + """The reverse of the id_from_string() method. Given a supplied numeric + identifier for an attribute, we look up the corresponding string + representation, via a DB lookup. The results of these DB lookups are + cached since the lookups are so frequent. + + :param attr_id: The numeric representation of the attribute to look + up a string identifier for. + :returns: String identifier for the attribute. + :raises An instances of the subclass' _not_found exception if attr_id + cannot be found in the DB. + """ + attr_str = self._str_cache.get(attr_id) + if attr_str is not None: + return attr_str + + # Otherwise, check the database table + self._refresh_from_db(self._ctx) + if attr_id in self._str_cache: + return self._str_cache[attr_id] + raise self._not_found(name=attr_id) + + def get_all(self): + """Return an iterator of all the resources in the cache with all their + attributes. + + In Python3 the return value is a generator. + """ + if not self._all_cache: + self._refresh_from_db(self._ctx) + return self._all_cache.values() + + @db_api.placement_context_manager.reader + def _refresh_from_db(self, ctx): + """Grabs all resource classes or traits from the respective DB table + and populates the supplied cache object's internal integer and string + identifier dicts. + + :param ctx: RequestContext with the the database session. + """ + table = self._table + sel = sa.select([table.c.id, table.c.name, table.c.updated_at, + table.c.created_at]) + res = ctx.session.execute(sel).fetchall() + self._id_cache = {r[1]: r[0] for r in res} + self._str_cache = {r[0]: r[1] for r in res} + self._all_cache = {r[1]: r for r in res} + + +class ResourceClassCache(_AttributeCache): + """An _AttributeCache for resource classes.""" + + _table = _RC_TBL + _not_found = exception.ResourceClassNotFound + + +class TraitCache(_AttributeCache): + """An _AttributeCache for traits.""" + + _table = _TRAIT_TBL + _not_found = exception.TraitNotFound diff --git a/placement/context.py b/placement/context.py index 787b625c5..fc1e30ad8 100644 --- a/placement/context.py +++ b/placement/context.py @@ -13,9 +13,9 @@ from oslo_context import context from oslo_db.sqlalchemy import enginefacade +from placement import attribute_cache from placement import exception from placement import policy -from placement import resource_class_cache as rc_cache @enginefacade.transaction_context_provider @@ -23,7 +23,8 @@ class RequestContext(context.RequestContext): def __init__(self, *args, **kwargs): self.config = kwargs.pop('config', None) - self.rc_cache = rc_cache.ensure(self) + self.rc_cache = attribute_cache.ResourceClassCache(self) + self.trait_cache = attribute_cache.TraitCache(self) super(RequestContext, self).__init__(*args, **kwargs) def can(self, action, target=None, fatal=True): diff --git a/placement/exception.py b/placement/exception.py index 308c424de..11a779c48 100644 --- a/placement/exception.py +++ b/placement/exception.py @@ -156,7 +156,7 @@ class ResourceClassInUse(_BaseException): class ResourceClassNotFound(NotFound): - msg_fmt = "No such resource class %(resource_class)s." + msg_fmt = "No such resource class %(name)s." class ResourceProviderInUse(_BaseException): @@ -176,7 +176,7 @@ class TraitInUse(_BaseException): class TraitNotFound(NotFound): - msg_fmt = "No such trait(s): %(names)s." + msg_fmt = "No such trait(s): %(name)s." class ProjectNotFound(NotFound): diff --git a/placement/objects/research_context.py b/placement/objects/research_context.py index a45abd79d..e965227ed 100644 --- a/placement/objects/research_context.py +++ b/placement/objects/research_context.py @@ -26,7 +26,6 @@ from placement.objects import trait as trait_obj # TODO(tetsuro): Move these public symbols in a central place. -_TRAIT_TBL = models.Trait.__table__ _ALLOC_TBL = models.Allocation.__table__ _INV_TBL = models.Inventory.__table__ _RP_TBL = models.ResourceProvider.__table__ diff --git a/placement/objects/resource_class.py b/placement/objects/resource_class.py index 7c8ff9077..fd5c2e32f 100644 --- a/placement/objects/resource_class.py +++ b/placement/objects/resource_class.py @@ -135,6 +135,7 @@ class ResourceClass(object): {'name': self.name}) msg = "creating resource class %s" % self.name raise exception.MaxDBRetriesExceeded(action=msg) + self._context.rc_cache.clear() @staticmethod @db_api.placement_context_manager.writer @@ -212,10 +213,9 @@ def ensure_sync(ctx): _RESOURCE_CLASSES_SYNCED = True -@db_api.placement_context_manager.reader def get_all(context): """Get a list of all the resource classes in the database.""" - resource_classes = context.session.query(models.ResourceClass).all() + resource_classes = context.rc_cache.get_all() return [ResourceClass(context, **rc) for rc in resource_classes] diff --git a/placement/objects/resource_provider.py b/placement/objects/resource_provider.py index 663896fe8..1a2e9f7d7 100644 --- a/placement/objects/resource_provider.py +++ b/placement/objects/resource_provider.py @@ -33,7 +33,6 @@ from placement.objects import inventory as inv_obj from placement.objects import research_context as res_ctx from placement.objects import trait as trait_obj -_TRAIT_TBL = models.Trait.__table__ _ALLOC_TBL = models.Allocation.__table__ _INV_TBL = models.Inventory.__table__ _RP_TBL = models.ResourceProvider.__table__ diff --git a/placement/objects/trait.py b/placement/objects/trait.py index f7f8db0be..2a021bd32 100644 --- a/placement/objects/trait.py +++ b/placement/objects/trait.py @@ -86,6 +86,7 @@ class Trait(object): raise exception.TraitExists(name=self.name) self._from_db_object(self._context, self, db_trait) + self._context.trait_cache.clear() @staticmethod @db_api.placement_context_manager.reader @@ -93,7 +94,7 @@ class Trait(object): result = context.session.query(models.Trait).filter_by( name=name).first() if not result: - raise exception.TraitNotFound(names=name) + raise exception.TraitNotFound(name=name) return result @classmethod @@ -112,7 +113,7 @@ class Trait(object): res = context.session.query(models.Trait).filter_by( name=name).delete() if not res: - raise exception.TraitNotFound(names=name) + raise exception.TraitNotFound(name=name) def destroy(self): if not self.name: @@ -127,6 +128,7 @@ class Trait(object): reason='ID attribute not found') self._destroy_in_db(self._context, self.id, self.name) + self._context.trait_cache.clear() def ensure_sync(ctx): @@ -168,15 +170,14 @@ def get_all_by_resource_provider(context, rp): @db_api.placement_context_manager.reader def get_traits_by_provider_id(context, rp_id): - t = sa.alias(_TRAIT_TBL, name='t') - rpt = sa.alias(_RP_TRAIT_TBL, name='rpt') + rp_traits_id = _RP_TRAIT_TBL.c.resource_provider_id + trait_id = _RP_TRAIT_TBL.c.trait_id + trait_cache = context.trait_cache - join_cond = sa.and_(t.c.id == rpt.c.trait_id, - rpt.c.resource_provider_id == rp_id) - join = sa.join(t, rpt, join_cond) - sel = sa.select([t.c.id, t.c.name, - t.c.created_at, t.c.updated_at]).select_from(join) - return [dict(r) for r in context.session.execute(sel).fetchall()] + sel = sa.select([trait_id]).where(rp_traits_id == rp_id) + return [ + trait_cache.all_from_string(trait_cache.string_from_id(r.trait_id)) + for r in context.session.execute(sel).fetchall()] @db_api.placement_context_manager.reader @@ -196,14 +197,13 @@ def get_traits_by_provider_tree(ctx, root_ids): rpt = sa.alias(_RP_TBL, name='rpt') rptt = sa.alias(_RP_TRAIT_TBL, name='rptt') - tt = sa.alias(_TRAIT_TBL, name='t') rpt_rptt = sa.join(rpt, rptt, rpt.c.id == rptt.c.resource_provider_id) - j = sa.join(rpt_rptt, tt, rptt.c.trait_id == tt.c.id) - sel = sa.select([rptt.c.resource_provider_id, tt.c.name]).select_from(j) + sel = sa.select([rptt.c.resource_provider_id, rptt.c.trait_id]) + sel = sel.select_from(rpt_rptt) sel = sel.where(rpt.c.root_provider_id.in_(root_ids)) res = collections.defaultdict(list) for r in ctx.session.execute(sel): - res[r[0]].append(r[1]) + res[r[0]].append(ctx.trait_cache.string_from_id(r[1])) return res @@ -222,21 +222,18 @@ def ids_from_names(ctx, names): raise ValueError("Expected names to be a list of string trait " "names, but got an empty list.") - # Avoid SAWarnings about unicode types... - unames = map(six.text_type, names) - tt = sa.alias(_TRAIT_TBL, name='t') - sel = sa.select([tt.c.name, tt.c.id]).where(tt.c.name.in_(unames)) - trait_map = {r[0]: r[1] for r in ctx.session.execute(sel)} - if len(trait_map) != len(names): - missing = names - set(trait_map) - raise exception.TraitNotFound(names=', '.join(missing)) - return trait_map + return {name: ctx.trait_cache.id_from_string(name) for name in names} + + +def _get_all_from_db(context, filters): + # If no filters are required, returns everything from the cache. + if not filters: + return context.trait_cache.get_all() + return _get_all_filtered_from_db(context, filters) @db_api.placement_context_manager.reader -def _get_all_from_db(context, filters): - if not filters: - filters = {} +def _get_all_filtered_from_db(context, filters): query = context.session.query(models.Trait) if 'name_in' in filters: diff --git a/placement/resource_class_cache.py b/placement/resource_class_cache.py deleted file mode 100644 index 1bb7dac54..000000000 --- a/placement/resource_class_cache.py +++ /dev/null @@ -1,132 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sa - -from placement.db.sqlalchemy import models -from placement import db_api -from placement import exception - -_RC_TBL = models.ResourceClass.__table__ - - -@db_api.placement_context_manager.reader -def ensure(ctx): - """Ensures that a resource class cache has been created for the provided - context. - - :param ctx: `placement.context.RequestContext` that may be used to grab a - DB connection. - """ - return ResourceClassCache(ctx) - - -@db_api.placement_context_manager.reader -def _refresh_from_db(ctx, cache): - """Grabs all resource classes from the DB table and populates the - supplied cache object's internal integer and string identifier dicts. - - :param cache: ResourceClassCache object to refresh. - """ - sel = sa.select([_RC_TBL.c.id, _RC_TBL.c.name, _RC_TBL.c.updated_at, - _RC_TBL.c.created_at]) - res = ctx.session.execute(sel).fetchall() - cache.id_cache = {r[1]: r[0] for r in res} - cache.str_cache = {r[0]: r[1] for r in res} - cache.all_cache = {r[1]: r for r in res} - - -class ResourceClassCache(object): - """A cache of integer and string lookup values for resource classes.""" - - def __init__(self, ctx): - """Initialize the cache of resource class identifiers. - - :param ctx: `placement.context.RequestContext` from which we can grab a - `SQLAlchemy.Connection` object to use for any DB lookups. - """ - self.ctx = ctx - self.id_cache = {} - self.str_cache = {} - self.all_cache = {} - - def clear(self): - self.id_cache = {} - self.str_cache = {} - self.all_cache = {} - - def id_from_string(self, rc_str): - """Given a string representation of a resource class -- e.g. "DISK_GB" - or "CUSTOM_IRON_SILVER" -- return the integer code for the resource - class by doing a DB lookup into the resource_classes table; however, - the results of these DB lookups are cached since the lookups are so - frequent. - - :param rc_str: The string representation of the resource class to look - up a numeric identifier for. - :returns Integer identifier for the resource class. - :raises `exception.ResourceClassNotFound` if rc_str cannot be found in - the DB. - """ - rc_id = self.id_cache.get(rc_str) - if rc_id is not None: - return rc_id - - # Otherwise, check the database table - _refresh_from_db(self.ctx, self) - if rc_str in self.id_cache: - return self.id_cache[rc_str] - raise exception.ResourceClassNotFound(resource_class=rc_str) - - def all_from_string(self, rc_str): - """Given a string representation of a resource class -- e.g. "DISK_GB" - or "CUSTOM_IRON_SILVER" -- return all the resource class info. - - :param rc_str: The string representation of the resource class for - which to look up a resource_class. - :returns: dict representing the resource class fields, if the - resource class was found in the resource_classes database - table. - :raises: `exception.ResourceClassNotFound` if rc_str cannot be found in - the DB. - """ - rc_id_str = self.all_cache.get(rc_str) - if rc_id_str is not None: - return rc_id_str - - # Otherwise, check the database table - _refresh_from_db(self.ctx, self) - if rc_str in self.all_cache: - return self.all_cache[rc_str] - raise exception.ResourceClassNotFound(resource_class=rc_str) - - def string_from_id(self, rc_id): - """The reverse of the id_from_string() method. Given a supplied numeric - identifier for a resource class, we look up the corresponding string - representation, via a DB lookup. The results of these DB lookups are - cached since the lookups are so frequent. - - :param rc_id: The numeric representation of the resource class to look - up a string identifier for. - :returns: String identifier for the resource class. - :raises `exception.ResourceClassNotFound` if rc_id cannot be found in - the DB. - """ - rc_str = self.str_cache.get(rc_id) - if rc_str is not None: - return rc_str - - # Otherwise, check the database table - _refresh_from_db(self.ctx, self) - if rc_id in self.str_cache: - return self.str_cache[rc_id] - raise exception.ResourceClassNotFound(resource_class=rc_id) diff --git a/placement/tests/functional/db/test_resource_class_cache.py b/placement/tests/functional/db/test_attribute_cache.py similarity index 83% rename from placement/tests/functional/db/test_resource_class_cache.py rename to placement/tests/functional/db/test_attribute_cache.py index bba1161c6..6a13d0e8e 100644 --- a/placement/tests/functional/db/test_resource_class_cache.py +++ b/placement/tests/functional/db/test_attribute_cache.py @@ -15,11 +15,21 @@ import mock from oslo_utils import timeutils +from placement import attribute_cache from placement import exception -from placement import resource_class_cache as rc_cache from placement.tests.functional import base +class TestAttributeCache(base.TestCase): + + def test_no_super_instance(self): + """Test that we can't create an _AttributeCache.""" + exc = self.assertRaises( + AssertionError, attribute_cache._AttributeCache, + self.context) + self.assertIn('_table must be defined', str(exc)) + + class TestResourceClassCache(base.TestCase): def test_rc_cache_std_db(self): @@ -27,8 +37,8 @@ class TestResourceClassCache(base.TestCase): cache for a standardized resource class doesn't result in a DB call once the cache is initialized """ - cache = rc_cache.ResourceClassCache(self.context) - rc_cache._refresh_from_db(self.context, cache) + cache = attribute_cache.ResourceClassCache(self.context) + cache._refresh_from_db(self.context) with mock.patch('sqlalchemy.select') as sel_mock: self.assertEqual('VCPU', cache.string_from_id(0)) @@ -39,7 +49,7 @@ class TestResourceClassCache(base.TestCase): self.assertFalse(sel_mock.called) def test_standard_has_time_fields(self): - cache = rc_cache.ResourceClassCache(self.context) + cache = attribute_cache.ResourceClassCache(self.context) vcpu_class = dict(cache.all_from_string('VCPU')) expected = {'id': 0, 'name': 'VCPU', 'updated_at': None, @@ -54,7 +64,7 @@ class TestResourceClassCache(base.TestCase): return appropriate results, caching the results after a single query. """ - cache = rc_cache.ResourceClassCache(self.context) + cache = attribute_cache.ResourceClassCache(self.context) # Haven't added anything to the DB yet, so should raise # ResourceClassNotFound @@ -65,7 +75,7 @@ class TestResourceClassCache(base.TestCase): # Now add to the database and verify appropriate results... with self.placement_db.get_engine().connect() as conn: - ins_stmt = rc_cache._RC_TBL.insert().values( + ins_stmt = attribute_cache._RC_TBL.insert().values( id=1001, name='IRON_NFV' ) @@ -94,13 +104,13 @@ class TestResourceClassCache(base.TestCase): # the automatic timestamp handling provided by the oslo_db # TimestampMixin is not provided. created_at is a default # but updated_at is an onupdate. - upd_stmt = rc_cache._RC_TBL.update().where( - rc_cache._RC_TBL.c.id == 1001).values( + upd_stmt = attribute_cache._RC_TBL.update().where( + attribute_cache._RC_TBL.c.id == 1001).values( name='IRON_NFV', updated_at=timeutils.utcnow()) conn.execute(upd_stmt) # reset cache - cache = rc_cache.ResourceClassCache(self.context) + cache = attribute_cache.ResourceClassCache(self.context) iron_nfv_class = cache.all_from_string('IRON_NFV') # updated_at set on update @@ -110,7 +120,7 @@ class TestResourceClassCache(base.TestCase): """Test that we raise ResourceClassNotFound if an unknown resource class ID or string is searched for. """ - cache = rc_cache.ResourceClassCache(self.context) + cache = attribute_cache.ResourceClassCache(self.context) self.assertRaises(exception.ResourceClassNotFound, cache.string_from_id, 99999999) self.assertRaises(exception.ResourceClassNotFound, diff --git a/placement/tests/functional/gabbits/resource-classes.yaml b/placement/tests/functional/gabbits/resource-classes.yaml index 0e42d4c62..0c4445d96 100644 --- a/placement/tests/functional/gabbits/resource-classes.yaml +++ b/placement/tests/functional/gabbits/resource-classes.yaml @@ -51,7 +51,6 @@ tests: GET: /resource_classes response_json_paths: $.resource_classes.`len`: 18 # Number of standard resource classes - $.resource_classes[0].name: VCPU - name: non admin forbidden GET: /resource_classes @@ -138,8 +137,6 @@ tests: GET: /resource_classes response_json_paths: $.resource_classes.`len`: 19 # 18 standard plus 1 custom - $.resource_classes[18].name: $ENVIRON['CUSTOM_RES_CLASS'] - $.resource_classes[18].links[?rel = "self"].href: /resource_classes/$ENVIRON['CUSTOM_RES_CLASS'] - name: update standard resource class bad json PUT: /resource_classes/VCPU diff --git a/placement/tests/unit/base.py b/placement/tests/unit/base.py index d982034c9..f7ef0e92d 100644 --- a/placement/tests/unit/base.py +++ b/placement/tests/unit/base.py @@ -15,10 +15,12 @@ import testtools class ContextTestCase(testtools.TestCase): - """Base class for tests that need a mocked resource_class_cache on Context. + """Base class for tests that need mocked attribute caches on Context. """ def setUp(self): super(ContextTestCase, self).setUp() self.useFixture( - fixtures.MockPatch('placement.resource_class_cache.ensure')) + fixtures.MockPatch('placement.attribute_cache.ResourceClassCache')) + self.useFixture( + fixtures.MockPatch('placement.attribute_cache.TraitCache'))