From c46b469eb731287368081b719cbf30423b9e2d4c Mon Sep 17 00:00:00 2001 From: Jay Pipes Date: Wed, 28 Sep 2016 18:19:37 -0400 Subject: [PATCH] placement: add cache for resource classes Adds a simple caching mechanism for resource class string and integer codes. The cache is initialized with a RequestContext in order to fetch a DB connection but only uses the DB connection to look up string names or integer IDs if the requested key (string or integer) isn't either in the fields.ResourceClass.ALL collection or not already looked up in the DB. The next patch in this series adds code to the resource_providers.py module that changes all occurrences of fields.ResourceClass.index() and fields.ResourceClass.from_index() to utilize the resource class cache. In this way, we will transparently move from a situation with resource classes represented only as Enum-based constants to a DB-backed solution where custom resource classes can co-exist with the standardized Enum constants. Change-Id: I9b5b59a1e9715d61f55dc0f6189bcf8596052e2c blueprint: custom-resource-classes --- nova/db/sqlalchemy/resource_class_cache.py | 97 +++++++++++++++++++ .../db/test_resource_class_cache.py | 71 ++++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 nova/db/sqlalchemy/resource_class_cache.py create mode 100644 nova/tests/functional/db/test_resource_class_cache.py diff --git a/nova/db/sqlalchemy/resource_class_cache.py b/nova/db/sqlalchemy/resource_class_cache.py new file mode 100644 index 000000000000..25d8701a9e3c --- /dev/null +++ b/nova/db/sqlalchemy/resource_class_cache.py @@ -0,0 +1,97 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy as sa + +from nova.db.sqlalchemy import api as db_api +from nova.db.sqlalchemy import api_models as models +from nova.objects import fields + +_RC_TBL = models.ResourceClass.__table__ + + +@db_api.api_context_manager.reader +def _refresh_from_db(ctx, cache): + """Grabs all custom resource classes from the DB table and populates the + supplied cache object's internal integer and string identifier dicts. + + :param cache: ResourceClassCache object to refresh. + """ + with ctx.session.connection() as conn: + sel = sa.select([_RC_TBL.c.id, _RC_TBL.c.name]) + res = conn.execute(sel).fetchall() + cache.id_cache = {r[1]: r[0] for r in res} + cache.str_cache = {r[0]: r[1] for r in res} + + +class ResourceClassCache(object): + """A cache of integer and string lookup values for resource classes.""" + + def __init__(self, ctx): + """Initialize the cache of resource class identifiers. + + :param ctx: `nova.context.RequestContext` from which we can grab a + `SQLAlchemy.Connection` object to use for any DB lookups. + """ + self.ctx = ctx + self.id_cache = {} + self.str_cache = {} + + def id_from_string(self, rc_str): + """Given a string representation of a resource class -- e.g. "DISK_GB" + or "IRON_SILVER" -- return the integer code for the resource class. For + standard resource classes, this integer code will match the list of + resource classes on the fields.ResourceClass field type. Other custom + resource classes will cause a DB lookup into the resource_classes + table, however the results of these DB lookups are cached since the + lookups are so frequent. + + :param rc_str: The string representation of the resource class to look + up a numeric identifier for. + :returns integer identifier for the resource class, or None, if no such + resource class was found in the list of standard resource + classes or the resource_classes database table. + """ + if rc_str in self.id_cache: + return self.id_cache[rc_str] + + # First check the standard resource classes + if rc_str in fields.ResourceClass.ALL: + return fields.ResourceClass.ALL.index(rc_str) + else: + # Otherwise, check the database table + _refresh_from_db(self.ctx, self) + return self.id_cache.get(rc_str) + + def string_from_id(self, rc_id): + """The reverse of the id_from_string() method. Given a supplied numeric + identifier for a resource class, we look up the corresponding string + representation, either in the list of standard resource classes or via + a DB lookup. The results of these DB lookups are cached since the + lookups are so frequent. + + :param rc_id: The numeric representation of the resource class to look + up a string identifier for. + :returns string identifier for the resource class, or None, if no such + resource class was found in the list of standard resource + classes or the resource_classes database table. + """ + if rc_id in self.str_cache: + return self.str_cache[rc_id] + + # First check the fields.ResourceClass.ALL enum + try: + return fields.ResourceClass.ALL[rc_id] + except IndexError: + # Otherwise, check the database table + _refresh_from_db(self.ctx, self) + return self.str_cache.get(rc_id) diff --git a/nova/tests/functional/db/test_resource_class_cache.py b/nova/tests/functional/db/test_resource_class_cache.py new file mode 100644 index 000000000000..aae9984d17c3 --- /dev/null +++ b/nova/tests/functional/db/test_resource_class_cache.py @@ -0,0 +1,71 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from nova.db.sqlalchemy import resource_class_cache as rc_cache +from nova import test +from nova.tests import fixtures + + +class TestResourceClassCache(test.TestCase): + + def setUp(self): + super(TestResourceClassCache, self).setUp() + self.db = self.useFixture(fixtures.Database(database='api')) + self.context = mock.Mock() + sess_mock = mock.Mock() + sess_mock.connection.side_effect = self.db.get_engine().connect + self.context.session = sess_mock + + @mock.patch('sqlalchemy.select') + def test_rc_cache_std_no_db(self, sel_mock): + """Test that looking up either an ID or a string in the resource class + cache for a standardized resource class does not result in a DB + call. + """ + cache = rc_cache.ResourceClassCache(self.context) + + self.assertEqual('VCPU', cache.string_from_id(0)) + self.assertEqual('MEMORY_MB', cache.string_from_id(1)) + self.assertEqual(0, cache.id_from_string('VCPU')) + self.assertEqual(1, cache.id_from_string('MEMORY_MB')) + + self.assertFalse(sel_mock.called) + + def test_rc_cache_custom(self): + """Test that non-standard, custom resource classes hit the database and + return appropriate results, caching the results after a single + query. + """ + cache = rc_cache.ResourceClassCache(self.context) + + # Haven't added anything to the DB yet, so should return None + self.assertIsNone(cache.string_from_id(1001)) + self.assertIsNone(cache.id_from_string("IRON_NFV")) + + # Now add to the database and verify appropriate results... + with self.context.session.connection() as conn: + ins_stmt = rc_cache._RC_TBL.insert().values( + id=1001, + name='IRON_NFV' + ) + conn.execute(ins_stmt) + + self.assertEqual('IRON_NFV', cache.string_from_id(1001)) + self.assertEqual(1001, cache.id_from_string('IRON_NFV')) + + # Try same again and verify we don't hit the DB. + with mock.patch('sqlalchemy.select') as sel_mock: + self.assertEqual('IRON_NFV', cache.string_from_id(1001)) + self.assertEqual(1001, cache.id_from_string('IRON_NFV')) + self.assertFalse(sel_mock.called)