From a8c1d06bd0badf1a24ede8194804c7bee06a9aa0 Mon Sep 17 00:00:00 2001 From: Kaifeng Wang Date: Thu, 19 Jul 2018 19:14:15 +0800 Subject: [PATCH] introspection data backend: implements db Configurable introspection data storage backend [1] is proposed to provide flexible extension of introspection data storage instead of the single support of Swift storage backend. This patch adds database support for using ironic-inspector database as the storage backend. A table named ``introspection_data`` is created to serve as the storage for introspected data. [1] http://specs.openstack.org/openstack/ironic-inspector-specs/specs/configurable-introspection-data-backends.html Change-Id: I8b29b7b86d90823d29b921ebf64acddbcd2d8d0d Story: 1726713 Task: 11373 --- ironic_inspector/db.py | 8 ++++ ...8dec16023c_add_introspection_data_table.py | 42 ++++++++++++++++++ ironic_inspector/node_cache.py | 43 ++++++++++++++++++- ironic_inspector/test/unit/test_migrations.py | 13 ++++++ ironic_inspector/test/unit/test_node_cache.py | 29 +++++++++++++ ironic_inspector/utils.py | 4 ++ 6 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 ironic_inspector/migrations/versions/bf8dec16023c_add_introspection_data_table.py diff --git a/ironic_inspector/db.py b/ironic_inspector/db.py index 94f0effce..7d7a11111 100644 --- a/ironic_inspector/db.py +++ b/ironic_inspector/db.py @@ -134,6 +134,14 @@ class RuleAction(Base): return res +class IntrospectionData(Base): + __tablename__ = 'introspection_data' + uuid = Column(String(36), ForeignKey('nodes.uuid'), primary_key=True) + processed = Column(Boolean, default=False) + data = Column(db_types.JsonEncodedDict(mysql_as_long=True), + nullable=True) + + def init(): """Initialize the database. diff --git a/ironic_inspector/migrations/versions/bf8dec16023c_add_introspection_data_table.py b/ironic_inspector/migrations/versions/bf8dec16023c_add_introspection_data_table.py new file mode 100644 index 000000000..bceefac29 --- /dev/null +++ b/ironic_inspector/migrations/versions/bf8dec16023c_add_introspection_data_table.py @@ -0,0 +1,42 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""add_introspection_data_table + +Revision ID: bf8dec16023c +Revises: 2970d2d44edc +Create Date: 2018-07-19 18:51:38.124614 + +""" + +from alembic import op +from oslo_db.sqlalchemy import types as db_types +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'bf8dec16023c' +down_revision = '2970d2d44edc' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'introspection_data', + sa.Column('uuid', sa.String(36), sa.ForeignKey('nodes.uuid'), + primary_key=True), + sa.Column('processed', sa.Boolean, default=False), + sa.Column('data', db_types.JsonEncodedDict(mysql_as_long=True).impl, + nullable=True), + mysql_ENGINE='InnoDB', + mysql_DEFAULT_CHARSET='UTF8' + ) diff --git a/ironic_inspector/node_cache.py b/ironic_inspector/node_cache.py index 9365222b6..0cf7932d6 100644 --- a/ironic_inspector/node_cache.py +++ b/ironic_inspector/node_cache.py @@ -755,7 +755,7 @@ def _delete_node(uuid, session=None): with db.ensure_transaction(session) as session: db.model_query(db.Attribute, session=session).filter_by( node_uuid=uuid).delete() - for model in (db.Option, db.Node): + for model in (db.Option, db.IntrospectionData, db.Node): db.model_query(model, session=session).filter_by(uuid=uuid).delete() @@ -979,3 +979,44 @@ def get_node_list(ironic=None, marker=None, limit=None): ('started_at', 'uuid'), marker=marker, sort_dir='desc') return [NodeInfo.from_row(row, ironic=ironic) for row in rows] + + +def store_introspection_data(node_id, introspection_data, processed=True): + """Store introspection data for this node. + + :param node_id: node UUID. + :param introspection_data: A dictionary of introspection data + :param processed: Specify the type of introspected data, set to False + indicates the data is unprocessed. + """ + with db.ensure_transaction() as session: + record = db.model_query(db.IntrospectionData, + session=session).filter_by( + uuid=node_id, processed=processed).first() + if record is None: + row = db.IntrospectionData() + row.update({'uuid': node_id, 'processed': processed, + 'data': introspection_data}) + session.add(row) + else: + record.update({'data': introspection_data}) + session.flush() + + +def get_introspection_data(node_id, processed=True): + """Get introspection data for this node. + + :param node_id: node UUID. + :param processed: Specify the type of introspected data, set to False + indicates retrieving the unprocessed data. + :return: A dictionary representation of intropsected data + """ + try: + ref = db.model_query(db.IntrospectionData).filter_by( + uuid=node_id, processed=processed).one() + return ref['data'] + except orm_errors.NoResultFound: + msg = _('Introspection data not found for node %(node)s, ' + 'processed=%(processed)s') % {'node': node_id, + 'processed': processed} + raise utils.IntrospectionDataNotFound(msg) diff --git a/ironic_inspector/test/unit/test_migrations.py b/ironic_inspector/test/unit/test_migrations.py index 5e5fd4bae..aa005cefc 100644 --- a/ironic_inspector/test/unit/test_migrations.py +++ b/ironic_inspector/test/unit/test_migrations.py @@ -441,6 +441,19 @@ class MigrationCheckersMixin(object): n = nodes.select(nodes.c.uuid == 'abcd').execute().first() self.assertIsNone(n['manage_boot']) + def _check_bf8dec16023c(self, engine, data): + introspection_data = db_utils.get_table(engine, 'introspection_data') + col_names = [column.name for column in introspection_data.c] + self.assertIn('uuid', col_names) + self.assertIn('processed', col_names) + self.assertIn('data', col_names) + self.assertIsInstance(introspection_data.c.uuid.type, + sqlalchemy.types.String) + self.assertIsInstance(introspection_data.c.processed.type, + sqlalchemy.types.Boolean) + self.assertIsInstance(introspection_data.c.data.type, + sqlalchemy.types.Text) + def test_upgrade_and_version(self): with patch_with_engine(self.engine): self.migration_ext.upgrade('head') diff --git a/ironic_inspector/test/unit/test_node_cache.py b/ironic_inspector/test/unit/test_node_cache.py index 9ad9b6ff5..4e6bed5a1 100644 --- a/ironic_inspector/test/unit/test_node_cache.py +++ b/ironic_inspector/test/unit/test_node_cache.py @@ -331,6 +331,8 @@ class TestNodeCacheCleanUp(test_base.NodeTest): value=v, node_uuid=self.uuid).save(session) db.Option(uuid=self.uuid, name='foo', value='bar').save( session) + db.IntrospectionData(uuid=self.uuid, processed=False, + data={'fake': 'data'}).save(session) def test_no_timeout(self): CONF.set_override('timeout', 0) @@ -358,6 +360,7 @@ class TestNodeCacheCleanUp(test_base.NodeTest): self.assertEqual(len(self.macs), db.model_query(db.Attribute).count()) self.assertEqual(1, db.model_query(db.Option).count()) + self.assertEqual(1, db.model_query(db.IntrospectionData).count()) self.assertFalse(get_lock_mock.called) @mock.patch.object(node_cache, '_get_lock', autospec=True) @@ -1256,3 +1259,29 @@ class TestStartIntrospection(test_base.NodeTest): node_cache.start_introspection, self.node_info.uuid) self.assertFalse(add_node_mock.called) + + +class TestIntrospectionDataDbStore(test_base.NodeTest): + def setUp(self): + super(TestIntrospectionDataDbStore, self).setUp() + node_cache.add_node(self.node.uuid, + istate.States.processing, + bmc_address='1.2.3.4') + + def _test_store_and_get(self, processed=False): + node_cache.store_introspection_data(self.node.uuid, + copy.deepcopy(self.data), + processed=processed) + stored_data = node_cache.get_introspection_data(self.node.uuid, + processed=processed) + self.assertEqual(stored_data, self.data) + + def test_store_and_get_unprocessed(self): + self._test_store_and_get(processed=False) + + def test_store_and_get_processed(self): + self._test_store_and_get(processed=True) + + def test_get_no_data_available(self): + self.assertRaises(utils.IntrospectionDataNotFound, + node_cache.get_introspection_data, self.node.uuid) diff --git a/ironic_inspector/utils.py b/ironic_inspector/utils.py index 7fecf6edc..69244a4f8 100644 --- a/ironic_inspector/utils.py +++ b/ironic_inspector/utils.py @@ -140,6 +140,10 @@ class NodeStateInvalidEvent(Error): """Invalid event attempted.""" +class IntrospectionDataNotFound(NotFoundInCacheError): + """Introspection data not found.""" + + def executor(): """Return the current futures executor.""" global _EXECUTOR