introspection data backend: implements db

Configurable introspection data storage backend [1] is
proposed to provide flexible extension of introspection
data storage instead of the single support of Swift storage
backend.

This patch adds database support for using ironic-inspector
database as the storage backend.

A table named ``introspection_data`` is created to serve as
the storage for introspected data.

[1] http://specs.openstack.org/openstack/ironic-inspector-specs/specs/configurable-introspection-data-backends.html

Change-Id: I8b29b7b86d90823d29b921ebf64acddbcd2d8d0d
Story: 1726713
Task: 11373

ironic_inspector/db.py

@@ -134,6 +134,14 @@ class RuleAction(Base):
         return res
 
 
+class IntrospectionData(Base):
+    __tablename__ = 'introspection_data'
+    uuid = Column(String(36), ForeignKey('nodes.uuid'), primary_key=True)
+    processed = Column(Boolean, default=False)
+    data = Column(db_types.JsonEncodedDict(mysql_as_long=True),
+                  nullable=True)
+
+
 def init():
     """Initialize the database.
 

ironic_inspector/migrations/versions/bf8dec16023c_add_introspection_data_table.py

@@ -0,0 +1,42 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""add_introspection_data_table
+
+Revision ID: bf8dec16023c
+Revises: 2970d2d44edc
+Create Date: 2018-07-19 18:51:38.124614
+
+"""
+
+from alembic import op
+from oslo_db.sqlalchemy import types as db_types
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'bf8dec16023c'
+down_revision = '2970d2d44edc'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        'introspection_data',
+        sa.Column('uuid', sa.String(36), sa.ForeignKey('nodes.uuid'),
+                  primary_key=True),
+        sa.Column('processed', sa.Boolean, default=False),
+        sa.Column('data', db_types.JsonEncodedDict(mysql_as_long=True).impl,
+                  nullable=True),
+        mysql_ENGINE='InnoDB',
+        mysql_DEFAULT_CHARSET='UTF8'
+    )

ironic_inspector/node_cache.py

@@ -755,7 +755,7 @@ def _delete_node(uuid, session=None):
     with db.ensure_transaction(session) as session:
         db.model_query(db.Attribute, session=session).filter_by(
             node_uuid=uuid).delete()
-        for model in (db.Option, db.Node):
+        for model in (db.Option, db.IntrospectionData, db.Node):
             db.model_query(model,
                            session=session).filter_by(uuid=uuid).delete()
 
@@ -979,3 +979,44 @@ def get_node_list(ironic=None, marker=None, limit=None):
                                    ('started_at', 'uuid'),
                                    marker=marker, sort_dir='desc')
     return [NodeInfo.from_row(row, ironic=ironic) for row in rows]
+
+
+def store_introspection_data(node_id, introspection_data, processed=True):
+    """Store introspection data for this node.
+
+    :param node_id: node UUID.
+    :param introspection_data: A dictionary of introspection data
+    :param processed: Specify the type of introspected data, set to False
+                      indicates the data is unprocessed.
+    """
+    with db.ensure_transaction() as session:
+        record = db.model_query(db.IntrospectionData,
+                                session=session).filter_by(
+            uuid=node_id, processed=processed).first()
+        if record is None:
+            row = db.IntrospectionData()
+            row.update({'uuid': node_id, 'processed': processed,
+                        'data': introspection_data})
+            session.add(row)
+        else:
+            record.update({'data': introspection_data})
+        session.flush()
+
+
+def get_introspection_data(node_id, processed=True):
+    """Get introspection data for this node.
+
+    :param node_id: node UUID.
+    :param processed: Specify the type of introspected data, set to False
+                      indicates retrieving the unprocessed data.
+    :return: A dictionary representation of intropsected data
+    """
+    try:
+        ref = db.model_query(db.IntrospectionData).filter_by(
+            uuid=node_id, processed=processed).one()
+        return ref['data']
+    except orm_errors.NoResultFound:
+        msg = _('Introspection data not found for node %(node)s, '
+                'processed=%(processed)s') % {'node': node_id,
+                                              'processed': processed}
+        raise utils.IntrospectionDataNotFound(msg)

ironic_inspector/test/unit/test_migrations.py

@@ -441,6 +441,19 @@ class MigrationCheckersMixin(object):
         n = nodes.select(nodes.c.uuid == 'abcd').execute().first()
         self.assertIsNone(n['manage_boot'])
 
+    def _check_bf8dec16023c(self, engine, data):
+        introspection_data = db_utils.get_table(engine, 'introspection_data')
+        col_names = [column.name for column in introspection_data.c]
+        self.assertIn('uuid', col_names)
+        self.assertIn('processed', col_names)
+        self.assertIn('data', col_names)
+        self.assertIsInstance(introspection_data.c.uuid.type,
+                              sqlalchemy.types.String)
+        self.assertIsInstance(introspection_data.c.processed.type,
+                              sqlalchemy.types.Boolean)
+        self.assertIsInstance(introspection_data.c.data.type,
+                              sqlalchemy.types.Text)
+
     def test_upgrade_and_version(self):
         with patch_with_engine(self.engine):
             self.migration_ext.upgrade('head')

ironic_inspector/test/unit/test_node_cache.py

@@ -331,6 +331,8 @@ class TestNodeCacheCleanUp(test_base.NodeTest):
                              value=v, node_uuid=self.uuid).save(session)
             db.Option(uuid=self.uuid, name='foo', value='bar').save(
                 session)
+            db.IntrospectionData(uuid=self.uuid, processed=False,
+                                 data={'fake': 'data'}).save(session)
 
     def test_no_timeout(self):
         CONF.set_override('timeout', 0)
@@ -358,6 +360,7 @@ class TestNodeCacheCleanUp(test_base.NodeTest):
         self.assertEqual(len(self.macs),
                          db.model_query(db.Attribute).count())
         self.assertEqual(1, db.model_query(db.Option).count())
+        self.assertEqual(1, db.model_query(db.IntrospectionData).count())
         self.assertFalse(get_lock_mock.called)
 
     @mock.patch.object(node_cache, '_get_lock', autospec=True)
@@ -1256,3 +1259,29 @@ class TestStartIntrospection(test_base.NodeTest):
                               node_cache.start_introspection,
                               self.node_info.uuid)
         self.assertFalse(add_node_mock.called)
+
+
+class TestIntrospectionDataDbStore(test_base.NodeTest):
+    def setUp(self):
+        super(TestIntrospectionDataDbStore, self).setUp()
+        node_cache.add_node(self.node.uuid,
+                            istate.States.processing,
+                            bmc_address='1.2.3.4')
+
+    def _test_store_and_get(self, processed=False):
+        node_cache.store_introspection_data(self.node.uuid,
+                                            copy.deepcopy(self.data),
+                                            processed=processed)
+        stored_data = node_cache.get_introspection_data(self.node.uuid,
+                                                        processed=processed)
+        self.assertEqual(stored_data, self.data)
+
+    def test_store_and_get_unprocessed(self):
+        self._test_store_and_get(processed=False)
+
+    def test_store_and_get_processed(self):
+        self._test_store_and_get(processed=True)
+
+    def test_get_no_data_available(self):
+        self.assertRaises(utils.IntrospectionDataNotFound,
+                          node_cache.get_introspection_data, self.node.uuid)

ironic_inspector/utils.py

@@ -140,6 +140,10 @@ class NodeStateInvalidEvent(Error):
     """Invalid event attempted."""
 
 
+class IntrospectionDataNotFound(NotFoundInCacheError):
+    """Introspection data not found."""
+
+
 def executor():
     """Return the current futures executor."""
     global _EXECUTOR