introspection data backend: plugin layer

Configurable introspection data storage backend [1] is proposed
to provide flexible extension of introspection data storage
instead of the single support of Swift storage backend.

This patch adds plugin mechanism for loading introspection
storage, creates database backend and moves Swift storage
into a plugin.

[1] http://specs.openstack.org/openstack/ironic-inspector-specs/specs/configurable-introspection-data-backends.html

Story: 1726713
Task: 11373

Co-Authored-By: Kaifeng Wang <kaifeng.w@gmail.com>
Change-Id: Ie4d09dc0afc441b20a1e5e3bd8e742b1df918954

ironic_inspector/conductor/manager.py

@@ -22,6 +22,7 @@ from oslo_log import log
 import oslo_messaging as messaging
 from oslo_utils import reflection
 
+from ironic_inspector.common.i18n import _
 from ironic_inspector.common import ironic as ir_utils
 from ironic_inspector import db
 from ironic_inspector import introspect
@@ -126,7 +127,15 @@ class ConductorManager(object):
 
     @messaging.expected_exceptions(utils.Error)
     def do_reapply(self, context, node_id, token=None):
-        process.reapply(node_id)
+        try:
+            data = process.get_introspection_data(node_id, processed=False,
+                                                  get_json=True)
+        except utils.IntrospectionDataStoreDisabled:
+            raise utils.Error(_('Inspector is not configured to store '
+                                'data. Set the [processing]store_data '
+                                'configuration option to change this.'),
+                              code=400)
+        process.reapply(node_id, data)
 
 
 def periodic_clean_up():  # pragma: no cover

ironic_inspector/conf/processing.py

@@ -18,7 +18,6 @@ from ironic_inspector.common.i18n import _
 
 VALID_ADD_PORTS_VALUES = ('all', 'active', 'pxe', 'disabled')
 VALID_KEEP_PORTS_VALUES = ('all', 'present', 'added')
-VALID_STORE_DATA_VALUES = ('none', 'swift')
 
 
 _OPTS = [
@@ -75,9 +74,10 @@ _OPTS = [
                       'aware of. This hook is ignored by default.')),
     cfg.StrOpt('store_data',
                default='none',
-               choices=VALID_STORE_DATA_VALUES,
-               help=_('Method for storing introspection data. If set to \'none'
-                      '\', introspection data will not be stored.')),
+               help=_('The storage backend for storing introspection data. '
+                      'Possible values are: \'none\', \'database\' and '
+                      '\'swift\'. If set to \'none\', introspection data will '
+                      'not be stored.')),
     cfg.StrOpt('store_data_location',
                help=_('Name of the key to store the location of stored data '
                       'in the extra column of the Ironic database.')),

ironic_inspector/main.py

@@ -25,7 +25,6 @@ from ironic_inspector.common import context
 from ironic_inspector.common.i18n import _
 from ironic_inspector.common import ironic as ir_utils
 from ironic_inspector.common import rpc
-from ironic_inspector.common import swift
 import ironic_inspector.conf
 from ironic_inspector.conf import opts as conf_opts
 from ironic_inspector import node_cache
@@ -289,13 +288,13 @@ def api_introspection_abort(node_id):
 @api('/v1/introspection/<node_id>/data', rule="introspection:data",
      methods=['GET'])
 def api_introspection_data(node_id):
-    if CONF.processing.store_data == 'swift':
+    try:
         if not uuidutils.is_uuid_like(node_id):
             node = ir_utils.get_node(node_id, fields=['uuid'])
             node_id = node.uuid
-        res = swift.get_introspection_data(node_id)
+        res = process.get_introspection_data(node_id)
         return res, 200, {'Content-Type': 'application/json'}
-    else:
+    except utils.IntrospectionDataStoreDisabled:
         return error_response(_('Inspector is not configured to store data. '
                                 'Set the [processing]store_data '
                                 'configuration option to change this.'),
@@ -309,15 +308,9 @@ def api_introspection_reapply(node_id):
         return error_response(_('User data processing is not '
                                 'supported yet'), code=400)
 
-    if CONF.processing.store_data == 'swift':
     client = rpc.get_client()
     client.call({}, 'do_reapply', node_id=node_id)
     return '', 202
-    else:
-        return error_response(_('Inspector is not configured to store'
-                                ' data. Set the [processing] '
-                                'store_data configuration option to '
-                                'change this.'), code=400)
 
 
 def rule_repr(rule, short):

ironic_inspector/plugins/base.py

@@ -142,6 +142,7 @@ _HOOKS_MGR = None
 _NOT_FOUND_HOOK_MGR = None
 _CONDITIONS_MGR = None
 _ACTIONS_MGR = None
+_INTROSPECTION_DATA_MGR = None
 
 
 def missing_entrypoints_callback(names):
@@ -229,3 +230,12 @@ def rule_actions_manager():
             'ironic_inspector.rules.actions',
             invoke_on_load=True)
     return _ACTIONS_MGR
+
+
+def introspection_data_manager():
+    global _INTROSPECTION_DATA_MGR
+    if _INTROSPECTION_DATA_MGR is None:
+        _INTROSPECTION_DATA_MGR = stevedore.ExtensionManager(
+            'ironic_inspector.introspection_data.store',
+            invoke_on_load=True)
+    return _INTROSPECTION_DATA_MGR

ironic_inspector/plugins/introspection_data.py

@@ -0,0 +1,123 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Backends for storing introspection data."""
+
+import abc
+import json
+
+from oslo_config import cfg
+from oslo_utils import excutils
+import six
+
+from ironic_inspector.common import swift
+from ironic_inspector import node_cache
+from ironic_inspector import utils
+
+
+CONF = cfg.CONF
+
+LOG = utils.getProcessingLogger(__name__)
+
+_STORAGE_EXCLUDED_KEYS = {'logs'}
+_UNPROCESSED_DATA_STORE_SUFFIX = 'UNPROCESSED'
+
+
+def _filter_data_excluded_keys(data):
+    return {k: v for k, v in data.items()
+            if k not in _STORAGE_EXCLUDED_KEYS}
+
+
+@six.add_metaclass(abc.ABCMeta)
+class BaseStorageBackend(object):
+
+    @abc.abstractmethod
+    def get(self, node_id, processed=True, get_json=False):
+        """Get introspected data from storage backend.
+
+        :param node_id: node UUID or name.
+        :param processed: Specify whether the data to be retrieved is
+                          processed or not.
+        :param get_json: Specify whether return the introspection data in json
+                         format, string value is returned if False.
+        :returns: the introspection data.
+        :raises: IntrospectionDataStoreDisabled if storage backend is disabled.
+        """
+
+    @abc.abstractmethod
+    def save(self, node_info, data, processed=True):
+        """Save introspected data to storage backend.
+
+        :param node_info: a NodeInfo object.
+        :param data: the introspected data to be saved, in dict format.
+        :param processed: Specify whether the data to be saved is processed or
+                          not.
+        :raises: IntrospectionDataStoreDisabled if storage backend is disabled.
+        """
+
+
+class NoStore(BaseStorageBackend):
+    def get(self, node_id, processed=True, get_json=False):
+        raise utils.IntrospectionDataStoreDisabled(
+            'Introspection data storage is disabled')
+
+    def save(self, node_info, data, processed=True):
+        LOG.debug('Introspection data storage is disabled, the data will not '
+                  'be saved', node_info=node_info)
+
+
+class SwiftStore(object):
+    def get(self, node_id, processed=True, get_json=False):
+        suffix = None if processed else _UNPROCESSED_DATA_STORE_SUFFIX
+        LOG.debug('Fetching introspection data from Swift for %s', node_id)
+        data = swift.get_introspection_data(node_id, suffix=suffix)
+        if get_json:
+            return json.loads(data)
+        return data
+
+    def save(self, node_info, data, processed=True):
+        suffix = None if processed else _UNPROCESSED_DATA_STORE_SUFFIX
+        swift_object_name = swift.store_introspection_data(
+            _filter_data_excluded_keys(data),
+            node_info.uuid,
+            suffix=suffix
+        )
+        LOG.info('Introspection data was stored in Swift object %s',
+                 swift_object_name, node_info=node_info)
+        if CONF.processing.store_data_location:
+            node_info.patch([{'op': 'add', 'path': '/extra/%s' %
+                              CONF.processing.store_data_location,
+                              'value': swift_object_name}])
+
+
+class DatabaseStore(object):
+    def get(self, node_id, processed=True, get_json=False):
+        LOG.debug('Fetching introspection data from database for %(node)s',
+                  {'node': node_id})
+        data = node_cache.get_introspection_data(node_id, processed)
+        if get_json:
+            return data
+        return json.dumps(data)
+
+    def save(self, node_info, data, processed=True):
+        introspection_data = _filter_data_excluded_keys(data)
+        try:
+            node_cache.store_introspection_data(node_info.uuid,
+                                                introspection_data, processed)
+        except Exception as e:
+            with excutils.save_and_reraise_exception():
+                LOG.exception('Failed to store introspection data in '
+                              'database: %(exc)s', {'exc': e})
+        else:
+            LOG.info('Introspection data was stored in database',
+                     node_info=node_info)

ironic_inspector/process.py

@@ -15,7 +15,6 @@
 
 import copy
 import datetime
-import json
 import os
 
 from oslo_config import cfg
@@ -25,7 +24,6 @@ from oslo_utils import timeutils
 
 from ironic_inspector.common.i18n import _
 from ironic_inspector.common import ironic as ir_utils
-from ironic_inspector.common import swift
 from ironic_inspector import introspection_state as istate
 from ironic_inspector import node_cache
 from ironic_inspector.plugins import base as plugins_base
@@ -38,7 +36,6 @@ CONF = cfg.CONF
 LOG = utils.getProcessingLogger(__name__)
 
 _STORAGE_EXCLUDED_KEYS = {'logs'}
-_UNPROCESSED_DATA_STORE_SUFFIX = 'UNPROCESSED'
 
 
 def _store_logs(introspection_data, node_info):
@@ -143,48 +140,28 @@ def _filter_data_excluded_keys(data):
             if k not in _STORAGE_EXCLUDED_KEYS}
 
 
-def _store_data(node_info, data, suffix=None):
-    if CONF.processing.store_data != 'swift':
-        LOG.debug("Swift support is disabled, introspection data "
-                  "won't be stored", node_info=node_info)
-        return
-
-    swift_object_name = swift.store_introspection_data(
-        _filter_data_excluded_keys(data),
-        node_info.uuid,
-        suffix=suffix
-    )
-    LOG.info('Introspection data was stored in Swift in object '
-             '%s', swift_object_name, node_info=node_info)
-    if CONF.processing.store_data_location:
-        node_info.patch([{'op': 'add', 'path': '/extra/%s' %
-                          CONF.processing.store_data_location,
-                          'value': swift_object_name}])
+def _store_data(node_info, data, processed=True):
+    introspection_data_manager = plugins_base.introspection_data_manager()
+    store = CONF.processing.store_data
+    ext = introspection_data_manager[store].obj
+    ext.save(node_info, data, processed)
 
 
 def _store_unprocessed_data(node_info, data):
     # runs in background
     try:
-        _store_data(node_info, data,
-                    suffix=_UNPROCESSED_DATA_STORE_SUFFIX)
+        _store_data(node_info, data, processed=False)
     except Exception:
         LOG.exception('Encountered exception saving unprocessed '
                       'introspection data', node_info=node_info,
                       data=data)
 
 
-def _get_unprocessed_data(uuid):
-    if CONF.processing.store_data == 'swift':
-        LOG.debug('Fetching unprocessed introspection data from '
-                  'Swift for %s', uuid)
-        return json.loads(
-            swift.get_introspection_data(
-                uuid,
-                suffix=_UNPROCESSED_DATA_STORE_SUFFIX
-            )
-        )
-    else:
-        raise utils.Error(_('Swift support is disabled'), code=400)
+def get_introspection_data(uuid, processed=True, get_json=False):
+    introspection_data_manager = plugins_base.introspection_data_manager()
+    store = CONF.processing.store_data
+    ext = introspection_data_manager[store].obj
+    return ext.get(uuid, processed=processed, get_json=get_json)
 
 
 def process(introspection_data):
@@ -309,7 +286,7 @@ def _finish(node_info, ironic, introspection_data, power_off=True):
              node_info=node_info, data=introspection_data)
 
 
-def reapply(node_ident):
+def reapply(node_ident, data=None):
     """Re-apply introspection steps.
 
     Re-apply preprocessing, postprocessing and introspection rules on
@@ -331,15 +308,20 @@ def reapply(node_ident):
         raise utils.Error(_('Node locked, please, try again later'),
                           node_info=node_info, code=409)
 
-    utils.executor().submit(_reapply, node_info)
+    utils.executor().submit(_reapply, node_info, data)
 
 
-def _reapply(node_info):
+def _reapply(node_info, data=None):
     # runs in background
     try:
         node_info.started_at = timeutils.utcnow()
         node_info.commit()
-        introspection_data = _get_unprocessed_data(node_info.uuid)
+        if data:
+            introspection_data = data
+        else:
+            introspection_data = get_introspection_data(node_info.uuid,
+                                                        processed=False,
+                                                        get_json=True)
     except Exception as exc:
         LOG.exception('Encountered exception while fetching '
                       'stored introspection data',

ironic_inspector/test/unit/test_main.py

@@ -22,6 +22,7 @@ from oslo_utils import uuidutils
 
 from ironic_inspector.common import ironic as ir_utils
 from ironic_inspector.common import rpc
+from ironic_inspector.common import swift
 import ironic_inspector.conf
 from ironic_inspector.conf import opts as conf_opts
 from ironic_inspector import introspection_state as istate
@@ -29,6 +30,7 @@ from ironic_inspector import main
 from ironic_inspector import node_cache
 from ironic_inspector.plugins import base as plugins_base
 from ironic_inspector.plugins import example as example_plugin
+from ironic_inspector.plugins import introspection_data as intros_data_plugin
 from ironic_inspector import process
 from ironic_inspector import rules
 from ironic_inspector.test import base as test_base
@@ -297,10 +299,9 @@ class TestApiListStatus(GetStatusAPIBaseTest):
 
 
 class TestApiGetData(BaseAPITest):
-    @mock.patch.object(main.swift, 'SwiftAPI', autospec=True)
-    def test_get_introspection_data(self, swift_mock):
-        CONF.set_override('store_data', 'swift', 'processing')
-        data = {
+    def setUp(self):
+        super(TestApiGetData, self).setUp()
+        self.introspection_data = {
             'ipmi_address': '1.2.3.4',
             'cpus': 2,
             'cpu_arch': 'x86_64',
@@ -310,44 +311,48 @@ class TestApiGetData(BaseAPITest):
                 'em1': {'mac': '11:22:33:44:55:66', 'ip': '1.2.0.1'},
             }
         }
+
+    @mock.patch.object(swift, 'SwiftAPI', autospec=True)
+    def test_get_introspection_data_from_swift(self, swift_mock):
+        CONF.set_override('store_data', 'swift', 'processing')
         swift_conn = swift_mock.return_value
-        swift_conn.get_object.return_value = json.dumps(data)
+        swift_conn.get_object.return_value = json.dumps(
+            self.introspection_data)
         res = self.app.get('/v1/introspection/%s/data' % self.uuid)
         name = 'inspector_data-%s' % self.uuid
         swift_conn.get_object.assert_called_once_with(name)
         self.assertEqual(200, res.status_code)
-        self.assertEqual(data, json.loads(res.data.decode('utf-8')))
+        self.assertEqual(self.introspection_data,
+                         json.loads(res.data.decode('utf-8')))
 
-    @mock.patch.object(main.swift, 'SwiftAPI', autospec=True)
-    def test_introspection_data_not_stored(self, swift_mock):
-        CONF.set_override('store_data', 'none', 'processing')
-        swift_conn = swift_mock.return_value
+    @mock.patch.object(intros_data_plugin, 'DatabaseStore',
+                       autospec=True)
+    def test_get_introspection_data_from_db(self, db_mock):
+        CONF.set_override('store_data', 'database', 'processing')
+        db_store = db_mock.return_value
+        db_store.get.return_value = json.dumps(self.introspection_data)
+        res = self.app.get('/v1/introspection/%s/data' % self.uuid)
+        db_store.get.assert_called_once_with(self.uuid, processed=True,
+                                             get_json=False)
+        self.assertEqual(200, res.status_code)
+        self.assertEqual(self.introspection_data,
+                         json.loads(res.data.decode('utf-8')))
+
+    def test_introspection_data_not_stored(self):
+        CONF.set_override('store_data', 'none', 'processing')
         res = self.app.get('/v1/introspection/%s/data' % self.uuid)
-        self.assertFalse(swift_conn.get_object.called)
         self.assertEqual(404, res.status_code)
 
     @mock.patch.object(ir_utils, 'get_node', autospec=True)
-    @mock.patch.object(main.swift, 'SwiftAPI', autospec=True)
-    def test_with_name(self, swift_mock, get_mock):
+    @mock.patch.object(main.process, 'get_introspection_data', autospec=True)
+    def test_with_name(self, process_mock, get_mock):
         get_mock.return_value = mock.Mock(uuid=self.uuid)
         CONF.set_override('store_data', 'swift', 'processing')
-        data = {
-            'ipmi_address': '1.2.3.4',
-            'cpus': 2,
-            'cpu_arch': 'x86_64',
-            'memory_mb': 1024,
-            'local_gb': 20,
-            'interfaces': {
-                'em1': {'mac': '11:22:33:44:55:66', 'ip': '1.2.0.1'},
-            }
-        }
-        swift_conn = swift_mock.return_value
-        swift_conn.get_object.return_value = json.dumps(data)
+        process_mock.return_value = json.dumps(self.introspection_data)
         res = self.app.get('/v1/introspection/name1/data')
-        name = 'inspector_data-%s' % self.uuid
-        swift_conn.get_object.assert_called_once_with(name)
         self.assertEqual(200, res.status_code)
-        self.assertEqual(data, json.loads(res.data.decode('utf-8')))
+        self.assertEqual(self.introspection_data,
+                         json.loads(res.data.decode('utf-8')))
         get_mock.assert_called_once_with('name1', fields=['uuid'])
 
 
@@ -361,8 +366,7 @@ class TestApiReapply(BaseAPITest):
         self.rpc_get_client_mock.return_value = self.client_mock
         CONF.set_override('store_data', 'swift', 'processing')
 
-    def test_ok(self):
-
+    def test_api_ok(self):
         self.app.post('/v1/introspection/%s/data/unprocessed' %
                       self.uuid)
         self.client_mock.call.assert_called_once_with({}, 'do_reapply',
@@ -377,18 +381,18 @@ class TestApiReapply(BaseAPITest):
                          message)
         self.assertFalse(self.client_mock.call.called)
 
-    def test_swift_disabled(self):
-        CONF.set_override('store_data', 'none', 'processing')
+    def test_get_introspection_data_error(self):
+        exc = utils.Error('The store is crashed', code=404)
+        self.client_mock.call.side_effect = exc
 
         res = self.app.post('/v1/introspection/%s/data/unprocessed' %
                             self.uuid)
-        self.assertEqual(400, res.status_code)
+
+        self.assertEqual(404, res.status_code)
         message = json.loads(res.data.decode())['error']['message']
-        self.assertEqual('Inspector is not configured to store '
-                         'data. Set the [processing] store_data '
-                         'configuration option to change this.',
-                         message)
-        self.assertFalse(self.client_mock.call.called)
+        self.assertEqual(str(exc), message)
+        self.client_mock.call.assert_called_once_with({}, 'do_reapply',
+                                                      node_id=self.uuid)
 
     def test_generic_error(self):
         exc = utils.Error('Oops', code=400)

ironic_inspector/test/unit/test_manager.py

@@ -11,10 +11,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
+
 import fixtures
 import mock
 import oslo_messaging as messaging
 
+from ironic_inspector.common import swift
 from ironic_inspector.conductor import manager
 import ironic_inspector.conf
 from ironic_inspector import introspect
@@ -302,11 +305,17 @@ class TestManagerReapply(BaseManagerTest):
         super(TestManagerReapply, self).setUp()
         CONF.set_override('store_data', 'swift', 'processing')
 
-    def test_ok(self, reapply_mock):
+    @mock.patch.object(swift, 'store_introspection_data', autospec=True)
+    @mock.patch.object(swift, 'get_introspection_data', autospec=True)
+    def test_ok(self, swift_get_mock, swift_set_mock, reapply_mock):
+        swift_get_mock.return_value = json.dumps(self.data)
         self.manager.do_reapply(self.context, self.uuid)
-        reapply_mock.assert_called_once_with(self.uuid)
+        reapply_mock.assert_called_once_with(self.uuid, data=self.data)
 
-    def test_node_locked(self, reapply_mock):
+    @mock.patch.object(swift, 'store_introspection_data', autospec=True)
+    @mock.patch.object(swift, 'get_introspection_data', autospec=True)
+    def test_node_locked(self, swift_get_mock, swift_set_mock, reapply_mock):
+        swift_get_mock.return_value = json.dumps(self.data)
         exc = utils.Error('Locked.', code=409)
         reapply_mock.side_effect = exc
 
@@ -317,9 +326,13 @@ class TestManagerReapply(BaseManagerTest):
         self.assertEqual(utils.Error, exc.exc_info[0])
         self.assertIn('Locked.', str(exc.exc_info[1]))
         self.assertEqual(409, exc.exc_info[1].http_code)
-        reapply_mock.assert_called_once_with(self.uuid)
+        reapply_mock.assert_called_once_with(self.uuid, data=self.data)
 
-    def test_node_not_found(self, reapply_mock):
+    @mock.patch.object(swift, 'store_introspection_data', autospec=True)
+    @mock.patch.object(swift, 'get_introspection_data', autospec=True)
+    def test_node_not_found(self, swift_get_mock, swift_set_mock,
+                            reapply_mock):
+        swift_get_mock.return_value = json.dumps(self.data)
         exc = utils.Error('Not found.', code=404)
         reapply_mock.side_effect = exc
 
@@ -330,9 +343,11 @@ class TestManagerReapply(BaseManagerTest):
         self.assertEqual(utils.Error, exc.exc_info[0])
         self.assertIn('Not found.', str(exc.exc_info[1]))
         self.assertEqual(404, exc.exc_info[1].http_code)
-        reapply_mock.assert_called_once_with(self.uuid)
+        reapply_mock.assert_called_once_with(self.uuid, data=self.data)
 
-    def test_generic_error(self, reapply_mock):
+    @mock.patch.object(process, 'get_introspection_data', autospec=True)
+    def test_generic_error(self, get_data_mock, reapply_mock):
+        get_data_mock.return_value = self.data
         exc = utils.Error('Oops', code=400)
         reapply_mock.side_effect = exc
 
@@ -343,4 +358,52 @@ class TestManagerReapply(BaseManagerTest):
         self.assertEqual(utils.Error, exc.exc_info[0])
         self.assertIn('Oops', str(exc.exc_info[1]))
         self.assertEqual(400, exc.exc_info[1].http_code)
-        reapply_mock.assert_called_once_with(self.uuid)
+        reapply_mock.assert_called_once_with(self.uuid, data=self.data)
+        get_data_mock.assert_called_once_with(self.uuid, processed=False,
+                                              get_json=True)
+
+    @mock.patch.object(process, 'get_introspection_data', autospec=True)
+    def test_get_introspection_data_error(self, get_data_mock, reapply_mock):
+        exc = utils.Error('The store is empty', code=404)
+        get_data_mock.side_effect = exc
+
+        exc = self.assertRaises(messaging.rpc.ExpectedException,
+                                self.manager.do_reapply,
+                                self.context, self.uuid)
+
+        self.assertEqual(utils.Error, exc.exc_info[0])
+        self.assertIn('The store is empty', str(exc.exc_info[1]))
+        self.assertEqual(404, exc.exc_info[1].http_code)
+        get_data_mock.assert_called_once_with(self.uuid, processed=False,
+                                              get_json=True)
+        self.assertFalse(reapply_mock.called)
+
+    def test_store_data_disabled(self, reapply_mock):
+        CONF.set_override('store_data', 'none', 'processing')
+
+        exc = self.assertRaises(messaging.rpc.ExpectedException,
+                                self.manager.do_reapply,
+                                self.context, self.uuid)
+
+        self.assertEqual(utils.Error, exc.exc_info[0])
+        self.assertIn('Inspector is not configured to store data',
+                      str(exc.exc_info[1]))
+        self.assertEqual(400, exc.exc_info[1].http_code)
+        self.assertFalse(reapply_mock.called)
+
+    @mock.patch.object(process, 'get_introspection_data', autospec=True)
+    def test_ok_swift(self, get_data_mock, reapply_mock):
+        get_data_mock.return_value = self.data
+        self.manager.do_reapply(self.context, self.uuid)
+        reapply_mock.assert_called_once_with(self.uuid, data=self.data)
+        get_data_mock.assert_called_once_with(self.uuid, processed=False,
+                                              get_json=True)
+
+    @mock.patch.object(process, 'get_introspection_data', autospec=True)
+    def test_ok_db(self, get_data_mock, reapply_mock):
+        get_data_mock.return_value = self.data
+        CONF.set_override('store_data', 'database', 'processing')
+        self.manager.do_reapply(self.context, self.uuid)
+        reapply_mock.assert_called_once_with(self.uuid, data=self.data)
+        get_data_mock.assert_called_once_with(self.uuid, processed=False,
+                                              get_json=True)

ironic_inspector/test/unit/test_plugins_introspection_data.py

@@ -0,0 +1,108 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+import fixtures
+import mock
+from oslo_config import cfg
+
+from ironic_inspector.common import ironic as ir_utils
+from ironic_inspector import db
+from ironic_inspector import introspection_state as istate
+from ironic_inspector.plugins import introspection_data
+from ironic_inspector.test import base as test_base
+
+CONF = cfg.CONF
+
+
+class BaseTest(test_base.NodeTest):
+    data = {
+        'ipmi_address': '1.2.3.4',
+        'cpus': 2,
+        'cpu_arch': 'x86_64',
+        'memory_mb': 1024,
+        'local_gb': 20,
+        'interfaces': {
+            'em1': {'mac': '11:22:33:44:55:66', 'ip': '1.2.0.1'},
+        }
+    }
+
+    def setUp(self):
+        super(BaseTest, self).setUp()
+        self.cli_fixture = self.useFixture(
+            fixtures.MockPatchObject(ir_utils, 'get_client', autospec=True))
+        self.cli = self.cli_fixture.mock.return_value
+
+
+@mock.patch.object(introspection_data.swift, 'SwiftAPI', autospec=True)
+class TestSwiftStore(BaseTest):
+
+    def setUp(self):
+        super(TestSwiftStore, self).setUp()
+        self.driver = introspection_data.SwiftStore()
+
+    def test_get_data(self, swift_mock):
+        swift_conn = swift_mock.return_value
+        swift_conn.get_object.return_value = json.dumps(self.data)
+        name = 'inspector_data-%s' % self.uuid
+
+        res_data = self.driver.get(self.uuid)
+
+        swift_conn.get_object.assert_called_once_with(name)
+        self.assertEqual(self.data, json.loads(res_data))
+
+    def test_store_data(self, swift_mock):
+        swift_conn = swift_mock.return_value
+        name = 'inspector_data-%s' % self.uuid
+
+        self.driver.save(self.node_info, self.data)
+
+        data = introspection_data._filter_data_excluded_keys(self.data)
+        swift_conn.create_object.assert_called_once_with(name,
+                                                         json.dumps(data))
+
+    def test_store_data_location(self, swift_mock):
+        CONF.set_override('store_data_location', 'inspector_data_object',
+                          'processing')
+        swift_conn = swift_mock.return_value
+        name = 'inspector_data-%s' % self.uuid
+        patch = [{'path': '/extra/inspector_data_object',
+                  'value': name, 'op': 'add'}]
+        expected = self.data
+
+        self.driver.save(self.node_info, self.data)
+
+        data = introspection_data._filter_data_excluded_keys(self.data)
+        swift_conn.create_object.assert_called_once_with(name,
+                                                         json.dumps(data))
+        self.assertEqual(expected,
+                         json.loads(swift_conn.create_object.call_args[0][1]))
+        self.cli.node.update.assert_any_call(self.uuid, patch)
+
+
+class TestDatabaseStore(BaseTest):
+    def setUp(self):
+        super(TestDatabaseStore, self).setUp()
+        self.driver = introspection_data.DatabaseStore()
+        session = db.get_writer_session()
+        with session.begin():
+            db.Node(uuid=self.node_info.uuid,
+                    state=istate.States.starting).save(session)
+
+    def test_store_and_get_data(self):
+        self.driver.save(self.node_info, self.data)
+
+        res_data = self.driver.get(self.node_info.uuid)
+
+        self.assertEqual(self.data, json.loads(res_data))

ironic_inspector/test/unit/test_process.py

@@ -28,11 +28,13 @@ from oslo_utils import uuidutils
 import six
 
 from ironic_inspector.common import ironic as ir_utils
+from ironic_inspector.common import swift
 from ironic_inspector import db
 from ironic_inspector import introspection_state as istate
 from ironic_inspector import node_cache
 from ironic_inspector.plugins import base as plugins_base
 from ironic_inspector.plugins import example as example_plugin
+from ironic_inspector.plugins import introspection_data as intros_data_plugin
 from ironic_inspector import process
 from ironic_inspector.pxe_filter import base as pxe_filter
 from ironic_inspector.test import base as test_base
@@ -259,22 +261,13 @@ class TestUnprocessedData(BaseProcessTest):
 
         store_mock.assert_called_once_with(mock.ANY, expected)
 
-    @mock.patch.object(process.swift, 'SwiftAPI', autospec=True)
-    def test_save_unprocessed_data_failure(self, swift_mock):
+    def test_save_unprocessed_data_failure(self):
         CONF.set_override('store_data', 'swift', 'processing')
-        name = 'inspector_data-%s-%s' % (
-            self.uuid,
-            process._UNPROCESSED_DATA_STORE_SUFFIX
-        )
-
-        swift_conn = swift_mock.return_value
-        swift_conn.create_object.side_effect = utils.Error('Oops')
 
         res = process.process(self.data)
 
         # assert store failure doesn't break processing
         self.assertEqual(self.fake_result_json, res)
-        swift_conn.create_object.assert_called_once_with(name, mock.ANY)
 
 
 @mock.patch.object(example_plugin.ExampleProcessingHook, 'before_processing',
@@ -405,6 +398,7 @@ class TestProcessNode(BaseTest):
                 started_at=self.node_info.started_at,
                 finished_at=self.node_info.finished_at,
                 error=self.node_info.error).save(self.session)
+        plugins_base._INTROSPECTION_DATA_MGR = None
 
     def test_return_includes_uuid(self):
         ret_val = process._process_node(self.node_info, self.node, self.data)
@@ -485,8 +479,8 @@ class TestProcessNode(BaseTest):
         finished_mock.assert_called_once_with(
             self.node_info, istate.Events.finish)
 
-    @mock.patch.object(process.swift, 'SwiftAPI', autospec=True)
-    def test_store_data(self, swift_mock):
+    @mock.patch.object(swift, 'SwiftAPI', autospec=True)
+    def test_store_data_with_swift(self, swift_mock):
         CONF.set_override('store_data', 'swift', 'processing')
         swift_conn = swift_mock.return_value
         name = 'inspector_data-%s' % self.uuid
@@ -498,8 +492,8 @@ class TestProcessNode(BaseTest):
         self.assertEqual(expected,
                          json.loads(swift_conn.create_object.call_args[0][1]))
 
-    @mock.patch.object(process.swift, 'SwiftAPI', autospec=True)
-    def test_store_data_no_logs(self, swift_mock):
+    @mock.patch.object(swift, 'SwiftAPI', autospec=True)
+    def test_store_data_no_logs_with_swift(self, swift_mock):
         CONF.set_override('store_data', 'swift', 'processing')
         swift_conn = swift_mock.return_value
         name = 'inspector_data-%s' % self.uuid
@@ -511,8 +505,8 @@ class TestProcessNode(BaseTest):
         self.assertNotIn('logs',
                          json.loads(swift_conn.create_object.call_args[0][1]))
 
-    @mock.patch.object(process.swift, 'SwiftAPI', autospec=True)
-    def test_store_data_location(self, swift_mock):
+    @mock.patch.object(swift, 'SwiftAPI', autospec=True)
+    def test_store_data_location_with_swift(self, swift_mock):
         CONF.set_override('store_data', 'swift', 'processing')
         CONF.set_override('store_data_location', 'inspector_data_object',
                           'processing')
@@ -529,6 +523,28 @@ class TestProcessNode(BaseTest):
                          json.loads(swift_conn.create_object.call_args[0][1]))
         self.cli.node.update.assert_any_call(self.uuid, patch)
 
+    @mock.patch.object(node_cache, 'store_introspection_data', autospec=True)
+    def test_store_data_with_database(self, store_mock):
+        CONF.set_override('store_data', 'database', 'processing')
+
+        process._process_node(self.node_info, self.node, self.data)
+
+        data = intros_data_plugin._filter_data_excluded_keys(self.data)
+        store_mock.assert_called_once_with(self.node_info.uuid, data, True)
+        self.assertEqual(data, store_mock.call_args[0][1])
+
+    @mock.patch.object(node_cache, 'store_introspection_data', autospec=True)
+    def test_store_data_no_logs_with_database(self, store_mock):
+        CONF.set_override('store_data', 'database', 'processing')
+
+        self.data['logs'] = 'something'
+
+        process._process_node(self.node_info, self.node, self.data)
+
+        data = intros_data_plugin._filter_data_excluded_keys(self.data)
+        store_mock.assert_called_once_with(self.node_info.uuid, data, True)
+        self.assertNotIn('logs', store_mock.call_args[0][1])
+
 
 @mock.patch.object(process, '_reapply', autospec=True)
 @mock.patch.object(node_cache, 'get_node', autospec=True)
@@ -558,7 +574,7 @@ class TestReapply(BaseTest):
             blocking=False
         )
 
-        reapply_mock.assert_called_once_with(pop_mock.return_value)
+        reapply_mock.assert_called_once_with(pop_mock.return_value, data=None)
 
     @prepare_mocks
     def test_locking_failed(self, pop_mock, reapply_mock):
@@ -575,7 +591,7 @@ class TestReapply(BaseTest):
 
 @mock.patch.object(example_plugin.ExampleProcessingHook, 'before_update')
 @mock.patch.object(process.rules, 'apply', autospec=True)
-@mock.patch.object(process.swift, 'SwiftAPI', autospec=True)
+@mock.patch.object(swift, 'SwiftAPI', autospec=True)
 @mock.patch.object(node_cache.NodeInfo, 'finished', autospec=True)
 @mock.patch.object(node_cache.NodeInfo, 'release_lock', autospec=True)
 class TestReapplyNode(BaseTest):

ironic_inspector/utils.py

@@ -140,6 +140,10 @@ class NodeStateInvalidEvent(Error):
     """Invalid event attempted."""
 
 
+class IntrospectionDataStoreDisabled(Error):
+    """Introspection data store is disabled."""
+
+
 class IntrospectionDataNotFound(NotFoundInCacheError):
     """Introspection data not found."""
 

releasenotes/notes/introspection-data-db-store-0586292de05cbfd7.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Adds the support to store introspection data in ironic-inspector database.
+    Set the option ``[processing]store_data`` to ``database`` to use this
+    feature.

setup.cfg

@@ -43,6 +43,10 @@ ironic_inspector.hooks.processing =
 ironic_inspector.hooks.node_not_found =
     example = ironic_inspector.plugins.example:example_not_found_hook
     enroll = ironic_inspector.plugins.discovery:enroll_node_not_found_hook
+ironic_inspector.introspection_data.store =
+    none = ironic_inspector.plugins.introspection_data:NoStore
+    swift = ironic_inspector.plugins.introspection_data:SwiftStore
+    database = ironic_inspector.plugins.introspection_data:DatabaseStore
 ironic_inspector.rules.conditions =
     eq = ironic_inspector.plugins.rules:EqCondition
     lt = ironic_inspector.plugins.rules:LtCondition