Browse Source

introspection data backend: plugin layer

Configurable introspection data storage backend [1] is proposed
to provide flexible extension of introspection data storage
instead of the single support of Swift storage backend.

This patch adds plugin mechanism for loading introspection
storage, creates database backend and moves Swift storage
into a plugin.

[1] http://specs.openstack.org/openstack/ironic-inspector-specs/specs/configurable-introspection-data-backends.html

Story: 1726713
Task: 11373

Co-Authored-By: Kaifeng Wang <kaifeng.w@gmail.com>
Change-Id: Ie4d09dc0afc441b20a1e5e3bd8e742b1df918954
tags/8.2.0
space 1 year ago
parent
commit
d278bb6f77

+ 10
- 1
ironic_inspector/conductor/manager.py View File

@@ -22,6 +22,7 @@ from oslo_log import log
22 22
 import oslo_messaging as messaging
23 23
 from oslo_utils import reflection
24 24
 
25
+from ironic_inspector.common.i18n import _
25 26
 from ironic_inspector.common import ironic as ir_utils
26 27
 from ironic_inspector import db
27 28
 from ironic_inspector import introspect
@@ -126,7 +127,15 @@ class ConductorManager(object):
126 127
 
127 128
     @messaging.expected_exceptions(utils.Error)
128 129
     def do_reapply(self, context, node_id, token=None):
129
-        process.reapply(node_id)
130
+        try:
131
+            data = process.get_introspection_data(node_id, processed=False,
132
+                                                  get_json=True)
133
+        except utils.IntrospectionDataStoreDisabled:
134
+            raise utils.Error(_('Inspector is not configured to store '
135
+                                'data. Set the [processing]store_data '
136
+                                'configuration option to change this.'),
137
+                              code=400)
138
+        process.reapply(node_id, data)
130 139
 
131 140
 
132 141
 def periodic_clean_up():  # pragma: no cover

+ 4
- 4
ironic_inspector/conf/processing.py View File

@@ -18,7 +18,6 @@ from ironic_inspector.common.i18n import _
18 18
 
19 19
 VALID_ADD_PORTS_VALUES = ('all', 'active', 'pxe', 'disabled')
20 20
 VALID_KEEP_PORTS_VALUES = ('all', 'present', 'added')
21
-VALID_STORE_DATA_VALUES = ('none', 'swift')
22 21
 
23 22
 
24 23
 _OPTS = [
@@ -75,9 +74,10 @@ _OPTS = [
75 74
                       'aware of. This hook is ignored by default.')),
76 75
     cfg.StrOpt('store_data',
77 76
                default='none',
78
-               choices=VALID_STORE_DATA_VALUES,
79
-               help=_('Method for storing introspection data. If set to \'none'
80
-                      '\', introspection data will not be stored.')),
77
+               help=_('The storage backend for storing introspection data. '
78
+                      'Possible values are: \'none\', \'database\' and '
79
+                      '\'swift\'. If set to \'none\', introspection data will '
80
+                      'not be stored.')),
81 81
     cfg.StrOpt('store_data_location',
82 82
                help=_('Name of the key to store the location of stored data '
83 83
                       'in the extra column of the Ironic database.')),

+ 7
- 14
ironic_inspector/main.py View File

@@ -25,7 +25,6 @@ from ironic_inspector.common import context
25 25
 from ironic_inspector.common.i18n import _
26 26
 from ironic_inspector.common import ironic as ir_utils
27 27
 from ironic_inspector.common import rpc
28
-from ironic_inspector.common import swift
29 28
 import ironic_inspector.conf
30 29
 from ironic_inspector.conf import opts as conf_opts
31 30
 from ironic_inspector import node_cache
@@ -289,15 +288,15 @@ def api_introspection_abort(node_id):
289 288
 @api('/v1/introspection/<node_id>/data', rule="introspection:data",
290 289
      methods=['GET'])
291 290
 def api_introspection_data(node_id):
292
-    if CONF.processing.store_data == 'swift':
291
+    try:
293 292
         if not uuidutils.is_uuid_like(node_id):
294 293
             node = ir_utils.get_node(node_id, fields=['uuid'])
295 294
             node_id = node.uuid
296
-        res = swift.get_introspection_data(node_id)
295
+        res = process.get_introspection_data(node_id)
297 296
         return res, 200, {'Content-Type': 'application/json'}
298
-    else:
297
+    except utils.IntrospectionDataStoreDisabled:
299 298
         return error_response(_('Inspector is not configured to store data. '
300
-                                'Set the [processing] store_data '
299
+                                'Set the [processing]store_data '
301 300
                                 'configuration option to change this.'),
302 301
                               code=404)
303 302
 
@@ -309,15 +308,9 @@ def api_introspection_reapply(node_id):
309 308
         return error_response(_('User data processing is not '
310 309
                                 'supported yet'), code=400)
311 310
 
312
-    if CONF.processing.store_data == 'swift':
313
-        client = rpc.get_client()
314
-        client.call({}, 'do_reapply', node_id=node_id)
315
-        return '', 202
316
-    else:
317
-        return error_response(_('Inspector is not configured to store'
318
-                                ' data. Set the [processing] '
319
-                                'store_data configuration option to '
320
-                                'change this.'), code=400)
311
+    client = rpc.get_client()
312
+    client.call({}, 'do_reapply', node_id=node_id)
313
+    return '', 202
321 314
 
322 315
 
323 316
 def rule_repr(rule, short):

+ 10
- 0
ironic_inspector/plugins/base.py View File

@@ -142,6 +142,7 @@ _HOOKS_MGR = None
142 142
 _NOT_FOUND_HOOK_MGR = None
143 143
 _CONDITIONS_MGR = None
144 144
 _ACTIONS_MGR = None
145
+_INTROSPECTION_DATA_MGR = None
145 146
 
146 147
 
147 148
 def missing_entrypoints_callback(names):
@@ -229,3 +230,12 @@ def rule_actions_manager():
229 230
             'ironic_inspector.rules.actions',
230 231
             invoke_on_load=True)
231 232
     return _ACTIONS_MGR
233
+
234
+
235
+def introspection_data_manager():
236
+    global _INTROSPECTION_DATA_MGR
237
+    if _INTROSPECTION_DATA_MGR is None:
238
+        _INTROSPECTION_DATA_MGR = stevedore.ExtensionManager(
239
+            'ironic_inspector.introspection_data.store',
240
+            invoke_on_load=True)
241
+    return _INTROSPECTION_DATA_MGR

+ 123
- 0
ironic_inspector/plugins/introspection_data.py View File

@@ -0,0 +1,123 @@
1
+# Licensed under the Apache License, Version 2.0 (the "License");
2
+# you may not use this file except in compliance with the License.
3
+# You may obtain a copy of the License at
4
+#
5
+#    http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+# Unless required by applicable law or agreed to in writing, software
8
+# distributed under the License is distributed on an "AS IS" BASIS,
9
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
10
+# implied.
11
+# See the License for the specific language governing permissions and
12
+# limitations under the License.
13
+
14
+"""Backends for storing introspection data."""
15
+
16
+import abc
17
+import json
18
+
19
+from oslo_config import cfg
20
+from oslo_utils import excutils
21
+import six
22
+
23
+from ironic_inspector.common import swift
24
+from ironic_inspector import node_cache
25
+from ironic_inspector import utils
26
+
27
+
28
+CONF = cfg.CONF
29
+
30
+LOG = utils.getProcessingLogger(__name__)
31
+
32
+_STORAGE_EXCLUDED_KEYS = {'logs'}
33
+_UNPROCESSED_DATA_STORE_SUFFIX = 'UNPROCESSED'
34
+
35
+
36
+def _filter_data_excluded_keys(data):
37
+    return {k: v for k, v in data.items()
38
+            if k not in _STORAGE_EXCLUDED_KEYS}
39
+
40
+
41
+@six.add_metaclass(abc.ABCMeta)
42
+class BaseStorageBackend(object):
43
+
44
+    @abc.abstractmethod
45
+    def get(self, node_id, processed=True, get_json=False):
46
+        """Get introspected data from storage backend.
47
+
48
+        :param node_id: node UUID or name.
49
+        :param processed: Specify whether the data to be retrieved is
50
+                          processed or not.
51
+        :param get_json: Specify whether return the introspection data in json
52
+                         format, string value is returned if False.
53
+        :returns: the introspection data.
54
+        :raises: IntrospectionDataStoreDisabled if storage backend is disabled.
55
+        """
56
+
57
+    @abc.abstractmethod
58
+    def save(self, node_info, data, processed=True):
59
+        """Save introspected data to storage backend.
60
+
61
+        :param node_info: a NodeInfo object.
62
+        :param data: the introspected data to be saved, in dict format.
63
+        :param processed: Specify whether the data to be saved is processed or
64
+                          not.
65
+        :raises: IntrospectionDataStoreDisabled if storage backend is disabled.
66
+        """
67
+
68
+
69
+class NoStore(BaseStorageBackend):
70
+    def get(self, node_id, processed=True, get_json=False):
71
+        raise utils.IntrospectionDataStoreDisabled(
72
+            'Introspection data storage is disabled')
73
+
74
+    def save(self, node_info, data, processed=True):
75
+        LOG.debug('Introspection data storage is disabled, the data will not '
76
+                  'be saved', node_info=node_info)
77
+
78
+
79
+class SwiftStore(object):
80
+    def get(self, node_id, processed=True, get_json=False):
81
+        suffix = None if processed else _UNPROCESSED_DATA_STORE_SUFFIX
82
+        LOG.debug('Fetching introspection data from Swift for %s', node_id)
83
+        data = swift.get_introspection_data(node_id, suffix=suffix)
84
+        if get_json:
85
+            return json.loads(data)
86
+        return data
87
+
88
+    def save(self, node_info, data, processed=True):
89
+        suffix = None if processed else _UNPROCESSED_DATA_STORE_SUFFIX
90
+        swift_object_name = swift.store_introspection_data(
91
+            _filter_data_excluded_keys(data),
92
+            node_info.uuid,
93
+            suffix=suffix
94
+        )
95
+        LOG.info('Introspection data was stored in Swift object %s',
96
+                 swift_object_name, node_info=node_info)
97
+        if CONF.processing.store_data_location:
98
+            node_info.patch([{'op': 'add', 'path': '/extra/%s' %
99
+                              CONF.processing.store_data_location,
100
+                              'value': swift_object_name}])
101
+
102
+
103
+class DatabaseStore(object):
104
+    def get(self, node_id, processed=True, get_json=False):
105
+        LOG.debug('Fetching introspection data from database for %(node)s',
106
+                  {'node': node_id})
107
+        data = node_cache.get_introspection_data(node_id, processed)
108
+        if get_json:
109
+            return data
110
+        return json.dumps(data)
111
+
112
+    def save(self, node_info, data, processed=True):
113
+        introspection_data = _filter_data_excluded_keys(data)
114
+        try:
115
+            node_cache.store_introspection_data(node_info.uuid,
116
+                                                introspection_data, processed)
117
+        except Exception as e:
118
+            with excutils.save_and_reraise_exception():
119
+                LOG.exception('Failed to store introspection data in '
120
+                              'database: %(exc)s', {'exc': e})
121
+        else:
122
+            LOG.info('Introspection data was stored in database',
123
+                     node_info=node_info)

+ 20
- 38
ironic_inspector/process.py View File

@@ -15,7 +15,6 @@
15 15
 
16 16
 import copy
17 17
 import datetime
18
-import json
19 18
 import os
20 19
 
21 20
 from oslo_config import cfg
@@ -25,7 +24,6 @@ from oslo_utils import timeutils
25 24
 
26 25
 from ironic_inspector.common.i18n import _
27 26
 from ironic_inspector.common import ironic as ir_utils
28
-from ironic_inspector.common import swift
29 27
 from ironic_inspector import introspection_state as istate
30 28
 from ironic_inspector import node_cache
31 29
 from ironic_inspector.plugins import base as plugins_base
@@ -38,7 +36,6 @@ CONF = cfg.CONF
38 36
 LOG = utils.getProcessingLogger(__name__)
39 37
 
40 38
 _STORAGE_EXCLUDED_KEYS = {'logs'}
41
-_UNPROCESSED_DATA_STORE_SUFFIX = 'UNPROCESSED'
42 39
 
43 40
 
44 41
 def _store_logs(introspection_data, node_info):
@@ -143,48 +140,28 @@ def _filter_data_excluded_keys(data):
143 140
             if k not in _STORAGE_EXCLUDED_KEYS}
144 141
 
145 142
 
146
-def _store_data(node_info, data, suffix=None):
147
-    if CONF.processing.store_data != 'swift':
148
-        LOG.debug("Swift support is disabled, introspection data "
149
-                  "won't be stored", node_info=node_info)
150
-        return
151
-
152
-    swift_object_name = swift.store_introspection_data(
153
-        _filter_data_excluded_keys(data),
154
-        node_info.uuid,
155
-        suffix=suffix
156
-    )
157
-    LOG.info('Introspection data was stored in Swift in object '
158
-             '%s', swift_object_name, node_info=node_info)
159
-    if CONF.processing.store_data_location:
160
-        node_info.patch([{'op': 'add', 'path': '/extra/%s' %
161
-                          CONF.processing.store_data_location,
162
-                          'value': swift_object_name}])
143
+def _store_data(node_info, data, processed=True):
144
+    introspection_data_manager = plugins_base.introspection_data_manager()
145
+    store = CONF.processing.store_data
146
+    ext = introspection_data_manager[store].obj
147
+    ext.save(node_info, data, processed)
163 148
 
164 149
 
165 150
 def _store_unprocessed_data(node_info, data):
166 151
     # runs in background
167 152
     try:
168
-        _store_data(node_info, data,
169
-                    suffix=_UNPROCESSED_DATA_STORE_SUFFIX)
153
+        _store_data(node_info, data, processed=False)
170 154
     except Exception:
171 155
         LOG.exception('Encountered exception saving unprocessed '
172 156
                       'introspection data', node_info=node_info,
173 157
                       data=data)
174 158
 
175 159
 
176
-def _get_unprocessed_data(uuid):
177
-    if CONF.processing.store_data == 'swift':
178
-        LOG.debug('Fetching unprocessed introspection data from '
179
-                  'Swift for %s', uuid)
180
-        return json.loads(
181
-            swift.get_introspection_data(
182
-                uuid,
183
-                suffix=_UNPROCESSED_DATA_STORE_SUFFIX
184
-            )
185
-        )
186
-    else:
187
-        raise utils.Error(_('Swift support is disabled'), code=400)
160
+def get_introspection_data(uuid, processed=True, get_json=False):
161
+    introspection_data_manager = plugins_base.introspection_data_manager()
162
+    store = CONF.processing.store_data
163
+    ext = introspection_data_manager[store].obj
164
+    return ext.get(uuid, processed=processed, get_json=get_json)
188 165
 
189 166
 
190 167
 def process(introspection_data):
@@ -309,7 +286,7 @@ def _finish(node_info, ironic, introspection_data, power_off=True):
309 286
              node_info=node_info, data=introspection_data)
310 287
 
311 288
 
312
-def reapply(node_ident):
289
+def reapply(node_ident, data=None):
313 290
     """Re-apply introspection steps.
314 291
 
315 292
     Re-apply preprocessing, postprocessing and introspection rules on
@@ -331,15 +308,20 @@ def reapply(node_ident):
331 308
         raise utils.Error(_('Node locked, please, try again later'),
332 309
                           node_info=node_info, code=409)
333 310
 
334
-    utils.executor().submit(_reapply, node_info)
311
+    utils.executor().submit(_reapply, node_info, data)
335 312
 
336 313
 
337
-def _reapply(node_info):
314
+def _reapply(node_info, data=None):
338 315
     # runs in background
339 316
     try:
340 317
         node_info.started_at = timeutils.utcnow()
341 318
         node_info.commit()
342
-        introspection_data = _get_unprocessed_data(node_info.uuid)
319
+        if data:
320
+            introspection_data = data
321
+        else:
322
+            introspection_data = get_introspection_data(node_info.uuid,
323
+                                                        processed=False,
324
+                                                        get_json=True)
343 325
     except Exception as exc:
344 326
         LOG.exception('Encountered exception while fetching '
345 327
                       'stored introspection data',

+ 41
- 37
ironic_inspector/test/unit/test_main.py View File

@@ -22,6 +22,7 @@ from oslo_utils import uuidutils
22 22
 
23 23
 from ironic_inspector.common import ironic as ir_utils
24 24
 from ironic_inspector.common import rpc
25
+from ironic_inspector.common import swift
25 26
 import ironic_inspector.conf
26 27
 from ironic_inspector.conf import opts as conf_opts
27 28
 from ironic_inspector import introspection_state as istate
@@ -29,6 +30,7 @@ from ironic_inspector import main
29 30
 from ironic_inspector import node_cache
30 31
 from ironic_inspector.plugins import base as plugins_base
31 32
 from ironic_inspector.plugins import example as example_plugin
33
+from ironic_inspector.plugins import introspection_data as intros_data_plugin
32 34
 from ironic_inspector import process
33 35
 from ironic_inspector import rules
34 36
 from ironic_inspector.test import base as test_base
@@ -297,10 +299,9 @@ class TestApiListStatus(GetStatusAPIBaseTest):
297 299
 
298 300
 
299 301
 class TestApiGetData(BaseAPITest):
300
-    @mock.patch.object(main.swift, 'SwiftAPI', autospec=True)
301
-    def test_get_introspection_data(self, swift_mock):
302
-        CONF.set_override('store_data', 'swift', 'processing')
303
-        data = {
302
+    def setUp(self):
303
+        super(TestApiGetData, self).setUp()
304
+        self.introspection_data = {
304 305
             'ipmi_address': '1.2.3.4',
305 306
             'cpus': 2,
306 307
             'cpu_arch': 'x86_64',
@@ -310,44 +311,48 @@ class TestApiGetData(BaseAPITest):
310 311
                 'em1': {'mac': '11:22:33:44:55:66', 'ip': '1.2.0.1'},
311 312
             }
312 313
         }
314
+
315
+    @mock.patch.object(swift, 'SwiftAPI', autospec=True)
316
+    def test_get_introspection_data_from_swift(self, swift_mock):
317
+        CONF.set_override('store_data', 'swift', 'processing')
313 318
         swift_conn = swift_mock.return_value
314
-        swift_conn.get_object.return_value = json.dumps(data)
319
+        swift_conn.get_object.return_value = json.dumps(
320
+            self.introspection_data)
315 321
         res = self.app.get('/v1/introspection/%s/data' % self.uuid)
316 322
         name = 'inspector_data-%s' % self.uuid
317 323
         swift_conn.get_object.assert_called_once_with(name)
318 324
         self.assertEqual(200, res.status_code)
319
-        self.assertEqual(data, json.loads(res.data.decode('utf-8')))
325
+        self.assertEqual(self.introspection_data,
326
+                         json.loads(res.data.decode('utf-8')))
327
+
328
+    @mock.patch.object(intros_data_plugin, 'DatabaseStore',
329
+                       autospec=True)
330
+    def test_get_introspection_data_from_db(self, db_mock):
331
+        CONF.set_override('store_data', 'database', 'processing')
332
+        db_store = db_mock.return_value
333
+        db_store.get.return_value = json.dumps(self.introspection_data)
334
+        res = self.app.get('/v1/introspection/%s/data' % self.uuid)
335
+        db_store.get.assert_called_once_with(self.uuid, processed=True,
336
+                                             get_json=False)
337
+        self.assertEqual(200, res.status_code)
338
+        self.assertEqual(self.introspection_data,
339
+                         json.loads(res.data.decode('utf-8')))
320 340
 
321
-    @mock.patch.object(main.swift, 'SwiftAPI', autospec=True)
322
-    def test_introspection_data_not_stored(self, swift_mock):
341
+    def test_introspection_data_not_stored(self):
323 342
         CONF.set_override('store_data', 'none', 'processing')
324
-        swift_conn = swift_mock.return_value
325 343
         res = self.app.get('/v1/introspection/%s/data' % self.uuid)
326
-        self.assertFalse(swift_conn.get_object.called)
327 344
         self.assertEqual(404, res.status_code)
328 345
 
329 346
     @mock.patch.object(ir_utils, 'get_node', autospec=True)
330
-    @mock.patch.object(main.swift, 'SwiftAPI', autospec=True)
331
-    def test_with_name(self, swift_mock, get_mock):
347
+    @mock.patch.object(main.process, 'get_introspection_data', autospec=True)
348
+    def test_with_name(self, process_mock, get_mock):
332 349
         get_mock.return_value = mock.Mock(uuid=self.uuid)
333 350
         CONF.set_override('store_data', 'swift', 'processing')
334
-        data = {
335
-            'ipmi_address': '1.2.3.4',
336
-            'cpus': 2,
337
-            'cpu_arch': 'x86_64',
338
-            'memory_mb': 1024,
339
-            'local_gb': 20,
340
-            'interfaces': {
341
-                'em1': {'mac': '11:22:33:44:55:66', 'ip': '1.2.0.1'},
342
-            }
343
-        }
344
-        swift_conn = swift_mock.return_value
345
-        swift_conn.get_object.return_value = json.dumps(data)
351
+        process_mock.return_value = json.dumps(self.introspection_data)
346 352
         res = self.app.get('/v1/introspection/name1/data')
347
-        name = 'inspector_data-%s' % self.uuid
348
-        swift_conn.get_object.assert_called_once_with(name)
349 353
         self.assertEqual(200, res.status_code)
350
-        self.assertEqual(data, json.loads(res.data.decode('utf-8')))
354
+        self.assertEqual(self.introspection_data,
355
+                         json.loads(res.data.decode('utf-8')))
351 356
         get_mock.assert_called_once_with('name1', fields=['uuid'])
352 357
 
353 358
 
@@ -361,8 +366,7 @@ class TestApiReapply(BaseAPITest):
361 366
         self.rpc_get_client_mock.return_value = self.client_mock
362 367
         CONF.set_override('store_data', 'swift', 'processing')
363 368
 
364
-    def test_ok(self):
365
-
369
+    def test_api_ok(self):
366 370
         self.app.post('/v1/introspection/%s/data/unprocessed' %
367 371
                       self.uuid)
368 372
         self.client_mock.call.assert_called_once_with({}, 'do_reapply',
@@ -377,18 +381,18 @@ class TestApiReapply(BaseAPITest):
377 381
                          message)
378 382
         self.assertFalse(self.client_mock.call.called)
379 383
 
380
-    def test_swift_disabled(self):
381
-        CONF.set_override('store_data', 'none', 'processing')
384
+    def test_get_introspection_data_error(self):
385
+        exc = utils.Error('The store is crashed', code=404)
386
+        self.client_mock.call.side_effect = exc
382 387
 
383 388
         res = self.app.post('/v1/introspection/%s/data/unprocessed' %
384 389
                             self.uuid)
385
-        self.assertEqual(400, res.status_code)
390
+
391
+        self.assertEqual(404, res.status_code)
386 392
         message = json.loads(res.data.decode())['error']['message']
387
-        self.assertEqual('Inspector is not configured to store '
388
-                         'data. Set the [processing] store_data '
389
-                         'configuration option to change this.',
390
-                         message)
391
-        self.assertFalse(self.client_mock.call.called)
393
+        self.assertEqual(str(exc), message)
394
+        self.client_mock.call.assert_called_once_with({}, 'do_reapply',
395
+                                                      node_id=self.uuid)
392 396
 
393 397
     def test_generic_error(self):
394 398
         exc = utils.Error('Oops', code=400)

+ 71
- 8
ironic_inspector/test/unit/test_manager.py View File

@@ -11,10 +11,13 @@
11 11
 # See the License for the specific language governing permissions and
12 12
 # limitations under the License.
13 13
 
14
+import json
15
+
14 16
 import fixtures
15 17
 import mock
16 18
 import oslo_messaging as messaging
17 19
 
20
+from ironic_inspector.common import swift
18 21
 from ironic_inspector.conductor import manager
19 22
 import ironic_inspector.conf
20 23
 from ironic_inspector import introspect
@@ -302,11 +305,17 @@ class TestManagerReapply(BaseManagerTest):
302 305
         super(TestManagerReapply, self).setUp()
303 306
         CONF.set_override('store_data', 'swift', 'processing')
304 307
 
305
-    def test_ok(self, reapply_mock):
308
+    @mock.patch.object(swift, 'store_introspection_data', autospec=True)
309
+    @mock.patch.object(swift, 'get_introspection_data', autospec=True)
310
+    def test_ok(self, swift_get_mock, swift_set_mock, reapply_mock):
311
+        swift_get_mock.return_value = json.dumps(self.data)
306 312
         self.manager.do_reapply(self.context, self.uuid)
307
-        reapply_mock.assert_called_once_with(self.uuid)
313
+        reapply_mock.assert_called_once_with(self.uuid, data=self.data)
308 314
 
309
-    def test_node_locked(self, reapply_mock):
315
+    @mock.patch.object(swift, 'store_introspection_data', autospec=True)
316
+    @mock.patch.object(swift, 'get_introspection_data', autospec=True)
317
+    def test_node_locked(self, swift_get_mock, swift_set_mock, reapply_mock):
318
+        swift_get_mock.return_value = json.dumps(self.data)
310 319
         exc = utils.Error('Locked.', code=409)
311 320
         reapply_mock.side_effect = exc
312 321
 
@@ -317,9 +326,13 @@ class TestManagerReapply(BaseManagerTest):
317 326
         self.assertEqual(utils.Error, exc.exc_info[0])
318 327
         self.assertIn('Locked.', str(exc.exc_info[1]))
319 328
         self.assertEqual(409, exc.exc_info[1].http_code)
320
-        reapply_mock.assert_called_once_with(self.uuid)
329
+        reapply_mock.assert_called_once_with(self.uuid, data=self.data)
321 330
 
322
-    def test_node_not_found(self, reapply_mock):
331
+    @mock.patch.object(swift, 'store_introspection_data', autospec=True)
332
+    @mock.patch.object(swift, 'get_introspection_data', autospec=True)
333
+    def test_node_not_found(self, swift_get_mock, swift_set_mock,
334
+                            reapply_mock):
335
+        swift_get_mock.return_value = json.dumps(self.data)
323 336
         exc = utils.Error('Not found.', code=404)
324 337
         reapply_mock.side_effect = exc
325 338
 
@@ -330,9 +343,11 @@ class TestManagerReapply(BaseManagerTest):
330 343
         self.assertEqual(utils.Error, exc.exc_info[0])
331 344
         self.assertIn('Not found.', str(exc.exc_info[1]))
332 345
         self.assertEqual(404, exc.exc_info[1].http_code)
333
-        reapply_mock.assert_called_once_with(self.uuid)
346
+        reapply_mock.assert_called_once_with(self.uuid, data=self.data)
334 347
 
335
-    def test_generic_error(self, reapply_mock):
348
+    @mock.patch.object(process, 'get_introspection_data', autospec=True)
349
+    def test_generic_error(self, get_data_mock, reapply_mock):
350
+        get_data_mock.return_value = self.data
336 351
         exc = utils.Error('Oops', code=400)
337 352
         reapply_mock.side_effect = exc
338 353
 
@@ -343,4 +358,52 @@ class TestManagerReapply(BaseManagerTest):
343 358
         self.assertEqual(utils.Error, exc.exc_info[0])
344 359
         self.assertIn('Oops', str(exc.exc_info[1]))
345 360
         self.assertEqual(400, exc.exc_info[1].http_code)
346
-        reapply_mock.assert_called_once_with(self.uuid)
361
+        reapply_mock.assert_called_once_with(self.uuid, data=self.data)
362
+        get_data_mock.assert_called_once_with(self.uuid, processed=False,
363
+                                              get_json=True)
364
+
365
+    @mock.patch.object(process, 'get_introspection_data', autospec=True)
366
+    def test_get_introspection_data_error(self, get_data_mock, reapply_mock):
367
+        exc = utils.Error('The store is empty', code=404)
368
+        get_data_mock.side_effect = exc
369
+
370
+        exc = self.assertRaises(messaging.rpc.ExpectedException,
371
+                                self.manager.do_reapply,
372
+                                self.context, self.uuid)
373
+
374
+        self.assertEqual(utils.Error, exc.exc_info[0])
375
+        self.assertIn('The store is empty', str(exc.exc_info[1]))
376
+        self.assertEqual(404, exc.exc_info[1].http_code)
377
+        get_data_mock.assert_called_once_with(self.uuid, processed=False,
378
+                                              get_json=True)
379
+        self.assertFalse(reapply_mock.called)
380
+
381
+    def test_store_data_disabled(self, reapply_mock):
382
+        CONF.set_override('store_data', 'none', 'processing')
383
+
384
+        exc = self.assertRaises(messaging.rpc.ExpectedException,
385
+                                self.manager.do_reapply,
386
+                                self.context, self.uuid)
387
+
388
+        self.assertEqual(utils.Error, exc.exc_info[0])
389
+        self.assertIn('Inspector is not configured to store data',
390
+                      str(exc.exc_info[1]))
391
+        self.assertEqual(400, exc.exc_info[1].http_code)
392
+        self.assertFalse(reapply_mock.called)
393
+
394
+    @mock.patch.object(process, 'get_introspection_data', autospec=True)
395
+    def test_ok_swift(self, get_data_mock, reapply_mock):
396
+        get_data_mock.return_value = self.data
397
+        self.manager.do_reapply(self.context, self.uuid)
398
+        reapply_mock.assert_called_once_with(self.uuid, data=self.data)
399
+        get_data_mock.assert_called_once_with(self.uuid, processed=False,
400
+                                              get_json=True)
401
+
402
+    @mock.patch.object(process, 'get_introspection_data', autospec=True)
403
+    def test_ok_db(self, get_data_mock, reapply_mock):
404
+        get_data_mock.return_value = self.data
405
+        CONF.set_override('store_data', 'database', 'processing')
406
+        self.manager.do_reapply(self.context, self.uuid)
407
+        reapply_mock.assert_called_once_with(self.uuid, data=self.data)
408
+        get_data_mock.assert_called_once_with(self.uuid, processed=False,
409
+                                              get_json=True)

+ 108
- 0
ironic_inspector/test/unit/test_plugins_introspection_data.py View File

@@ -0,0 +1,108 @@
1
+# Licensed under the Apache License, Version 2.0 (the "License");
2
+# you may not use this file except in compliance with the License.
3
+# You may obtain a copy of the License at
4
+#
5
+#    http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+# Unless required by applicable law or agreed to in writing, software
8
+# distributed under the License is distributed on an "AS IS" BASIS,
9
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
10
+# implied.
11
+# See the License for the specific language governing permissions and
12
+# limitations under the License.
13
+
14
+import json
15
+
16
+import fixtures
17
+import mock
18
+from oslo_config import cfg
19
+
20
+from ironic_inspector.common import ironic as ir_utils
21
+from ironic_inspector import db
22
+from ironic_inspector import introspection_state as istate
23
+from ironic_inspector.plugins import introspection_data
24
+from ironic_inspector.test import base as test_base
25
+
26
+CONF = cfg.CONF
27
+
28
+
29
+class BaseTest(test_base.NodeTest):
30
+    data = {
31
+        'ipmi_address': '1.2.3.4',
32
+        'cpus': 2,
33
+        'cpu_arch': 'x86_64',
34
+        'memory_mb': 1024,
35
+        'local_gb': 20,
36
+        'interfaces': {
37
+            'em1': {'mac': '11:22:33:44:55:66', 'ip': '1.2.0.1'},
38
+        }
39
+    }
40
+
41
+    def setUp(self):
42
+        super(BaseTest, self).setUp()
43
+        self.cli_fixture = self.useFixture(
44
+            fixtures.MockPatchObject(ir_utils, 'get_client', autospec=True))
45
+        self.cli = self.cli_fixture.mock.return_value
46
+
47
+
48
+@mock.patch.object(introspection_data.swift, 'SwiftAPI', autospec=True)
49
+class TestSwiftStore(BaseTest):
50
+
51
+    def setUp(self):
52
+        super(TestSwiftStore, self).setUp()
53
+        self.driver = introspection_data.SwiftStore()
54
+
55
+    def test_get_data(self, swift_mock):
56
+        swift_conn = swift_mock.return_value
57
+        swift_conn.get_object.return_value = json.dumps(self.data)
58
+        name = 'inspector_data-%s' % self.uuid
59
+
60
+        res_data = self.driver.get(self.uuid)
61
+
62
+        swift_conn.get_object.assert_called_once_with(name)
63
+        self.assertEqual(self.data, json.loads(res_data))
64
+
65
+    def test_store_data(self, swift_mock):
66
+        swift_conn = swift_mock.return_value
67
+        name = 'inspector_data-%s' % self.uuid
68
+
69
+        self.driver.save(self.node_info, self.data)
70
+
71
+        data = introspection_data._filter_data_excluded_keys(self.data)
72
+        swift_conn.create_object.assert_called_once_with(name,
73
+                                                         json.dumps(data))
74
+
75
+    def test_store_data_location(self, swift_mock):
76
+        CONF.set_override('store_data_location', 'inspector_data_object',
77
+                          'processing')
78
+        swift_conn = swift_mock.return_value
79
+        name = 'inspector_data-%s' % self.uuid
80
+        patch = [{'path': '/extra/inspector_data_object',
81
+                  'value': name, 'op': 'add'}]
82
+        expected = self.data
83
+
84
+        self.driver.save(self.node_info, self.data)
85
+
86
+        data = introspection_data._filter_data_excluded_keys(self.data)
87
+        swift_conn.create_object.assert_called_once_with(name,
88
+                                                         json.dumps(data))
89
+        self.assertEqual(expected,
90
+                         json.loads(swift_conn.create_object.call_args[0][1]))
91
+        self.cli.node.update.assert_any_call(self.uuid, patch)
92
+
93
+
94
+class TestDatabaseStore(BaseTest):
95
+    def setUp(self):
96
+        super(TestDatabaseStore, self).setUp()
97
+        self.driver = introspection_data.DatabaseStore()
98
+        session = db.get_writer_session()
99
+        with session.begin():
100
+            db.Node(uuid=self.node_info.uuid,
101
+                    state=istate.States.starting).save(session)
102
+
103
+    def test_store_and_get_data(self):
104
+        self.driver.save(self.node_info, self.data)
105
+
106
+        res_data = self.driver.get(self.node_info.uuid)
107
+
108
+        self.assertEqual(self.data, json.loads(res_data))

+ 34
- 18
ironic_inspector/test/unit/test_process.py View File

@@ -28,11 +28,13 @@ from oslo_utils import uuidutils
28 28
 import six
29 29
 
30 30
 from ironic_inspector.common import ironic as ir_utils
31
+from ironic_inspector.common import swift
31 32
 from ironic_inspector import db
32 33
 from ironic_inspector import introspection_state as istate
33 34
 from ironic_inspector import node_cache
34 35
 from ironic_inspector.plugins import base as plugins_base
35 36
 from ironic_inspector.plugins import example as example_plugin
37
+from ironic_inspector.plugins import introspection_data as intros_data_plugin
36 38
 from ironic_inspector import process
37 39
 from ironic_inspector.pxe_filter import base as pxe_filter
38 40
 from ironic_inspector.test import base as test_base
@@ -259,22 +261,13 @@ class TestUnprocessedData(BaseProcessTest):
259 261
 
260 262
         store_mock.assert_called_once_with(mock.ANY, expected)
261 263
 
262
-    @mock.patch.object(process.swift, 'SwiftAPI', autospec=True)
263
-    def test_save_unprocessed_data_failure(self, swift_mock):
264
+    def test_save_unprocessed_data_failure(self):
264 265
         CONF.set_override('store_data', 'swift', 'processing')
265
-        name = 'inspector_data-%s-%s' % (
266
-            self.uuid,
267
-            process._UNPROCESSED_DATA_STORE_SUFFIX
268
-        )
269
-
270
-        swift_conn = swift_mock.return_value
271
-        swift_conn.create_object.side_effect = utils.Error('Oops')
272 266
 
273 267
         res = process.process(self.data)
274 268
 
275 269
         # assert store failure doesn't break processing
276 270
         self.assertEqual(self.fake_result_json, res)
277
-        swift_conn.create_object.assert_called_once_with(name, mock.ANY)
278 271
 
279 272
 
280 273
 @mock.patch.object(example_plugin.ExampleProcessingHook, 'before_processing',
@@ -405,6 +398,7 @@ class TestProcessNode(BaseTest):
405 398
                 started_at=self.node_info.started_at,
406 399
                 finished_at=self.node_info.finished_at,
407 400
                 error=self.node_info.error).save(self.session)
401
+        plugins_base._INTROSPECTION_DATA_MGR = None
408 402
 
409 403
     def test_return_includes_uuid(self):
410 404
         ret_val = process._process_node(self.node_info, self.node, self.data)
@@ -485,8 +479,8 @@ class TestProcessNode(BaseTest):
485 479
         finished_mock.assert_called_once_with(
486 480
             self.node_info, istate.Events.finish)
487 481
 
488
-    @mock.patch.object(process.swift, 'SwiftAPI', autospec=True)
489
-    def test_store_data(self, swift_mock):
482
+    @mock.patch.object(swift, 'SwiftAPI', autospec=True)
483
+    def test_store_data_with_swift(self, swift_mock):
490 484
         CONF.set_override('store_data', 'swift', 'processing')
491 485
         swift_conn = swift_mock.return_value
492 486
         name = 'inspector_data-%s' % self.uuid
@@ -498,8 +492,8 @@ class TestProcessNode(BaseTest):
498 492
         self.assertEqual(expected,
499 493
                          json.loads(swift_conn.create_object.call_args[0][1]))
500 494
 
501
-    @mock.patch.object(process.swift, 'SwiftAPI', autospec=True)
502
-    def test_store_data_no_logs(self, swift_mock):
495
+    @mock.patch.object(swift, 'SwiftAPI', autospec=True)
496
+    def test_store_data_no_logs_with_swift(self, swift_mock):
503 497
         CONF.set_override('store_data', 'swift', 'processing')
504 498
         swift_conn = swift_mock.return_value
505 499
         name = 'inspector_data-%s' % self.uuid
@@ -511,8 +505,8 @@ class TestProcessNode(BaseTest):
511 505
         self.assertNotIn('logs',
512 506
                          json.loads(swift_conn.create_object.call_args[0][1]))
513 507
 
514
-    @mock.patch.object(process.swift, 'SwiftAPI', autospec=True)
515
-    def test_store_data_location(self, swift_mock):
508
+    @mock.patch.object(swift, 'SwiftAPI', autospec=True)
509
+    def test_store_data_location_with_swift(self, swift_mock):
516 510
         CONF.set_override('store_data', 'swift', 'processing')
517 511
         CONF.set_override('store_data_location', 'inspector_data_object',
518 512
                           'processing')
@@ -529,6 +523,28 @@ class TestProcessNode(BaseTest):
529 523
                          json.loads(swift_conn.create_object.call_args[0][1]))
530 524
         self.cli.node.update.assert_any_call(self.uuid, patch)
531 525
 
526
+    @mock.patch.object(node_cache, 'store_introspection_data', autospec=True)
527
+    def test_store_data_with_database(self, store_mock):
528
+        CONF.set_override('store_data', 'database', 'processing')
529
+
530
+        process._process_node(self.node_info, self.node, self.data)
531
+
532
+        data = intros_data_plugin._filter_data_excluded_keys(self.data)
533
+        store_mock.assert_called_once_with(self.node_info.uuid, data, True)
534
+        self.assertEqual(data, store_mock.call_args[0][1])
535
+
536
+    @mock.patch.object(node_cache, 'store_introspection_data', autospec=True)
537
+    def test_store_data_no_logs_with_database(self, store_mock):
538
+        CONF.set_override('store_data', 'database', 'processing')
539
+
540
+        self.data['logs'] = 'something'
541
+
542
+        process._process_node(self.node_info, self.node, self.data)
543
+
544
+        data = intros_data_plugin._filter_data_excluded_keys(self.data)
545
+        store_mock.assert_called_once_with(self.node_info.uuid, data, True)
546
+        self.assertNotIn('logs', store_mock.call_args[0][1])
547
+
532 548
 
533 549
 @mock.patch.object(process, '_reapply', autospec=True)
534 550
 @mock.patch.object(node_cache, 'get_node', autospec=True)
@@ -558,7 +574,7 @@ class TestReapply(BaseTest):
558 574
             blocking=False
559 575
         )
560 576
 
561
-        reapply_mock.assert_called_once_with(pop_mock.return_value)
577
+        reapply_mock.assert_called_once_with(pop_mock.return_value, data=None)
562 578
 
563 579
     @prepare_mocks
564 580
     def test_locking_failed(self, pop_mock, reapply_mock):
@@ -575,7 +591,7 @@ class TestReapply(BaseTest):
575 591
 
576 592
 @mock.patch.object(example_plugin.ExampleProcessingHook, 'before_update')
577 593
 @mock.patch.object(process.rules, 'apply', autospec=True)
578
-@mock.patch.object(process.swift, 'SwiftAPI', autospec=True)
594
+@mock.patch.object(swift, 'SwiftAPI', autospec=True)
579 595
 @mock.patch.object(node_cache.NodeInfo, 'finished', autospec=True)
580 596
 @mock.patch.object(node_cache.NodeInfo, 'release_lock', autospec=True)
581 597
 class TestReapplyNode(BaseTest):

+ 4
- 0
ironic_inspector/utils.py View File

@@ -140,6 +140,10 @@ class NodeStateInvalidEvent(Error):
140 140
     """Invalid event attempted."""
141 141
 
142 142
 
143
+class IntrospectionDataStoreDisabled(Error):
144
+    """Introspection data store is disabled."""
145
+
146
+
143 147
 class IntrospectionDataNotFound(NotFoundInCacheError):
144 148
     """Introspection data not found."""
145 149
 

+ 6
- 0
releasenotes/notes/introspection-data-db-store-0586292de05cbfd7.yaml View File

@@ -0,0 +1,6 @@
1
+---
2
+features:
3
+  - |
4
+    Adds the support to store introspection data in ironic-inspector database.
5
+    Set the option ``[processing]store_data`` to ``database`` to use this
6
+    feature.

+ 4
- 0
setup.cfg View File

@@ -43,6 +43,10 @@ ironic_inspector.hooks.processing =
43 43
 ironic_inspector.hooks.node_not_found =
44 44
     example = ironic_inspector.plugins.example:example_not_found_hook
45 45
     enroll = ironic_inspector.plugins.discovery:enroll_node_not_found_hook
46
+ironic_inspector.introspection_data.store =
47
+    none = ironic_inspector.plugins.introspection_data:NoStore
48
+    swift = ironic_inspector.plugins.introspection_data:SwiftStore
49
+    database = ironic_inspector.plugins.introspection_data:DatabaseStore
46 50
 ironic_inspector.rules.conditions =
47 51
     eq = ironic_inspector.plugins.rules:EqCondition
48 52
     lt = ironic_inspector.plugins.rules:LtCondition

Loading…
Cancel
Save