Browse Source

Merge "introspection data backend: plugin layer"

Zuul 2 months ago
parent
commit
13e70283b1

+ 10
- 1
ironic_inspector/conductor/manager.py View File

@@ -22,6 +22,7 @@ from oslo_log import log
22 22
 import oslo_messaging as messaging
23 23
 from oslo_utils import reflection
24 24
 
25
+from ironic_inspector.common.i18n import _
25 26
 from ironic_inspector.common import ironic as ir_utils
26 27
 from ironic_inspector import db
27 28
 from ironic_inspector import introspect
@@ -126,7 +127,15 @@ class ConductorManager(object):
126 127
 
127 128
     @messaging.expected_exceptions(utils.Error)
128 129
     def do_reapply(self, context, node_id, token=None):
129
-        process.reapply(node_id)
130
+        try:
131
+            data = process.get_introspection_data(node_id, processed=False,
132
+                                                  get_json=True)
133
+        except utils.IntrospectionDataStoreDisabled:
134
+            raise utils.Error(_('Inspector is not configured to store '
135
+                                'data. Set the [processing]store_data '
136
+                                'configuration option to change this.'),
137
+                              code=400)
138
+        process.reapply(node_id, data)
130 139
 
131 140
 
132 141
 def periodic_clean_up():  # pragma: no cover

+ 4
- 4
ironic_inspector/conf/processing.py View File

@@ -18,7 +18,6 @@ from ironic_inspector.common.i18n import _
18 18
 
19 19
 VALID_ADD_PORTS_VALUES = ('all', 'active', 'pxe', 'disabled')
20 20
 VALID_KEEP_PORTS_VALUES = ('all', 'present', 'added')
21
-VALID_STORE_DATA_VALUES = ('none', 'swift')
22 21
 
23 22
 
24 23
 _OPTS = [
@@ -75,9 +74,10 @@ _OPTS = [
75 74
                       'aware of. This hook is ignored by default.')),
76 75
     cfg.StrOpt('store_data',
77 76
                default='none',
78
-               choices=VALID_STORE_DATA_VALUES,
79
-               help=_('Method for storing introspection data. If set to \'none'
80
-                      '\', introspection data will not be stored.')),
77
+               help=_('The storage backend for storing introspection data. '
78
+                      'Possible values are: \'none\', \'database\' and '
79
+                      '\'swift\'. If set to \'none\', introspection data will '
80
+                      'not be stored.')),
81 81
     cfg.StrOpt('store_data_location',
82 82
                help=_('Name of the key to store the location of stored data '
83 83
                       'in the extra column of the Ironic database.')),

+ 7
- 14
ironic_inspector/main.py View File

@@ -25,7 +25,6 @@ from ironic_inspector.common import context
25 25
 from ironic_inspector.common.i18n import _
26 26
 from ironic_inspector.common import ironic as ir_utils
27 27
 from ironic_inspector.common import rpc
28
-from ironic_inspector.common import swift
29 28
 import ironic_inspector.conf
30 29
 from ironic_inspector.conf import opts as conf_opts
31 30
 from ironic_inspector import node_cache
@@ -289,15 +288,15 @@ def api_introspection_abort(node_id):
289 288
 @api('/v1/introspection/<node_id>/data', rule="introspection:data",
290 289
      methods=['GET'])
291 290
 def api_introspection_data(node_id):
292
-    if CONF.processing.store_data == 'swift':
291
+    try:
293 292
         if not uuidutils.is_uuid_like(node_id):
294 293
             node = ir_utils.get_node(node_id, fields=['uuid'])
295 294
             node_id = node.uuid
296
-        res = swift.get_introspection_data(node_id)
295
+        res = process.get_introspection_data(node_id)
297 296
         return res, 200, {'Content-Type': 'application/json'}
298
-    else:
297
+    except utils.IntrospectionDataStoreDisabled:
299 298
         return error_response(_('Inspector is not configured to store data. '
300
-                                'Set the [processing] store_data '
299
+                                'Set the [processing]store_data '
301 300
                                 'configuration option to change this.'),
302 301
                               code=404)
303 302
 
@@ -309,15 +308,9 @@ def api_introspection_reapply(node_id):
309 308
         return error_response(_('User data processing is not '
310 309
                                 'supported yet'), code=400)
311 310
 
312
-    if CONF.processing.store_data == 'swift':
313
-        client = rpc.get_client()
314
-        client.call({}, 'do_reapply', node_id=node_id)
315
-        return '', 202
316
-    else:
317
-        return error_response(_('Inspector is not configured to store'
318
-                                ' data. Set the [processing] '
319
-                                'store_data configuration option to '
320
-                                'change this.'), code=400)
311
+    client = rpc.get_client()
312
+    client.call({}, 'do_reapply', node_id=node_id)
313
+    return '', 202
321 314
 
322 315
 
323 316
 def rule_repr(rule, short):

+ 10
- 0
ironic_inspector/plugins/base.py View File

@@ -142,6 +142,7 @@ _HOOKS_MGR = None
142 142
 _NOT_FOUND_HOOK_MGR = None
143 143
 _CONDITIONS_MGR = None
144 144
 _ACTIONS_MGR = None
145
+_INTROSPECTION_DATA_MGR = None
145 146
 
146 147
 
147 148
 def missing_entrypoints_callback(names):
@@ -229,3 +230,12 @@ def rule_actions_manager():
229 230
             'ironic_inspector.rules.actions',
230 231
             invoke_on_load=True)
231 232
     return _ACTIONS_MGR
233
+
234
+
235
+def introspection_data_manager():
236
+    global _INTROSPECTION_DATA_MGR
237
+    if _INTROSPECTION_DATA_MGR is None:
238
+        _INTROSPECTION_DATA_MGR = stevedore.ExtensionManager(
239
+            'ironic_inspector.introspection_data.store',
240
+            invoke_on_load=True)
241
+    return _INTROSPECTION_DATA_MGR

+ 123
- 0
ironic_inspector/plugins/introspection_data.py View File

@@ -0,0 +1,123 @@
1
+# Licensed under the Apache License, Version 2.0 (the "License");
2
+# you may not use this file except in compliance with the License.
3
+# You may obtain a copy of the License at
4
+#
5
+#    http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+# Unless required by applicable law or agreed to in writing, software
8
+# distributed under the License is distributed on an "AS IS" BASIS,
9
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
10
+# implied.
11
+# See the License for the specific language governing permissions and
12
+# limitations under the License.
13
+
14
+"""Backends for storing introspection data."""
15
+
16
+import abc
17
+import json
18
+
19
+from oslo_config import cfg
20
+from oslo_utils import excutils
21
+import six
22
+
23
+from ironic_inspector.common import swift
24
+from ironic_inspector import node_cache
25
+from ironic_inspector import utils
26
+
27
+
28
+CONF = cfg.CONF
29
+
30
+LOG = utils.getProcessingLogger(__name__)
31
+
32
+_STORAGE_EXCLUDED_KEYS = {'logs'}
33
+_UNPROCESSED_DATA_STORE_SUFFIX = 'UNPROCESSED'
34
+
35
+
36
+def _filter_data_excluded_keys(data):
37
+    return {k: v for k, v in data.items()
38
+            if k not in _STORAGE_EXCLUDED_KEYS}
39
+
40
+
41
+@six.add_metaclass(abc.ABCMeta)
42
+class BaseStorageBackend(object):
43
+
44
+    @abc.abstractmethod
45
+    def get(self, node_id, processed=True, get_json=False):
46
+        """Get introspected data from storage backend.
47
+
48
+        :param node_id: node UUID or name.
49
+        :param processed: Specify whether the data to be retrieved is
50
+                          processed or not.
51
+        :param get_json: Specify whether return the introspection data in json
52
+                         format, string value is returned if False.
53
+        :returns: the introspection data.
54
+        :raises: IntrospectionDataStoreDisabled if storage backend is disabled.
55
+        """
56
+
57
+    @abc.abstractmethod
58
+    def save(self, node_info, data, processed=True):
59
+        """Save introspected data to storage backend.
60
+
61
+        :param node_info: a NodeInfo object.
62
+        :param data: the introspected data to be saved, in dict format.
63
+        :param processed: Specify whether the data to be saved is processed or
64
+                          not.
65
+        :raises: IntrospectionDataStoreDisabled if storage backend is disabled.
66
+        """
67
+
68
+
69
+class NoStore(BaseStorageBackend):
70
+    def get(self, node_id, processed=True, get_json=False):
71
+        raise utils.IntrospectionDataStoreDisabled(
72
+            'Introspection data storage is disabled')
73
+
74
+    def save(self, node_info, data, processed=True):
75
+        LOG.debug('Introspection data storage is disabled, the data will not '
76
+                  'be saved', node_info=node_info)
77
+
78
+
79
+class SwiftStore(object):
80
+    def get(self, node_id, processed=True, get_json=False):
81
+        suffix = None if processed else _UNPROCESSED_DATA_STORE_SUFFIX
82
+        LOG.debug('Fetching introspection data from Swift for %s', node_id)
83
+        data = swift.get_introspection_data(node_id, suffix=suffix)
84
+        if get_json:
85
+            return json.loads(data)
86
+        return data
87
+
88
+    def save(self, node_info, data, processed=True):
89
+        suffix = None if processed else _UNPROCESSED_DATA_STORE_SUFFIX
90
+        swift_object_name = swift.store_introspection_data(
91
+            _filter_data_excluded_keys(data),
92
+            node_info.uuid,
93
+            suffix=suffix
94
+        )
95
+        LOG.info('Introspection data was stored in Swift object %s',
96
+                 swift_object_name, node_info=node_info)
97
+        if CONF.processing.store_data_location:
98
+            node_info.patch([{'op': 'add', 'path': '/extra/%s' %
99
+                              CONF.processing.store_data_location,
100
+                              'value': swift_object_name}])
101
+
102
+
103
+class DatabaseStore(object):
104
+    def get(self, node_id, processed=True, get_json=False):
105
+        LOG.debug('Fetching introspection data from database for %(node)s',
106
+                  {'node': node_id})
107
+        data = node_cache.get_introspection_data(node_id, processed)
108
+        if get_json:
109
+            return data
110
+        return json.dumps(data)
111
+
112
+    def save(self, node_info, data, processed=True):
113
+        introspection_data = _filter_data_excluded_keys(data)
114
+        try:
115
+            node_cache.store_introspection_data(node_info.uuid,
116
+                                                introspection_data, processed)
117
+        except Exception as e:
118
+            with excutils.save_and_reraise_exception():
119
+                LOG.exception('Failed to store introspection data in '
120
+                              'database: %(exc)s', {'exc': e})
121
+        else:
122
+            LOG.info('Introspection data was stored in database',
123
+                     node_info=node_info)

+ 20
- 38
ironic_inspector/process.py View File

@@ -15,7 +15,6 @@
15 15
 
16 16
 import copy
17 17
 import datetime
18
-import json
19 18
 import os
20 19
 
21 20
 from oslo_config import cfg
@@ -25,7 +24,6 @@ from oslo_utils import timeutils
25 24
 
26 25
 from ironic_inspector.common.i18n import _
27 26
 from ironic_inspector.common import ironic as ir_utils
28
-from ironic_inspector.common import swift
29 27
 from ironic_inspector import introspection_state as istate
30 28
 from ironic_inspector import node_cache
31 29
 from ironic_inspector.plugins import base as plugins_base
@@ -38,7 +36,6 @@ CONF = cfg.CONF
38 36
 LOG = utils.getProcessingLogger(__name__)
39 37
 
40 38
 _STORAGE_EXCLUDED_KEYS = {'logs'}
41
-_UNPROCESSED_DATA_STORE_SUFFIX = 'UNPROCESSED'
42 39
 
43 40
 
44 41
 def _store_logs(introspection_data, node_info):
@@ -143,48 +140,28 @@ def _filter_data_excluded_keys(data):
143 140
             if k not in _STORAGE_EXCLUDED_KEYS}
144 141
 
145 142
 
146
-def _store_data(node_info, data, suffix=None):
147
-    if CONF.processing.store_data != 'swift':
148
-        LOG.debug("Swift support is disabled, introspection data "
149
-                  "won't be stored", node_info=node_info)
150
-        return
151
-
152
-    swift_object_name = swift.store_introspection_data(
153
-        _filter_data_excluded_keys(data),
154
-        node_info.uuid,
155
-        suffix=suffix
156
-    )
157
-    LOG.info('Introspection data was stored in Swift in object '
158
-             '%s', swift_object_name, node_info=node_info)
159
-    if CONF.processing.store_data_location:
160
-        node_info.patch([{'op': 'add', 'path': '/extra/%s' %
161
-                          CONF.processing.store_data_location,
162
-                          'value': swift_object_name}])
143
+def _store_data(node_info, data, processed=True):
144
+    introspection_data_manager = plugins_base.introspection_data_manager()
145
+    store = CONF.processing.store_data
146
+    ext = introspection_data_manager[store].obj
147
+    ext.save(node_info, data, processed)
163 148
 
164 149
 
165 150
 def _store_unprocessed_data(node_info, data):
166 151
     # runs in background
167 152
     try:
168
-        _store_data(node_info, data,
169
-                    suffix=_UNPROCESSED_DATA_STORE_SUFFIX)
153
+        _store_data(node_info, data, processed=False)
170 154
     except Exception:
171 155
         LOG.exception('Encountered exception saving unprocessed '
172 156
                       'introspection data', node_info=node_info,
173 157
                       data=data)
174 158
 
175 159
 
176
-def _get_unprocessed_data(uuid):
177
-    if CONF.processing.store_data == 'swift':
178
-        LOG.debug('Fetching unprocessed introspection data from '
179
-                  'Swift for %s', uuid)
180
-        return json.loads(
181
-            swift.get_introspection_data(
182
-                uuid,
183
-                suffix=_UNPROCESSED_DATA_STORE_SUFFIX
184
-            )
185
-        )
186
-    else:
187
-        raise utils.Error(_('Swift support is disabled'), code=400)
160
+def get_introspection_data(uuid, processed=True, get_json=False):
161
+    introspection_data_manager = plugins_base.introspection_data_manager()
162
+    store = CONF.processing.store_data
163
+    ext = introspection_data_manager[store].obj
164
+    return ext.get(uuid, processed=processed, get_json=get_json)
188 165
 
189 166
 
190 167
 def process(introspection_data):
@@ -309,7 +286,7 @@ def _finish(node_info, ironic, introspection_data, power_off=True):
309 286
              node_info=node_info, data=introspection_data)
310 287
 
311 288
 
312
-def reapply(node_ident):
289
+def reapply(node_ident, data=None):
313 290
     """Re-apply introspection steps.
314 291
 
315 292
     Re-apply preprocessing, postprocessing and introspection rules on
@@ -331,15 +308,20 @@ def reapply(node_ident):
331 308
         raise utils.Error(_('Node locked, please, try again later'),
332 309
                           node_info=node_info, code=409)
333 310
 
334
-    utils.executor().submit(_reapply, node_info)
311
+    utils.executor().submit(_reapply, node_info, data)
335 312
 
336 313
 
337
-def _reapply(node_info):
314
+def _reapply(node_info, data=None):
338 315
     # runs in background
339 316
     try:
340 317
         node_info.started_at = timeutils.utcnow()
341 318
         node_info.commit()
342
-        introspection_data = _get_unprocessed_data(node_info.uuid)
319
+        if data:
320
+            introspection_data = data
321
+        else:
322
+            introspection_data = get_introspection_data(node_info.uuid,
323
+                                                        processed=False,
324
+                                                        get_json=True)
343 325
     except Exception as exc:
344 326
         LOG.exception('Encountered exception while fetching '
345 327
                       'stored introspection data',

+ 41
- 37
ironic_inspector/test/unit/test_main.py View File

@@ -22,6 +22,7 @@ from oslo_utils import uuidutils
22 22
 
23 23
 from ironic_inspector.common import ironic as ir_utils
24 24
 from ironic_inspector.common import rpc
25
+from ironic_inspector.common import swift
25 26
 import ironic_inspector.conf
26 27
 from ironic_inspector.conf import opts as conf_opts
27 28
 from ironic_inspector import introspection_state as istate
@@ -29,6 +30,7 @@ from ironic_inspector import main
29 30
 from ironic_inspector import node_cache
30 31
 from ironic_inspector.plugins import base as plugins_base
31 32
 from ironic_inspector.plugins import example as example_plugin
33
+from ironic_inspector.plugins import introspection_data as intros_data_plugin
32 34
 from ironic_inspector import process
33 35
 from ironic_inspector import rules
34 36
 from ironic_inspector.test import base as test_base
@@ -297,10 +299,9 @@ class TestApiListStatus(GetStatusAPIBaseTest):
297 299
 
298 300
 
299 301
 class TestApiGetData(BaseAPITest):
300
-    @mock.patch.object(main.swift, 'SwiftAPI', autospec=True)
301
-    def test_get_introspection_data(self, swift_mock):
302
-        CONF.set_override('store_data', 'swift', 'processing')
303
-        data = {
302
+    def setUp(self):
303
+        super(TestApiGetData, self).setUp()
304
+        self.introspection_data = {
304 305
             'ipmi_address': '1.2.3.4',
305 306
             'cpus': 2,
306 307
             'cpu_arch': 'x86_64',
@@ -310,44 +311,48 @@ class TestApiGetData(BaseAPITest):
310 311
                 'em1': {'mac': '11:22:33:44:55:66', 'ip': '1.2.0.1'},
311 312
             }
312 313
         }
314
+
315
+    @mock.patch.object(swift, 'SwiftAPI', autospec=True)
316
+    def test_get_introspection_data_from_swift(self, swift_mock):
317
+        CONF.set_override('store_data', 'swift', 'processing')
313 318
         swift_conn = swift_mock.return_value
314
-        swift_conn.get_object.return_value = json.dumps(data)
319
+        swift_conn.get_object.return_value = json.dumps(
320
+            self.introspection_data)
315 321
         res = self.app.get('/v1/introspection/%s/data' % self.uuid)
316 322
         name = 'inspector_data-%s' % self.uuid
317 323
         swift_conn.get_object.assert_called_once_with(name)
318 324
         self.assertEqual(200, res.status_code)
319
-        self.assertEqual(data, json.loads(res.data.decode('utf-8')))
325
+        self.assertEqual(self.introspection_data,
326
+                         json.loads(res.data.decode('utf-8')))
327
+
328
+    @mock.patch.object(intros_data_plugin, 'DatabaseStore',
329
+                       autospec=True)
330
+    def test_get_introspection_data_from_db(self, db_mock):
331
+        CONF.set_override('store_data', 'database', 'processing')
332
+        db_store = db_mock.return_value
333
+        db_store.get.return_value = json.dumps(self.introspection_data)
334
+        res = self.app.get('/v1/introspection/%s/data' % self.uuid)
335
+        db_store.get.assert_called_once_with(self.uuid, processed=True,
336
+                                             get_json=False)
337
+        self.assertEqual(200, res.status_code)
338
+        self.assertEqual(self.introspection_data,
339
+                         json.loads(res.data.decode('utf-8')))
320 340
 
321
-    @mock.patch.object(main.swift, 'SwiftAPI', autospec=True)
322
-    def test_introspection_data_not_stored(self, swift_mock):
341
+    def test_introspection_data_not_stored(self):
323 342
         CONF.set_override('store_data', 'none', 'processing')
324
-        swift_conn = swift_mock.return_value
325 343
         res = self.app.get('/v1/introspection/%s/data' % self.uuid)
326
-        self.assertFalse(swift_conn.get_object.called)
327 344
         self.assertEqual(404, res.status_code)
328 345
 
329 346
     @mock.patch.object(ir_utils, 'get_node', autospec=True)
330
-    @mock.patch.object(main.swift, 'SwiftAPI', autospec=True)
331
-    def test_with_name(self, swift_mock, get_mock):
347
+    @mock.patch.object(main.process, 'get_introspection_data', autospec=True)
348
+    def test_with_name(self, process_mock, get_mock):
332 349
         get_mock.return_value = mock.Mock(uuid=self.uuid)
333 350
         CONF.set_override('store_data', 'swift', 'processing')
334
-        data = {
335
-            'ipmi_address': '1.2.3.4',
336
-            'cpus': 2,
337
-            'cpu_arch': 'x86_64',
338
-            'memory_mb': 1024,
339
-            'local_gb': 20,
340
-            'interfaces': {
341
-                'em1': {'mac': '11:22:33:44:55:66', 'ip': '1.2.0.1'},
342
-            }
343
-        }
344
-        swift_conn = swift_mock.return_value
345
-        swift_conn.get_object.return_value = json.dumps(data)
351
+        process_mock.return_value = json.dumps(self.introspection_data)
346 352
         res = self.app.get('/v1/introspection/name1/data')
347
-        name = 'inspector_data-%s' % self.uuid
348
-        swift_conn.get_object.assert_called_once_with(name)
349 353
         self.assertEqual(200, res.status_code)
350
-        self.assertEqual(data, json.loads(res.data.decode('utf-8')))
354
+        self.assertEqual(self.introspection_data,
355
+                         json.loads(res.data.decode('utf-8')))
351 356
         get_mock.assert_called_once_with('name1', fields=['uuid'])
352 357
 
353 358
 
@@ -361,8 +366,7 @@ class TestApiReapply(BaseAPITest):
361 366
         self.rpc_get_client_mock.return_value = self.client_mock
362 367
         CONF.set_override('store_data', 'swift', 'processing')
363 368
 
364
-    def test_ok(self):
365
-
369
+    def test_api_ok(self):
366 370
         self.app.post('/v1/introspection/%s/data/unprocessed' %
367 371
                       self.uuid)
368 372
         self.client_mock.call.assert_called_once_with({}, 'do_reapply',
@@ -377,18 +381,18 @@ class TestApiReapply(BaseAPITest):
377 381
                          message)
378 382
         self.assertFalse(self.client_mock.call.called)
379 383
 
380
-    def test_swift_disabled(self):
381
-        CONF.set_override('store_data', 'none', 'processing')
384
+    def test_get_introspection_data_error(self):
385
+        exc = utils.Error('The store is crashed', code=404)
386
+        self.client_mock.call.side_effect = exc
382 387
 
383 388
         res = self.app.post('/v1/introspection/%s/data/unprocessed' %
384 389
                             self.uuid)
385
-        self.assertEqual(400, res.status_code)
390
+
391
+        self.assertEqual(404, res.status_code)
386 392
         message = json.loads(res.data.decode())['error']['message']
387
-        self.assertEqual('Inspector is not configured to store '
388
-                         'data. Set the [processing] store_data '
389
-                         'configuration option to change this.',
390
-                         message)
391
-        self.assertFalse(self.client_mock.call.called)
393
+        self.assertEqual(str(exc), message)
394
+        self.client_mock.call.assert_called_once_with({}, 'do_reapply',
395
+                                                      node_id=self.uuid)
392 396
 
393 397
     def test_generic_error(self):
394 398
         exc = utils.Error('Oops', code=400)

+ 71
- 8
ironic_inspector/test/unit/test_manager.py View File

@@ -11,10 +11,13 @@
11 11
 # See the License for the specific language governing permissions and
12 12
 # limitations under the License.
13 13
 
14
+import json
15
+
14 16
 import fixtures
15 17
 import mock
16 18
 import oslo_messaging as messaging
17 19
 
20
+from ironic_inspector.common import swift
18 21
 from ironic_inspector.conductor import manager
19 22
 import ironic_inspector.conf
20 23
 from ironic_inspector import introspect
@@ -302,11 +305,17 @@ class TestManagerReapply(BaseManagerTest):
302 305
         super(TestManagerReapply, self).setUp()
303 306
         CONF.set_override('store_data', 'swift', 'processing')
304 307
 
305
-    def test_ok(self, reapply_mock):
308
+    @mock.patch.object(swift, 'store_introspection_data', autospec=True)
309
+    @mock.patch.object(swift, 'get_introspection_data', autospec=True)
310
+    def test_ok(self, swift_get_mock, swift_set_mock, reapply_mock):
311
+        swift_get_mock.return_value = json.dumps(self.data)
306 312
         self.manager.do_reapply(self.context, self.uuid)
307
-        reapply_mock.assert_called_once_with(self.uuid)
313
+        reapply_mock.assert_called_once_with(self.uuid, data=self.data)
308 314
 
309
-    def test_node_locked(self, reapply_mock):
315
+    @mock.patch.object(swift, 'store_introspection_data', autospec=True)
316
+    @mock.patch.object(swift, 'get_introspection_data', autospec=True)
317
+    def test_node_locked(self, swift_get_mock, swift_set_mock, reapply_mock):
318
+        swift_get_mock.return_value = json.dumps(self.data)
310 319
         exc = utils.Error('Locked.', code=409)
311 320
         reapply_mock.side_effect = exc
312 321
 
@@ -317,9 +326,13 @@ class TestManagerReapply(BaseManagerTest):
317 326
         self.assertEqual(utils.Error, exc.exc_info[0])
318 327
         self.assertIn('Locked.', str(exc.exc_info[1]))
319 328
         self.assertEqual(409, exc.exc_info[1].http_code)
320
-        reapply_mock.assert_called_once_with(self.uuid)
329
+        reapply_mock.assert_called_once_with(self.uuid, data=self.data)
321 330
 
322
-    def test_node_not_found(self, reapply_mock):
331
+    @mock.patch.object(swift, 'store_introspection_data', autospec=True)
332
+    @mock.patch.object(swift, 'get_introspection_data', autospec=True)
333
+    def test_node_not_found(self, swift_get_mock, swift_set_mock,
334
+                            reapply_mock):
335
+        swift_get_mock.return_value = json.dumps(self.data)
323 336
         exc = utils.Error('Not found.', code=404)
324 337
         reapply_mock.side_effect = exc
325 338
 
@@ -330,9 +343,11 @@ class TestManagerReapply(BaseManagerTest):
330 343
         self.assertEqual(utils.Error, exc.exc_info[0])
331 344
         self.assertIn('Not found.', str(exc.exc_info[1]))
332 345
         self.assertEqual(404, exc.exc_info[1].http_code)
333
-        reapply_mock.assert_called_once_with(self.uuid)
346
+        reapply_mock.assert_called_once_with(self.uuid, data=self.data)
334 347
 
335
-    def test_generic_error(self, reapply_mock):
348
+    @mock.patch.object(process, 'get_introspection_data', autospec=True)
349
+    def test_generic_error(self, get_data_mock, reapply_mock):
350
+        get_data_mock.return_value = self.data
336 351
         exc = utils.Error('Oops', code=400)
337 352
         reapply_mock.side_effect = exc
338 353
 
@@ -343,4 +358,52 @@ class TestManagerReapply(BaseManagerTest):
343 358
         self.assertEqual(utils.Error, exc.exc_info[0])
344 359
         self.assertIn('Oops', str(exc.exc_info[1]))
345 360
         self.assertEqual(400, exc.exc_info[1].http_code)
346
-        reapply_mock.assert_called_once_with(self.uuid)
361
+        reapply_mock.assert_called_once_with(self.uuid, data=self.data)
362
+        get_data_mock.assert_called_once_with(self.uuid, processed=False,
363
+                                              get_json=True)
364
+
365
+    @mock.patch.object(process, 'get_introspection_data', autospec=True)
366
+    def test_get_introspection_data_error(self, get_data_mock, reapply_mock):
367
+        exc = utils.Error('The store is empty', code=404)
368
+        get_data_mock.side_effect = exc
369
+
370
+        exc = self.assertRaises(messaging.rpc.ExpectedException,
371
+                                self.manager.do_reapply,
372
+                                self.context, self.uuid)
373
+
374
+        self.assertEqual(utils.Error, exc.exc_info[0])
375
+        self.assertIn('The store is empty', str(exc.exc_info[1]))
376
+        self.assertEqual(404, exc.exc_info[1].http_code)
377
+        get_data_mock.assert_called_once_with(self.uuid, processed=False,
378
+                                              get_json=True)
379
+        self.assertFalse(reapply_mock.called)
380
+
381
+    def test_store_data_disabled(self, reapply_mock):
382
+        CONF.set_override('store_data', 'none', 'processing')
383
+
384
+        exc = self.assertRaises(messaging.rpc.ExpectedException,
385
+                                self.manager.do_reapply,
386
+                                self.context, self.uuid)
387
+
388
+        self.assertEqual(utils.Error, exc.exc_info[0])
389
+        self.assertIn('Inspector is not configured to store data',
390
+                      str(exc.exc_info[1]))
391
+        self.assertEqual(400, exc.exc_info[1].http_code)
392
+        self.assertFalse(reapply_mock.called)
393
+
394
+    @mock.patch.object(process, 'get_introspection_data', autospec=True)
395
+    def test_ok_swift(self, get_data_mock, reapply_mock):
396
+        get_data_mock.return_value = self.data
397
+        self.manager.do_reapply(self.context, self.uuid)
398
+        reapply_mock.assert_called_once_with(self.uuid, data=self.data)
399
+        get_data_mock.assert_called_once_with(self.uuid, processed=False,
400
+                                              get_json=True)
401
+
402
+    @mock.patch.object(process, 'get_introspection_data', autospec=True)
403
+    def test_ok_db(self, get_data_mock, reapply_mock):
404
+        get_data_mock.return_value = self.data
405
+        CONF.set_override('store_data', 'database', 'processing')
406
+        self.manager.do_reapply(self.context, self.uuid)
407
+        reapply_mock.assert_called_once_with(self.uuid, data=self.data)
408
+        get_data_mock.assert_called_once_with(self.uuid, processed=False,
409
+                                              get_json=True)

+ 108
- 0
ironic_inspector/test/unit/test_plugins_introspection_data.py View File

@@ -0,0 +1,108 @@
1
+# Licensed under the Apache License, Version 2.0 (the "License");
2
+# you may not use this file except in compliance with the License.
3
+# You may obtain a copy of the License at
4
+#
5
+#    http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+# Unless required by applicable law or agreed to in writing, software
8
+# distributed under the License is distributed on an "AS IS" BASIS,
9
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
10
+# implied.
11
+# See the License for the specific language governing permissions and
12
+# limitations under the License.
13
+
14
+import json
15
+
16
+import fixtures
17
+import mock
18
+from oslo_config import cfg
19
+
20
+from ironic_inspector.common import ironic as ir_utils
21
+from ironic_inspector import db
22
+from ironic_inspector import introspection_state as istate
23
+from ironic_inspector.plugins import introspection_data
24
+from ironic_inspector.test import base as test_base
25
+
26
+CONF = cfg.CONF
27
+
28
+
29
+class BaseTest(test_base.NodeTest):
30
+    data = {
31
+        'ipmi_address': '1.2.3.4',
32
+        'cpus': 2,
33
+        'cpu_arch': 'x86_64',
34
+        'memory_mb': 1024,
35
+        'local_gb': 20,
36
+        'interfaces': {
37
+            'em1': {'mac': '11:22:33:44:55:66', 'ip': '1.2.0.1'},
38
+        }
39
+    }
40
+
41
+    def setUp(self):
42
+        super(BaseTest, self).setUp()
43
+        self.cli_fixture = self.useFixture(
44
+            fixtures.MockPatchObject(ir_utils, 'get_client', autospec=True))
45
+        self.cli = self.cli_fixture.mock.return_value
46
+
47
+
48
+@mock.patch.object(introspection_data.swift, 'SwiftAPI', autospec=True)
49
+class TestSwiftStore(BaseTest):
50
+
51
+    def setUp(self):
52
+        super(TestSwiftStore, self).setUp()
53
+        self.driver = introspection_data.SwiftStore()
54
+
55
+    def test_get_data(self, swift_mock):
56
+        swift_conn = swift_mock.return_value
57
+        swift_conn.get_object.return_value = json.dumps(self.data)
58
+        name = 'inspector_data-%s' % self.uuid
59
+
60
+        res_data = self.driver.get(self.uuid)
61
+
62
+        swift_conn.get_object.assert_called_once_with(name)
63
+        self.assertEqual(self.data, json.loads(res_data))
64
+
65
+    def test_store_data(self, swift_mock):
66
+        swift_conn = swift_mock.return_value
67
+        name = 'inspector_data-%s' % self.uuid
68
+
69
+        self.driver.save(self.node_info, self.data)
70
+
71
+        data = introspection_data._filter_data_excluded_keys(self.data)
72
+        swift_conn.create_object.assert_called_once_with(name,
73
+                                                         json.dumps(data))
74
+
75
+    def test_store_data_location(self, swift_mock):
76
+        CONF.set_override('store_data_location', 'inspector_data_object',
77
+                          'processing')
78
+        swift_conn = swift_mock.return_value
79
+        name = 'inspector_data-%s' % self.uuid
80
+        patch = [{'path': '/extra/inspector_data_object',
81
+                  'value': name, 'op': 'add'}]
82
+        expected = self.data
83
+
84
+        self.driver.save(self.node_info, self.data)
85
+
86
+        data = introspection_data._filter_data_excluded_keys(self.data)
87
+        swift_conn.create_object.assert_called_once_with(name,
88
+                                                         json.dumps(data))
89
+        self.assertEqual(expected,
90
+                         json.loads(swift_conn.create_object.call_args[0][1]))
91
+        self.cli.node.update.assert_any_call(self.uuid, patch)
92
+
93
+
94
+class TestDatabaseStore(BaseTest):
95
+    def setUp(self):
96
+        super(TestDatabaseStore, self).setUp()
97
+        self.driver = introspection_data.DatabaseStore()
98
+        session = db.get_writer_session()
99
+        with session.begin():
100
+            db.Node(uuid=self.node_info.uuid,
101
+                    state=istate.States.starting).save(session)
102
+
103
+    def test_store_and_get_data(self):
104
+        self.driver.save(self.node_info, self.data)
105
+
106
+        res_data = self.driver.get(self.node_info.uuid)
107
+
108
+        self.assertEqual(self.data, json.loads(res_data))

+ 34
- 18
ironic_inspector/test/unit/test_process.py View File

@@ -28,11 +28,13 @@ from oslo_utils import uuidutils
28 28
 import six
29 29
 
30 30
 from ironic_inspector.common import ironic as ir_utils
31
+from ironic_inspector.common import swift
31 32
 from ironic_inspector import db
32 33
 from ironic_inspector import introspection_state as istate
33 34
 from ironic_inspector import node_cache
34 35
 from ironic_inspector.plugins import base as plugins_base
35 36
 from ironic_inspector.plugins import example as example_plugin
37
+from ironic_inspector.plugins import introspection_data as intros_data_plugin
36 38
 from ironic_inspector import process
37 39
 from ironic_inspector.pxe_filter import base as pxe_filter
38 40
 from ironic_inspector.test import base as test_base
@@ -259,22 +261,13 @@ class TestUnprocessedData(BaseProcessTest):
259 261
 
260 262
         store_mock.assert_called_once_with(mock.ANY, expected)
261 263
 
262
-    @mock.patch.object(process.swift, 'SwiftAPI', autospec=True)
263
-    def test_save_unprocessed_data_failure(self, swift_mock):
264
+    def test_save_unprocessed_data_failure(self):
264 265
         CONF.set_override('store_data', 'swift', 'processing')
265
-        name = 'inspector_data-%s-%s' % (
266
-            self.uuid,
267
-            process._UNPROCESSED_DATA_STORE_SUFFIX
268
-        )
269
-
270
-        swift_conn = swift_mock.return_value
271
-        swift_conn.create_object.side_effect = utils.Error('Oops')
272 266
 
273 267
         res = process.process(self.data)
274 268
 
275 269
         # assert store failure doesn't break processing
276 270
         self.assertEqual(self.fake_result_json, res)
277
-        swift_conn.create_object.assert_called_once_with(name, mock.ANY)
278 271
 
279 272
 
280 273
 @mock.patch.object(example_plugin.ExampleProcessingHook, 'before_processing',
@@ -405,6 +398,7 @@ class TestProcessNode(BaseTest):
405 398
                 started_at=self.node_info.started_at,
406 399
                 finished_at=self.node_info.finished_at,
407 400
                 error=self.node_info.error).save(self.session)
401
+        plugins_base._INTROSPECTION_DATA_MGR = None
408 402
 
409 403
     def test_return_includes_uuid(self):
410 404
         ret_val = process._process_node(self.node_info, self.node, self.data)
@@ -485,8 +479,8 @@ class TestProcessNode(BaseTest):
485 479
         finished_mock.assert_called_once_with(
486 480
             self.node_info, istate.Events.finish)
487 481
 
488
-    @mock.patch.object(process.swift, 'SwiftAPI', autospec=True)
489
-    def test_store_data(self, swift_mock):
482
+    @mock.patch.object(swift, 'SwiftAPI', autospec=True)
483
+    def test_store_data_with_swift(self, swift_mock):
490 484
         CONF.set_override('store_data', 'swift', 'processing')
491 485
         swift_conn = swift_mock.return_value
492 486
         name = 'inspector_data-%s' % self.uuid
@@ -498,8 +492,8 @@ class TestProcessNode(BaseTest):
498 492
         self.assertEqual(expected,
499 493
                          json.loads(swift_conn.create_object.call_args[0][1]))
500 494
 
501
-    @mock.patch.object(process.swift, 'SwiftAPI', autospec=True)
502
-    def test_store_data_no_logs(self, swift_mock):
495
+    @mock.patch.object(swift, 'SwiftAPI', autospec=True)
496
+    def test_store_data_no_logs_with_swift(self, swift_mock):
503 497
         CONF.set_override('store_data', 'swift', 'processing')
504 498
         swift_conn = swift_mock.return_value
505 499
         name = 'inspector_data-%s' % self.uuid
@@ -511,8 +505,8 @@ class TestProcessNode(BaseTest):
511 505
         self.assertNotIn('logs',
512 506
                          json.loads(swift_conn.create_object.call_args[0][1]))
513 507
 
514
-    @mock.patch.object(process.swift, 'SwiftAPI', autospec=True)
515
-    def test_store_data_location(self, swift_mock):
508
+    @mock.patch.object(swift, 'SwiftAPI', autospec=True)
509
+    def test_store_data_location_with_swift(self, swift_mock):
516 510
         CONF.set_override('store_data', 'swift', 'processing')
517 511
         CONF.set_override('store_data_location', 'inspector_data_object',
518 512
                           'processing')
@@ -529,6 +523,28 @@ class TestProcessNode(BaseTest):
529 523
                          json.loads(swift_conn.create_object.call_args[0][1]))
530 524
         self.cli.node.update.assert_any_call(self.uuid, patch)
531 525
 
526
+    @mock.patch.object(node_cache, 'store_introspection_data', autospec=True)
527
+    def test_store_data_with_database(self, store_mock):
528
+        CONF.set_override('store_data', 'database', 'processing')
529
+
530
+        process._process_node(self.node_info, self.node, self.data)
531
+
532
+        data = intros_data_plugin._filter_data_excluded_keys(self.data)
533
+        store_mock.assert_called_once_with(self.node_info.uuid, data, True)
534
+        self.assertEqual(data, store_mock.call_args[0][1])
535
+
536
+    @mock.patch.object(node_cache, 'store_introspection_data', autospec=True)
537
+    def test_store_data_no_logs_with_database(self, store_mock):
538
+        CONF.set_override('store_data', 'database', 'processing')
539
+
540
+        self.data['logs'] = 'something'
541
+
542
+        process._process_node(self.node_info, self.node, self.data)
543
+
544
+        data = intros_data_plugin._filter_data_excluded_keys(self.data)
545
+        store_mock.assert_called_once_with(self.node_info.uuid, data, True)
546
+        self.assertNotIn('logs', store_mock.call_args[0][1])
547
+
532 548
 
533 549
 @mock.patch.object(process, '_reapply', autospec=True)
534 550
 @mock.patch.object(node_cache, 'get_node', autospec=True)
@@ -558,7 +574,7 @@ class TestReapply(BaseTest):
558 574
             blocking=False
559 575
         )
560 576
 
561
-        reapply_mock.assert_called_once_with(pop_mock.return_value)
577
+        reapply_mock.assert_called_once_with(pop_mock.return_value, data=None)
562 578
 
563 579
     @prepare_mocks
564 580
     def test_locking_failed(self, pop_mock, reapply_mock):
@@ -575,7 +591,7 @@ class TestReapply(BaseTest):
575 591
 
576 592
 @mock.patch.object(example_plugin.ExampleProcessingHook, 'before_update')
577 593
 @mock.patch.object(process.rules, 'apply', autospec=True)
578
-@mock.patch.object(process.swift, 'SwiftAPI', autospec=True)
594
+@mock.patch.object(swift, 'SwiftAPI', autospec=True)
579 595
 @mock.patch.object(node_cache.NodeInfo, 'finished', autospec=True)
580 596
 @mock.patch.object(node_cache.NodeInfo, 'release_lock', autospec=True)
581 597
 class TestReapplyNode(BaseTest):

+ 4
- 0
ironic_inspector/utils.py View File

@@ -140,6 +140,10 @@ class NodeStateInvalidEvent(Error):
140 140
     """Invalid event attempted."""
141 141
 
142 142
 
143
+class IntrospectionDataStoreDisabled(Error):
144
+    """Introspection data store is disabled."""
145
+
146
+
143 147
 class IntrospectionDataNotFound(NotFoundInCacheError):
144 148
     """Introspection data not found."""
145 149
 

+ 6
- 0
releasenotes/notes/introspection-data-db-store-0586292de05cbfd7.yaml View File

@@ -0,0 +1,6 @@
1
+---
2
+features:
3
+  - |
4
+    Adds the support to store introspection data in ironic-inspector database.
5
+    Set the option ``[processing]store_data`` to ``database`` to use this
6
+    feature.

+ 4
- 0
setup.cfg View File

@@ -43,6 +43,10 @@ ironic_inspector.hooks.processing =
43 43
 ironic_inspector.hooks.node_not_found =
44 44
     example = ironic_inspector.plugins.example:example_not_found_hook
45 45
     enroll = ironic_inspector.plugins.discovery:enroll_node_not_found_hook
46
+ironic_inspector.introspection_data.store =
47
+    none = ironic_inspector.plugins.introspection_data:NoStore
48
+    swift = ironic_inspector.plugins.introspection_data:SwiftStore
49
+    database = ironic_inspector.plugins.introspection_data:DatabaseStore
46 50
 ironic_inspector.rules.conditions =
47 51
     eq = ironic_inspector.plugins.rules:EqCondition
48 52
     lt = ironic_inspector.plugins.rules:LtCondition

Loading…
Cancel
Save