From 29d8515f500bafe34e7fd566e59cb68b03f5472d Mon Sep 17 00:00:00 2001 From: Kaifeng Wang Date: Thu, 24 Jan 2019 15:46:03 +0800 Subject: [PATCH] Support reapply with supplied introspection data This patch adds support to provide unprocessed introspection data to reapply a node. The provided introspection data will be save to current introspection storage backend. Change-Id: I969ae9c32f53f89c006a64a006388ddea9542aa5 Story: 1564863 Task: 11344 --- ...ection-api-v1-introspection-management.inc | 20 ++++++++----- doc/source/user/http-api.rst | 2 +- ironic_inspector/common/rpc.py | 4 +-- ironic_inspector/conductor/manager.py | 27 ++++++++++------- ironic_inspector/main.py | 19 +++++++++--- ironic_inspector/process.py | 29 +++++++++++++++---- ironic_inspector/test/functional.py | 8 +++++ ironic_inspector/test/unit/test_main.py | 24 +++++++++++---- ironic_inspector/test/unit/test_manager.py | 13 +++++++-- ironic_inspector/test/unit/test_process.py | 10 +++++++ ...t-introspection-data-9cdd39a3de446e92.yaml | 7 +++++ 11 files changed, 123 insertions(+), 40 deletions(-) create mode 100644 releasenotes/notes/post-introspection-data-9cdd39a3de446e92.yaml diff --git a/api-ref/source/introspection-api-v1-introspection-management.inc b/api-ref/source/introspection-api-v1-introspection-management.inc index 1b5cb357d..69acb7aac 100644 --- a/api-ref/source/introspection-api-v1-introspection-management.inc +++ b/api-ref/source/introspection-api-v1-introspection-management.inc @@ -70,28 +70,32 @@ The response will contain introspection data in the form of json string. :language: javascript -Reapply Introspection on stored data -==================================== +Reapply Introspection on data +============================= .. rest_method:: POST /v1/introspection/{node_id}/data/unprocessed -This method triggers introspection on stored unprocessed data. -No data is allowed to be sent along with the request. +This method triggers introspection on either stored introspection data or raw +introspection data provided in the request. If the introspection data is +provided in the request body, it should be a valid JSON with content similar to +ramdisk callback request. + +.. versionadded:: 1.15 + Unprocessed introspection data can be sent via request body. .. note:: - - Requires enabling introspection storage backend via ``[processing]store_data``. + Reapplying introspection on stored data is only possible when a storage + backend is enabled via ``[processing]store_data``. Normal response codes: 202 Error codes: -* 400 - bad request or store not configured +* 400 - bad request, store not configured or malformed data in request body * 401, 403 - missing or invalid authentication * 404 - node not found for Node ID * 409 - inspector locked node for processing - Request ------- diff --git a/doc/source/user/http-api.rst b/doc/source/user/http-api.rst index eaaf610f6..debb9b13f 100644 --- a/doc/source/user/http-api.rst +++ b/doc/source/user/http-api.rst @@ -398,4 +398,4 @@ Version History * **1.13** adds ``manage_boot`` parameter for the introspection API. * **1.14** allows formatting to be applied to strings nested in dicts and lists in the actions of introspection rules. - +* **1.15** allows reapply with provided introspection data from request. diff --git a/ironic_inspector/common/rpc.py b/ironic_inspector/common/rpc.py index 31e531102..94f49efad 100644 --- a/ironic_inspector/common/rpc.py +++ b/ironic_inspector/common/rpc.py @@ -33,7 +33,7 @@ def get_transport(): def get_client(): """Get a RPC client instance.""" target = messaging.Target(topic=manager.MANAGER_TOPIC, server=CONF.host, - version='1.1') + version='1.2') transport = get_transport() return messaging.RPCClient(transport, target) @@ -43,7 +43,7 @@ def get_server(endpoints): transport = get_transport() target = messaging.Target(topic=manager.MANAGER_TOPIC, server=CONF.host, - version='1.1') + version='1.2') return messaging.get_rpc_server( transport, target, endpoints, executor='eventlet', access_policy=dispatcher.DefaultRPCAccessPolicy) diff --git a/ironic_inspector/conductor/manager.py b/ironic_inspector/conductor/manager.py index b0bfebadd..29ca30ef0 100644 --- a/ironic_inspector/conductor/manager.py +++ b/ironic_inspector/conductor/manager.py @@ -39,7 +39,7 @@ MANAGER_TOPIC = 'ironic-inspector-conductor' class ConductorManager(object): """ironic inspector conductor manager""" - RPC_API_VERSION = '1.1' + RPC_API_VERSION = '1.2' target = messaging.Target(version=RPC_API_VERSION) @@ -126,16 +126,21 @@ class ConductorManager(object): introspect.abort(node_id, token=token) @messaging.expected_exceptions(utils.Error) - def do_reapply(self, context, node_uuid, token=None): - try: - data = process.get_introspection_data(node_uuid, processed=False, - get_json=True) - except utils.IntrospectionDataStoreDisabled: - raise utils.Error(_('Inspector is not configured to store ' - 'data. Set the [processing]store_data ' - 'configuration option to change this.'), - code=400) - process.reapply(node_uuid, data) + def do_reapply(self, context, node_uuid, token=None, data=None): + if not data: + try: + data = process.get_introspection_data(node_uuid, + processed=False, + get_json=True) + except utils.IntrospectionDataStoreDisabled: + raise utils.Error(_('Inspector is not configured to store ' + 'introspection data. Set the ' + '[processing]store_data configuration ' + 'option to change this.')) + else: + process.store_introspection_data(node_uuid, data, processed=False) + + process.reapply(node_uuid, data=data) def periodic_clean_up(): # pragma: no cover diff --git a/ironic_inspector/main.py b/ironic_inspector/main.py index 6a7842ccd..44f1ec169 100644 --- a/ironic_inspector/main.py +++ b/ironic_inspector/main.py @@ -39,7 +39,7 @@ app = flask.Flask(__name__) LOG = utils.getProcessingLogger(__name__) MINIMUM_API_VERSION = (1, 0) -CURRENT_API_VERSION = (1, 14) +CURRENT_API_VERSION = (1, 15) DEFAULT_API_VERSION = CURRENT_API_VERSION _LOGGING_EXCLUDED_KEYS = ('logs',) @@ -307,14 +307,25 @@ def api_introspection_data(node_id): @api('/v1/introspection//data/unprocessed', rule="introspection:reapply", methods=['POST']) def api_introspection_reapply(node_id): + data = None if flask.request.content_length: - return error_response(_('User data processing is not ' - 'supported yet'), code=400) + try: + data = flask.request.get_json(force=True) + except Exception: + raise utils.Error( + _('Invalid data: expected a JSON object, got %s') % data) + if not isinstance(data, dict): + raise utils.Error( + _('Invalid data: expected a JSON object, got %s') % + data.__class__.__name__) + LOG.debug("Received reapply data from request", data=data) + if not uuidutils.is_uuid_like(node_id): node = ir_utils.get_node(node_id, fields=['uuid']) node_id = node.uuid + client = rpc.get_client() - client.call({}, 'do_reapply', node_uuid=node_id) + client.call({}, 'do_reapply', node_uuid=node_id, data=data) return '', 202 diff --git a/ironic_inspector/process.py b/ironic_inspector/process.py index 8d476cfef..6810b80f9 100644 --- a/ironic_inspector/process.py +++ b/ironic_inspector/process.py @@ -140,7 +140,15 @@ def _filter_data_excluded_keys(data): if k not in _STORAGE_EXCLUDED_KEYS} -def _store_data(node_uuid, data, processed=True): +def store_introspection_data(node_uuid, data, processed=True): + """Store introspection data to the storage backend. + + :param node_uuid: node UUID + :param data: Introspection data to be saved + :param processed: The type of introspection data, set to True means the + introspection data is processed, otherwise unprocessed. + :raises: utils.Error + """ introspection_data_manager = plugins_base.introspection_data_manager() store = CONF.processing.store_data ext = introspection_data_manager[store].obj @@ -150,13 +158,22 @@ def _store_data(node_uuid, data, processed=True): def _store_unprocessed_data(node_uuid, data): # runs in background try: - _store_data(node_uuid, data, processed=False) + store_introspection_data(node_uuid, data, processed=False) except Exception: LOG.exception('Encountered exception saving unprocessed ' 'introspection data for node %s', node_uuid, data=data) def get_introspection_data(uuid, processed=True, get_json=False): + """Get introspection data from the storage backend. + + :param uuid: node UUID + :param processed: Indicates the type of introspection data to be read, + set True to request processed introspection data. + :param get_json: Specify whether return the introspection data in json + format, string value is returned if False. + :raises: utils.Error + """ introspection_data_manager = plugins_base.introspection_data_manager() store = CONF.processing.store_data ext = introspection_data_manager[store].obj @@ -242,7 +259,7 @@ def _process_node(node_info, node, introspection_data): # NOTE(dtantsur): repeat the check in case something changed ir_utils.check_provision_state(node) _run_post_hooks(node_info, introspection_data) - _store_data(node_info.uuid, introspection_data) + store_introspection_data(node_info.uuid, introspection_data) ironic = ir_utils.get_client() pxe_filter.driver().sync(ironic) @@ -292,8 +309,8 @@ def reapply(node_uuid, data=None): stored data. :param node_uuid: node UUID + :param data: unprocessed introspection data to be reapplied :raises: utils.Error - """ LOG.debug('Processing re-apply introspection request for node ' @@ -307,7 +324,7 @@ def reapply(node_uuid, data=None): raise utils.Error(_('Node locked, please, try again later'), node_info=node_info, code=409) - utils.executor().submit(_reapply, node_info, data) + utils.executor().submit(_reapply, node_info, introspection_data=data) def _reapply(node_info, introspection_data=None): @@ -350,6 +367,6 @@ def _reapply_with_data(node_info, introspection_data): '\n'.join(failures), node_info=node_info) _run_post_hooks(node_info, introspection_data) - _store_data(node_info.uuid, introspection_data) + store_introspection_data(node_info.uuid, introspection_data) node_info.invalidate_cache() rules.apply(node_info, introspection_data) diff --git a/ironic_inspector/test/functional.py b/ironic_inspector/test/functional.py index cdae214e9..f7edfc00b 100644 --- a/ironic_inspector/test/functional.py +++ b/ironic_inspector/test/functional.py @@ -634,6 +634,14 @@ class Test(Base): self.assertEqual(store_processing_call, store_mock.call_args_list[-1]) + # Reapply with provided data + res = self.call_reapply(self.uuid, data=copy.deepcopy(self.data)) + self.assertEqual(202, res.status_code) + self.assertEqual('', res.text) + eventlet.greenthread.sleep(DEFAULT_SLEEP) + + self.check_status(status, finished=True, state=istate.States.finished) + @mock.patch.object(swift, 'store_introspection_data', autospec=True) @mock.patch.object(swift, 'get_introspection_data', autospec=True) def test_edge_state_transitions(self, get_mock, store_mock): diff --git a/ironic_inspector/test/unit/test_main.py b/ironic_inspector/test/unit/test_main.py index 690f8ce70..ce0a50337 100644 --- a/ironic_inspector/test/unit/test_main.py +++ b/ironic_inspector/test/unit/test_main.py @@ -370,17 +370,26 @@ class TestApiReapply(BaseAPITest): self.app.post('/v1/introspection/%s/data/unprocessed' % self.uuid) self.client_mock.call.assert_called_once_with({}, 'do_reapply', - node_uuid=self.uuid) + node_uuid=self.uuid, + data=None) def test_user_data(self): res = self.app.post('/v1/introspection/%s/data/unprocessed' % self.uuid, data='some data') self.assertEqual(400, res.status_code) message = json.loads(res.data.decode())['error']['message'] - self.assertEqual('User data processing is not supported yet', - message) + self.assertIn('Invalid data: expected a JSON object', message) self.assertFalse(self.client_mock.call.called) + def test_user_data_valid(self): + data = {"foo": "bar"} + res = self.app.post('/v1/introspection/%s/data/unprocessed' % + self.uuid, data=json.dumps(data)) + self.assertEqual(202, res.status_code) + self.client_mock.call.assert_called_once_with({}, 'do_reapply', + node_uuid=self.uuid, + data=data) + def test_get_introspection_data_error(self): exc = utils.Error('The store is crashed', code=404) self.client_mock.call.side_effect = exc @@ -392,7 +401,8 @@ class TestApiReapply(BaseAPITest): message = json.loads(res.data.decode())['error']['message'] self.assertEqual(str(exc), message) self.client_mock.call.assert_called_once_with({}, 'do_reapply', - node_uuid=self.uuid) + node_uuid=self.uuid, + data=None) def test_generic_error(self): exc = utils.Error('Oops', code=400) @@ -405,7 +415,8 @@ class TestApiReapply(BaseAPITest): message = json.loads(res.data.decode())['error']['message'] self.assertEqual(str(exc), message) self.client_mock.call.assert_called_once_with({}, 'do_reapply', - node_uuid=self.uuid) + node_uuid=self.uuid, + data=None) @mock.patch.object(ir_utils, 'get_node', autospec=True) def test_reapply_with_node_name(self, get_mock): @@ -413,7 +424,8 @@ class TestApiReapply(BaseAPITest): self.app.post('/v1/introspection/%s/data/unprocessed' % 'fake-node') self.client_mock.call.assert_called_once_with({}, 'do_reapply', - node_uuid=self.uuid) + node_uuid=self.uuid, + data=None) get_mock.assert_called_once_with('fake-node', fields=['uuid']) diff --git a/ironic_inspector/test/unit/test_manager.py b/ironic_inspector/test/unit/test_manager.py index e374441db..35f0473f2 100644 --- a/ironic_inspector/test/unit/test_manager.py +++ b/ironic_inspector/test/unit/test_manager.py @@ -386,8 +386,8 @@ class TestManagerReapply(BaseManagerTest): self.context, self.uuid) self.assertEqual(utils.Error, exc.exc_info[0]) - self.assertIn('Inspector is not configured to store data', - str(exc.exc_info[1])) + self.assertIn('Inspector is not configured to store introspection ' + 'data', str(exc.exc_info[1])) self.assertEqual(400, exc.exc_info[1].http_code) self.assertFalse(reapply_mock.called) @@ -407,3 +407,12 @@ class TestManagerReapply(BaseManagerTest): reapply_mock.assert_called_once_with(self.uuid, data=self.data) get_data_mock.assert_called_once_with(self.uuid, processed=False, get_json=True) + + @mock.patch.object(process, 'store_introspection_data', autospec=True) + @mock.patch.object(process, 'get_introspection_data', autospec=True) + def test_reapply_with_data(self, get_mock, store_mock, reapply_mock): + self.manager.do_reapply(self.context, self.uuid, data=self.data) + reapply_mock.assert_called_once_with(self.uuid, data=self.data) + store_mock.assert_called_once_with(self.uuid, self.data, + processed=False) + self.assertFalse(get_mock.called) diff --git a/ironic_inspector/test/unit/test_process.py b/ironic_inspector/test/unit/test_process.py index 3d89630ac..8ea360649 100644 --- a/ironic_inspector/test/unit/test_process.py +++ b/ironic_inspector/test/unit/test_process.py @@ -584,6 +584,16 @@ class TestReapply(BaseTest): blocking=False ) + @prepare_mocks + def test_reapply_with_data(self, pop_mock, reapply_mock): + process.reapply(self.uuid, data=self.data) + pop_mock.assert_called_once_with(self.uuid, locked=False) + pop_mock.return_value.acquire_lock.assert_called_once_with( + blocking=False + ) + reapply_mock.assert_called_once_with(pop_mock.return_value, + introspection_data=self.data) + @mock.patch.object(example_plugin.ExampleProcessingHook, 'before_update') @mock.patch.object(process.rules, 'apply', autospec=True) diff --git a/releasenotes/notes/post-introspection-data-9cdd39a3de446e92.yaml b/releasenotes/notes/post-introspection-data-9cdd39a3de446e92.yaml new file mode 100644 index 000000000..c686425ea --- /dev/null +++ b/releasenotes/notes/post-introspection-data-9cdd39a3de446e92.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Adds support to reapply with provided unprocessed introspection data. The + introspection data is supplied in the body of POST request to + ``/v1/introspection//data/unprocessed``. The introspection data + will also be saved to storage backend.