openstack/ironic-inspector
Code Issues Proposed changes

Support reapply with supplied introspection data

Browse Source 
This patch adds support to provide unprocessed introspection data
to reapply a node. The provided introspection data will be save to
current introspection storage backend.

Change-Id: I969ae9c32f53f89c006a64a006388ddea9542aa5
Story: 1564863
Task: 11344
This commit is contained in:
Kaifeng Wang 2019-01-24 15:46:03 +08:00
parent 47b9468838
commit 29d8515f50
11 changed files with 123 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
api-ref/source/introspection-api-v1-introspection-management.inc
View File

@ -70,28 +70,32 @@ The response will contain introspection data in the form of json string.
:language: javascript
Reapply Introspection on stored data
====================================
Reapply Introspection on data
=============================
.. rest_method:: POST /v1/introspection/{node_id}/data/unprocessed
This method triggers introspection on stored unprocessed data.
No data is allowed to be sent along with the request.
This method triggers introspection on either stored introspection data or raw
introspection data provided in the request. If the introspection data is
provided in the request body, it should be a valid JSON with content similar to
ramdisk callback request.
.. versionadded:: 1.15
Unprocessed introspection data can be sent via request body.
.. note::
Requires enabling introspection storage backend via ``[processing]store_data``.
Reapplying introspection on stored data is only possible when a storage
backend is enabled via ``[processing]store_data``.
Normal response codes: 202
Error codes:
* 400 - bad request or store not configured
* 400 - bad request, store not configured or malformed data in request body
* 401, 403 - missing or invalid authentication
* 404 - node not found for Node ID
* 409 - inspector locked node for processing
Request
-------

2
doc/source/user/http-api.rst
View File

@ -398,4 +398,4 @@ Version History
* **1.13** adds ``manage_boot`` parameter for the introspection API.
* **1.14** allows formatting to be applied to strings nested in dicts and lists
in the actions of introspection rules.
* **1.15** allows reapply with provided introspection data from request.

4
ironic_inspector/common/rpc.py
View File

@ -33,7 +33,7 @@ def get_transport():
def get_client():
"""Get a RPC client instance."""
target = messaging.Target(topic=manager.MANAGER_TOPIC, server=CONF.host,
version='1.1')
version='1.2')
transport = get_transport()
return messaging.RPCClient(transport, target)
@ -43,7 +43,7 @@ def get_server(endpoints):
transport = get_transport()
target = messaging.Target(topic=manager.MANAGER_TOPIC, server=CONF.host,
version='1.1')
version='1.2')
return messaging.get_rpc_server(
transport, target, endpoints, executor='eventlet',
access_policy=dispatcher.DefaultRPCAccessPolicy)

27
ironic_inspector/conductor/manager.py
View File

@ -39,7 +39,7 @@ MANAGER_TOPIC = 'ironic-inspector-conductor'
class ConductorManager(object):
"""ironic inspector conductor manager"""
RPC_API_VERSION = '1.1'
RPC_API_VERSION = '1.2'
target = messaging.Target(version=RPC_API_VERSION)
@ -126,16 +126,21 @@ class ConductorManager(object):
introspect.abort(node_id, token=token)
@messaging.expected_exceptions(utils.Error)
def do_reapply(self, context, node_uuid, token=None):
try:
data = process.get_introspection_data(node_uuid, processed=False,
get_json=True)
except utils.IntrospectionDataStoreDisabled:
raise utils.Error(_('Inspector is not configured to store '
'data. Set the [processing]store_data '
'configuration option to change this.'),
code=400)
process.reapply(node_uuid, data)
def do_reapply(self, context, node_uuid, token=None, data=None):
if not data:
try:
data = process.get_introspection_data(node_uuid,
processed=False,
get_json=True)
except utils.IntrospectionDataStoreDisabled:
raise utils.Error(_('Inspector is not configured to store '
'introspection data. Set the '
'[processing]store_data configuration '
'option to change this.'))
else:
process.store_introspection_data(node_uuid, data, processed=False)
process.reapply(node_uuid, data=data)
def periodic_clean_up(): # pragma: no cover

19
ironic_inspector/main.py
View File

@ -39,7 +39,7 @@ app = flask.Flask(__name__)
LOG = utils.getProcessingLogger(__name__)
MINIMUM_API_VERSION = (1, 0)
CURRENT_API_VERSION = (1, 14)
CURRENT_API_VERSION = (1, 15)
DEFAULT_API_VERSION = CURRENT_API_VERSION
_LOGGING_EXCLUDED_KEYS = ('logs',)
@ -307,14 +307,25 @@ def api_introspection_data(node_id):
@api('/v1/introspection/<node_id>/data/unprocessed',
rule="introspection:reapply", methods=['POST'])
def api_introspection_reapply(node_id):
data = None
if flask.request.content_length:
return error_response(_('User data processing is not '
'supported yet'), code=400)
try:
data = flask.request.get_json(force=True)
except Exception:
raise utils.Error(
_('Invalid data: expected a JSON object, got %s') % data)
if not isinstance(data, dict):
raise utils.Error(
_('Invalid data: expected a JSON object, got %s') %
data.__class__.__name__)
LOG.debug("Received reapply data from request", data=data)
if not uuidutils.is_uuid_like(node_id):
node = ir_utils.get_node(node_id, fields=['uuid'])
node_id = node.uuid
client = rpc.get_client()
client.call({}, 'do_reapply', node_uuid=node_id)
client.call({}, 'do_reapply', node_uuid=node_id, data=data)
return '', 202

29
ironic_inspector/process.py
View File

@ -140,7 +140,15 @@ def _filter_data_excluded_keys(data):
if k not in _STORAGE_EXCLUDED_KEYS}
def _store_data(node_uuid, data, processed=True):
def store_introspection_data(node_uuid, data, processed=True):
"""Store introspection data to the storage backend.
:param node_uuid: node UUID
:param data: Introspection data to be saved
:param processed: The type of introspection data, set to True means the
introspection data is processed, otherwise unprocessed.
:raises: utils.Error
"""
introspection_data_manager = plugins_base.introspection_data_manager()
store = CONF.processing.store_data
ext = introspection_data_manager[store].obj
@ -150,13 +158,22 @@ def _store_data(node_uuid, data, processed=True):
def _store_unprocessed_data(node_uuid, data):
# runs in background
try:
_store_data(node_uuid, data, processed=False)
store_introspection_data(node_uuid, data, processed=False)
except Exception:
LOG.exception('Encountered exception saving unprocessed '
'introspection data for node %s', node_uuid, data=data)
def get_introspection_data(uuid, processed=True, get_json=False):
"""Get introspection data from the storage backend.
:param uuid: node UUID
:param processed: Indicates the type of introspection data to be read,
set True to request processed introspection data.
:param get_json: Specify whether return the introspection data in json
format, string value is returned if False.
:raises: utils.Error
"""
introspection_data_manager = plugins_base.introspection_data_manager()
store = CONF.processing.store_data
ext = introspection_data_manager[store].obj
@ -242,7 +259,7 @@ def _process_node(node_info, node, introspection_data):
# NOTE(dtantsur): repeat the check in case something changed
ir_utils.check_provision_state(node)
_run_post_hooks(node_info, introspection_data)
_store_data(node_info.uuid, introspection_data)
store_introspection_data(node_info.uuid, introspection_data)
ironic = ir_utils.get_client()
pxe_filter.driver().sync(ironic)
@ -292,8 +309,8 @@ def reapply(node_uuid, data=None):
stored data.
:param node_uuid: node UUID
:param data: unprocessed introspection data to be reapplied
:raises: utils.Error
"""
LOG.debug('Processing re-apply introspection request for node '
@ -307,7 +324,7 @@ def reapply(node_uuid, data=None):
raise utils.Error(_('Node locked, please, try again later'),
node_info=node_info, code=409)
utils.executor().submit(_reapply, node_info, data)
utils.executor().submit(_reapply, node_info, introspection_data=data)
def _reapply(node_info, introspection_data=None):
@ -350,6 +367,6 @@ def _reapply_with_data(node_info, introspection_data):
'\n'.join(failures), node_info=node_info)
_run_post_hooks(node_info, introspection_data)
_store_data(node_info.uuid, introspection_data)
store_introspection_data(node_info.uuid, introspection_data)
node_info.invalidate_cache()
rules.apply(node_info, introspection_data)

8
ironic_inspector/test/functional.py
View File

@ -634,6 +634,14 @@ class Test(Base):
self.assertEqual(store_processing_call,
store_mock.call_args_list[-1])
# Reapply with provided data
res = self.call_reapply(self.uuid, data=copy.deepcopy(self.data))
self.assertEqual(202, res.status_code)
self.assertEqual('', res.text)
eventlet.greenthread.sleep(DEFAULT_SLEEP)
self.check_status(status, finished=True, state=istate.States.finished)
@mock.patch.object(swift, 'store_introspection_data', autospec=True)
@mock.patch.object(swift, 'get_introspection_data', autospec=True)
def test_edge_state_transitions(self, get_mock, store_mock):

24
ironic_inspector/test/unit/test_main.py
View File

@ -370,17 +370,26 @@ class TestApiReapply(BaseAPITest):
self.app.post('/v1/introspection/%s/data/unprocessed' %
self.uuid)
self.client_mock.call.assert_called_once_with({}, 'do_reapply',
node_uuid=self.uuid)
node_uuid=self.uuid,
data=None)
def test_user_data(self):
res = self.app.post('/v1/introspection/%s/data/unprocessed' %
self.uuid, data='some data')
self.assertEqual(400, res.status_code)
message = json.loads(res.data.decode())['error']['message']
self.assertEqual('User data processing is not supported yet',
message)
self.assertIn('Invalid data: expected a JSON object', message)
self.assertFalse(self.client_mock.call.called)
def test_user_data_valid(self):
data = {"foo": "bar"}
res = self.app.post('/v1/introspection/%s/data/unprocessed' %
self.uuid, data=json.dumps(data))
self.assertEqual(202, res.status_code)
self.client_mock.call.assert_called_once_with({}, 'do_reapply',
node_uuid=self.uuid,
data=data)
def test_get_introspection_data_error(self):
exc = utils.Error('The store is crashed', code=404)
self.client_mock.call.side_effect = exc
@ -392,7 +401,8 @@ class TestApiReapply(BaseAPITest):
message = json.loads(res.data.decode())['error']['message']
self.assertEqual(str(exc), message)
self.client_mock.call.assert_called_once_with({}, 'do_reapply',
node_uuid=self.uuid)
node_uuid=self.uuid,
data=None)
def test_generic_error(self):
exc = utils.Error('Oops', code=400)
@ -405,7 +415,8 @@ class TestApiReapply(BaseAPITest):
message = json.loads(res.data.decode())['error']['message']
self.assertEqual(str(exc), message)
self.client_mock.call.assert_called_once_with({}, 'do_reapply',
node_uuid=self.uuid)
node_uuid=self.uuid,
data=None)
@mock.patch.object(ir_utils, 'get_node', autospec=True)
def test_reapply_with_node_name(self, get_mock):
@ -413,7 +424,8 @@ class TestApiReapply(BaseAPITest):
self.app.post('/v1/introspection/%s/data/unprocessed' %
'fake-node')
self.client_mock.call.assert_called_once_with({}, 'do_reapply',
node_uuid=self.uuid)
node_uuid=self.uuid,
data=None)
get_mock.assert_called_once_with('fake-node', fields=['uuid'])

13
ironic_inspector/test/unit/test_manager.py
View File

@ -386,8 +386,8 @@ class TestManagerReapply(BaseManagerTest):
self.context, self.uuid)
self.assertEqual(utils.Error, exc.exc_info[0])
self.assertIn('Inspector is not configured to store data',
str(exc.exc_info[1]))
self.assertIn('Inspector is not configured to store introspection '
'data', str(exc.exc_info[1]))
self.assertEqual(400, exc.exc_info[1].http_code)
self.assertFalse(reapply_mock.called)
@ -407,3 +407,12 @@ class TestManagerReapply(BaseManagerTest):
reapply_mock.assert_called_once_with(self.uuid, data=self.data)
get_data_mock.assert_called_once_with(self.uuid, processed=False,
get_json=True)
@mock.patch.object(process, 'store_introspection_data', autospec=True)
@mock.patch.object(process, 'get_introspection_data', autospec=True)
def test_reapply_with_data(self, get_mock, store_mock, reapply_mock):
self.manager.do_reapply(self.context, self.uuid, data=self.data)
reapply_mock.assert_called_once_with(self.uuid, data=self.data)
store_mock.assert_called_once_with(self.uuid, self.data,
processed=False)
self.assertFalse(get_mock.called)

10
ironic_inspector/test/unit/test_process.py
View File

@ -584,6 +584,16 @@ class TestReapply(BaseTest):
blocking=False
)
@prepare_mocks
def test_reapply_with_data(self, pop_mock, reapply_mock):
process.reapply(self.uuid, data=self.data)
pop_mock.assert_called_once_with(self.uuid, locked=False)
pop_mock.return_value.acquire_lock.assert_called_once_with(
blocking=False
)
reapply_mock.assert_called_once_with(pop_mock.return_value,
introspection_data=self.data)
@mock.patch.object(example_plugin.ExampleProcessingHook, 'before_update')
@mock.patch.object(process.rules, 'apply', autospec=True)

7
releasenotes/notes/post-introspection-data-9cdd39a3de446e92.yaml Normal file
View File

@ -0,0 +1,7 @@
---
features:
- |
Adds support to reapply with provided unprocessed introspection data. The
introspection data is supplied in the body of POST request to
``/v1/introspection/<node_id>/data/unprocessed``. The introspection data
will also be saved to storage backend.