Support reapply with supplied introspection data
This patch adds support to provide unprocessed introspection data to reapply a node. The provided introspection data will be save to current introspection storage backend. Change-Id: I969ae9c32f53f89c006a64a006388ddea9542aa5 Story: 1564863 Task: 11344
This commit is contained in:
parent
47b9468838
commit
29d8515f50
@ -70,28 +70,32 @@ The response will contain introspection data in the form of json string.
|
|||||||
:language: javascript
|
:language: javascript
|
||||||
|
|
||||||
|
|
||||||
Reapply Introspection on stored data
|
Reapply Introspection on data
|
||||||
====================================
|
=============================
|
||||||
|
|
||||||
.. rest_method:: POST /v1/introspection/{node_id}/data/unprocessed
|
.. rest_method:: POST /v1/introspection/{node_id}/data/unprocessed
|
||||||
|
|
||||||
This method triggers introspection on stored unprocessed data.
|
This method triggers introspection on either stored introspection data or raw
|
||||||
No data is allowed to be sent along with the request.
|
introspection data provided in the request. If the introspection data is
|
||||||
|
provided in the request body, it should be a valid JSON with content similar to
|
||||||
|
ramdisk callback request.
|
||||||
|
|
||||||
|
.. versionadded:: 1.15
|
||||||
|
Unprocessed introspection data can be sent via request body.
|
||||||
|
|
||||||
.. note::
|
.. note::
|
||||||
|
Reapplying introspection on stored data is only possible when a storage
|
||||||
Requires enabling introspection storage backend via ``[processing]store_data``.
|
backend is enabled via ``[processing]store_data``.
|
||||||
|
|
||||||
Normal response codes: 202
|
Normal response codes: 202
|
||||||
|
|
||||||
Error codes:
|
Error codes:
|
||||||
|
|
||||||
* 400 - bad request or store not configured
|
* 400 - bad request, store not configured or malformed data in request body
|
||||||
* 401, 403 - missing or invalid authentication
|
* 401, 403 - missing or invalid authentication
|
||||||
* 404 - node not found for Node ID
|
* 404 - node not found for Node ID
|
||||||
* 409 - inspector locked node for processing
|
* 409 - inspector locked node for processing
|
||||||
|
|
||||||
|
|
||||||
Request
|
Request
|
||||||
-------
|
-------
|
||||||
|
|
||||||
|
@ -398,4 +398,4 @@ Version History
|
|||||||
* **1.13** adds ``manage_boot`` parameter for the introspection API.
|
* **1.13** adds ``manage_boot`` parameter for the introspection API.
|
||||||
* **1.14** allows formatting to be applied to strings nested in dicts and lists
|
* **1.14** allows formatting to be applied to strings nested in dicts and lists
|
||||||
in the actions of introspection rules.
|
in the actions of introspection rules.
|
||||||
|
* **1.15** allows reapply with provided introspection data from request.
|
||||||
|
@ -33,7 +33,7 @@ def get_transport():
|
|||||||
def get_client():
|
def get_client():
|
||||||
"""Get a RPC client instance."""
|
"""Get a RPC client instance."""
|
||||||
target = messaging.Target(topic=manager.MANAGER_TOPIC, server=CONF.host,
|
target = messaging.Target(topic=manager.MANAGER_TOPIC, server=CONF.host,
|
||||||
version='1.1')
|
version='1.2')
|
||||||
transport = get_transport()
|
transport = get_transport()
|
||||||
return messaging.RPCClient(transport, target)
|
return messaging.RPCClient(transport, target)
|
||||||
|
|
||||||
@ -43,7 +43,7 @@ def get_server(endpoints):
|
|||||||
|
|
||||||
transport = get_transport()
|
transport = get_transport()
|
||||||
target = messaging.Target(topic=manager.MANAGER_TOPIC, server=CONF.host,
|
target = messaging.Target(topic=manager.MANAGER_TOPIC, server=CONF.host,
|
||||||
version='1.1')
|
version='1.2')
|
||||||
return messaging.get_rpc_server(
|
return messaging.get_rpc_server(
|
||||||
transport, target, endpoints, executor='eventlet',
|
transport, target, endpoints, executor='eventlet',
|
||||||
access_policy=dispatcher.DefaultRPCAccessPolicy)
|
access_policy=dispatcher.DefaultRPCAccessPolicy)
|
||||||
|
@ -39,7 +39,7 @@ MANAGER_TOPIC = 'ironic-inspector-conductor'
|
|||||||
|
|
||||||
class ConductorManager(object):
|
class ConductorManager(object):
|
||||||
"""ironic inspector conductor manager"""
|
"""ironic inspector conductor manager"""
|
||||||
RPC_API_VERSION = '1.1'
|
RPC_API_VERSION = '1.2'
|
||||||
|
|
||||||
target = messaging.Target(version=RPC_API_VERSION)
|
target = messaging.Target(version=RPC_API_VERSION)
|
||||||
|
|
||||||
@ -126,16 +126,21 @@ class ConductorManager(object):
|
|||||||
introspect.abort(node_id, token=token)
|
introspect.abort(node_id, token=token)
|
||||||
|
|
||||||
@messaging.expected_exceptions(utils.Error)
|
@messaging.expected_exceptions(utils.Error)
|
||||||
def do_reapply(self, context, node_uuid, token=None):
|
def do_reapply(self, context, node_uuid, token=None, data=None):
|
||||||
try:
|
if not data:
|
||||||
data = process.get_introspection_data(node_uuid, processed=False,
|
try:
|
||||||
get_json=True)
|
data = process.get_introspection_data(node_uuid,
|
||||||
except utils.IntrospectionDataStoreDisabled:
|
processed=False,
|
||||||
raise utils.Error(_('Inspector is not configured to store '
|
get_json=True)
|
||||||
'data. Set the [processing]store_data '
|
except utils.IntrospectionDataStoreDisabled:
|
||||||
'configuration option to change this.'),
|
raise utils.Error(_('Inspector is not configured to store '
|
||||||
code=400)
|
'introspection data. Set the '
|
||||||
process.reapply(node_uuid, data)
|
'[processing]store_data configuration '
|
||||||
|
'option to change this.'))
|
||||||
|
else:
|
||||||
|
process.store_introspection_data(node_uuid, data, processed=False)
|
||||||
|
|
||||||
|
process.reapply(node_uuid, data=data)
|
||||||
|
|
||||||
|
|
||||||
def periodic_clean_up(): # pragma: no cover
|
def periodic_clean_up(): # pragma: no cover
|
||||||
|
@ -39,7 +39,7 @@ app = flask.Flask(__name__)
|
|||||||
LOG = utils.getProcessingLogger(__name__)
|
LOG = utils.getProcessingLogger(__name__)
|
||||||
|
|
||||||
MINIMUM_API_VERSION = (1, 0)
|
MINIMUM_API_VERSION = (1, 0)
|
||||||
CURRENT_API_VERSION = (1, 14)
|
CURRENT_API_VERSION = (1, 15)
|
||||||
DEFAULT_API_VERSION = CURRENT_API_VERSION
|
DEFAULT_API_VERSION = CURRENT_API_VERSION
|
||||||
_LOGGING_EXCLUDED_KEYS = ('logs',)
|
_LOGGING_EXCLUDED_KEYS = ('logs',)
|
||||||
|
|
||||||
@ -307,14 +307,25 @@ def api_introspection_data(node_id):
|
|||||||
@api('/v1/introspection/<node_id>/data/unprocessed',
|
@api('/v1/introspection/<node_id>/data/unprocessed',
|
||||||
rule="introspection:reapply", methods=['POST'])
|
rule="introspection:reapply", methods=['POST'])
|
||||||
def api_introspection_reapply(node_id):
|
def api_introspection_reapply(node_id):
|
||||||
|
data = None
|
||||||
if flask.request.content_length:
|
if flask.request.content_length:
|
||||||
return error_response(_('User data processing is not '
|
try:
|
||||||
'supported yet'), code=400)
|
data = flask.request.get_json(force=True)
|
||||||
|
except Exception:
|
||||||
|
raise utils.Error(
|
||||||
|
_('Invalid data: expected a JSON object, got %s') % data)
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
raise utils.Error(
|
||||||
|
_('Invalid data: expected a JSON object, got %s') %
|
||||||
|
data.__class__.__name__)
|
||||||
|
LOG.debug("Received reapply data from request", data=data)
|
||||||
|
|
||||||
if not uuidutils.is_uuid_like(node_id):
|
if not uuidutils.is_uuid_like(node_id):
|
||||||
node = ir_utils.get_node(node_id, fields=['uuid'])
|
node = ir_utils.get_node(node_id, fields=['uuid'])
|
||||||
node_id = node.uuid
|
node_id = node.uuid
|
||||||
|
|
||||||
client = rpc.get_client()
|
client = rpc.get_client()
|
||||||
client.call({}, 'do_reapply', node_uuid=node_id)
|
client.call({}, 'do_reapply', node_uuid=node_id, data=data)
|
||||||
return '', 202
|
return '', 202
|
||||||
|
|
||||||
|
|
||||||
|
@ -140,7 +140,15 @@ def _filter_data_excluded_keys(data):
|
|||||||
if k not in _STORAGE_EXCLUDED_KEYS}
|
if k not in _STORAGE_EXCLUDED_KEYS}
|
||||||
|
|
||||||
|
|
||||||
def _store_data(node_uuid, data, processed=True):
|
def store_introspection_data(node_uuid, data, processed=True):
|
||||||
|
"""Store introspection data to the storage backend.
|
||||||
|
|
||||||
|
:param node_uuid: node UUID
|
||||||
|
:param data: Introspection data to be saved
|
||||||
|
:param processed: The type of introspection data, set to True means the
|
||||||
|
introspection data is processed, otherwise unprocessed.
|
||||||
|
:raises: utils.Error
|
||||||
|
"""
|
||||||
introspection_data_manager = plugins_base.introspection_data_manager()
|
introspection_data_manager = plugins_base.introspection_data_manager()
|
||||||
store = CONF.processing.store_data
|
store = CONF.processing.store_data
|
||||||
ext = introspection_data_manager[store].obj
|
ext = introspection_data_manager[store].obj
|
||||||
@ -150,13 +158,22 @@ def _store_data(node_uuid, data, processed=True):
|
|||||||
def _store_unprocessed_data(node_uuid, data):
|
def _store_unprocessed_data(node_uuid, data):
|
||||||
# runs in background
|
# runs in background
|
||||||
try:
|
try:
|
||||||
_store_data(node_uuid, data, processed=False)
|
store_introspection_data(node_uuid, data, processed=False)
|
||||||
except Exception:
|
except Exception:
|
||||||
LOG.exception('Encountered exception saving unprocessed '
|
LOG.exception('Encountered exception saving unprocessed '
|
||||||
'introspection data for node %s', node_uuid, data=data)
|
'introspection data for node %s', node_uuid, data=data)
|
||||||
|
|
||||||
|
|
||||||
def get_introspection_data(uuid, processed=True, get_json=False):
|
def get_introspection_data(uuid, processed=True, get_json=False):
|
||||||
|
"""Get introspection data from the storage backend.
|
||||||
|
|
||||||
|
:param uuid: node UUID
|
||||||
|
:param processed: Indicates the type of introspection data to be read,
|
||||||
|
set True to request processed introspection data.
|
||||||
|
:param get_json: Specify whether return the introspection data in json
|
||||||
|
format, string value is returned if False.
|
||||||
|
:raises: utils.Error
|
||||||
|
"""
|
||||||
introspection_data_manager = plugins_base.introspection_data_manager()
|
introspection_data_manager = plugins_base.introspection_data_manager()
|
||||||
store = CONF.processing.store_data
|
store = CONF.processing.store_data
|
||||||
ext = introspection_data_manager[store].obj
|
ext = introspection_data_manager[store].obj
|
||||||
@ -242,7 +259,7 @@ def _process_node(node_info, node, introspection_data):
|
|||||||
# NOTE(dtantsur): repeat the check in case something changed
|
# NOTE(dtantsur): repeat the check in case something changed
|
||||||
ir_utils.check_provision_state(node)
|
ir_utils.check_provision_state(node)
|
||||||
_run_post_hooks(node_info, introspection_data)
|
_run_post_hooks(node_info, introspection_data)
|
||||||
_store_data(node_info.uuid, introspection_data)
|
store_introspection_data(node_info.uuid, introspection_data)
|
||||||
|
|
||||||
ironic = ir_utils.get_client()
|
ironic = ir_utils.get_client()
|
||||||
pxe_filter.driver().sync(ironic)
|
pxe_filter.driver().sync(ironic)
|
||||||
@ -292,8 +309,8 @@ def reapply(node_uuid, data=None):
|
|||||||
stored data.
|
stored data.
|
||||||
|
|
||||||
:param node_uuid: node UUID
|
:param node_uuid: node UUID
|
||||||
|
:param data: unprocessed introspection data to be reapplied
|
||||||
:raises: utils.Error
|
:raises: utils.Error
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
LOG.debug('Processing re-apply introspection request for node '
|
LOG.debug('Processing re-apply introspection request for node '
|
||||||
@ -307,7 +324,7 @@ def reapply(node_uuid, data=None):
|
|||||||
raise utils.Error(_('Node locked, please, try again later'),
|
raise utils.Error(_('Node locked, please, try again later'),
|
||||||
node_info=node_info, code=409)
|
node_info=node_info, code=409)
|
||||||
|
|
||||||
utils.executor().submit(_reapply, node_info, data)
|
utils.executor().submit(_reapply, node_info, introspection_data=data)
|
||||||
|
|
||||||
|
|
||||||
def _reapply(node_info, introspection_data=None):
|
def _reapply(node_info, introspection_data=None):
|
||||||
@ -350,6 +367,6 @@ def _reapply_with_data(node_info, introspection_data):
|
|||||||
'\n'.join(failures), node_info=node_info)
|
'\n'.join(failures), node_info=node_info)
|
||||||
|
|
||||||
_run_post_hooks(node_info, introspection_data)
|
_run_post_hooks(node_info, introspection_data)
|
||||||
_store_data(node_info.uuid, introspection_data)
|
store_introspection_data(node_info.uuid, introspection_data)
|
||||||
node_info.invalidate_cache()
|
node_info.invalidate_cache()
|
||||||
rules.apply(node_info, introspection_data)
|
rules.apply(node_info, introspection_data)
|
||||||
|
@ -634,6 +634,14 @@ class Test(Base):
|
|||||||
self.assertEqual(store_processing_call,
|
self.assertEqual(store_processing_call,
|
||||||
store_mock.call_args_list[-1])
|
store_mock.call_args_list[-1])
|
||||||
|
|
||||||
|
# Reapply with provided data
|
||||||
|
res = self.call_reapply(self.uuid, data=copy.deepcopy(self.data))
|
||||||
|
self.assertEqual(202, res.status_code)
|
||||||
|
self.assertEqual('', res.text)
|
||||||
|
eventlet.greenthread.sleep(DEFAULT_SLEEP)
|
||||||
|
|
||||||
|
self.check_status(status, finished=True, state=istate.States.finished)
|
||||||
|
|
||||||
@mock.patch.object(swift, 'store_introspection_data', autospec=True)
|
@mock.patch.object(swift, 'store_introspection_data', autospec=True)
|
||||||
@mock.patch.object(swift, 'get_introspection_data', autospec=True)
|
@mock.patch.object(swift, 'get_introspection_data', autospec=True)
|
||||||
def test_edge_state_transitions(self, get_mock, store_mock):
|
def test_edge_state_transitions(self, get_mock, store_mock):
|
||||||
|
@ -370,17 +370,26 @@ class TestApiReapply(BaseAPITest):
|
|||||||
self.app.post('/v1/introspection/%s/data/unprocessed' %
|
self.app.post('/v1/introspection/%s/data/unprocessed' %
|
||||||
self.uuid)
|
self.uuid)
|
||||||
self.client_mock.call.assert_called_once_with({}, 'do_reapply',
|
self.client_mock.call.assert_called_once_with({}, 'do_reapply',
|
||||||
node_uuid=self.uuid)
|
node_uuid=self.uuid,
|
||||||
|
data=None)
|
||||||
|
|
||||||
def test_user_data(self):
|
def test_user_data(self):
|
||||||
res = self.app.post('/v1/introspection/%s/data/unprocessed' %
|
res = self.app.post('/v1/introspection/%s/data/unprocessed' %
|
||||||
self.uuid, data='some data')
|
self.uuid, data='some data')
|
||||||
self.assertEqual(400, res.status_code)
|
self.assertEqual(400, res.status_code)
|
||||||
message = json.loads(res.data.decode())['error']['message']
|
message = json.loads(res.data.decode())['error']['message']
|
||||||
self.assertEqual('User data processing is not supported yet',
|
self.assertIn('Invalid data: expected a JSON object', message)
|
||||||
message)
|
|
||||||
self.assertFalse(self.client_mock.call.called)
|
self.assertFalse(self.client_mock.call.called)
|
||||||
|
|
||||||
|
def test_user_data_valid(self):
|
||||||
|
data = {"foo": "bar"}
|
||||||
|
res = self.app.post('/v1/introspection/%s/data/unprocessed' %
|
||||||
|
self.uuid, data=json.dumps(data))
|
||||||
|
self.assertEqual(202, res.status_code)
|
||||||
|
self.client_mock.call.assert_called_once_with({}, 'do_reapply',
|
||||||
|
node_uuid=self.uuid,
|
||||||
|
data=data)
|
||||||
|
|
||||||
def test_get_introspection_data_error(self):
|
def test_get_introspection_data_error(self):
|
||||||
exc = utils.Error('The store is crashed', code=404)
|
exc = utils.Error('The store is crashed', code=404)
|
||||||
self.client_mock.call.side_effect = exc
|
self.client_mock.call.side_effect = exc
|
||||||
@ -392,7 +401,8 @@ class TestApiReapply(BaseAPITest):
|
|||||||
message = json.loads(res.data.decode())['error']['message']
|
message = json.loads(res.data.decode())['error']['message']
|
||||||
self.assertEqual(str(exc), message)
|
self.assertEqual(str(exc), message)
|
||||||
self.client_mock.call.assert_called_once_with({}, 'do_reapply',
|
self.client_mock.call.assert_called_once_with({}, 'do_reapply',
|
||||||
node_uuid=self.uuid)
|
node_uuid=self.uuid,
|
||||||
|
data=None)
|
||||||
|
|
||||||
def test_generic_error(self):
|
def test_generic_error(self):
|
||||||
exc = utils.Error('Oops', code=400)
|
exc = utils.Error('Oops', code=400)
|
||||||
@ -405,7 +415,8 @@ class TestApiReapply(BaseAPITest):
|
|||||||
message = json.loads(res.data.decode())['error']['message']
|
message = json.loads(res.data.decode())['error']['message']
|
||||||
self.assertEqual(str(exc), message)
|
self.assertEqual(str(exc), message)
|
||||||
self.client_mock.call.assert_called_once_with({}, 'do_reapply',
|
self.client_mock.call.assert_called_once_with({}, 'do_reapply',
|
||||||
node_uuid=self.uuid)
|
node_uuid=self.uuid,
|
||||||
|
data=None)
|
||||||
|
|
||||||
@mock.patch.object(ir_utils, 'get_node', autospec=True)
|
@mock.patch.object(ir_utils, 'get_node', autospec=True)
|
||||||
def test_reapply_with_node_name(self, get_mock):
|
def test_reapply_with_node_name(self, get_mock):
|
||||||
@ -413,7 +424,8 @@ class TestApiReapply(BaseAPITest):
|
|||||||
self.app.post('/v1/introspection/%s/data/unprocessed' %
|
self.app.post('/v1/introspection/%s/data/unprocessed' %
|
||||||
'fake-node')
|
'fake-node')
|
||||||
self.client_mock.call.assert_called_once_with({}, 'do_reapply',
|
self.client_mock.call.assert_called_once_with({}, 'do_reapply',
|
||||||
node_uuid=self.uuid)
|
node_uuid=self.uuid,
|
||||||
|
data=None)
|
||||||
get_mock.assert_called_once_with('fake-node', fields=['uuid'])
|
get_mock.assert_called_once_with('fake-node', fields=['uuid'])
|
||||||
|
|
||||||
|
|
||||||
|
@ -386,8 +386,8 @@ class TestManagerReapply(BaseManagerTest):
|
|||||||
self.context, self.uuid)
|
self.context, self.uuid)
|
||||||
|
|
||||||
self.assertEqual(utils.Error, exc.exc_info[0])
|
self.assertEqual(utils.Error, exc.exc_info[0])
|
||||||
self.assertIn('Inspector is not configured to store data',
|
self.assertIn('Inspector is not configured to store introspection '
|
||||||
str(exc.exc_info[1]))
|
'data', str(exc.exc_info[1]))
|
||||||
self.assertEqual(400, exc.exc_info[1].http_code)
|
self.assertEqual(400, exc.exc_info[1].http_code)
|
||||||
self.assertFalse(reapply_mock.called)
|
self.assertFalse(reapply_mock.called)
|
||||||
|
|
||||||
@ -407,3 +407,12 @@ class TestManagerReapply(BaseManagerTest):
|
|||||||
reapply_mock.assert_called_once_with(self.uuid, data=self.data)
|
reapply_mock.assert_called_once_with(self.uuid, data=self.data)
|
||||||
get_data_mock.assert_called_once_with(self.uuid, processed=False,
|
get_data_mock.assert_called_once_with(self.uuid, processed=False,
|
||||||
get_json=True)
|
get_json=True)
|
||||||
|
|
||||||
|
@mock.patch.object(process, 'store_introspection_data', autospec=True)
|
||||||
|
@mock.patch.object(process, 'get_introspection_data', autospec=True)
|
||||||
|
def test_reapply_with_data(self, get_mock, store_mock, reapply_mock):
|
||||||
|
self.manager.do_reapply(self.context, self.uuid, data=self.data)
|
||||||
|
reapply_mock.assert_called_once_with(self.uuid, data=self.data)
|
||||||
|
store_mock.assert_called_once_with(self.uuid, self.data,
|
||||||
|
processed=False)
|
||||||
|
self.assertFalse(get_mock.called)
|
||||||
|
@ -584,6 +584,16 @@ class TestReapply(BaseTest):
|
|||||||
blocking=False
|
blocking=False
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@prepare_mocks
|
||||||
|
def test_reapply_with_data(self, pop_mock, reapply_mock):
|
||||||
|
process.reapply(self.uuid, data=self.data)
|
||||||
|
pop_mock.assert_called_once_with(self.uuid, locked=False)
|
||||||
|
pop_mock.return_value.acquire_lock.assert_called_once_with(
|
||||||
|
blocking=False
|
||||||
|
)
|
||||||
|
reapply_mock.assert_called_once_with(pop_mock.return_value,
|
||||||
|
introspection_data=self.data)
|
||||||
|
|
||||||
|
|
||||||
@mock.patch.object(example_plugin.ExampleProcessingHook, 'before_update')
|
@mock.patch.object(example_plugin.ExampleProcessingHook, 'before_update')
|
||||||
@mock.patch.object(process.rules, 'apply', autospec=True)
|
@mock.patch.object(process.rules, 'apply', autospec=True)
|
||||||
|
@ -0,0 +1,7 @@
|
|||||||
|
---
|
||||||
|
features:
|
||||||
|
- |
|
||||||
|
Adds support to reapply with provided unprocessed introspection data. The
|
||||||
|
introspection data is supplied in the body of POST request to
|
||||||
|
``/v1/introspection/<node_id>/data/unprocessed``. The introspection data
|
||||||
|
will also be saved to storage backend.
|
Loading…
Reference in New Issue
Block a user