From 32ade7a24342781dc6887aae78d83d2119f0c8cd Mon Sep 17 00:00:00 2001 From: Steve Baker Date: Fri, 3 Jun 2016 12:58:30 +1200 Subject: [PATCH] Implement event list nested-depth The GET call to list events will support a nested_depth parameter. The response will have an additional links url with the ref 'root_stack' to indicate that this API supports nested_depth queries. This has the following consequences for old/new combinations of client/server - new heatclient, new server - nested_depth param is set, server returns nested events - new heatclient, old server - nested_depth param is set, server returns events with no root_stack, heatclient falls back to recursive event fetch - old heatclient, new server - nested_depth param is never set, recursive event fetch works as before Here are some timings for a TripleO overcloud stack with ~700 events. Current heat and python-heatclient master: time openstack stack event list --nested-depth 4 overcloud |wc -l 744 real 0m17.500s This change, with heatclient 31278ff5f77b152b5ef7a4197e15c441c72ff163: time openstack stack event list --nested-depth 4 overcloud |wc -l 608 real 0m1.725s The difference in event count (744 vs 608) is due to the stack events being filtered out for stacks with zero resources - these are a source of unnecessary noise so their removal should be considered an improvement. Closes-Bug: #1588561 Change-Id: I27e1ffb770e00a7f929c081b2a505e2007f5d584 --- heat/api/openstack/v1/events.py | 32 +++++---- heat/engine/api.py | 4 +- heat/engine/service.py | 60 +++++++++++++---- heat/rpc/api.py | 4 +- heat/rpc/client.py | 10 ++- heat/tests/api/cfn/test_api_cfn_v1.py | 10 ++- heat/tests/api/openstack_v1/test_events.py | 65 +++++++++++++------ .../engine/service/test_service_engine.py | 2 +- .../tests/engine/service/test_stack_events.py | 15 +++++ heat/tests/test_rpc_client.py | 3 +- 10 files changed, 151 insertions(+), 54 deletions(-) diff --git a/heat/api/openstack/v1/events.py b/heat/api/openstack/v1/events.py index d1d40a561e..9cfc14f3f8 100644 --- a/heat/api/openstack/v1/events.py +++ b/heat/api/openstack/v1/events.py @@ -69,9 +69,15 @@ def format_event(req, event, keys=None): else: yield (key, value) - return dict(itertools.chain.from_iterable( + ev = dict(itertools.chain.from_iterable( transform(k, v) for k, v in event.items())) + root_stack_id = event.get(rpc_api.EVENT_ROOT_STACK_ID) + if root_stack_id: + root_identifier = identifier.HeatIdentifier(**root_stack_id) + ev['links'].append(util.make_link(req, root_identifier, 'root_stack')) + return ev + class EventController(object): """WSGI controller for Events in Heat v1 API. @@ -86,14 +92,16 @@ class EventController(object): self.rpc_client = rpc_client.EngineClient() def _event_list(self, req, identity, detail=False, filters=None, - limit=None, marker=None, sort_keys=None, sort_dir=None): + limit=None, marker=None, sort_keys=None, sort_dir=None, + nested_depth=None): events = self.rpc_client.list_events(req.context, identity, filters=filters, limit=limit, marker=marker, sort_keys=sort_keys, - sort_dir=sort_dir) + sort_dir=sort_dir, + nested_depth=nested_depth) keys = None if detail else summary_keys return [format_event(req, e, keys) for e in events] @@ -106,6 +114,7 @@ class EventController(object): 'marker': util.PARAM_TYPE_SINGLE, 'sort_dir': util.PARAM_TYPE_SINGLE, 'sort_keys': util.PARAM_TYPE_MULTI, + 'nested_depth': util.PARAM_TYPE_SINGLE, } filter_whitelist = { 'resource_status': util.PARAM_TYPE_MIXED, @@ -115,14 +124,15 @@ class EventController(object): } params = util.get_allowed_params(req.params, whitelist) filter_params = util.get_allowed_params(req.params, filter_whitelist) - key = rpc_api.PARAM_LIMIT - if key in params: - try: - limit = param_utils.extract_int(key, params[key], - allow_zero=True) - except ValueError as e: - raise exc.HTTPBadRequest(six.text_type(e)) - params[key] = limit + + int_params = (rpc_api.PARAM_LIMIT, rpc_api.PARAM_NESTED_DEPTH) + try: + for key in int_params: + if key in params: + params[key] = param_utils.extract_int( + key, params[key], allow_zero=True) + except ValueError as e: + raise exc.HTTPBadRequest(six.text_type(e)) if resource_name is None: if not filter_params: diff --git a/heat/engine/api.py b/heat/engine/api.py index 125f062aed..4db710d944 100644 --- a/heat/engine/api.py +++ b/heat/engine/api.py @@ -379,7 +379,7 @@ def format_stack_preview(stack): return fmt_stack -def format_event(event, stack_identifier): +def format_event(event, stack_identifier, root_stack_identifier=None): result = { rpc_api.EVENT_ID: dict(event.identifier(stack_identifier)), rpc_api.EVENT_STACK_ID: dict(stack_identifier), @@ -393,6 +393,8 @@ def format_event(event, stack_identifier): rpc_api.EVENT_RES_TYPE: event.resource_type, rpc_api.EVENT_RES_PROPERTIES: event.resource_properties, } + if root_stack_identifier: + result[rpc_api.EVENT_ROOT_STACK_ID] = dict(root_stack_identifier) return result diff --git a/heat/engine/service.py b/heat/engine/service.py index dda984f532..04bacc9e3d 100644 --- a/heat/engine/service.py +++ b/heat/engine/service.py @@ -297,7 +297,7 @@ class EngineService(service.Service): by the RPC caller. """ - RPC_API_VERSION = '1.30' + RPC_API_VERSION = '1.31' def __init__(self, host, topic): super(EngineService, self).__init__() @@ -1575,7 +1575,8 @@ class EngineService(service.Service): @context.request_context def list_events(self, cnxt, stack_identity, filters=None, limit=None, - marker=None, sort_keys=None, sort_dir=None): + marker=None, sort_keys=None, sort_dir=None, + nested_depth=None): """Lists all events associated with a given stack. It supports pagination (``limit`` and ``marker``), @@ -1589,21 +1590,52 @@ class EngineService(service.Service): :param marker: the ID of the last event in the previous page :param sort_keys: an array of fields used to sort the list :param sort_dir: the direction of the sort ('asc' or 'desc'). + :param nested_depth: Levels of nested stacks to list events for. """ stack_identifiers = None - if stack_identity is not None: + root_stack_identifier = None + if stack_identity: st = self._get_stack(cnxt, stack_identity, show_deleted=True) - events = list(event_object.Event.get_all_by_stack( - cnxt, - st.id, - limit=limit, - marker=marker, - sort_keys=sort_keys, - sort_dir=sort_dir, - filters=filters)) - stack_identifiers = {st.id: st.identifier()} + if nested_depth: + root_stack_identifier = st.identifier() + # find all resources associated with a root stack + all_r = resource_objects.Resource.get_all_by_root_stack( + cnxt, st.id, None) + + # find stacks to the requested nested_depth + stack_ids = {r.stack_id for r in six.itervalues(all_r)} + stack_filters = { + 'id': stack_ids, + 'nested_depth': list(range(nested_depth + 1)) + } + + stacks = stack_object.Stack.get_all(cnxt, + filters=stack_filters, + show_nested=True) + stack_identifiers = {s.id: s.identifier() for s in stacks} + + if filters is None: + filters = {} + filters['stack_id'] = list(stack_identifiers.keys()) + events = list(event_object.Event.get_all_by_tenant( + cnxt, limit=limit, + marker=marker, + sort_keys=sort_keys, + sort_dir=sort_dir, + filters=filters)) + + else: + events = list(event_object.Event.get_all_by_stack( + cnxt, + st.id, + limit=limit, + marker=marker, + sort_keys=sort_keys, + sort_dir=sort_dir, + filters=filters)) + stack_identifiers = {st.id: st.identifier()} else: events = list(event_object.Event.get_all_by_tenant( cnxt, limit=limit, @@ -1618,8 +1650,8 @@ class EngineService(service.Service): show_nested=True) stack_identifiers = {s.id: s.identifier() for s in stacks} - return [api.format_event(e, stack_identifiers.get(e.stack_id)) - for e in events] + return [api.format_event(e, stack_identifiers.get(e.stack_id), + root_stack_identifier) for e in events] def _authorize_stack_user(self, cnxt, stack, resource_name): """Filter access to describe_stack_resource for in-instance users. diff --git a/heat/rpc/api.py b/heat/rpc/api.py index 3600926f64..760209dec3 100644 --- a/heat/rpc/api.py +++ b/heat/rpc/api.py @@ -89,14 +89,14 @@ EVENT_KEYS = ( EVENT_TIMESTAMP, EVENT_RES_NAME, EVENT_RES_PHYSICAL_ID, EVENT_RES_ACTION, EVENT_RES_STATUS, EVENT_RES_STATUS_DATA, EVENT_RES_TYPE, - EVENT_RES_PROPERTIES, + EVENT_RES_PROPERTIES, EVENT_ROOT_STACK_ID ) = ( 'event_identity', STACK_ID, STACK_NAME, 'event_time', RES_NAME, RES_PHYSICAL_ID, RES_ACTION, RES_STATUS, RES_STATUS_DATA, RES_TYPE, - 'resource_properties', + 'resource_properties', 'root_stack_id' ) NOTIFY_KEYS = ( diff --git a/heat/rpc/client.py b/heat/rpc/client.py index 1f73101e23..804f2cae34 100644 --- a/heat/rpc/client.py +++ b/heat/rpc/client.py @@ -51,6 +51,8 @@ class EngineClient(object): 1.28 - Add environment_show call 1.29 - Add template_id to create_stack/update_stack 1.30 - Add possibility to resource_type_* return descriptions + 1.31 - Add nested_depth to list_events, when nested_depth is specified + add root_stack_id to response """ BASE_RPC_API_VERSION = '1.0' @@ -494,7 +496,8 @@ class EngineClient(object): version='1.9') def list_events(self, ctxt, stack_identity, filters=None, limit=None, - marker=None, sort_keys=None, sort_dir=None,): + marker=None, sort_keys=None, sort_dir=None, + nested_depth=None): """Lists all events associated with a given stack. It supports pagination (``limit`` and ``marker``), @@ -508,6 +511,7 @@ class EngineClient(object): :param marker: the ID of the last event in the previous page :param sort_keys: an array of fields used to sort the list :param sort_dir: the direction of the sort ('asc' or 'desc'). + :param nested_depth: Levels of nested stacks to list events for. """ return self.call(ctxt, self.make_msg('list_events', stack_identity=stack_identity, @@ -515,7 +519,9 @@ class EngineClient(object): limit=limit, marker=marker, sort_keys=sort_keys, - sort_dir=sort_dir)) + sort_dir=sort_dir, + nested_depth=nested_depth), + version='1.31') def describe_stack_resource(self, ctxt, stack_identity, resource_name, with_attr=False): diff --git a/heat/tests/api/cfn/test_api_cfn_v1.py b/heat/tests/api/cfn/test_api_cfn_v1.py index 345921ac44..38a193b60c 100644 --- a/heat/tests/api/cfn/test_api_cfn_v1.py +++ b/heat/tests/api/cfn/test_api_cfn_v1.py @@ -1216,7 +1216,7 @@ class CfnStackControllerTest(common.HeatTestCase): u'resource_properties': {u'UserData': u'blah'}, u'resource_type': u'AWS::EC2::Instance'}] - kwargs = {'stack_identity': identity, + kwargs = {'stack_identity': identity, 'nested_depth': None, 'limit': None, 'sort_keys': None, 'marker': None, 'sort_dir': None, 'filters': None} self.m.StubOutWithMock(rpc_client.EngineClient, 'call') @@ -1224,7 +1224,9 @@ class CfnStackControllerTest(common.HeatTestCase): dummy_req.context, ('identify_stack', {'stack_name': stack_name}) ).AndReturn(identity) rpc_client.EngineClient.call( - dummy_req.context, ('list_events', kwargs) + dummy_req.context, + ('list_events', kwargs), + version='1.31' ).AndReturn(engine_resp) self.m.ReplayAll() @@ -1262,7 +1264,9 @@ class CfnStackControllerTest(common.HeatTestCase): dummy_req.context, ('identify_stack', {'stack_name': stack_name}) ).AndReturn(identity) rpc_client.EngineClient.call( - dummy_req.context, ('list_events', {'stack_identity': identity}) + dummy_req.context, + ('list_events', {'stack_identity': identity}), + version='1.31' ).AndRaise(Exception()) self.m.ReplayAll() diff --git a/heat/tests/api/openstack_v1/test_events.py b/heat/tests/api/openstack_v1/test_events.py index 6a86f1bcfe..754354d36a 100644 --- a/heat/tests/api/openstack_v1/test_events.py +++ b/heat/tests/api/openstack_v1/test_events.py @@ -50,9 +50,16 @@ class EventControllerTest(tools.ControllerTest, common.HeatTestCase): self._test_resource_index('a3455d8c-9f88-404d-a85b-5315293e67de', mock_enforce) - def _test_resource_index(self, event_id, mock_enforce): + def test_resource_index_nested_depth(self, mock_enforce): + self._test_resource_index('a3455d8c-9f88-404d-a85b-5315293e67de', + mock_enforce, nested_depth=1) + + def _test_resource_index(self, event_id, mock_enforce, nested_depth=None): self._mock_enforce_setup(mock_enforce, 'index', True) res_name = 'WikiDatabase' + params = {} + if nested_depth: + params['nested_depth'] = nested_depth stack_identity = identifier.HeatIdentifier(self.tenant, 'wordpress', '6') res_identity = identifier.ResourceIdentifier(resource_name=res_name, @@ -61,9 +68,11 @@ class EventControllerTest(tools.ControllerTest, common.HeatTestCase): **res_identity) req = self._get(stack_identity._tenant_path() + - '/resources/' + res_name + '/events') + '/resources/' + res_name + '/events', + params=params) kwargs = {'stack_identity': stack_identity, + 'nested_depth': nested_depth, 'limit': None, 'sort_keys': None, 'marker': None, 'sort_dir': None, 'filters': {'resource_name': res_name}} @@ -82,9 +91,14 @@ class EventControllerTest(tools.ControllerTest, common.HeatTestCase): u'resource_type': u'AWS::EC2::Instance', } ] + if nested_depth: + engine_resp[0]['root_stack_id'] = dict(stack_identity) + self.m.StubOutWithMock(rpc_client.EngineClient, 'call') rpc_client.EngineClient.call( - req.context, ('list_events', kwargs) + req.context, + ('list_events', kwargs), + version='1.31' ).AndReturn(engine_resp) self.m.ReplayAll() @@ -111,6 +125,10 @@ class EventControllerTest(tools.ControllerTest, common.HeatTestCase): } ] } + if nested_depth: + expected['events'][0]['links'].append( + {'href': self._url(stack_identity), 'rel': 'root_stack'} + ) self.assertEqual(expected, result) self.m.VerifyAll() @@ -155,7 +173,7 @@ class EventControllerTest(tools.ControllerTest, common.HeatTestCase): rpc_call_args, _ = mock_call.call_args engine_args = rpc_call_args[1][1] - self.assertEqual(6, len(engine_args)) + self.assertEqual(7, len(engine_args)) self.assertIn('filters', engine_args) self.assertIn('resource_name', engine_args['filters']) self.assertEqual(res_name, engine_args['filters']['resource_name']) @@ -202,7 +220,7 @@ class EventControllerTest(tools.ControllerTest, common.HeatTestCase): rpc_call_args, _ = mock_call.call_args engine_args = rpc_call_args[1][1] - self.assertEqual(6, len(engine_args)) + self.assertEqual(7, len(engine_args)) self.assertIn('filters', engine_args) self.assertIn('resource_name', engine_args['filters']) self.assertIn('resource1', engine_args['filters']['resource_name']) @@ -227,7 +245,7 @@ class EventControllerTest(tools.ControllerTest, common.HeatTestCase): req = self._get(stack_identity._tenant_path() + '/events') - kwargs = {'stack_identity': stack_identity, + kwargs = {'stack_identity': stack_identity, 'nested_depth': None, 'limit': None, 'sort_keys': None, 'marker': None, 'sort_dir': None, 'filters': {'resource_name': res_name}} @@ -249,7 +267,8 @@ class EventControllerTest(tools.ControllerTest, common.HeatTestCase): self.m.StubOutWithMock(rpc_client.EngineClient, 'call') rpc_client.EngineClient.call( req.context, - ('list_events', kwargs) + ('list_events', kwargs), + version='1.31' ).AndReturn(engine_resp) self.m.ReplayAll() @@ -287,7 +306,7 @@ class EventControllerTest(tools.ControllerTest, common.HeatTestCase): req = self._get(stack_identity._tenant_path() + '/events') - kwargs = {'stack_identity': stack_identity, + kwargs = {'stack_identity': stack_identity, 'nested_depth': None, 'limit': None, 'sort_keys': None, 'marker': None, 'sort_dir': None, 'filters': None} @@ -295,7 +314,8 @@ class EventControllerTest(tools.ControllerTest, common.HeatTestCase): self.m.StubOutWithMock(rpc_client.EngineClient, 'call') rpc_client.EngineClient.call( req.context, - ('list_events', kwargs) + ('list_events', kwargs), + version='1.31' ).AndRaise(tools.to_remote_error(error)) self.m.ReplayAll() @@ -336,7 +356,7 @@ class EventControllerTest(tools.ControllerTest, common.HeatTestCase): req = self._get(stack_identity._tenant_path() + '/resources/' + res_name + '/events') - kwargs = {'stack_identity': stack_identity, + kwargs = {'stack_identity': stack_identity, 'nested_depth': None, 'limit': None, 'sort_keys': None, 'marker': None, 'sort_dir': None, 'filters': {'resource_name': res_name}} @@ -344,7 +364,8 @@ class EventControllerTest(tools.ControllerTest, common.HeatTestCase): self.m.StubOutWithMock(rpc_client.EngineClient, 'call') rpc_client.EngineClient.call( req.context, - ('list_events', kwargs) + ('list_events', kwargs), + version='1.31' ).AndReturn(engine_resp) self.m.ReplayAll() @@ -380,7 +401,7 @@ class EventControllerTest(tools.ControllerTest, common.HeatTestCase): rpc_call_args, _ = mock_call.call_args engine_args = rpc_call_args[1][1] - self.assertEqual(6, len(engine_args)) + self.assertEqual(7, len(engine_args)) self.assertIn('limit', engine_args) self.assertEqual(10, engine_args['limit']) self.assertIn('sort_keys', engine_args) @@ -469,7 +490,7 @@ class EventControllerTest(tools.ControllerTest, common.HeatTestCase): kwargs = {'stack_identity': stack_identity, 'limit': None, 'sort_keys': None, 'marker': None, - 'sort_dir': None, + 'sort_dir': None, 'nested_depth': None, 'filters': {'resource_name': res_name, 'uuid': event_id}} engine_resp = [ @@ -491,7 +512,8 @@ class EventControllerTest(tools.ControllerTest, common.HeatTestCase): self.m.StubOutWithMock(rpc_client.EngineClient, 'call') rpc_client.EngineClient.call( req.context, - ('list_events', kwargs) + ('list_events', kwargs), + version='1.31' ).AndReturn(engine_resp) self.m.ReplayAll() @@ -536,13 +558,16 @@ class EventControllerTest(tools.ControllerTest, common.HeatTestCase): kwargs = {'stack_identity': stack_identity, 'limit': None, 'sort_keys': None, 'marker': None, - 'sort_dir': None, + 'sort_dir': None, 'nested_depth': None, 'filters': {'resource_name': res_name, 'uuid': '42'}} engine_resp = [] self.m.StubOutWithMock(rpc_client.EngineClient, 'call') rpc_client.EngineClient.call( - req.context, ('list_events', kwargs)).AndReturn(engine_resp) + req.context, + ('list_events', kwargs), + version='1.31' + ).AndReturn(engine_resp) self.m.ReplayAll() self.assertRaises(webob.exc.HTTPNotFound, @@ -565,13 +590,15 @@ class EventControllerTest(tools.ControllerTest, common.HeatTestCase): kwargs = {'stack_identity': stack_identity, 'limit': None, 'sort_keys': None, 'marker': None, - 'sort_dir': None, + 'sort_dir': None, 'nested_depth': None, 'filters': {'resource_name': res_name, 'uuid': '42'}} error = heat_exc.EntityNotFound(entity='Stack', name='a') self.m.StubOutWithMock(rpc_client.EngineClient, 'call') rpc_client.EngineClient.call( - req.context, ('list_events', kwargs) + req.context, + ('list_events', kwargs), + version='1.31' ).AndRaise(tools.to_remote_error(error)) self.m.ReplayAll() @@ -647,7 +674,7 @@ class EventControllerTest(tools.ControllerTest, common.HeatTestCase): rpc_call_args, _ = mock_call.call_args engine_args = rpc_call_args[1][1] - self.assertEqual(6, len(engine_args)) + self.assertEqual(7, len(engine_args)) self.assertIn('filters', engine_args) self.assertIn('resource_name', engine_args['filters']) self.assertIn(res_name, engine_args['filters']['resource_name']) diff --git a/heat/tests/engine/service/test_service_engine.py b/heat/tests/engine/service/test_service_engine.py index 786bf38be5..fdd7b525c0 100644 --- a/heat/tests/engine/service/test_service_engine.py +++ b/heat/tests/engine/service/test_service_engine.py @@ -40,7 +40,7 @@ class ServiceEngineTest(common.HeatTestCase): def test_make_sure_rpc_version(self): self.assertEqual( - '1.30', + '1.31', service.EngineService.RPC_API_VERSION, ('RPC version is changed, please update this test to new version ' 'and make sure additional test cases are added for RPC APIs ' diff --git a/heat/tests/engine/service/test_stack_events.py b/heat/tests/engine/service/test_stack_events.py index bfb082dda6..1b0fbe8038 100644 --- a/heat/tests/engine/service/test_stack_events.py +++ b/heat/tests/engine/service/test_stack_events.py @@ -43,6 +43,7 @@ class StackEventTest(common.HeatTestCase): self.assertEqual(4, len(events)) for ev in events: + self.assertNotIn('root_stack_id', ev) self.assertIn('event_identity', ev) self.assertIsInstance(ev['event_identity'], dict) self.assertTrue(ev['event_identity']['path'].rsplit('/', 1)[1]) @@ -87,6 +88,20 @@ class StackEventTest(common.HeatTestCase): mock_get.assert_called_once_with(self.ctx, self.stack.identifier(), show_deleted=True) + @tools.stack_context('service_event_list_test_stack') + @mock.patch.object(service.EngineService, '_get_stack') + def test_event_list_nested_depth(self, mock_get): + mock_get.return_value = stack_object.Stack.get_by_id(self.ctx, + self.stack.id) + events = self.eng.list_events(self.ctx, self.stack.identifier(), + nested_depth=1) + + self.assertEqual(4, len(events)) + for ev in events: + self.assertIn('root_stack_id', ev) + mock_get.assert_called_once_with(self.ctx, self.stack.identifier(), + show_deleted=True) + @tools.stack_context('service_event_list_deleted_resource') @mock.patch.object(instances.Instance, 'handle_delete') def test_event_list_deleted_resource(self, mock_delete): diff --git a/heat/tests/test_rpc_client.py b/heat/tests/test_rpc_client.py index 5e1baaf476..0fc0a5c173 100644 --- a/heat/tests/test_rpc_client.py +++ b/heat/tests/test_rpc_client.py @@ -239,7 +239,8 @@ class EngineRpcAPITestCase(common.HeatTestCase): 'marker': None, 'sort_keys': None, 'sort_dir': None, - 'filters': None} + 'filters': None, + 'nested_depth': None} self._test_engine_api('list_events', 'call', **kwargs) def test_describe_stack_resource(self):