From d1b5a3eb6b9d1eb91f7a1732ecfb2a203adfc099 Mon Sep 17 00:00:00 2001 From: Idan Hefetz Date: Tue, 25 Dec 2018 13:22:08 +0000 Subject: [PATCH] Enhance Vitrage resource APIs Add a new API for retrieving the number of resources, similar to 'vitrage alarm count'. The API includes filter and group-by options. Add filter option to 'vitrage resource list' Story: 2004669 Task: 28650 Task: 28656 Change-Id: I3b9f2be581dcbd4a539b8040a387512b55597953 --- doc/source/contributor/vitrage-api.rst | 146 +++++++++++++++++- .../resource_count-1e3184e5f1f413ab.yaml | 4 + .../resource_list_query-b3fbcbe01090f64a.yaml | 5 + vitrage/api/controllers/v1/alarm.py | 2 +- vitrage/api/controllers/v1/count.py | 38 ++++- vitrage/api/controllers/v1/resource.py | 68 +++++--- vitrage/api_handler/apis/resource.py | 43 +++++- vitrage/common/policies/resource.py | 25 +++ vitrage/graph/driver/networkx_graph.py | 9 ++ .../tests/functional/api_handler/test_apis.py | 140 +++++++++++++++++ 10 files changed, 444 insertions(+), 36 deletions(-) create mode 100644 releasenotes/notes/resource_count-1e3184e5f1f413ab.yaml create mode 100644 releasenotes/notes/resource_list_query-b3fbcbe01090f64a.yaml diff --git a/doc/source/contributor/vitrage-api.rst b/doc/source/contributor/vitrage-api.rst index 19af5ce87..531172f15 100644 --- a/doc/source/contributor/vitrage-api.rst +++ b/doc/source/contributor/vitrage-api.rst @@ -1259,8 +1259,8 @@ Resource list ^^^^^^^^^^^^^ List the resources with specified type or all the resources. -GET /v1/resources/ -~~~~~~~~~~~~~~~~~~ +POST /v1/resources/ +~~~~~~~~~~~~~~~~~~~ Headers ======= @@ -1278,20 +1278,21 @@ None. Query Parameters ================ -* resource_type - (string, optional) the type of resource, defaults to return all resources. -* all_tenants - (boolean, optional) shows the resources of all tenants (in case the user has the permissions). +None. Request Body ============ -None. +* resource_type - (string, optional) the type of resource, defaults to return all resources. +* all_tenants - (boolean, optional) shows the resources of all tenants (in case the user has the permissions). +* query - (string, optional) a json query to filter the resources by Request Examples ================ :: - GET /v1/resources/?all_tenants=False&resource_type=nova.host + POST /v1/resources/ Host: 135.248.18.122:8999 User-Agent: keystoneauth1/2.3.0 python-requests/2.9.1 CPython/2.7.6 Content-Type: application/json @@ -1305,6 +1306,37 @@ Response Status code - 200 - OK - 404 - Bad request +Query example +============= + +:: + + POST /v1/resources/ + Host: 135.248.19.18:8999 + Content-Type: application/json + X-Auth-Token: 2b8882ba2ec44295bf300aecb2caa4f7 + + { + "query" :" + { + \"or\": [ + { + \"==\": { + \"state\": \"OK\" + } + }, + { + \"==\": { + \"state\": \"SUBOPTIMAL\" + } + } + ] + + }", + "resource_type" : "nova.host" + "all_tenants" : True + } + Response Body ============= @@ -1400,6 +1432,108 @@ Response Examples "vitrage_id": "11680c27-86a2-41a7-89db-863e68b1c2c9" } +Resource count +^^^^^^^^^^^^^^ +Count resources + +POST /v1/resources/count +~~~~~~~~~~~~~~~~~~~~~~~~ + +Headers +======= + +- X-Auth-Token (string, required) - Keystone auth token +- Accept (string) - application/json +- User-Agent (String) +- Content-Type (String): application/json + +Path Parameters +=============== + +None. + +Query Parameters +================ + +None. + +Request Body +============ + +* resource_type - (string, optional) the type of resource, defaults to return all resources. +* all_tenants - (boolean, optional) shows the resources of all tenants (in case the user has the permissions). +* query - (string, optional) a json query to filter the resources by +* group_by - (string, optional) a resource data field, to group by its values + +Request Examples +================ + +:: + + POST /v1/resources/count/ + Host: 127.0.0.1:8999 + User-Agent: keystoneauth1/2.3.0 python-requests/2.9.1 CPython/2.7.6 + Accept: application/json + X-Auth-Token: 2b8882ba2ec44295bf300aecb2caa4f7 + + +Response Status code +==================== + +- 200 - OK +- 404 - Bad request + +Response Body +============= + +Returns counts of the requested resource, grouped by the selected field + +Query example +============= + +:: + + POST /v1/resources/count/ + Host: 135.248.19.18:8999 + Content-Type: application/json + X-Auth-Token: 2b8882ba2ec44295bf300aecb2caa4f7 + + { + "query" :" + { + \"or\": [ + { + \"==\": { + \"state\": \"OK\" + } + }, + { + \"==\": { + \"state\": \"SUBOPTIMAL\" + } + } + ] + + }", + "group_by" : "vitrage_operational_status", + "resource_type" : "nova.instance" + "all_tenants" : True + } + + +Response Examples +================= + +For the above request, will count all instances with status OK or SUBOPTIMAL, +group by the status field. + +:: + + { + "OK": 157, + "SUBOPTIMAL": 3, + } + Webhook List ^^^^^^^^^^^^ List all webhooks. diff --git a/releasenotes/notes/resource_count-1e3184e5f1f413ab.yaml b/releasenotes/notes/resource_count-1e3184e5f1f413ab.yaml new file mode 100644 index 000000000..5825a57b7 --- /dev/null +++ b/releasenotes/notes/resource_count-1e3184e5f1f413ab.yaml @@ -0,0 +1,4 @@ +--- +features: + - Resource count new API with support for queries and group-by. + Allows retrieving quick summaries of graph nodes. diff --git a/releasenotes/notes/resource_list_query-b3fbcbe01090f64a.yaml b/releasenotes/notes/resource_list_query-b3fbcbe01090f64a.yaml new file mode 100644 index 000000000..0f5453e3c --- /dev/null +++ b/releasenotes/notes/resource_list_query-b3fbcbe01090f64a.yaml @@ -0,0 +1,5 @@ +--- +features: + - Resource list API now supports using a query +deprecations: + - Resource list GET is deprecated, use POST instead. diff --git a/vitrage/api/controllers/v1/alarm.py b/vitrage/api/controllers/v1/alarm.py index de04851ef..5e054f21e 100755 --- a/vitrage/api/controllers/v1/alarm.py +++ b/vitrage/api/controllers/v1/alarm.py @@ -35,7 +35,7 @@ LOG = log.getLogger(__name__) @profiler.trace_cls("alarm controller", info={}, hide_args=False, trace_private=False) class AlarmsController(BaseAlarmsController): - count = count.CountsController() + count = count.AlarmCountsController() history = history.HistoryController() @pecan.expose('json') diff --git a/vitrage/api/controllers/v1/count.py b/vitrage/api/controllers/v1/count.py index 9f74dba2f..e07750d4d 100755 --- a/vitrage/api/controllers/v1/count.py +++ b/vitrage/api/controllers/v1/count.py @@ -25,7 +25,7 @@ LOG = log.getLogger(__name__) # noinspection PyBroadException -class CountsController(RootRestController): +class AlarmCountsController(RootRestController): @pecan.expose('json') def index(self, all_tenants=False): @@ -53,3 +53,39 @@ class CountsController(RootRestController): except Exception: LOG.exception('failed to get alarm count.') abort(404, 'Failed to get alarm count.') + + +class ResourceCountsController(RootRestController): + + @pecan.expose('json') + def post(self, **kwargs): + resource_type = kwargs.get('resource_type', None) + all_tenants = kwargs.get('all_tenants', False) + all_tenants = bool_from_string(all_tenants) + query = kwargs.get('query') + group_by = kwargs.get('group_by') + if query: + query = json.loads(query) + + if all_tenants: + enforce("count resources:all_tenants", pecan.request.headers, + pecan.request.enforcer, {}) + else: + enforce("count resources", pecan.request.headers, + pecan.request.enforcer, {}) + + LOG.info('received get resource counts') + + try: + resource_counts_json = pecan.request.client.call( + pecan.request.context, 'count_resources', + resource_type=resource_type, + all_tenants=all_tenants, + query=query, + group_by=group_by) + + return json.loads(resource_counts_json) + + except Exception: + LOG.exception('failed to get resource count.') + abort(404, 'Failed to get resource count.') diff --git a/vitrage/api/controllers/v1/resource.py b/vitrage/api/controllers/v1/resource.py index a6aca30b6..dbbaacf6a 100644 --- a/vitrage/api/controllers/v1/resource.py +++ b/vitrage/api/controllers/v1/resource.py @@ -13,11 +13,13 @@ import json import pecan from oslo_log import log +from oslo_log import versionutils from oslo_utils.strutils import bool_from_string from osprofiler import profiler from pecan.core import abort from vitrage.api.controllers.rest import RootRestController +from vitrage.api.controllers.v1 import count from vitrage.api.policy import enforce from vitrage.common.utils import decompress_obj @@ -28,13 +30,47 @@ LOG = log.getLogger(__name__) @profiler.trace_cls("resource controller", info={}, hide_args=False, trace_private=False) class ResourcesController(RootRestController): + count = count.ResourceCountsController() + @pecan.expose('json') + def post(self, **kwargs): + LOG.info('post list resource with args: %s', kwargs) + + resource_type = kwargs.get('resource_type', None) + all_tenants = kwargs.get('all_tenants', False) + all_tenants = bool_from_string(all_tenants) + query = kwargs.get('query') + try: + return self._get_resources(resource_type, all_tenants, query) + except Exception: + LOG.exception('Failed to list resources.') + abort(404, 'Failed to list resources.') + + @pecan.expose('json') + @versionutils.deprecated( + as_of=versionutils.deprecated.STEIN, + what='rca:list_resources GET', + in_favor_of='rca:list_resources POST', + ) def get_all(self, **kwargs): LOG.info('get list resource with args: %s', kwargs) resource_type = kwargs.get('resource_type', None) all_tenants = kwargs.get('all_tenants', False) all_tenants = bool_from_string(all_tenants) + query = kwargs.get('query') + + try: + return self._get_resources(resource_type, all_tenants, query) + except Exception: + LOG.exception('Failed to list resources.') + abort(404, 'Failed to list resources.') + + @staticmethod + def _get_resources(resource_type=None, all_tenants=False, query=None): + if query: + query = json.loads(query) + if all_tenants: enforce('list resources:all_tenants', pecan.request.headers, pecan.request.enforcer, {}) @@ -42,29 +78,17 @@ class ResourcesController(RootRestController): enforce('list resources', pecan.request.headers, pecan.request.enforcer, {}) - LOG.info('received resources list with filter %s', resource_type) + LOG.info('get resources with type: %s, all_tenants: %s, query: %s', + resource_type, all_tenants, str(query)) - try: - return self._get_resources(resource_type, all_tenants) - except Exception: - LOG.exception('Failed to list resources.') - abort(404, 'Failed to list resources.') - - @staticmethod - def _get_resources(resource_type=None, all_tenants=False): - LOG.info('get_resources with type: %s, all_tenants: %s', - resource_type, all_tenants) - try: - resources = \ - pecan.request.client.call(pecan.request.context, - 'get_resources', - resource_type=resource_type, - all_tenants=all_tenants) - resources = decompress_obj(resources)['resources'] - return resources - except Exception: - LOG.exception('Failed to get resources.') - abort(404, 'Failed to list resources.') + resources = pecan.request.client.call( + pecan.request.context, + 'get_resources', + resource_type=resource_type, + all_tenants=all_tenants, + query=query) + resources = decompress_obj(resources)['resources'] + return resources @pecan.expose('json') def get(self, vitrage_id): diff --git a/vitrage/api_handler/apis/resource.py b/vitrage/api_handler/apis/resource.py index 8c84df7b9..d922ca6e8 100644 --- a/vitrage/api_handler/apis/resource.py +++ b/vitrage/api_handler/apis/resource.py @@ -36,10 +36,41 @@ class ResourceApis(base.EntityGraphApisBase): @timed_method(log_results=True) @base.lock_graph - def get_resources(self, ctx, resource_type=None, all_tenants=False): - LOG.debug('ResourceApis get_resources - resource_type: %s,' - 'all_tenants: %s', str(resource_type), all_tenants) + def get_resources(self, ctx, resource_type=None, all_tenants=False, + query=None): + LOG.debug( + 'ResourceApis get_resources - resource_type: %s, all_tenants: %s,' + ' query: %s', + str(resource_type), + all_tenants, + str(query)) + query = self._get_query(ctx, resource_type, all_tenants, query) + resources = self.entity_graph.get_vertices(query_dict=query) + data = {'resources': [r.properties for r in resources]} + return compress_obj(data, level=1) + + @timed_method(log_results=True) + @base.lock_graph + def count_resources(self, ctx, resource_type=None, all_tenants=False, + query=None, group_by=None): + LOG.debug( + 'ResourceApis count_resources - type: %s, all_tenants: %s,' + ' query: %s, group_by: %s', + str(resource_type), + all_tenants, + str(query), + str(group_by)) + + query = self._get_query(ctx, resource_type, all_tenants, query) + if group_by is None: + group_by = VProps.VITRAGE_TYPE + counts = self.entity_graph.get_vertices_count(query_dict=query, + group_by=group_by) + + return json.dumps(counts) + + def _get_query(self, ctx, resource_type, all_tenants, query_dict): project_id = ctx.get(TenantProps.TENANT, None) is_admin_project = ctx.get(TenantProps.IS_ADMIN, False) @@ -56,9 +87,9 @@ class ResourceApis(base.EntityGraphApisBase): type_query = {'==': {VProps.VITRAGE_TYPE: resource_type}} query['and'].append(type_query) - resources = self.entity_graph.get_vertices(query_dict=query) - data = {'resources': [r.properties for r in resources]} - return compress_obj(data, level=1) + if query_dict: + query['and'].append(query_dict) + return query @base.lock_graph def show_resource(self, ctx, vitrage_id): diff --git a/vitrage/common/policies/resource.py b/vitrage/common/policies/resource.py index 9f0a037ed..a2ec07a1d 100644 --- a/vitrage/common/policies/resource.py +++ b/vitrage/common/policies/resource.py @@ -50,6 +50,31 @@ rules = [ 'method': 'GET' } ] + ), + policy.DocumentedRuleDefault( + name='count resources', + check_str=base.UNPROTECTED, + description='count the resources with the specified type, or all the ' + 'resources', + operations=[ + { + 'path': '/resources/count', + 'method': 'GET' + } + ] + ), + policy.DocumentedRuleDefault( + name='count resources:all_tenants', + check_str=base.ROLE_ADMIN, + description='Count the resources with the specified type, or all the ' + 'resources. Include resources of all tenants (if the user' + ' has the permissions)', + operations=[ + { + 'path': '/resources/count', + 'method': 'GET' + } + ] ) ] diff --git a/vitrage/graph/driver/networkx_graph.py b/vitrage/graph/driver/networkx_graph.py index 1d63de691..87fefbad1 100644 --- a/vitrage/graph/driver/networkx_graph.py +++ b/vitrage/graph/driver/networkx_graph.py @@ -12,6 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. +from collections import defaultdict import copy import json import networkx as nx @@ -262,6 +263,14 @@ class NXGraph(Graph): vertices_ids.add(node) return vertices_ids + def get_vertices_count(self, query_dict, group_by): + vertices_counts = defaultdict(int) + match_func = create_predicate(query_dict) if query_dict else None + for node, node_data in self._g.nodes(data=True): + if match_func is None or match_func(node_data): + vertices_counts[node_data.get(group_by, '')] += 1 + return vertices_counts + def get_vertices_by_key(self, key_values_hash): if key_values_hash in self.key_to_vertex_ids: diff --git a/vitrage/tests/functional/api_handler/test_apis.py b/vitrage/tests/functional/api_handler/test_apis.py index 2958365a5..82d28fa6a 100755 --- a/vitrage/tests/functional/api_handler/test_apis.py +++ b/vitrage/tests/functional/api_handler/test_apis.py @@ -31,6 +31,7 @@ from vitrage.common.utils import decompress_obj from vitrage.datasources import NOVA_HOST_DATASOURCE from vitrage.datasources import NOVA_INSTANCE_DATASOURCE from vitrage.datasources import NOVA_ZONE_DATASOURCE +from vitrage.datasources import OPENSTACK_CLUSTER from vitrage.datasources.transformer_base \ import create_cluster_placeholder_vertex from vitrage.entity_graph.mappings.operational_alarm_severity import \ @@ -268,6 +269,23 @@ class TestApis(TestEntityGraphUnitBase, TestConfiguration): # Test assertions self.assertThat(resources, matchers.HasLength(5)) + def test_resource_list_with_admin_project_and_query(self): + # Setup + graph = self._create_graph() + apis = ResourceApis(graph, None, self.api_lock) + ctx = {'tenant': 'project_2', 'is_admin': True} + + # Action + resources = apis.get_resources( + ctx, + resource_type=NOVA_INSTANCE_DATASOURCE, + all_tenants=False, + query={'==': {'id': 'instance_3'}}) + resources = decompress_obj(resources)['resources'] + + # Test assertions + self.assertThat(resources, matchers.HasLength(1)) + def test_resource_list_with_not_admin_project(self): # Setup graph = self._create_graph() @@ -332,6 +350,128 @@ class TestApis(TestEntityGraphUnitBase, TestConfiguration): # Test assertions self.assertThat(resources, matchers.HasLength(7)) + def test_resource_count_with_admin_project(self): + # Setup + graph = self._create_graph() + apis = ResourceApis(graph, None, self.api_lock) + ctx = {'tenant': 'project_2', 'is_admin': True} + + # Action + resources = apis.count_resources( + ctx, + resource_type=None, + all_tenants=False) + resources = json.loads(resources) + + # Test assertions + self.assertEqual(2, resources[NOVA_INSTANCE_DATASOURCE]) + self.assertEqual(1, resources[NOVA_ZONE_DATASOURCE]) + self.assertEqual(1, resources[OPENSTACK_CLUSTER]) + self.assertEqual(1, resources[NOVA_HOST_DATASOURCE]) + + def test_resource_count_with_admin_project_and_query(self): + # Setup + graph = self._create_graph() + apis = ResourceApis(graph, None, self.api_lock) + ctx = {'tenant': 'project_2', 'is_admin': True} + + # Action + resources = apis.count_resources( + ctx, + resource_type=NOVA_INSTANCE_DATASOURCE, + all_tenants=False, + query={'==': {'id': 'instance_3'}}) + resources = json.loads(resources) + + # Test assertions + self.assertEqual(1, resources[NOVA_INSTANCE_DATASOURCE]) + + def test_resource_count_with_not_admin_project(self): + # Setup + graph = self._create_graph() + apis = ResourceApis(graph, None, self.api_lock) + ctx = {'tenant': 'project_2', 'is_admin': False} + + # Action + resources = apis.count_resources( + ctx, + resource_type=None, + all_tenants=False) + resources = json.loads(resources) + + # Test assertions + self.assertEqual(2, resources[NOVA_INSTANCE_DATASOURCE]) + + def test_resource_count_with_not_admin_project_and_no_existing_type(self): + # Setup + graph = self._create_graph() + apis = ResourceApis(graph, None, self.api_lock) + ctx = {'tenant': 'project_2', 'is_admin': False} + + # Action + resources = apis.count_resources( + ctx, + resource_type=NOVA_HOST_DATASOURCE, + all_tenants=False) + resources = json.loads(resources) + + # Test assertions + self.assertThat(resources.items(), matchers.HasLength(0)) + + def test_resource_count_with_not_admin_project_and_existing_type(self): + # Setup + graph = self._create_graph() + apis = ResourceApis(graph, None, self.api_lock) + ctx = {'tenant': 'project_2', 'is_admin': False} + + # Action + resources = apis.count_resources( + ctx, + resource_type=NOVA_INSTANCE_DATASOURCE, + all_tenants=False) + resources = json.loads(resources) + + # Test assertions + self.assertEqual(2, resources[NOVA_INSTANCE_DATASOURCE]) + + def test_resource_count_with_all_tenants(self): + # Setup + graph = self._create_graph() + apis = ResourceApis(graph, None, self.api_lock) + ctx = {'tenant': 'project_1', 'is_admin': False} + + # Action + resources = apis.count_resources( + ctx, + resource_type=None, + all_tenants=True) + resources = json.loads(resources) + + # Test assertions + self.assertEqual(4, resources[NOVA_INSTANCE_DATASOURCE]) + self.assertEqual(1, resources[NOVA_ZONE_DATASOURCE]) + self.assertEqual(1, resources[OPENSTACK_CLUSTER]) + self.assertEqual(1, resources[NOVA_HOST_DATASOURCE]) + + def test_resource_count_with_all_tenants_and_group_by(self): + # Setup + graph = self._create_graph() + apis = ResourceApis(graph, None, self.api_lock) + ctx = {'tenant': 'project_1', 'is_admin': False} + + # Action + resources = apis.count_resources( + ctx, + resource_type=None, + all_tenants=True, + group_by=VProps.PROJECT_ID) + resources = json.loads(resources) + + # Test assertions + self.assertEqual(2, resources['project_1']) + self.assertEqual(2, resources['project_2']) + self.assertEqual(3, resources['']) + def test_resource_show_with_admin_and_no_project_resource(self): # Setup graph = self._create_graph()