openstack/vitrage
Code Issues Proposed changes

Enhance Vitrage resource APIs

Browse Source 
Add a new API for retrieving the number of resources,
 similar to 'vitrage alarm count'.
The API includes filter and group-by options.
Add filter option to 'vitrage resource list'

Story: 2004669
Task: 28650
Task: 28656

Change-Id: I3b9f2be581dcbd4a539b8040a387512b55597953
This commit is contained in:
Idan Hefetz 2018-12-25 13:22:08 +00:00
parent d7d4188d06
commit d1b5a3eb6b
10 changed files with 444 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

146
doc/source/contributor/vitrage-api.rst
View File

@ -1259,8 +1259,8 @@ Resource list
^^^^^^^^^^^^^
List the resources with specified type or all the resources.
GET /v1/resources/
~~~~~~~~~~~~~~~~~~
POST /v1/resources/
~~~~~~~~~~~~~~~~~~~
Headers
=======
@ -1278,20 +1278,21 @@ None.
Query Parameters
================
* resource_type - (string, optional) the type of resource, defaults to return all resources.
* all_tenants - (boolean, optional) shows the resources of all tenants (in case the user has the permissions).
None.
Request Body
============
None.
* resource_type - (string, optional) the type of resource, defaults to return all resources.
* all_tenants - (boolean, optional) shows the resources of all tenants (in case the user has the permissions).
* query - (string, optional) a json query to filter the resources by
Request Examples
================
::
GET /v1/resources/?all_tenants=False&resource_type=nova.host
POST /v1/resources/
Host: 135.248.18.122:8999
User-Agent: keystoneauth1/2.3.0 python-requests/2.9.1 CPython/2.7.6
Content-Type: application/json
@ -1305,6 +1306,37 @@ Response Status code
- 200 - OK
- 404 - Bad request
Query example
=============
::
POST /v1/resources/
Host: 135.248.19.18:8999
Content-Type: application/json
X-Auth-Token: 2b8882ba2ec44295bf300aecb2caa4f7
{
"query" :"
{
\"or\": [
{
\"==\": {
\"state\": \"OK\"
}
},
{
\"==\": {
\"state\": \"SUBOPTIMAL\"
}
}
]
}",
"resource_type" : "nova.host"
"all_tenants" : True
}
Response Body
=============
@ -1400,6 +1432,108 @@ Response Examples
"vitrage_id": "11680c27-86a2-41a7-89db-863e68b1c2c9"
}
Resource count
^^^^^^^^^^^^^^
Count resources
POST /v1/resources/count
~~~~~~~~~~~~~~~~~~~~~~~~
Headers
=======
- X-Auth-Token (string, required) - Keystone auth token
- Accept (string) - application/json
- User-Agent (String)
- Content-Type (String): application/json
Path Parameters
===============
None.
Query Parameters
================
None.
Request Body
============
* resource_type - (string, optional) the type of resource, defaults to return all resources.
* all_tenants - (boolean, optional) shows the resources of all tenants (in case the user has the permissions).
* query - (string, optional) a json query to filter the resources by
* group_by - (string, optional) a resource data field, to group by its values
Request Examples
================
::
POST /v1/resources/count/
Host: 127.0.0.1:8999
User-Agent: keystoneauth1/2.3.0 python-requests/2.9.1 CPython/2.7.6
Accept: application/json
X-Auth-Token: 2b8882ba2ec44295bf300aecb2caa4f7
Response Status code
====================
- 200 - OK
- 404 - Bad request
Response Body
=============
Returns counts of the requested resource, grouped by the selected field
Query example
=============
::
POST /v1/resources/count/
Host: 135.248.19.18:8999
Content-Type: application/json
X-Auth-Token: 2b8882ba2ec44295bf300aecb2caa4f7
{
"query" :"
{
\"or\": [
{
\"==\": {
\"state\": \"OK\"
}
},
{
\"==\": {
\"state\": \"SUBOPTIMAL\"
}
}
]
}",
"group_by" : "vitrage_operational_status",
"resource_type" : "nova.instance"
"all_tenants" : True
}
Response Examples
=================
For the above request, will count all instances with status OK or SUBOPTIMAL,
group by the status field.
::
{
"OK": 157,
"SUBOPTIMAL": 3,
}
Webhook List
^^^^^^^^^^^^
List all webhooks.

4
releasenotes/notes/resource_count-1e3184e5f1f413ab.yaml Normal file
View File

@ -0,0 +1,4 @@
---
features:
- Resource count new API with support for queries and group-by.
Allows retrieving quick summaries of graph nodes.

5
releasenotes/notes/resource_list_query-b3fbcbe01090f64a.yaml Normal file
View File

@ -0,0 +1,5 @@
---
features:
- Resource list API now supports using a query
deprecations:
- Resource list GET is deprecated, use POST instead.

2
vitrage/api/controllers/v1/alarm.py
View File

@ -35,7 +35,7 @@ LOG = log.getLogger(__name__)
@profiler.trace_cls("alarm controller",
info={}, hide_args=False, trace_private=False)
class AlarmsController(BaseAlarmsController):
count = count.CountsController()
count = count.AlarmCountsController()
history = history.HistoryController()
@pecan.expose('json')

38
vitrage/api/controllers/v1/count.py
View File

@ -25,7 +25,7 @@ LOG = log.getLogger(__name__)
# noinspection PyBroadException
class CountsController(RootRestController):
class AlarmCountsController(RootRestController):
@pecan.expose('json')
def index(self, all_tenants=False):
@ -53,3 +53,39 @@ class CountsController(RootRestController):
except Exception:
LOG.exception('failed to get alarm count.')
abort(404, 'Failed to get alarm count.')
class ResourceCountsController(RootRestController):
@pecan.expose('json')
def post(self, **kwargs):
resource_type = kwargs.get('resource_type', None)
all_tenants = kwargs.get('all_tenants', False)
all_tenants = bool_from_string(all_tenants)
query = kwargs.get('query')
group_by = kwargs.get('group_by')
if query:
query = json.loads(query)
if all_tenants:
enforce("count resources:all_tenants", pecan.request.headers,
pecan.request.enforcer, {})
else:
enforce("count resources", pecan.request.headers,
pecan.request.enforcer, {})
LOG.info('received get resource counts')
try:
resource_counts_json = pecan.request.client.call(
pecan.request.context, 'count_resources',
resource_type=resource_type,
all_tenants=all_tenants,
query=query,
group_by=group_by)
return json.loads(resource_counts_json)
except Exception:
LOG.exception('failed to get resource count.')
abort(404, 'Failed to get resource count.')

68
vitrage/api/controllers/v1/resource.py
View File

@ -13,11 +13,13 @@ import json
import pecan
from oslo_log import log
from oslo_log import versionutils
from oslo_utils.strutils import bool_from_string
from osprofiler import profiler
from pecan.core import abort
from vitrage.api.controllers.rest import RootRestController
from vitrage.api.controllers.v1 import count
from vitrage.api.policy import enforce
from vitrage.common.utils import decompress_obj
@ -28,13 +30,47 @@ LOG = log.getLogger(__name__)
@profiler.trace_cls("resource controller",
info={}, hide_args=False, trace_private=False)
class ResourcesController(RootRestController):
count = count.ResourceCountsController()
@pecan.expose('json')
def post(self, **kwargs):
LOG.info('post list resource with args: %s', kwargs)
resource_type = kwargs.get('resource_type', None)
all_tenants = kwargs.get('all_tenants', False)
all_tenants = bool_from_string(all_tenants)
query = kwargs.get('query')
try:
return self._get_resources(resource_type, all_tenants, query)
except Exception:
LOG.exception('Failed to list resources.')
abort(404, 'Failed to list resources.')
@pecan.expose('json')
@versionutils.deprecated(
as_of=versionutils.deprecated.STEIN,
what='rca:list_resources GET',
in_favor_of='rca:list_resources POST',
)
def get_all(self, **kwargs):
LOG.info('get list resource with args: %s', kwargs)
resource_type = kwargs.get('resource_type', None)
all_tenants = kwargs.get('all_tenants', False)
all_tenants = bool_from_string(all_tenants)
query = kwargs.get('query')
try:
return self._get_resources(resource_type, all_tenants, query)
except Exception:
LOG.exception('Failed to list resources.')
abort(404, 'Failed to list resources.')
@staticmethod
def _get_resources(resource_type=None, all_tenants=False, query=None):
if query:
query = json.loads(query)
if all_tenants:
enforce('list resources:all_tenants', pecan.request.headers,
pecan.request.enforcer, {})
@ -42,29 +78,17 @@ class ResourcesController(RootRestController):
enforce('list resources', pecan.request.headers,
pecan.request.enforcer, {})
LOG.info('received resources list with filter %s', resource_type)
LOG.info('get resources with type: %s, all_tenants: %s, query: %s',
resource_type, all_tenants, str(query))
try:
return self._get_resources(resource_type, all_tenants)
except Exception:
LOG.exception('Failed to list resources.')
abort(404, 'Failed to list resources.')
@staticmethod
def _get_resources(resource_type=None, all_tenants=False):
LOG.info('get_resources with type: %s, all_tenants: %s',
resource_type, all_tenants)
try:
resources = \
pecan.request.client.call(pecan.request.context,
'get_resources',
resource_type=resource_type,
all_tenants=all_tenants)
resources = decompress_obj(resources)['resources']
return resources
except Exception:
LOG.exception('Failed to get resources.')
abort(404, 'Failed to list resources.')
resources = pecan.request.client.call(
pecan.request.context,
'get_resources',
resource_type=resource_type,
all_tenants=all_tenants,
query=query)
resources = decompress_obj(resources)['resources']
return resources
@pecan.expose('json')
def get(self, vitrage_id):

43
vitrage/api_handler/apis/resource.py
View File

@ -36,10 +36,41 @@ class ResourceApis(base.EntityGraphApisBase):
@timed_method(log_results=True)
@base.lock_graph
def get_resources(self, ctx, resource_type=None, all_tenants=False):
LOG.debug('ResourceApis get_resources - resource_type: %s,'
'all_tenants: %s', str(resource_type), all_tenants)
def get_resources(self, ctx, resource_type=None, all_tenants=False,
query=None):
LOG.debug(
'ResourceApis get_resources - resource_type: %s, all_tenants: %s,'
' query: %s',
str(resource_type),
all_tenants,
str(query))
query = self._get_query(ctx, resource_type, all_tenants, query)
resources = self.entity_graph.get_vertices(query_dict=query)
data = {'resources': [r.properties for r in resources]}
return compress_obj(data, level=1)
@timed_method(log_results=True)
@base.lock_graph
def count_resources(self, ctx, resource_type=None, all_tenants=False,
query=None, group_by=None):
LOG.debug(
'ResourceApis count_resources - type: %s, all_tenants: %s,'
' query: %s, group_by: %s',
str(resource_type),
all_tenants,
str(query),
str(group_by))
query = self._get_query(ctx, resource_type, all_tenants, query)
if group_by is None:
group_by = VProps.VITRAGE_TYPE
counts = self.entity_graph.get_vertices_count(query_dict=query,
group_by=group_by)
return json.dumps(counts)
def _get_query(self, ctx, resource_type, all_tenants, query_dict):
project_id = ctx.get(TenantProps.TENANT, None)
is_admin_project = ctx.get(TenantProps.IS_ADMIN, False)
@ -56,9 +87,9 @@ class ResourceApis(base.EntityGraphApisBase):
type_query = {'==': {VProps.VITRAGE_TYPE: resource_type}}
query['and'].append(type_query)
resources = self.entity_graph.get_vertices(query_dict=query)
data = {'resources': [r.properties for r in resources]}
return compress_obj(data, level=1)
if query_dict:
query['and'].append(query_dict)
return query
@base.lock_graph
def show_resource(self, ctx, vitrage_id):

25
vitrage/common/policies/resource.py
View File

@ -50,6 +50,31 @@ rules = [
'method': 'GET'
}
]
),
policy.DocumentedRuleDefault(
name='count resources',
check_str=base.UNPROTECTED,
description='count the resources with the specified type, or all the '
'resources',
operations=[
{
'path': '/resources/count',
'method': 'GET'
}
]
),
policy.DocumentedRuleDefault(
name='count resources:all_tenants',
check_str=base.ROLE_ADMIN,
description='Count the resources with the specified type, or all the '
'resources. Include resources of all tenants (if the user'
' has the permissions)',
operations=[
{
'path': '/resources/count',
'method': 'GET'
}
]
)
]

9
vitrage/graph/driver/networkx_graph.py
View File

@ -12,6 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
from collections import defaultdict
import copy
import json
import networkx as nx
@ -262,6 +263,14 @@ class NXGraph(Graph):
vertices_ids.add(node)
return vertices_ids
def get_vertices_count(self, query_dict, group_by):
vertices_counts = defaultdict(int)
match_func = create_predicate(query_dict) if query_dict else None
for node, node_data in self._g.nodes(data=True):
if match_func is None or match_func(node_data):
vertices_counts[node_data.get(group_by, '')] += 1
return vertices_counts
def get_vertices_by_key(self, key_values_hash):
if key_values_hash in self.key_to_vertex_ids:

140
vitrage/tests/functional/api_handler/test_apis.py
View File

@ -31,6 +31,7 @@ from vitrage.common.utils import decompress_obj
from vitrage.datasources import NOVA_HOST_DATASOURCE
from vitrage.datasources import NOVA_INSTANCE_DATASOURCE
from vitrage.datasources import NOVA_ZONE_DATASOURCE
from vitrage.datasources import OPENSTACK_CLUSTER
from vitrage.datasources.transformer_base \
import create_cluster_placeholder_vertex
from vitrage.entity_graph.mappings.operational_alarm_severity import \
@ -268,6 +269,23 @@ class TestApis(TestEntityGraphUnitBase, TestConfiguration):
# Test assertions
self.assertThat(resources, matchers.HasLength(5))
def test_resource_list_with_admin_project_and_query(self):
# Setup
graph = self._create_graph()
apis = ResourceApis(graph, None, self.api_lock)
ctx = {'tenant': 'project_2', 'is_admin': True}
# Action
resources = apis.get_resources(
ctx,
resource_type=NOVA_INSTANCE_DATASOURCE,
all_tenants=False,
query={'==': {'id': 'instance_3'}})
resources = decompress_obj(resources)['resources']
# Test assertions
self.assertThat(resources, matchers.HasLength(1))
def test_resource_list_with_not_admin_project(self):
# Setup
graph = self._create_graph()
@ -332,6 +350,128 @@ class TestApis(TestEntityGraphUnitBase, TestConfiguration):
# Test assertions
self.assertThat(resources, matchers.HasLength(7))
def test_resource_count_with_admin_project(self):
# Setup
graph = self._create_graph()
apis = ResourceApis(graph, None, self.api_lock)
ctx = {'tenant': 'project_2', 'is_admin': True}
# Action
resources = apis.count_resources(
ctx,
resource_type=None,
all_tenants=False)
resources = json.loads(resources)
# Test assertions
self.assertEqual(2, resources[NOVA_INSTANCE_DATASOURCE])
self.assertEqual(1, resources[NOVA_ZONE_DATASOURCE])
self.assertEqual(1, resources[OPENSTACK_CLUSTER])
self.assertEqual(1, resources[NOVA_HOST_DATASOURCE])
def test_resource_count_with_admin_project_and_query(self):
# Setup
graph = self._create_graph()
apis = ResourceApis(graph, None, self.api_lock)
ctx = {'tenant': 'project_2', 'is_admin': True}
# Action
resources = apis.count_resources(
ctx,
resource_type=NOVA_INSTANCE_DATASOURCE,
all_tenants=False,
query={'==': {'id': 'instance_3'}})
resources = json.loads(resources)
# Test assertions
self.assertEqual(1, resources[NOVA_INSTANCE_DATASOURCE])
def test_resource_count_with_not_admin_project(self):
# Setup
graph = self._create_graph()
apis = ResourceApis(graph, None, self.api_lock)
ctx = {'tenant': 'project_2', 'is_admin': False}
# Action
resources = apis.count_resources(
ctx,
resource_type=None,
all_tenants=False)
resources = json.loads(resources)
# Test assertions
self.assertEqual(2, resources[NOVA_INSTANCE_DATASOURCE])
def test_resource_count_with_not_admin_project_and_no_existing_type(self):
# Setup
graph = self._create_graph()
apis = ResourceApis(graph, None, self.api_lock)
ctx = {'tenant': 'project_2', 'is_admin': False}
# Action
resources = apis.count_resources(
ctx,
resource_type=NOVA_HOST_DATASOURCE,
all_tenants=False)
resources = json.loads(resources)
# Test assertions
self.assertThat(resources.items(), matchers.HasLength(0))
def test_resource_count_with_not_admin_project_and_existing_type(self):
# Setup
graph = self._create_graph()
apis = ResourceApis(graph, None, self.api_lock)
ctx = {'tenant': 'project_2', 'is_admin': False}
# Action
resources = apis.count_resources(
ctx,
resource_type=NOVA_INSTANCE_DATASOURCE,
all_tenants=False)
resources = json.loads(resources)
# Test assertions
self.assertEqual(2, resources[NOVA_INSTANCE_DATASOURCE])
def test_resource_count_with_all_tenants(self):
# Setup
graph = self._create_graph()
apis = ResourceApis(graph, None, self.api_lock)
ctx = {'tenant': 'project_1', 'is_admin': False}
# Action
resources = apis.count_resources(
ctx,
resource_type=None,
all_tenants=True)
resources = json.loads(resources)
# Test assertions
self.assertEqual(4, resources[NOVA_INSTANCE_DATASOURCE])
self.assertEqual(1, resources[NOVA_ZONE_DATASOURCE])
self.assertEqual(1, resources[OPENSTACK_CLUSTER])
self.assertEqual(1, resources[NOVA_HOST_DATASOURCE])
def test_resource_count_with_all_tenants_and_group_by(self):
# Setup
graph = self._create_graph()
apis = ResourceApis(graph, None, self.api_lock)
ctx = {'tenant': 'project_1', 'is_admin': False}
# Action
resources = apis.count_resources(
ctx,
resource_type=None,
all_tenants=True,
group_by=VProps.PROJECT_ID)
resources = json.loads(resources)
# Test assertions
self.assertEqual(2, resources['project_1'])
self.assertEqual(2, resources['project_2'])
self.assertEqual(3, resources[''])
def test_resource_show_with_admin_and_no_project_resource(self):
# Setup
graph = self._create_graph()