openstack
/
monasca-api
Code Issues Proposed changes

Merge "Support time range to query dimension names/values"

This commit is contained in:
Zuul 2019-09-27 10:48:51 +00:00 committed by Gerrit Code Review
parent 24611dcc2a 233ea9c51b
commit b2c9e1551f
7 changed files with 128 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.zuul.yaml
View File

@ -103,6 +103,7 @@
MONASCA_PERSISTER_IMPLEMENTATION_LANG: python
MONASCA_METRICS_DB: cassandra
TEMPEST_PLUGINS: /opt/stack/monasca-tempest-plugin
tempest_test_regex: (?!.*\[.*\btimerange\b.*\])(^monasca_tempest_tests.tests.api)
- job:
name: monasca-tempest-python3-cassandra
@ -115,6 +116,7 @@
MONASCA_PERSISTER_IMPLEMENTATION_LANG: python
MONASCA_METRICS_DB: cassandra
TEMPEST_PLUGINS: /opt/stack/monasca-tempest-plugin
tempest_test_regex: (?!.*\[.*\btimerange\b.*\])(^monasca_tempest_tests.tests.api)
- job:
name: monasca-tempest-python2-java-cassandra
@ -127,6 +129,7 @@
MONASCA_PERSISTER_IMPLEMENTATION_LANG: java
MONASCA_METRICS_DB: cassandra
TEMPEST_PLUGINS: /opt/stack/monasca-tempest-plugin
tempest_test_regex: (?!.*\[.*\btimerange\b.*\])(^monasca_tempest_tests.tests.api)
- job:
name: monasca-tempest-python3-java-cassandra
@ -139,6 +142,7 @@
MONASCA_PERSISTER_IMPLEMENTATION_LANG: java
MONASCA_METRICS_DB: cassandra
TEMPEST_PLUGINS: /opt/stack/monasca-tempest-plugin
tempest_test_regex: (?!.*\[.*\btimerange\b.*\])(^monasca_tempest_tests.tests.api)
- project:

4
docs/monasca-api-spec.md
View File

@ -1229,6 +1229,8 @@ None.
* tenant_id (string, optional, restricted) - Tenant ID to from which to get dimension values. This parameter can be used to get dimension values from a tenant other than the tenant the request auth token is scoped to. Usage of this query parameter is restricted to users with the monasca admin role, as defined in the monasca api configuration file, which defaults to `monasca-admin`.
* metric_name (string(255), optional) - A metric name to filter dimension values by.
* dimension_name (string(255), required) - A dimension name to filter dimension values by.
* start_time (string, optional) - The start time in ISO 8601 combined date and time format in UTC.
* end_time (string, optional) - The end time in ISO 8601 combined date and time format in UTC.
* offset (string(255), optional) - The dimension values are returned in alphabetic order, and the offset is the dimension name after which to return in the next pagination request.
* limit (integer, optional)
@ -1291,6 +1293,8 @@ None.
#### Query Parameters
* tenant_id (string, optional, restricted) - Tenant ID from which to get dimension names. This parameter can be used to get dimension names from a tenant other than the tenant the request auth token is scoped to. Usage of this query parameter is restricted to users with the monasca admin role, as defined in the monasca api configuration file, which defaults to `monasca-admin`.
* metric_name (string(255), optional) - A metric name to filter dimension names by.
* start_time (string, optional) - The start time in ISO 8601 combined date and time format in UTC.
* end_time (string, optional) - The end time in ISO 8601 combined date and time format in UTC.
* offset (string(255), optional) - The dimension names are returned in alphabetic order, and the offset is the dimension name after which will return in the next pagination request.
* limit (integer, optional)

16
monasca_api/common/repositories/cassandra/metrics_repository.py
View File

@ -157,7 +157,13 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository):
self.epoch = datetime.utcfromtimestamp(0)
def list_dimension_values(self, tenant_id, region, metric_name,
dimension_name):
dimension_name, start_timestamp=None,
end_timestamp=None):
if start_timestamp or end_timestamp:
# NOTE(brtknr): For more details, see story
# https://storyboard.openstack.org/#!/story/2006204
LOG.info("Scoping by timestamp not implemented for cassandra.")
try:
if metric_name:
@ -185,7 +191,13 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository):
return json_dim_value_list
def list_dimension_names(self, tenant_id, region, metric_name):
def list_dimension_names(self, tenant_id, region, metric_name,
start_timestamp=None, end_timestamp=None):
if start_timestamp or end_timestamp:
# NOTE(brtknr): For more details, see story
# https://storyboard.openstack.org/#!/story/2006204
LOG.info("Scoping by timestamp not implemented for cassandra.")
try:
if metric_name:

26
monasca_api/common/repositories/influxdb/metrics_repository.py
View File

@ -159,7 +159,8 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository):
return query
def _build_show_tag_values_query(self, metric_name, dimension_name,
tenant_id, region):
tenant_id, region, start_timestamp,
end_timestamp):
from_with_clause = ''
if metric_name:
from_with_clause += ' from "{}"'.format(metric_name)
@ -167,18 +168,21 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository):
if dimension_name:
from_with_clause += ' with key = "{}"'.format(dimension_name)
where_clause = self._build_where_clause(None, None, tenant_id, region)
where_clause = self._build_where_clause(None, None, tenant_id, region,
start_timestamp, end_timestamp)
query = 'show tag values' + from_with_clause + where_clause
return query
def _build_show_tag_keys_query(self, metric_name, tenant_id, region):
def _build_show_tag_keys_query(self, metric_name, tenant_id, region,
start_timestamp, end_timestamp):
from_with_clause = ''
if metric_name:
from_with_clause += ' from "{}"'.format(metric_name)
where_clause = self._build_where_clause(None, None, tenant_id, region)
where_clause = self._build_where_clause(None, None, tenant_id, region,
start_timestamp, end_timestamp)
query = 'show tag keys' + from_with_clause + where_clause
@ -919,11 +923,14 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository):
return int((dt - datetime(1970, 1, 1)).total_seconds() * 1000)
def list_dimension_values(self, tenant_id, region, metric_name,
dimension_name):
dimension_name, start_timestamp=None,
end_timestamp=None):
try:
query = self._build_show_tag_values_query(metric_name,
dimension_name,
tenant_id, region)
tenant_id, region,
start_timestamp,
end_timestamp)
result = self.influxdb_client.query(query)
json_dim_name_list = self._build_serie_dimension_values(
result, dimension_name)
@ -932,10 +939,13 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository):
LOG.exception(ex)
raise exceptions.RepositoryException(ex)
def list_dimension_names(self, tenant_id, region, metric_name):
def list_dimension_names(self, tenant_id, region, metric_name,
start_timestamp=None, end_timestamp=None):
try:
query = self._build_show_tag_keys_query(metric_name,
tenant_id, region)
tenant_id, region,
start_timestamp,
end_timestamp)
result = self.influxdb_client.query(query)
json_dim_name_list = self._build_serie_dimension_names(result)
return json_dim_name_list

88
monasca_api/tests/test_repositories.py
View File

@ -192,53 +192,76 @@ class TestRepoMetricsInfluxDB(base.BaseTestCase):
@patch("monasca_api.common.repositories.influxdb."
"metrics_repository.client.InfluxDBClient")
def test_list_dimension_values(self, influxdb_client_mock):
def test_list_dimension_values(self, influxdb_client_mock, timestamp=True):
mock_client = influxdb_client_mock.return_value
tenant = u'38dc2a2549f94d2e9a4fa1cc45a4970c'
region = u'useast'
metric = u'custom_metric'
column = u'hostname'
hostname = u'custom_host'
start_timestamp = 1571917171275
end_timestamp = 1572917171275
mock_client.query.return_value.raw = {
u'series': [
{
u'values': [[u'custom_host']],
u'name': u'custom_metric',
u'columns': [u'hostname']
}]
u'series': [{
u'values': [[hostname]],
u'name': metric,
u'columns': [column]
}]
}
repo = influxdb_repo.MetricsRepository()
mock_client.query.reset_mock()
if timestamp:
result = repo.list_dimension_values(tenant, region, metric, column,
start_timestamp, end_timestamp)
else:
result = repo.list_dimension_values(tenant, region, metric, column)
result = repo.list_dimension_values(
"38dc2a2549f94d2e9a4fa1cc45a4970c",
"useast",
"custom_metric",
"hostname")
self.assertEqual(result, [{u'dimension_value': hostname}])
self.assertEqual(result, [{u'dimension_value': u'custom_host'}])
expected_query = ('show tag values from "{metric}"'
' with key = "{column}"'
' where _tenant_id = \'{tenant}\''
' and _region = \'{region}\' '
.format(tenant=tenant, region=region,
metric=metric, column=column))
expected_query += (' and time >= {start_timestamp}000000u'
' and time < {end_timestamp}000000u'
.format(start_timestamp=start_timestamp,
end_timestamp=end_timestamp)
if timestamp else '')
mock_client.query.assert_called_once_with(expected_query)
mock_client.query.assert_called_once_with(
'show tag values from "custom_metric" with key = "hostname"'
' where _tenant_id = \'{tenant}\''
' and _region = \'{region}\' '.format(tenant='38dc2a2549f94d2e9a4fa1cc45a4970c',
region='useast'))
def test_list_dimension_values_with_timestamp(self):
self.test_list_dimension_values(timestamp=True)
@patch("monasca_api.common.repositories.influxdb."
"metrics_repository.client.InfluxDBClient")
def test_list_dimension_names(self, influxdb_client_mock):
def test_list_dimension_names(self, influxdb_client_mock, timestamp=False):
mock_client = influxdb_client_mock.return_value
tenant = u'38dc2a2549f94d2e9a4fa1cc45a4970c'
region = u'useast'
metric = u'custom_metric'
start_timestamp = 1571917171275
end_timestamp = 1572917171275
mock_client.query.return_value.raw = {
u'series': [{
u'values': [[u'_region'], [u'_tenant_id'], [u'hostname'],
[u'service']],
u'name': u'custom_metric',
u'name': metric,
u'columns': [u'tagKey']
}]
}
repo = influxdb_repo.MetricsRepository()
result = repo.list_dimension_names(
"38dc2a2549f94d2e9a4fa1cc45a4970c",
"useast",
"custom_metric")
mock_client.query.reset_mock()
if timestamp:
result = repo.list_dimension_names(tenant, region, metric,
start_timestamp, end_timestamp)
else:
result = repo.list_dimension_names(tenant, region, metric)
self.assertEqual(result,
[
@ -246,6 +269,21 @@ class TestRepoMetricsInfluxDB(base.BaseTestCase):
{u'dimension_name': u'service'}
])
expected_query = ('show tag keys from "{metric}"'
' where _tenant_id = \'{tenant}\''
' and _region = \'{region}\' '
.format(tenant=tenant, region=region, metric=metric))
expected_query += (' and time >= {start_timestamp}000000u'
' and time < {end_timestamp}000000u'
.format(start_timestamp=start_timestamp,
end_timestamp=end_timestamp)
if timestamp else '')
mock_client.query.assert_called_once_with(expected_query)
def test_list_dimension_names_with_timestamp(self):
self.test_list_dimension_names(timestamp=True)
@patch("monasca_api.common.repositories.influxdb."
"metrics_repository.requests.head")
def test_check_status(self, head_mock):

24
monasca_api/v2/reference/metrics.py
View File

@ -289,18 +289,24 @@ class DimensionValues(metrics_api_v2.DimensionValuesV2API):
dimension_name = helpers.get_query_param(req, 'dimension_name',
required=True)
offset = helpers.get_query_param(req, 'offset')
start_timestamp = helpers.get_query_starttime_timestamp(req, False)
end_timestamp = helpers.get_query_endtime_timestamp(req, False)
result = self._dimension_values(tenant_id, req.uri, metric_name,
dimension_name, offset, req.limit)
dimension_name, offset, req.limit,
start_timestamp, end_timestamp)
res.body = helpers.to_json(result)
res.status = falcon.HTTP_200
def _dimension_values(self, tenant_id, req_uri, metric_name,
dimension_name, offset, limit):
dimension_name, offset, limit, start_timestamp,
end_timestamp):
result = self._metrics_repo.list_dimension_values(tenant_id,
self._region,
metric_name,
dimension_name)
dimension_name,
start_timestamp,
end_timestamp)
return helpers.paginate_with_no_id(result, req_uri, offset, limit)
@ -324,15 +330,21 @@ class DimensionNames(metrics_api_v2.DimensionNamesV2API):
tenant_id = helpers.get_x_tenant_or_tenant_id(req, ['api:delegate'])
metric_name = helpers.get_query_param(req, 'metric_name')
offset = helpers.get_query_param(req, 'offset')
start_timestamp = helpers.get_query_starttime_timestamp(req, False)
end_timestamp = helpers.get_query_endtime_timestamp(req, False)
result = self._dimension_names(tenant_id, req.uri, metric_name,
offset, req.limit)
offset, req.limit,
start_timestamp, end_timestamp)
res.body = helpers.to_json(result)
res.status = falcon.HTTP_200
def _dimension_names(self, tenant_id, req_uri, metric_name, offset, limit):
def _dimension_names(self, tenant_id, req_uri, metric_name, offset, limit,
start_timestamp, end_timestamp):
result = self._metrics_repo.list_dimension_names(tenant_id,
self._region,
metric_name)
metric_name,
start_timestamp,
end_timestamp)
return helpers.paginate_with_no_id(result, req_uri, offset, limit)

7
releasenotes/notes/support-timerange-for-dimension-names-and-values-e5a2ba64700dcd0b.yaml Normal file
View File

@ -0,0 +1,7 @@
---
features:
- |
Dimensions names and values can be scoped by a timerange which can make
dimension related queries to large databases much faster because only the
relevant shards are searched. Users that upgrade their Monasca Grafana
Datasource plugin to version 1.3.0 will benefit from this feature.