Merge "Support time range to query dimension names/values"
This commit is contained in:
commit
b2c9e1551f
|
@ -103,6 +103,7 @@
|
||||||
MONASCA_PERSISTER_IMPLEMENTATION_LANG: python
|
MONASCA_PERSISTER_IMPLEMENTATION_LANG: python
|
||||||
MONASCA_METRICS_DB: cassandra
|
MONASCA_METRICS_DB: cassandra
|
||||||
TEMPEST_PLUGINS: /opt/stack/monasca-tempest-plugin
|
TEMPEST_PLUGINS: /opt/stack/monasca-tempest-plugin
|
||||||
|
tempest_test_regex: (?!.*\[.*\btimerange\b.*\])(^monasca_tempest_tests.tests.api)
|
||||||
|
|
||||||
- job:
|
- job:
|
||||||
name: monasca-tempest-python3-cassandra
|
name: monasca-tempest-python3-cassandra
|
||||||
|
@ -115,6 +116,7 @@
|
||||||
MONASCA_PERSISTER_IMPLEMENTATION_LANG: python
|
MONASCA_PERSISTER_IMPLEMENTATION_LANG: python
|
||||||
MONASCA_METRICS_DB: cassandra
|
MONASCA_METRICS_DB: cassandra
|
||||||
TEMPEST_PLUGINS: /opt/stack/monasca-tempest-plugin
|
TEMPEST_PLUGINS: /opt/stack/monasca-tempest-plugin
|
||||||
|
tempest_test_regex: (?!.*\[.*\btimerange\b.*\])(^monasca_tempest_tests.tests.api)
|
||||||
|
|
||||||
- job:
|
- job:
|
||||||
name: monasca-tempest-python2-java-cassandra
|
name: monasca-tempest-python2-java-cassandra
|
||||||
|
@ -127,6 +129,7 @@
|
||||||
MONASCA_PERSISTER_IMPLEMENTATION_LANG: java
|
MONASCA_PERSISTER_IMPLEMENTATION_LANG: java
|
||||||
MONASCA_METRICS_DB: cassandra
|
MONASCA_METRICS_DB: cassandra
|
||||||
TEMPEST_PLUGINS: /opt/stack/monasca-tempest-plugin
|
TEMPEST_PLUGINS: /opt/stack/monasca-tempest-plugin
|
||||||
|
tempest_test_regex: (?!.*\[.*\btimerange\b.*\])(^monasca_tempest_tests.tests.api)
|
||||||
|
|
||||||
- job:
|
- job:
|
||||||
name: monasca-tempest-python3-java-cassandra
|
name: monasca-tempest-python3-java-cassandra
|
||||||
|
@ -139,6 +142,7 @@
|
||||||
MONASCA_PERSISTER_IMPLEMENTATION_LANG: java
|
MONASCA_PERSISTER_IMPLEMENTATION_LANG: java
|
||||||
MONASCA_METRICS_DB: cassandra
|
MONASCA_METRICS_DB: cassandra
|
||||||
TEMPEST_PLUGINS: /opt/stack/monasca-tempest-plugin
|
TEMPEST_PLUGINS: /opt/stack/monasca-tempest-plugin
|
||||||
|
tempest_test_regex: (?!.*\[.*\btimerange\b.*\])(^monasca_tempest_tests.tests.api)
|
||||||
|
|
||||||
|
|
||||||
- project:
|
- project:
|
||||||
|
|
|
@ -1229,6 +1229,8 @@ None.
|
||||||
* tenant_id (string, optional, restricted) - Tenant ID to from which to get dimension values. This parameter can be used to get dimension values from a tenant other than the tenant the request auth token is scoped to. Usage of this query parameter is restricted to users with the monasca admin role, as defined in the monasca api configuration file, which defaults to `monasca-admin`.
|
* tenant_id (string, optional, restricted) - Tenant ID to from which to get dimension values. This parameter can be used to get dimension values from a tenant other than the tenant the request auth token is scoped to. Usage of this query parameter is restricted to users with the monasca admin role, as defined in the monasca api configuration file, which defaults to `monasca-admin`.
|
||||||
* metric_name (string(255), optional) - A metric name to filter dimension values by.
|
* metric_name (string(255), optional) - A metric name to filter dimension values by.
|
||||||
* dimension_name (string(255), required) - A dimension name to filter dimension values by.
|
* dimension_name (string(255), required) - A dimension name to filter dimension values by.
|
||||||
|
* start_time (string, optional) - The start time in ISO 8601 combined date and time format in UTC.
|
||||||
|
* end_time (string, optional) - The end time in ISO 8601 combined date and time format in UTC.
|
||||||
* offset (string(255), optional) - The dimension values are returned in alphabetic order, and the offset is the dimension name after which to return in the next pagination request.
|
* offset (string(255), optional) - The dimension values are returned in alphabetic order, and the offset is the dimension name after which to return in the next pagination request.
|
||||||
* limit (integer, optional)
|
* limit (integer, optional)
|
||||||
|
|
||||||
|
@ -1291,6 +1293,8 @@ None.
|
||||||
#### Query Parameters
|
#### Query Parameters
|
||||||
* tenant_id (string, optional, restricted) - Tenant ID from which to get dimension names. This parameter can be used to get dimension names from a tenant other than the tenant the request auth token is scoped to. Usage of this query parameter is restricted to users with the monasca admin role, as defined in the monasca api configuration file, which defaults to `monasca-admin`.
|
* tenant_id (string, optional, restricted) - Tenant ID from which to get dimension names. This parameter can be used to get dimension names from a tenant other than the tenant the request auth token is scoped to. Usage of this query parameter is restricted to users with the monasca admin role, as defined in the monasca api configuration file, which defaults to `monasca-admin`.
|
||||||
* metric_name (string(255), optional) - A metric name to filter dimension names by.
|
* metric_name (string(255), optional) - A metric name to filter dimension names by.
|
||||||
|
* start_time (string, optional) - The start time in ISO 8601 combined date and time format in UTC.
|
||||||
|
* end_time (string, optional) - The end time in ISO 8601 combined date and time format in UTC.
|
||||||
* offset (string(255), optional) - The dimension names are returned in alphabetic order, and the offset is the dimension name after which will return in the next pagination request.
|
* offset (string(255), optional) - The dimension names are returned in alphabetic order, and the offset is the dimension name after which will return in the next pagination request.
|
||||||
* limit (integer, optional)
|
* limit (integer, optional)
|
||||||
|
|
||||||
|
|
|
@ -157,7 +157,13 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository):
|
||||||
self.epoch = datetime.utcfromtimestamp(0)
|
self.epoch = datetime.utcfromtimestamp(0)
|
||||||
|
|
||||||
def list_dimension_values(self, tenant_id, region, metric_name,
|
def list_dimension_values(self, tenant_id, region, metric_name,
|
||||||
dimension_name):
|
dimension_name, start_timestamp=None,
|
||||||
|
end_timestamp=None):
|
||||||
|
|
||||||
|
if start_timestamp or end_timestamp:
|
||||||
|
# NOTE(brtknr): For more details, see story
|
||||||
|
# https://storyboard.openstack.org/#!/story/2006204
|
||||||
|
LOG.info("Scoping by timestamp not implemented for cassandra.")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if metric_name:
|
if metric_name:
|
||||||
|
@ -185,7 +191,13 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository):
|
||||||
|
|
||||||
return json_dim_value_list
|
return json_dim_value_list
|
||||||
|
|
||||||
def list_dimension_names(self, tenant_id, region, metric_name):
|
def list_dimension_names(self, tenant_id, region, metric_name,
|
||||||
|
start_timestamp=None, end_timestamp=None):
|
||||||
|
|
||||||
|
if start_timestamp or end_timestamp:
|
||||||
|
# NOTE(brtknr): For more details, see story
|
||||||
|
# https://storyboard.openstack.org/#!/story/2006204
|
||||||
|
LOG.info("Scoping by timestamp not implemented for cassandra.")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if metric_name:
|
if metric_name:
|
||||||
|
|
|
@ -159,7 +159,8 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository):
|
||||||
return query
|
return query
|
||||||
|
|
||||||
def _build_show_tag_values_query(self, metric_name, dimension_name,
|
def _build_show_tag_values_query(self, metric_name, dimension_name,
|
||||||
tenant_id, region):
|
tenant_id, region, start_timestamp,
|
||||||
|
end_timestamp):
|
||||||
from_with_clause = ''
|
from_with_clause = ''
|
||||||
if metric_name:
|
if metric_name:
|
||||||
from_with_clause += ' from "{}"'.format(metric_name)
|
from_with_clause += ' from "{}"'.format(metric_name)
|
||||||
|
@ -167,18 +168,21 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository):
|
||||||
if dimension_name:
|
if dimension_name:
|
||||||
from_with_clause += ' with key = "{}"'.format(dimension_name)
|
from_with_clause += ' with key = "{}"'.format(dimension_name)
|
||||||
|
|
||||||
where_clause = self._build_where_clause(None, None, tenant_id, region)
|
where_clause = self._build_where_clause(None, None, tenant_id, region,
|
||||||
|
start_timestamp, end_timestamp)
|
||||||
|
|
||||||
query = 'show tag values' + from_with_clause + where_clause
|
query = 'show tag values' + from_with_clause + where_clause
|
||||||
|
|
||||||
return query
|
return query
|
||||||
|
|
||||||
def _build_show_tag_keys_query(self, metric_name, tenant_id, region):
|
def _build_show_tag_keys_query(self, metric_name, tenant_id, region,
|
||||||
|
start_timestamp, end_timestamp):
|
||||||
from_with_clause = ''
|
from_with_clause = ''
|
||||||
if metric_name:
|
if metric_name:
|
||||||
from_with_clause += ' from "{}"'.format(metric_name)
|
from_with_clause += ' from "{}"'.format(metric_name)
|
||||||
|
|
||||||
where_clause = self._build_where_clause(None, None, tenant_id, region)
|
where_clause = self._build_where_clause(None, None, tenant_id, region,
|
||||||
|
start_timestamp, end_timestamp)
|
||||||
|
|
||||||
query = 'show tag keys' + from_with_clause + where_clause
|
query = 'show tag keys' + from_with_clause + where_clause
|
||||||
|
|
||||||
|
@ -919,11 +923,14 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository):
|
||||||
return int((dt - datetime(1970, 1, 1)).total_seconds() * 1000)
|
return int((dt - datetime(1970, 1, 1)).total_seconds() * 1000)
|
||||||
|
|
||||||
def list_dimension_values(self, tenant_id, region, metric_name,
|
def list_dimension_values(self, tenant_id, region, metric_name,
|
||||||
dimension_name):
|
dimension_name, start_timestamp=None,
|
||||||
|
end_timestamp=None):
|
||||||
try:
|
try:
|
||||||
query = self._build_show_tag_values_query(metric_name,
|
query = self._build_show_tag_values_query(metric_name,
|
||||||
dimension_name,
|
dimension_name,
|
||||||
tenant_id, region)
|
tenant_id, region,
|
||||||
|
start_timestamp,
|
||||||
|
end_timestamp)
|
||||||
result = self.influxdb_client.query(query)
|
result = self.influxdb_client.query(query)
|
||||||
json_dim_name_list = self._build_serie_dimension_values(
|
json_dim_name_list = self._build_serie_dimension_values(
|
||||||
result, dimension_name)
|
result, dimension_name)
|
||||||
|
@ -932,10 +939,13 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository):
|
||||||
LOG.exception(ex)
|
LOG.exception(ex)
|
||||||
raise exceptions.RepositoryException(ex)
|
raise exceptions.RepositoryException(ex)
|
||||||
|
|
||||||
def list_dimension_names(self, tenant_id, region, metric_name):
|
def list_dimension_names(self, tenant_id, region, metric_name,
|
||||||
|
start_timestamp=None, end_timestamp=None):
|
||||||
try:
|
try:
|
||||||
query = self._build_show_tag_keys_query(metric_name,
|
query = self._build_show_tag_keys_query(metric_name,
|
||||||
tenant_id, region)
|
tenant_id, region,
|
||||||
|
start_timestamp,
|
||||||
|
end_timestamp)
|
||||||
result = self.influxdb_client.query(query)
|
result = self.influxdb_client.query(query)
|
||||||
json_dim_name_list = self._build_serie_dimension_names(result)
|
json_dim_name_list = self._build_serie_dimension_names(result)
|
||||||
return json_dim_name_list
|
return json_dim_name_list
|
||||||
|
|
|
@ -192,53 +192,76 @@ class TestRepoMetricsInfluxDB(base.BaseTestCase):
|
||||||
|
|
||||||
@patch("monasca_api.common.repositories.influxdb."
|
@patch("monasca_api.common.repositories.influxdb."
|
||||||
"metrics_repository.client.InfluxDBClient")
|
"metrics_repository.client.InfluxDBClient")
|
||||||
def test_list_dimension_values(self, influxdb_client_mock):
|
def test_list_dimension_values(self, influxdb_client_mock, timestamp=True):
|
||||||
mock_client = influxdb_client_mock.return_value
|
mock_client = influxdb_client_mock.return_value
|
||||||
|
|
||||||
|
tenant = u'38dc2a2549f94d2e9a4fa1cc45a4970c'
|
||||||
|
region = u'useast'
|
||||||
|
metric = u'custom_metric'
|
||||||
|
column = u'hostname'
|
||||||
|
hostname = u'custom_host'
|
||||||
|
start_timestamp = 1571917171275
|
||||||
|
end_timestamp = 1572917171275
|
||||||
mock_client.query.return_value.raw = {
|
mock_client.query.return_value.raw = {
|
||||||
u'series': [
|
u'series': [{
|
||||||
{
|
u'values': [[hostname]],
|
||||||
u'values': [[u'custom_host']],
|
u'name': metric,
|
||||||
u'name': u'custom_metric',
|
u'columns': [column]
|
||||||
u'columns': [u'hostname']
|
}]
|
||||||
}]
|
|
||||||
}
|
}
|
||||||
|
|
||||||
repo = influxdb_repo.MetricsRepository()
|
repo = influxdb_repo.MetricsRepository()
|
||||||
mock_client.query.reset_mock()
|
mock_client.query.reset_mock()
|
||||||
|
if timestamp:
|
||||||
|
result = repo.list_dimension_values(tenant, region, metric, column,
|
||||||
|
start_timestamp, end_timestamp)
|
||||||
|
else:
|
||||||
|
result = repo.list_dimension_values(tenant, region, metric, column)
|
||||||
|
|
||||||
result = repo.list_dimension_values(
|
self.assertEqual(result, [{u'dimension_value': hostname}])
|
||||||
"38dc2a2549f94d2e9a4fa1cc45a4970c",
|
|
||||||
"useast",
|
|
||||||
"custom_metric",
|
|
||||||
"hostname")
|
|
||||||
|
|
||||||
self.assertEqual(result, [{u'dimension_value': u'custom_host'}])
|
expected_query = ('show tag values from "{metric}"'
|
||||||
|
' with key = "{column}"'
|
||||||
|
' where _tenant_id = \'{tenant}\''
|
||||||
|
' and _region = \'{region}\' '
|
||||||
|
.format(tenant=tenant, region=region,
|
||||||
|
metric=metric, column=column))
|
||||||
|
expected_query += (' and time >= {start_timestamp}000000u'
|
||||||
|
' and time < {end_timestamp}000000u'
|
||||||
|
.format(start_timestamp=start_timestamp,
|
||||||
|
end_timestamp=end_timestamp)
|
||||||
|
if timestamp else '')
|
||||||
|
mock_client.query.assert_called_once_with(expected_query)
|
||||||
|
|
||||||
mock_client.query.assert_called_once_with(
|
def test_list_dimension_values_with_timestamp(self):
|
||||||
'show tag values from "custom_metric" with key = "hostname"'
|
self.test_list_dimension_values(timestamp=True)
|
||||||
' where _tenant_id = \'{tenant}\''
|
|
||||||
' and _region = \'{region}\' '.format(tenant='38dc2a2549f94d2e9a4fa1cc45a4970c',
|
|
||||||
region='useast'))
|
|
||||||
|
|
||||||
@patch("monasca_api.common.repositories.influxdb."
|
@patch("monasca_api.common.repositories.influxdb."
|
||||||
"metrics_repository.client.InfluxDBClient")
|
"metrics_repository.client.InfluxDBClient")
|
||||||
def test_list_dimension_names(self, influxdb_client_mock):
|
def test_list_dimension_names(self, influxdb_client_mock, timestamp=False):
|
||||||
mock_client = influxdb_client_mock.return_value
|
mock_client = influxdb_client_mock.return_value
|
||||||
|
|
||||||
|
tenant = u'38dc2a2549f94d2e9a4fa1cc45a4970c'
|
||||||
|
region = u'useast'
|
||||||
|
metric = u'custom_metric'
|
||||||
|
start_timestamp = 1571917171275
|
||||||
|
end_timestamp = 1572917171275
|
||||||
mock_client.query.return_value.raw = {
|
mock_client.query.return_value.raw = {
|
||||||
u'series': [{
|
u'series': [{
|
||||||
u'values': [[u'_region'], [u'_tenant_id'], [u'hostname'],
|
u'values': [[u'_region'], [u'_tenant_id'], [u'hostname'],
|
||||||
[u'service']],
|
[u'service']],
|
||||||
u'name': u'custom_metric',
|
u'name': metric,
|
||||||
u'columns': [u'tagKey']
|
u'columns': [u'tagKey']
|
||||||
}]
|
}]
|
||||||
}
|
}
|
||||||
|
|
||||||
repo = influxdb_repo.MetricsRepository()
|
repo = influxdb_repo.MetricsRepository()
|
||||||
|
mock_client.query.reset_mock()
|
||||||
result = repo.list_dimension_names(
|
if timestamp:
|
||||||
"38dc2a2549f94d2e9a4fa1cc45a4970c",
|
result = repo.list_dimension_names(tenant, region, metric,
|
||||||
"useast",
|
start_timestamp, end_timestamp)
|
||||||
"custom_metric")
|
else:
|
||||||
|
result = repo.list_dimension_names(tenant, region, metric)
|
||||||
|
|
||||||
self.assertEqual(result,
|
self.assertEqual(result,
|
||||||
[
|
[
|
||||||
|
@ -246,6 +269,21 @@ class TestRepoMetricsInfluxDB(base.BaseTestCase):
|
||||||
{u'dimension_name': u'service'}
|
{u'dimension_name': u'service'}
|
||||||
])
|
])
|
||||||
|
|
||||||
|
expected_query = ('show tag keys from "{metric}"'
|
||||||
|
' where _tenant_id = \'{tenant}\''
|
||||||
|
' and _region = \'{region}\' '
|
||||||
|
.format(tenant=tenant, region=region, metric=metric))
|
||||||
|
expected_query += (' and time >= {start_timestamp}000000u'
|
||||||
|
' and time < {end_timestamp}000000u'
|
||||||
|
.format(start_timestamp=start_timestamp,
|
||||||
|
end_timestamp=end_timestamp)
|
||||||
|
if timestamp else '')
|
||||||
|
|
||||||
|
mock_client.query.assert_called_once_with(expected_query)
|
||||||
|
|
||||||
|
def test_list_dimension_names_with_timestamp(self):
|
||||||
|
self.test_list_dimension_names(timestamp=True)
|
||||||
|
|
||||||
@patch("monasca_api.common.repositories.influxdb."
|
@patch("monasca_api.common.repositories.influxdb."
|
||||||
"metrics_repository.requests.head")
|
"metrics_repository.requests.head")
|
||||||
def test_check_status(self, head_mock):
|
def test_check_status(self, head_mock):
|
||||||
|
|
|
@ -289,18 +289,24 @@ class DimensionValues(metrics_api_v2.DimensionValuesV2API):
|
||||||
dimension_name = helpers.get_query_param(req, 'dimension_name',
|
dimension_name = helpers.get_query_param(req, 'dimension_name',
|
||||||
required=True)
|
required=True)
|
||||||
offset = helpers.get_query_param(req, 'offset')
|
offset = helpers.get_query_param(req, 'offset')
|
||||||
|
start_timestamp = helpers.get_query_starttime_timestamp(req, False)
|
||||||
|
end_timestamp = helpers.get_query_endtime_timestamp(req, False)
|
||||||
result = self._dimension_values(tenant_id, req.uri, metric_name,
|
result = self._dimension_values(tenant_id, req.uri, metric_name,
|
||||||
dimension_name, offset, req.limit)
|
dimension_name, offset, req.limit,
|
||||||
|
start_timestamp, end_timestamp)
|
||||||
res.body = helpers.to_json(result)
|
res.body = helpers.to_json(result)
|
||||||
res.status = falcon.HTTP_200
|
res.status = falcon.HTTP_200
|
||||||
|
|
||||||
def _dimension_values(self, tenant_id, req_uri, metric_name,
|
def _dimension_values(self, tenant_id, req_uri, metric_name,
|
||||||
dimension_name, offset, limit):
|
dimension_name, offset, limit, start_timestamp,
|
||||||
|
end_timestamp):
|
||||||
|
|
||||||
result = self._metrics_repo.list_dimension_values(tenant_id,
|
result = self._metrics_repo.list_dimension_values(tenant_id,
|
||||||
self._region,
|
self._region,
|
||||||
metric_name,
|
metric_name,
|
||||||
dimension_name)
|
dimension_name,
|
||||||
|
start_timestamp,
|
||||||
|
end_timestamp)
|
||||||
|
|
||||||
return helpers.paginate_with_no_id(result, req_uri, offset, limit)
|
return helpers.paginate_with_no_id(result, req_uri, offset, limit)
|
||||||
|
|
||||||
|
@ -324,15 +330,21 @@ class DimensionNames(metrics_api_v2.DimensionNamesV2API):
|
||||||
tenant_id = helpers.get_x_tenant_or_tenant_id(req, ['api:delegate'])
|
tenant_id = helpers.get_x_tenant_or_tenant_id(req, ['api:delegate'])
|
||||||
metric_name = helpers.get_query_param(req, 'metric_name')
|
metric_name = helpers.get_query_param(req, 'metric_name')
|
||||||
offset = helpers.get_query_param(req, 'offset')
|
offset = helpers.get_query_param(req, 'offset')
|
||||||
|
start_timestamp = helpers.get_query_starttime_timestamp(req, False)
|
||||||
|
end_timestamp = helpers.get_query_endtime_timestamp(req, False)
|
||||||
result = self._dimension_names(tenant_id, req.uri, metric_name,
|
result = self._dimension_names(tenant_id, req.uri, metric_name,
|
||||||
offset, req.limit)
|
offset, req.limit,
|
||||||
|
start_timestamp, end_timestamp)
|
||||||
res.body = helpers.to_json(result)
|
res.body = helpers.to_json(result)
|
||||||
res.status = falcon.HTTP_200
|
res.status = falcon.HTTP_200
|
||||||
|
|
||||||
def _dimension_names(self, tenant_id, req_uri, metric_name, offset, limit):
|
def _dimension_names(self, tenant_id, req_uri, metric_name, offset, limit,
|
||||||
|
start_timestamp, end_timestamp):
|
||||||
|
|
||||||
result = self._metrics_repo.list_dimension_names(tenant_id,
|
result = self._metrics_repo.list_dimension_names(tenant_id,
|
||||||
self._region,
|
self._region,
|
||||||
metric_name)
|
metric_name,
|
||||||
|
start_timestamp,
|
||||||
|
end_timestamp)
|
||||||
|
|
||||||
return helpers.paginate_with_no_id(result, req_uri, offset, limit)
|
return helpers.paginate_with_no_id(result, req_uri, offset, limit)
|
||||||
|
|
|
@ -0,0 +1,7 @@
|
||||||
|
---
|
||||||
|
features:
|
||||||
|
- |
|
||||||
|
Dimensions names and values can be scoped by a timerange which can make
|
||||||
|
dimension related queries to large databases much faster because only the
|
||||||
|
relevant shards are searched. Users that upgrade their Monasca Grafana
|
||||||
|
Datasource plugin to version 1.3.0 will benefit from this feature.
|
Loading…
Reference in New Issue