diff --git a/.zuul.yaml b/.zuul.yaml index 20c34a88e..bd03f6eca 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -101,6 +101,7 @@ MONASCA_PERSISTER_IMPLEMENTATION_LANG: python MONASCA_METRICS_DB: cassandra TEMPEST_PLUGINS: /opt/stack/monasca-tempest-plugin + tempest_test_regex: (?!.*\[.*\btimerange\b.*\])(^monasca_tempest_tests.tests.api) - job: name: monasca-tempest-python3-cassandra @@ -113,6 +114,7 @@ MONASCA_PERSISTER_IMPLEMENTATION_LANG: python MONASCA_METRICS_DB: cassandra TEMPEST_PLUGINS: /opt/stack/monasca-tempest-plugin + tempest_test_regex: (?!.*\[.*\btimerange\b.*\])(^monasca_tempest_tests.tests.api) - job: name: monasca-tempest-python2-java-cassandra @@ -123,6 +125,7 @@ MONASCA_PERSISTER_IMPLEMENTATION_LANG: java MONASCA_METRICS_DB: cassandra TEMPEST_PLUGINS: /opt/stack/monasca-tempest-plugin + tempest_test_regex: (?!.*\[.*\btimerange\b.*\])(^monasca_tempest_tests.tests.api) - job: name: monasca-tempest-python3-java-cassandra @@ -135,6 +138,7 @@ MONASCA_PERSISTER_IMPLEMENTATION_LANG: java MONASCA_METRICS_DB: cassandra TEMPEST_PLUGINS: /opt/stack/monasca-tempest-plugin + tempest_test_regex: (?!.*\[.*\btimerange\b.*\])(^monasca_tempest_tests.tests.api) - project: diff --git a/docs/monasca-api-spec.md b/docs/monasca-api-spec.md index 60887f5d6..822f56f85 100644 --- a/docs/monasca-api-spec.md +++ b/docs/monasca-api-spec.md @@ -1229,6 +1229,8 @@ None. * tenant_id (string, optional, restricted) - Tenant ID to from which to get dimension values. This parameter can be used to get dimension values from a tenant other than the tenant the request auth token is scoped to. Usage of this query parameter is restricted to users with the monasca admin role, as defined in the monasca api configuration file, which defaults to `monasca-admin`. * metric_name (string(255), optional) - A metric name to filter dimension values by. * dimension_name (string(255), required) - A dimension name to filter dimension values by. +* start_time (string, optional) - The start time in ISO 8601 combined date and time format in UTC. +* end_time (string, optional) - The end time in ISO 8601 combined date and time format in UTC. * offset (string(255), optional) - The dimension values are returned in alphabetic order, and the offset is the dimension name after which to return in the next pagination request. * limit (integer, optional) @@ -1291,6 +1293,8 @@ None. #### Query Parameters * tenant_id (string, optional, restricted) - Tenant ID from which to get dimension names. This parameter can be used to get dimension names from a tenant other than the tenant the request auth token is scoped to. Usage of this query parameter is restricted to users with the monasca admin role, as defined in the monasca api configuration file, which defaults to `monasca-admin`. * metric_name (string(255), optional) - A metric name to filter dimension names by. +* start_time (string, optional) - The start time in ISO 8601 combined date and time format in UTC. +* end_time (string, optional) - The end time in ISO 8601 combined date and time format in UTC. * offset (string(255), optional) - The dimension names are returned in alphabetic order, and the offset is the dimension name after which will return in the next pagination request. * limit (integer, optional) diff --git a/monasca_api/common/repositories/cassandra/metrics_repository.py b/monasca_api/common/repositories/cassandra/metrics_repository.py index 00b1b7053..ce1c42dcf 100644 --- a/monasca_api/common/repositories/cassandra/metrics_repository.py +++ b/monasca_api/common/repositories/cassandra/metrics_repository.py @@ -157,7 +157,13 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository): self.epoch = datetime.utcfromtimestamp(0) def list_dimension_values(self, tenant_id, region, metric_name, - dimension_name): + dimension_name, start_timestamp=None, + end_timestamp=None): + + if start_timestamp or end_timestamp: + # NOTE(brtknr): For more details, see story + # https://storyboard.openstack.org/#!/story/2006204 + LOG.info("Scoping by timestamp not implemented for cassandra.") try: if metric_name: @@ -185,7 +191,13 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository): return json_dim_value_list - def list_dimension_names(self, tenant_id, region, metric_name): + def list_dimension_names(self, tenant_id, region, metric_name, + start_timestamp=None, end_timestamp=None): + + if start_timestamp or end_timestamp: + # NOTE(brtknr): For more details, see story + # https://storyboard.openstack.org/#!/story/2006204 + LOG.info("Scoping by timestamp not implemented for cassandra.") try: if metric_name: diff --git a/monasca_api/common/repositories/influxdb/metrics_repository.py b/monasca_api/common/repositories/influxdb/metrics_repository.py index 477cae7e8..24b360dd3 100644 --- a/monasca_api/common/repositories/influxdb/metrics_repository.py +++ b/monasca_api/common/repositories/influxdb/metrics_repository.py @@ -159,7 +159,8 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository): return query def _build_show_tag_values_query(self, metric_name, dimension_name, - tenant_id, region): + tenant_id, region, start_timestamp, + end_timestamp): from_with_clause = '' if metric_name: from_with_clause += ' from "{}"'.format(metric_name) @@ -167,18 +168,21 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository): if dimension_name: from_with_clause += ' with key = "{}"'.format(dimension_name) - where_clause = self._build_where_clause(None, None, tenant_id, region) + where_clause = self._build_where_clause(None, None, tenant_id, region, + start_timestamp, end_timestamp) query = 'show tag values' + from_with_clause + where_clause return query - def _build_show_tag_keys_query(self, metric_name, tenant_id, region): + def _build_show_tag_keys_query(self, metric_name, tenant_id, region, + start_timestamp, end_timestamp): from_with_clause = '' if metric_name: from_with_clause += ' from "{}"'.format(metric_name) - where_clause = self._build_where_clause(None, None, tenant_id, region) + where_clause = self._build_where_clause(None, None, tenant_id, region, + start_timestamp, end_timestamp) query = 'show tag keys' + from_with_clause + where_clause @@ -919,11 +923,14 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository): return int((dt - datetime(1970, 1, 1)).total_seconds() * 1000) def list_dimension_values(self, tenant_id, region, metric_name, - dimension_name): + dimension_name, start_timestamp=None, + end_timestamp=None): try: query = self._build_show_tag_values_query(metric_name, dimension_name, - tenant_id, region) + tenant_id, region, + start_timestamp, + end_timestamp) result = self.influxdb_client.query(query) json_dim_name_list = self._build_serie_dimension_values( result, dimension_name) @@ -932,10 +939,13 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository): LOG.exception(ex) raise exceptions.RepositoryException(ex) - def list_dimension_names(self, tenant_id, region, metric_name): + def list_dimension_names(self, tenant_id, region, metric_name, + start_timestamp=None, end_timestamp=None): try: query = self._build_show_tag_keys_query(metric_name, - tenant_id, region) + tenant_id, region, + start_timestamp, + end_timestamp) result = self.influxdb_client.query(query) json_dim_name_list = self._build_serie_dimension_names(result) return json_dim_name_list diff --git a/monasca_api/tests/test_repositories.py b/monasca_api/tests/test_repositories.py index 46ba494b3..0a459ee99 100644 --- a/monasca_api/tests/test_repositories.py +++ b/monasca_api/tests/test_repositories.py @@ -192,53 +192,76 @@ class TestRepoMetricsInfluxDB(base.BaseTestCase): @patch("monasca_api.common.repositories.influxdb." "metrics_repository.client.InfluxDBClient") - def test_list_dimension_values(self, influxdb_client_mock): + def test_list_dimension_values(self, influxdb_client_mock, timestamp=True): mock_client = influxdb_client_mock.return_value + + tenant = u'38dc2a2549f94d2e9a4fa1cc45a4970c' + region = u'useast' + metric = u'custom_metric' + column = u'hostname' + hostname = u'custom_host' + start_timestamp = 1571917171275 + end_timestamp = 1572917171275 mock_client.query.return_value.raw = { - u'series': [ - { - u'values': [[u'custom_host']], - u'name': u'custom_metric', - u'columns': [u'hostname'] - }] + u'series': [{ + u'values': [[hostname]], + u'name': metric, + u'columns': [column] + }] } repo = influxdb_repo.MetricsRepository() mock_client.query.reset_mock() + if timestamp: + result = repo.list_dimension_values(tenant, region, metric, column, + start_timestamp, end_timestamp) + else: + result = repo.list_dimension_values(tenant, region, metric, column) - result = repo.list_dimension_values( - "38dc2a2549f94d2e9a4fa1cc45a4970c", - "useast", - "custom_metric", - "hostname") + self.assertEqual(result, [{u'dimension_value': hostname}]) - self.assertEqual(result, [{u'dimension_value': u'custom_host'}]) + expected_query = ('show tag values from "{metric}"' + ' with key = "{column}"' + ' where _tenant_id = \'{tenant}\'' + ' and _region = \'{region}\' ' + .format(tenant=tenant, region=region, + metric=metric, column=column)) + expected_query += (' and time >= {start_timestamp}000000u' + ' and time < {end_timestamp}000000u' + .format(start_timestamp=start_timestamp, + end_timestamp=end_timestamp) + if timestamp else '') + mock_client.query.assert_called_once_with(expected_query) - mock_client.query.assert_called_once_with( - 'show tag values from "custom_metric" with key = "hostname"' - ' where _tenant_id = \'{tenant}\'' - ' and _region = \'{region}\' '.format(tenant='38dc2a2549f94d2e9a4fa1cc45a4970c', - region='useast')) + def test_list_dimension_values_with_timestamp(self): + self.test_list_dimension_values(timestamp=True) @patch("monasca_api.common.repositories.influxdb." "metrics_repository.client.InfluxDBClient") - def test_list_dimension_names(self, influxdb_client_mock): + def test_list_dimension_names(self, influxdb_client_mock, timestamp=False): mock_client = influxdb_client_mock.return_value + + tenant = u'38dc2a2549f94d2e9a4fa1cc45a4970c' + region = u'useast' + metric = u'custom_metric' + start_timestamp = 1571917171275 + end_timestamp = 1572917171275 mock_client.query.return_value.raw = { u'series': [{ u'values': [[u'_region'], [u'_tenant_id'], [u'hostname'], [u'service']], - u'name': u'custom_metric', + u'name': metric, u'columns': [u'tagKey'] }] } repo = influxdb_repo.MetricsRepository() - - result = repo.list_dimension_names( - "38dc2a2549f94d2e9a4fa1cc45a4970c", - "useast", - "custom_metric") + mock_client.query.reset_mock() + if timestamp: + result = repo.list_dimension_names(tenant, region, metric, + start_timestamp, end_timestamp) + else: + result = repo.list_dimension_names(tenant, region, metric) self.assertEqual(result, [ @@ -246,6 +269,21 @@ class TestRepoMetricsInfluxDB(base.BaseTestCase): {u'dimension_name': u'service'} ]) + expected_query = ('show tag keys from "{metric}"' + ' where _tenant_id = \'{tenant}\'' + ' and _region = \'{region}\' ' + .format(tenant=tenant, region=region, metric=metric)) + expected_query += (' and time >= {start_timestamp}000000u' + ' and time < {end_timestamp}000000u' + .format(start_timestamp=start_timestamp, + end_timestamp=end_timestamp) + if timestamp else '') + + mock_client.query.assert_called_once_with(expected_query) + + def test_list_dimension_names_with_timestamp(self): + self.test_list_dimension_names(timestamp=True) + @patch("monasca_api.common.repositories.influxdb." "metrics_repository.requests.head") def test_check_status(self, head_mock): diff --git a/monasca_api/v2/reference/metrics.py b/monasca_api/v2/reference/metrics.py index c25f875b5..1d47fcdfd 100644 --- a/monasca_api/v2/reference/metrics.py +++ b/monasca_api/v2/reference/metrics.py @@ -289,18 +289,24 @@ class DimensionValues(metrics_api_v2.DimensionValuesV2API): dimension_name = helpers.get_query_param(req, 'dimension_name', required=True) offset = helpers.get_query_param(req, 'offset') + start_timestamp = helpers.get_query_starttime_timestamp(req, False) + end_timestamp = helpers.get_query_endtime_timestamp(req, False) result = self._dimension_values(tenant_id, req.uri, metric_name, - dimension_name, offset, req.limit) + dimension_name, offset, req.limit, + start_timestamp, end_timestamp) res.body = helpers.to_json(result) res.status = falcon.HTTP_200 def _dimension_values(self, tenant_id, req_uri, metric_name, - dimension_name, offset, limit): + dimension_name, offset, limit, start_timestamp, + end_timestamp): result = self._metrics_repo.list_dimension_values(tenant_id, self._region, metric_name, - dimension_name) + dimension_name, + start_timestamp, + end_timestamp) return helpers.paginate_with_no_id(result, req_uri, offset, limit) @@ -324,15 +330,21 @@ class DimensionNames(metrics_api_v2.DimensionNamesV2API): tenant_id = helpers.get_x_tenant_or_tenant_id(req, ['api:delegate']) metric_name = helpers.get_query_param(req, 'metric_name') offset = helpers.get_query_param(req, 'offset') + start_timestamp = helpers.get_query_starttime_timestamp(req, False) + end_timestamp = helpers.get_query_endtime_timestamp(req, False) result = self._dimension_names(tenant_id, req.uri, metric_name, - offset, req.limit) + offset, req.limit, + start_timestamp, end_timestamp) res.body = helpers.to_json(result) res.status = falcon.HTTP_200 - def _dimension_names(self, tenant_id, req_uri, metric_name, offset, limit): + def _dimension_names(self, tenant_id, req_uri, metric_name, offset, limit, + start_timestamp, end_timestamp): result = self._metrics_repo.list_dimension_names(tenant_id, self._region, - metric_name) + metric_name, + start_timestamp, + end_timestamp) return helpers.paginate_with_no_id(result, req_uri, offset, limit) diff --git a/releasenotes/notes/support-timerange-for-dimension-names-and-values-e5a2ba64700dcd0b.yaml b/releasenotes/notes/support-timerange-for-dimension-names-and-values-e5a2ba64700dcd0b.yaml new file mode 100644 index 000000000..65369a78f --- /dev/null +++ b/releasenotes/notes/support-timerange-for-dimension-names-and-values-e5a2ba64700dcd0b.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Dimensions names and values can be scoped by a timerange which can make + dimension related queries to large databases much faster because only the + relevant shards are searched. Users that upgrade their Monasca Grafana + Datasource plugin to version 1.3.0 will benefit from this feature.