diff --git a/doc/source/rest.j2 b/doc/source/rest.j2 index a37743acf..7e2d309b1 100644 --- a/doc/source/rest.j2 +++ b/doc/source/rest.j2 @@ -73,6 +73,17 @@ endpoint: {{ scenarios['get-measures']['doc'] }} +Depending on the driver, there may be some lag after POSTing measures before +they are processed and queryable. To ensure your query returns all measures +that have been POSTed, you can force any unprocessed measures to be handled: + +{{ scenarios['get-measures-refresh']['doc'] }} + +.. note:: + + Depending on the amount of data that is unprocessed, `refresh` may add + some overhead to your query. + The list of points returned is composed of tuples with (timestamp, granularity, value) sorted by timestamp. The granularity is the timespan covered by aggregation for this point. @@ -474,6 +485,10 @@ requested resource type, and the compute the aggregation: {{ scenarios['get-across-metrics-measures-by-attributes-lookup-groupby']['doc'] }} +Similar to retrieving measures for a single metric, the `refresh` parameter +can be provided to force all POSTed measures to be processed across all +metrics before computing the result. + Also aggregation across metrics have different behavior depending on if boundary are set ('start' and 'stop') and if 'needed_overlap' is set. diff --git a/doc/source/rest.yaml b/doc/source/rest.yaml index 46c35467f..8204394cb 100644 --- a/doc/source/rest.yaml +++ b/doc/source/rest.yaml @@ -214,6 +214,9 @@ - name: get-measures-granularity request: GET /v1/metric/{{ scenarios['create-metric']['response'].json['id'] }}/measures?granularity=1 HTTP/1.1 +- name: get-measures-refresh + request: GET /v1/metric/{{ scenarios['create-metric']['response'].json['id'] }}/measures?refresh=true HTTP/1.1 + - name: create-resource-generic request: | POST /v1/resource/generic HTTP/1.1 diff --git a/gnocchi/rest/__init__.py b/gnocchi/rest/__init__.py index ff996d30d..f80e0a0f9 100644 --- a/gnocchi/rest/__init__.py +++ b/gnocchi/rest/__init__.py @@ -449,7 +449,7 @@ class MetricController(rest.RestController): @pecan.expose('json') def get_measures(self, start=None, stop=None, aggregation='mean', - granularity=None, **param): + granularity=None, refresh=False, **param): self.enforce_metric("get measures") if not (aggregation in archive_policy.ArchivePolicy.VALID_AGGREGATION_METHODS @@ -473,6 +473,10 @@ class MetricController(rest.RestController): except Exception: abort(400, "Invalid value for stop") + if strutils.bool_from_string(refresh): + pecan.request.storage.process_new_measures( + pecan.request.indexer, [six.text_type(self.metric.id)], True) + try: if aggregation in self.custom_agg: measures = self.custom_agg[aggregation].compute( @@ -1244,9 +1248,8 @@ class AggregationResourceController(rest.RestController): @pecan.expose('json') def post(self, start=None, stop=None, aggregation='mean', - reaggregation=None, - granularity=None, needed_overlap=100.0, - groupby=None): + reaggregation=None, granularity=None, needed_overlap=100.0, + groupby=None, refresh=False): # First, set groupby in the right format: a sorted list of unique # strings. groupby = sorted(set(arg_to_list(groupby))) @@ -1270,7 +1273,7 @@ class AggregationResourceController(rest.RestController): for r in resources))) return AggregationController.get_cross_metric_measures_from_objs( metrics, start, stop, aggregation, reaggregation, - granularity, needed_overlap) + granularity, needed_overlap, refresh) def groupper(r): return tuple((attr, r[attr]) for attr in groupby) @@ -1284,7 +1287,7 @@ class AggregationResourceController(rest.RestController): "group": dict(key), "measures": AggregationController.get_cross_metric_measures_from_objs( # noqa metrics, start, stop, aggregation, reaggregation, - granularity, needed_overlap) + granularity, needed_overlap, refresh) }) return results @@ -1314,7 +1317,8 @@ class AggregationController(rest.RestController): aggregation='mean', reaggregation=None, granularity=None, - needed_overlap=100.0): + needed_overlap=100.0, + refresh=False): try: needed_overlap = float(needed_overlap) except ValueError: @@ -1344,14 +1348,18 @@ class AggregationController(rest.RestController): enforce("get metric", metric) number_of_metrics = len(metrics) + if number_of_metrics == 0: + return [] + if granularity is not None: + try: + granularity = float(granularity) + except ValueError as e: + abort(400, "granularity must be a float: %s" % e) try: - if number_of_metrics == 0: - return [] - if granularity is not None: - try: - granularity = float(granularity) - except ValueError as e: - abort(400, "granularity must be a float: %s" % e) + if strutils.bool_from_string(refresh): + pecan.request.storage.process_new_measures( + pecan.request.indexer, + [six.text_type(m.id) for m in metrics], True) if number_of_metrics == 1: # NOTE(sileht): don't do the aggregation if we only have one # metric @@ -1376,10 +1384,9 @@ class AggregationController(rest.RestController): abort(404, e) @pecan.expose('json') - def get_metric(self, metric=None, start=None, - stop=None, aggregation='mean', - reaggregation=None, - granularity=None, needed_overlap=100.0): + def get_metric(self, metric=None, start=None, stop=None, + aggregation='mean', reaggregation=None, granularity=None, + needed_overlap=100.0, refresh=False): # Check RBAC policy metric_ids = arg_to_list(metric) metrics = pecan.request.indexer.list_metrics(ids=metric_ids) @@ -1391,7 +1398,7 @@ class AggregationController(rest.RestController): missing_metric_ids.pop())) return self.get_cross_metric_measures_from_objs( metrics, start, stop, aggregation, reaggregation, - granularity, needed_overlap) + granularity, needed_overlap, refresh) class CapabilityController(rest.RestController): diff --git a/gnocchi/tests/gabbi/gabbits-live/live.yaml b/gnocchi/tests/gabbi/gabbits-live/live.yaml index bbc924fbe..226e4d695 100644 --- a/gnocchi/tests/gabbi/gabbits-live/live.yaml +++ b/gnocchi/tests/gabbi/gabbits-live/live.yaml @@ -617,6 +617,26 @@ tests: $[0][2]: 2 $[1][2]: 2 + - name: post some more measures to the metric on myresource + POST: /v1/resource/myresource/2ae35573-7f9f-4bb1-aae8-dad8dff5706e/metric/vcpus/measures + request_headers: + content-type: application/json + data: + - timestamp: "2015-03-06T14:34:15" + value: 5 + - timestamp: "2015-03-06T14:34:20" + value: 5 + status: 202 + + - name: get myresource measures with refresh + GET: /v1/resource/myresource/2ae35573-7f9f-4bb1-aae8-dad8dff5706e/metric/vcpus/measures?refresh=true + response_json_paths: + $[0][2]: 2 + $[1][2]: 4 + $[2][2]: 2 + $[3][2]: 2 + $[4][2]: 5 + $[5][2]: 5 # # Search for resources diff --git a/gnocchi/tests/gabbi/gabbits/aggregation.yaml b/gnocchi/tests/gabbi/gabbits/aggregation.yaml index f23663ede..6cb11d6cc 100644 --- a/gnocchi/tests/gabbi/gabbits/aggregation.yaml +++ b/gnocchi/tests/gabbi/gabbits/aggregation.yaml @@ -68,6 +68,16 @@ tests: GET: /v1/aggregation/metric?metric=$RESPONSE['$[0].id']&metric=$RESPONSE['$[1].id']&granularity=foobar status: 400 + - name: get metric list to get aggregates for get with refresh + GET: /v1/metric + + - name: get measure aggregates by granularity with refresh + GET: /v1/aggregation/metric?metric=$RESPONSE['$[0].id']&metric=$RESPONSE['$[1].id']&granularity=1&refresh=true + response_json_paths: + $: + - ['2015-03-06T14:33:57+00:00', 1.0, 23.1] + - ['2015-03-06T14:34:12+00:00', 1.0, 7.0] + - name: get metric list to get aggregates 2 GET: /v1/metric @@ -162,6 +172,17 @@ tests: value: 2 status: 202 + - name: get measure aggregates by granularity from resources with refresh + POST: /v1/aggregation/resource/generic/metric/agg_meter?granularity=1&refresh=true + request_headers: + x-user-id: 0fbb231484614b1a80131fc22f6afc9c + x-project-id: f3d41b770cc14f0bb94a1d5be9c0e3ea + content-type: application/json + response_json_paths: + $: + - ['2015-03-06T14:33:57+00:00', 1.0, 23.1] + - ['2015-03-06T14:34:12+00:00', 1.0, 7.0] + - name: get measure aggregates by granularity from resources POST: /v1/aggregation/resource/generic/metric/agg_meter?granularity=1 request_headers: