From 72a2091727431555eba65c6ef8ff89448f3432f0 Mon Sep 17 00:00:00 2001 From: gord chung Date: Fri, 5 Aug 2016 09:47:09 -0400 Subject: [PATCH] add support to process measures on GET depending on the backlog, the returned measures from get-measures may not include what sits in carbonara backlog. to allow for more flexibility on metricd deployments, this patch offers the ability to allow users to get all measures even if they don't have enough metricd workers, at the expense of responsiveness. refresh blocks until it can process measures Change-Id: I588ae6879474d780e8ec9e893d4ecc2b367b832e Closes-Bug: #1603495 --- doc/source/rest.j2 | 15 +++++++ doc/source/rest.yaml | 3 ++ gnocchi/rest/__init__.py | 45 +++++++++++--------- gnocchi/tests/gabbi/gabbits-live/live.yaml | 20 +++++++++ gnocchi/tests/gabbi/gabbits/aggregation.yaml | 21 +++++++++ 5 files changed, 85 insertions(+), 19 deletions(-) diff --git a/doc/source/rest.j2 b/doc/source/rest.j2 index a37743acf..7e2d309b1 100644 --- a/doc/source/rest.j2 +++ b/doc/source/rest.j2 @@ -73,6 +73,17 @@ endpoint: {{ scenarios['get-measures']['doc'] }} +Depending on the driver, there may be some lag after POSTing measures before +they are processed and queryable. To ensure your query returns all measures +that have been POSTed, you can force any unprocessed measures to be handled: + +{{ scenarios['get-measures-refresh']['doc'] }} + +.. note:: + + Depending on the amount of data that is unprocessed, `refresh` may add + some overhead to your query. + The list of points returned is composed of tuples with (timestamp, granularity, value) sorted by timestamp. The granularity is the timespan covered by aggregation for this point. @@ -474,6 +485,10 @@ requested resource type, and the compute the aggregation: {{ scenarios['get-across-metrics-measures-by-attributes-lookup-groupby']['doc'] }} +Similar to retrieving measures for a single metric, the `refresh` parameter +can be provided to force all POSTed measures to be processed across all +metrics before computing the result. + Also aggregation across metrics have different behavior depending on if boundary are set ('start' and 'stop') and if 'needed_overlap' is set. diff --git a/doc/source/rest.yaml b/doc/source/rest.yaml index 46c35467f..8204394cb 100644 --- a/doc/source/rest.yaml +++ b/doc/source/rest.yaml @@ -214,6 +214,9 @@ - name: get-measures-granularity request: GET /v1/metric/{{ scenarios['create-metric']['response'].json['id'] }}/measures?granularity=1 HTTP/1.1 +- name: get-measures-refresh + request: GET /v1/metric/{{ scenarios['create-metric']['response'].json['id'] }}/measures?refresh=true HTTP/1.1 + - name: create-resource-generic request: | POST /v1/resource/generic HTTP/1.1 diff --git a/gnocchi/rest/__init__.py b/gnocchi/rest/__init__.py index ff996d30d..f80e0a0f9 100644 --- a/gnocchi/rest/__init__.py +++ b/gnocchi/rest/__init__.py @@ -449,7 +449,7 @@ class MetricController(rest.RestController): @pecan.expose('json') def get_measures(self, start=None, stop=None, aggregation='mean', - granularity=None, **param): + granularity=None, refresh=False, **param): self.enforce_metric("get measures") if not (aggregation in archive_policy.ArchivePolicy.VALID_AGGREGATION_METHODS @@ -473,6 +473,10 @@ class MetricController(rest.RestController): except Exception: abort(400, "Invalid value for stop") + if strutils.bool_from_string(refresh): + pecan.request.storage.process_new_measures( + pecan.request.indexer, [six.text_type(self.metric.id)], True) + try: if aggregation in self.custom_agg: measures = self.custom_agg[aggregation].compute( @@ -1244,9 +1248,8 @@ class AggregationResourceController(rest.RestController): @pecan.expose('json') def post(self, start=None, stop=None, aggregation='mean', - reaggregation=None, - granularity=None, needed_overlap=100.0, - groupby=None): + reaggregation=None, granularity=None, needed_overlap=100.0, + groupby=None, refresh=False): # First, set groupby in the right format: a sorted list of unique # strings. groupby = sorted(set(arg_to_list(groupby))) @@ -1270,7 +1273,7 @@ class AggregationResourceController(rest.RestController): for r in resources))) return AggregationController.get_cross_metric_measures_from_objs( metrics, start, stop, aggregation, reaggregation, - granularity, needed_overlap) + granularity, needed_overlap, refresh) def groupper(r): return tuple((attr, r[attr]) for attr in groupby) @@ -1284,7 +1287,7 @@ class AggregationResourceController(rest.RestController): "group": dict(key), "measures": AggregationController.get_cross_metric_measures_from_objs( # noqa metrics, start, stop, aggregation, reaggregation, - granularity, needed_overlap) + granularity, needed_overlap, refresh) }) return results @@ -1314,7 +1317,8 @@ class AggregationController(rest.RestController): aggregation='mean', reaggregation=None, granularity=None, - needed_overlap=100.0): + needed_overlap=100.0, + refresh=False): try: needed_overlap = float(needed_overlap) except ValueError: @@ -1344,14 +1348,18 @@ class AggregationController(rest.RestController): enforce("get metric", metric) number_of_metrics = len(metrics) + if number_of_metrics == 0: + return [] + if granularity is not None: + try: + granularity = float(granularity) + except ValueError as e: + abort(400, "granularity must be a float: %s" % e) try: - if number_of_metrics == 0: - return [] - if granularity is not None: - try: - granularity = float(granularity) - except ValueError as e: - abort(400, "granularity must be a float: %s" % e) + if strutils.bool_from_string(refresh): + pecan.request.storage.process_new_measures( + pecan.request.indexer, + [six.text_type(m.id) for m in metrics], True) if number_of_metrics == 1: # NOTE(sileht): don't do the aggregation if we only have one # metric @@ -1376,10 +1384,9 @@ class AggregationController(rest.RestController): abort(404, e) @pecan.expose('json') - def get_metric(self, metric=None, start=None, - stop=None, aggregation='mean', - reaggregation=None, - granularity=None, needed_overlap=100.0): + def get_metric(self, metric=None, start=None, stop=None, + aggregation='mean', reaggregation=None, granularity=None, + needed_overlap=100.0, refresh=False): # Check RBAC policy metric_ids = arg_to_list(metric) metrics = pecan.request.indexer.list_metrics(ids=metric_ids) @@ -1391,7 +1398,7 @@ class AggregationController(rest.RestController): missing_metric_ids.pop())) return self.get_cross_metric_measures_from_objs( metrics, start, stop, aggregation, reaggregation, - granularity, needed_overlap) + granularity, needed_overlap, refresh) class CapabilityController(rest.RestController): diff --git a/gnocchi/tests/gabbi/gabbits-live/live.yaml b/gnocchi/tests/gabbi/gabbits-live/live.yaml index bbc924fbe..226e4d695 100644 --- a/gnocchi/tests/gabbi/gabbits-live/live.yaml +++ b/gnocchi/tests/gabbi/gabbits-live/live.yaml @@ -617,6 +617,26 @@ tests: $[0][2]: 2 $[1][2]: 2 + - name: post some more measures to the metric on myresource + POST: /v1/resource/myresource/2ae35573-7f9f-4bb1-aae8-dad8dff5706e/metric/vcpus/measures + request_headers: + content-type: application/json + data: + - timestamp: "2015-03-06T14:34:15" + value: 5 + - timestamp: "2015-03-06T14:34:20" + value: 5 + status: 202 + + - name: get myresource measures with refresh + GET: /v1/resource/myresource/2ae35573-7f9f-4bb1-aae8-dad8dff5706e/metric/vcpus/measures?refresh=true + response_json_paths: + $[0][2]: 2 + $[1][2]: 4 + $[2][2]: 2 + $[3][2]: 2 + $[4][2]: 5 + $[5][2]: 5 # # Search for resources diff --git a/gnocchi/tests/gabbi/gabbits/aggregation.yaml b/gnocchi/tests/gabbi/gabbits/aggregation.yaml index f23663ede..6cb11d6cc 100644 --- a/gnocchi/tests/gabbi/gabbits/aggregation.yaml +++ b/gnocchi/tests/gabbi/gabbits/aggregation.yaml @@ -68,6 +68,16 @@ tests: GET: /v1/aggregation/metric?metric=$RESPONSE['$[0].id']&metric=$RESPONSE['$[1].id']&granularity=foobar status: 400 + - name: get metric list to get aggregates for get with refresh + GET: /v1/metric + + - name: get measure aggregates by granularity with refresh + GET: /v1/aggregation/metric?metric=$RESPONSE['$[0].id']&metric=$RESPONSE['$[1].id']&granularity=1&refresh=true + response_json_paths: + $: + - ['2015-03-06T14:33:57+00:00', 1.0, 23.1] + - ['2015-03-06T14:34:12+00:00', 1.0, 7.0] + - name: get metric list to get aggregates 2 GET: /v1/metric @@ -162,6 +172,17 @@ tests: value: 2 status: 202 + - name: get measure aggregates by granularity from resources with refresh + POST: /v1/aggregation/resource/generic/metric/agg_meter?granularity=1&refresh=true + request_headers: + x-user-id: 0fbb231484614b1a80131fc22f6afc9c + x-project-id: f3d41b770cc14f0bb94a1d5be9c0e3ea + content-type: application/json + response_json_paths: + $: + - ['2015-03-06T14:33:57+00:00', 1.0, 23.1] + - ['2015-03-06T14:34:12+00:00', 1.0, 7.0] + - name: get measure aggregates by granularity from resources POST: /v1/aggregation/resource/generic/metric/agg_meter?granularity=1 request_headers: