rest: add batch/measures support

This allows to post measures for several metric in a single HTTP call.

Change-Id: Ibb8be5f82900663b7d35a989b1bc22e2b2e39fdf

doc/source/index.rst

@@ -42,6 +42,7 @@ Key Features
 - HTTP REST interface
 - Horizontal scalability
 - Metric aggregation
+- Measures batching support
 - Archiving policy
 - Metric value search
 - Structured resources

doc/source/rest.j2

@@ -77,6 +77,14 @@ to retrieve, rather than all the granularities available:
 {{ scenarios['get-measures-granularity']['doc'] }}
 
 
+Measures batching
+=================
+It is also possible to batch measures sending, i.e. send several measures for
+different metrics in a simple call:
+
+{{ scenarios['post-measures-batch']['doc'] }}
+
+
 Archive Policy
 ==============
 

doc/source/rest.yaml

@@ -77,6 +77,15 @@
       "archive_policy_name": "low"
     }
 
+- name: create-metric-2
+  request: |
+    POST /v1/metric HTTP/1.1
+    Content-Type: application/json
+
+    {
+      "archive_policy_name": "low"
+    }
+
 - name: create-archive-policy-rule
   request: |
     POST /v1/archive_policy_rule HTTP/1.1
@@ -135,6 +144,36 @@
       }
     ]
 
+- name: post-measures-batch
+  request: |
+    POST /v1/batch/measures HTTP/1.1
+    Content-Type: application/json
+
+    {
+      "{{ scenarios['create-metric']['response'].json['id'] }}":
+      [
+        {
+          "timestamp": "2014-10-06T14:34:12",
+          "value": 12
+        },
+        {
+          "timestamp": "2014-10-06T14:34:20",
+          "value": 2
+        }
+      ],
+      "{{ scenarios['create-metric-2']['response'].json['id'] }}":
+      [
+        {
+          "timestamp": "2014-10-06T16:12:12",
+          "value": 3
+        },
+        {
+          "timestamp": "2014-10-06T18:14:52",
+          "value": 4
+        }
+      ]
+    }
+
 - name: search-value-in-metric
   request: |
     POST /v1/search/metric?metric_id={{ scenarios['create-metric']['response'].json['id'] }} HTTP/1.1

gnocchi/rest/__init__.py

@@ -420,6 +420,21 @@ class AggregatedMetricController(rest.RestController):
             abort(404, e)
 
 
+def MeasureSchema(m):
+    # NOTE(sileht): don't use voluptuous for performance reasons
+    try:
+        value = float(m['value'])
+    except Exception:
+        abort(400, "Invalid input for a value")
+
+    try:
+        timestamp = utils.to_timestamp(m['timestamp'])
+    except Exception:
+        abort(400, "Invalid input for a timestamp")
+
+    return storage.Measure(timestamp, value)
+
+
 class MetricController(rest.RestController):
     _custom_actions = {
         'measures': ['POST', 'GET']
@@ -431,23 +446,6 @@ class MetricController(rest.RestController):
                                          invoke_on_load=True)
         self.custom_agg = dict((x.name, x.obj) for x in mgr)
 
-    @staticmethod
-    def to_measure(m):
-        # NOTE(sileht): we do the input validation
-        # during the iteration for not loop just for this
-        # and don't use voluptuous for performance reason
-        try:
-            value = float(m['value'])
-        except Exception:
-            abort(400, "Invalid input for a value")
-
-        try:
-            timestamp = utils.to_timestamp(m['timestamp'])
-        except Exception:
-            abort(400, "Invalid input for a timestamp")
-
-        return storage.Measure(timestamp, value)
-
     def enforce_metric(self, rule):
         enforce(rule, json.to_primitive(self.metric))
 
@@ -464,7 +462,7 @@ class MetricController(rest.RestController):
             abort(400, "Invalid input for measures")
         if params:
             pecan.request.storage.add_measures(
-                self.metric, six.moves.map(self.to_measure, params))
+                self.metric, six.moves.map(MeasureSchema, params))
         pecan.response.status = 202
 
     @pecan.expose('json')
@@ -1257,6 +1255,30 @@ class SearchMetricController(rest.RestController):
             abort(400, e)
 
 
+class MeasuresBatchController(rest.RestController):
+    MeasuresBatchSchema = voluptuous.Schema({
+        UUID: [MeasureSchema],
+    })
+
+    @pecan.expose()
+    def post(self):
+        body = deserialize_and_validate(self.MeasuresBatchSchema)
+        metrics = pecan.request.indexer.get_metrics(body.keys())
+
+        if len(metrics) != len(body):
+            missing_metrics = sorted(set(body) - set(m.id for m in metrics))
+            abort(400, "Unknown metrics: %s" % ", ".join(
+                six.moves.map(str, missing_metrics)))
+
+        for metric in metrics:
+            enforce("post measures", metric)
+
+        for metric in metrics:
+            pecan.request.storage.add_measures(metric, body[metric.id])
+
+        pecan.response.status = 202
+
+
 class SearchController(object):
     resource = SearchResourceController()
     metric = SearchMetricController()
@@ -1321,6 +1343,10 @@ class StatusController(rest.RestController):
         return {"storage": {"measures_to_process": report}}
 
 
+class BatchController(object):
+    measures = MeasuresBatchController()
+
+
 class V1Controller(object):
 
     def __init__(self):
@@ -1329,6 +1355,7 @@ class V1Controller(object):
             "archive_policy": ArchivePoliciesController(),
             "archive_policy_rule": ArchivePolicyRulesController(),
             "metric": MetricsController(),
+            "batch": BatchController(),
             "resource": ResourcesController(),
             "aggregation": Aggregation(),
             "capabilities": CapabilityController(),

gnocchi/tests/gabbi/gabbits/batch_measures.yaml

@@ -0,0 +1,88 @@
+fixtures:
+    - ConfigFixture
+
+tests:
+    - name: create archive policy
+      desc: for later use
+      url: /v1/archive_policy
+      method: POST
+      request_headers:
+          content-type: application/json
+          x-roles: admin
+      data:
+          name: simple
+          definition:
+              - granularity: 1 second
+      status: 201
+
+    - name: create metric
+      url: /v1/metric
+      request_headers:
+          content-type: application/json
+      method: post
+      data:
+          archive_policy_name: simple
+      status: 201
+
+    - name: push measurements to metric
+      url: /v1/batch/measures
+      request_headers:
+           content-type: application/json
+      method: post
+      data:
+        $RESPONSE['$.id']:
+          - timestamp: "2015-03-06T14:33:57"
+            value: 43.1
+          - timestamp: "2015-03-06T14:34:12"
+            value: 12
+      status: 202
+
+    - name: push measurements to unknown metrics
+      url: /v1/batch/measures
+      request_headers:
+           content-type: application/json
+      method: post
+      data:
+        37AEC8B7-C0D9-445B-8AB9-D3C6312DCF5C:
+          - timestamp: "2015-03-06T14:33:57"
+            value: 43.1
+          - timestamp: "2015-03-06T14:34:12"
+            value: 12
+        37AEC8B7-C0D9-445B-8AB9-D3C6312DCF5D:
+          - timestamp: "2015-03-06T14:33:57"
+            value: 43.1
+          - timestamp: "2015-03-06T14:34:12"
+            value: 12
+      status: 400
+      response_strings:
+        - "Unknown metrics: 37aec8b7-c0d9-445b-8ab9-d3c6312dcf5c, 37aec8b7-c0d9-445b-8ab9-d3c6312dcf5d"
+
+    - name: create second metric
+      url: /v1/metric
+      request_headers:
+          content-type: application/json
+      method: post
+      data:
+          archive_policy_name: simple
+      status: 201
+
+    - name: list metrics
+      url: /v1/metric
+
+    - name: push measurements to two metrics
+      url: /v1/batch/measures
+      request_headers:
+           content-type: application/json
+      method: post
+      data:
+        $RESPONSE['$[0].id']:
+          - timestamp: "2015-03-06T14:33:57"
+            value: 43.1
+          - timestamp: "2015-03-06T14:34:12"
+            value: 12
+        $RESPONSE['$[1].id']:
+          - timestamp: "2015-03-06T14:33:57"
+            value: 43.1
+          - timestamp: "2015-03-06T14:34:12"
+            value: 12
+      status: 202

gnocchi/tests/gabbi/gabbits/resource.yaml

@@ -54,9 +54,9 @@ tests:
       redirects: true
       response_json_paths:
           $.version: "1.0"
-          $.links.`len`: 9
+          $.links.`len`: 10
           $.links[0].href: $SCHEME://$NETLOC/v1
-          $.links[7].href: $SCHEME://$NETLOC/v1/search
+          $.links[7].href: $SCHEME://$NETLOC/v1/resource
 
     - name: root of resource
       url: /v1/resource