Add a v2 API endpoint to push DataFrame objects

A new endpoint has been made available to admin users on
``POST /v2/dataframes``. This will allow end users to push
DataFrames in the form of JSON objects into the CloudKitty
storage.

Documentation and unit tests are included in this commit.

Depends-On: https://review.opendev.org/#/c/668669/
Change-Id: I42641462ecbac89f400a257805fc99f4027903b3
Story: 2005890
Task: 35953

cloudkitty/api/v1/controllers/storage.py

@@ -80,7 +80,13 @@ class DataFramesController(rest.RestController):
                         volume=point.qty,
                         rating=point.price)
                     if frame_tenant is None:
-                        frame_tenant = point.desc[scope_key]
+                        # NOTE(jferrieu): Since DataFrame/DataPoint
+                        # implementation patch we cannot guarantee
+                        # anymore that a DataFrame does contain a scope_id
+                        # therefore the __UNDEF__ default value has been
+                        # retained to maintain backward compatibility
+                        # if it would occur being absent
+                        frame_tenant = point.desc.get(scope_key, '__UNDEF__')
                     resources.append(resource)
                 dataframe = storage_models.DataFrame(
                     begin=tzutils.local_to_utc(frame.start, naive=True),

cloudkitty/api/v2/__init__.py

@@ -33,6 +33,7 @@ RESOURCE_SCHEMA = voluptuous.Schema({
 
 API_MODULES = [
     'cloudkitty.api.v2.scope',
+    'cloudkitty.api.v2.dataframes',
     'cloudkitty.api.v2.summary',
 ]
 

cloudkitty/api/v2/dataframes/__init__.py

@@ -0,0 +1,26 @@
+# Copyright 2019 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+from cloudkitty.api.v2 import utils as api_utils
+
+
+def init(app):
+    api_utils.do_init(app, 'dataframes', [
+        {
+            'module': __name__ + '.' + 'dataframes',
+            'resource_class': 'DataFrameList',
+            'url': '',
+        },
+    ])
+    return app

cloudkitty/api/v2/dataframes/dataframes.py

@@ -0,0 +1,42 @@
+# Copyright 2019 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+import flask
+import voluptuous
+from werkzeug import exceptions as http_exceptions
+
+from cloudkitty.api.v2 import base
+from cloudkitty.api.v2 import utils as api_utils
+from cloudkitty.common import policy
+from cloudkitty import dataframe
+
+
+class DataFrameList(base.BaseResource):
+    @api_utils.add_input_schema('body', {
+        voluptuous.Required('dataframes'): [dataframe.DataFrame.from_dict],
+    })
+    def post(self, dataframes=[]):
+        policy.authorize(
+            flask.request.context,
+            'dataframes:add',
+            {},
+        )
+
+        if not dataframes:
+            raise http_exceptions.BadRequest(
+                "Parameter dataframes must not be empty.")
+
+        self._storage.push(dataframes)
+
+        return {}, 204

cloudkitty/common/policies/__init__.py

@@ -21,6 +21,7 @@ from cloudkitty.common.policies.v1 import info as v1_info
 from cloudkitty.common.policies.v1 import rating as v1_rating
 from cloudkitty.common.policies.v1 import report as v1_report
 from cloudkitty.common.policies.v1 import storage as v1_storage
+from cloudkitty.common.policies.v2 import dataframes as v2_dataframes
 from cloudkitty.common.policies.v2 import scope as v2_scope
 from cloudkitty.common.policies.v2 import summary as v2_summary
 
@@ -33,6 +34,7 @@ def list_rules():
         v1_rating.list_rules(),
         v1_report.list_rules(),
         v1_storage.list_rules(),
+        v2_dataframes.list_rules(),
         v2_scope.list_rules(),
         v2_summary.list_rules(),
     )

cloudkitty/common/policies/v2/dataframes.py

@@ -0,0 +1,31 @@
+# Copyright 2019 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+from oslo_policy import policy
+
+from cloudkitty.common.policies import base
+
+
+dataframes_policies = [
+    policy.DocumentedRuleDefault(
+        name='dataframes:add',
+        check_str=base.ROLE_ADMIN,
+        description='Add one or several DataFrames',
+        operations=[{'path': '/v2/dataframes',
+                     'method': 'POST'}]),
+]
+
+
+def list_rules():
+    return dataframes_policies

cloudkitty/tests/gabbi/gabbits/v2-dataframes.yaml

@@ -0,0 +1,186 @@
+fixtures:
+  - ConfigFixtureStorageV2
+  - InfluxStorageDataFixture
+
+tests:
+  - name: Push dataframes
+    url: /v2/dataframes
+    method: POST
+    status: 204
+    request_headers:
+      content-type: application/json
+    data:
+      dataframes:
+        - period:
+            begin: 20190723T122810Z
+            end: 20190723T132810Z
+          usage:
+            metric_one:
+              - vol:
+                  unit: GiB
+                  qty: 1.2
+                rating:
+                  price: 0.04
+                groupby:
+                  group_one: one
+                  group_two: two
+                metadata:
+                  attr_one: one
+                  attr_two: two
+            metric_two:
+              - vol:
+                  unit: GiB
+                  qty: 1.2
+                rating:
+                  price: 0.04
+                groupby:
+                  group_one: one
+                  group_two: two
+                metadata:
+                  attr_one: one
+                  attr_two: two
+        - period:
+            begin: 20190723T122810Z
+            end: 20190723T132810Z
+          usage:
+            metric_one:
+              - vol:
+                  unit: GiB
+                  qty: 1.2
+                rating:
+                  price: 0.04
+                groupby:
+                  group_one: one
+                  group_two: two
+                metadata:
+                  attr_one: one
+                  attr_two: two
+            metric_two:
+              - vol:
+                  unit: GiB
+                  qty: 1.2
+                rating:
+                  price: 0.04
+                groupby:
+                  group_one: one
+                  group_two: two
+                metadata:
+                  attr_one: one
+                  attr_two: two
+
+  - name: Push dataframes with empty dataframes
+    url: /v2/dataframes
+    method: POST
+    status: 400
+    request_headers:
+      content-type: application/json
+    data:
+      dataframes: []
+    response_strings:
+      - "Parameter dataframes must not be empty."
+
+  - name: Push dataframes with missing key
+    url: /v2/dataframes
+    method: POST
+    status: 400
+    request_headers:
+      content-type: application/json
+    data:
+      dataframes:
+        - period:
+            begin: 20190723T122810Z
+            end: 20190723T132810Z
+          usage:
+            metric_one:
+              - vol:
+                  unit: GiB
+                  qty: 1.2
+                rating:
+                  price: 0.04
+                groupby:
+                  group_one: one
+                  group_two: two
+                metadata:
+                  attr_one: one
+                  attr_two: two
+            metric_two:
+              - vol:
+                  unit: GiB
+                  qty: 1.2
+                rating:
+                  price: 0.04
+                groupby:
+                  group_one: one
+                  group_two: two
+                metadata:
+                  attr_one: one
+                  attr_two: two
+        - period:
+            begin: 20190723T122810Z
+            end: 20190723T132810Z
+
+  - name: Push dataframe with malformed datapoint
+    url: /v2/dataframes
+    method: POST
+    status: 400
+    request_headers:
+      content-type: application/json
+    data:
+      dataframes:
+        - period:
+            begin: 20190723T122810Z
+            end: 20190723T132810Z
+          usage:
+            metric_one:
+              - vol:
+                  unit: GiB
+                  qty: 1.2
+            metric_two:
+              - vol:
+                  unit: GiB
+                  qty: 1.2
+                rating:
+                  price: 0.04
+                groupby:
+                  group_one: one
+                  group_two: two
+                metadata:
+                  attr_one: one
+                  attr_two: two
+
+  - name: Push dataframe with malformed datetimes
+    url: /v2/dataframes
+    method: POST
+    status: 400
+    request_headers:
+      content-type: application/json
+    data:
+      dataframes:
+        - period:
+            begin: 20190723TZ
+            end: 20190723TZ
+          usage:
+            metric_one:
+              - vol:
+                  unit: GiB
+                  qty: 1.2
+                rating:
+                  price: 0.04
+                groupby:
+                  group_one: one
+                  group_two: two
+                metadata:
+                  attr_one: one
+                  attr_two: two
+            metric_two:
+              - vol:
+                  unit: GiB
+                  qty: 1.2
+                rating:
+                  price: 0.04
+                groupby:
+                  group_one: one
+                  group_two: two
+                metadata:
+                  attr_one: one
+                  attr_two: two

doc/source/_static/cloudkitty.policy.yaml.sample

@@ -84,6 +84,10 @@
 # GET  /v1/storage/dataframes
 #"storage:list_data_frames": "rule:admin_or_owner"
 
+# Add one or several DataFrames
+# POST  /v2/dataframes
+#"dataframes:add": "role:admin"
+
 # Get the state of one or several scopes
 # GET  /v2/scope
 #"scope:get_state": "role:admin"

doc/source/api-reference/v2/api_samples/dataframes/dataframes_post.json

@@ -0,0 +1,96 @@
+{
+    "dataframes": [
+        {
+            "period": {
+                "begin": "20190723T122810Z",
+                "end": "20190723T132810Z"
+            },
+            "usage": {
+                "metric_one": [
+                    {
+                        "vol": {
+                            "unit": "GiB",
+                            "qty": 1.2
+                        },
+                        "rating": {
+                            "price": 0.04
+                        },
+                        "groupby": {
+                            "group_one": "one",
+                            "group_two": "two"
+                        },
+                        "metadata": {
+                            "attr_one": "one",
+                            "attr_two": "two"
+                        }
+                    }
+                ],
+                "metric_two": [
+                    {
+                        "vol": {
+                            "unit": "MB",
+                            "qty": 200.4
+                        },
+                        "rating": {
+                            "price": 0.06
+                        },
+                        "groupby": {
+                            "group_one": "one",
+                            "group_two": "two"
+                        },
+                        "metadata": {
+                            "attr_one": "one",
+                            "attr_two": "two"
+                        }
+                    }
+                ]
+            }
+        },
+        {
+            "period": {
+                "begin": "20190823T122810Z",
+                "end": "20190823T132810Z"
+            },
+            "usage": {
+                "metric_one": [
+                    {
+                        "vol": {
+                            "unit": "GiB",
+                            "qty": 2.4
+                        },
+                        "rating": {
+                            "price": 0.08
+                        },
+                        "groupby": {
+                            "group_one": "one",
+                            "group_two": "two"
+                        },
+                        "metadata": {
+                            "attr_one": "one",
+                            "attr_two": "two"
+                        }
+                    }
+                ],
+                "metric_two": [
+                    {
+                        "vol": {
+                            "unit": "MB",
+                            "qty": 400.8
+                        },
+                        "rating": {
+                            "price": 0.12
+                        },
+                        "groupby": {
+                            "group_one": "one",
+                            "group_two": "two"
+                        },
+                        "metadata": {
+                            "attr_one": "one",
+                            "attr_two": "two"
+                        }
+                    }
+                ]
+            }
+        }
+    ]
+}

doc/source/api-reference/v2/dataframes/dataframes.inc

@@ -0,0 +1,41 @@
+===================
+Dataframes endpoint
+===================
+
+Add dataframes into the storage backend
+=======================================
+
+Add dataframes into the storage backend.
+
+.. rest_method::  POST /v2/dataframes
+
+.. rest_parameters:: dataframes/dataframes_parameters.yml
+
+   - dataframes: dataframes_body
+
+Request Example
+---------------
+
+In the body:
+
+.. literalinclude:: ./api_samples/dataframes/dataframes_post.json
+   :language: javascript
+
+Status codes
+------------
+
+.. rest_status_code:: success http_status.yml
+
+   - 204
+
+.. rest_status_code:: error http_status.yml
+
+   - 400
+   - 401
+   - 403
+   - 405
+
+Response
+--------
+
+No content is to be returned.

doc/source/api-reference/v2/dataframes/dataframes_parameters.yml

@@ -0,0 +1,6 @@
+dataframes_body:
+  in: body
+  description: |
+    List of dataframes to add
+  type: list
+  required: true

doc/source/api-reference/v2/dataframes/http_status.yml

@@ -0,0 +1 @@
+../http_status.yml

doc/source/api-reference/v2/http_status.yml

@@ -7,9 +7,15 @@
 202:
   default: Request has been accepted for asynchronous processing.
 
+204:
+  default: Request was successful even though no content is to be returned.
+
 400:
   default: Invalid request.
 
+401:
+  default: Unauthenticated user.
+
 403:
   default: Forbidden operation for the authentified user.
 

doc/source/api-reference/v2/index.rst

@@ -1,4 +1,5 @@
 .. rest_expand_all::
 
+.. include:: dataframes/dataframes.inc
 .. include:: scope/scope.inc
 .. include:: summary/summary.inc

releasenotes/notes/add-dataframes-v2-api-endpoint-601825c344ba0e2d.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Added a v2 API endpoint allowing to push dataframes into the CloudKitty
+    storage. This endpoint is available via a ``POST`` request on
+    ``/v2/dataframes``. Admin privileges are required to use this endpoint.