From 6e8efde43292b7ab0d8bc72d1c77b790ed1af954 Mon Sep 17 00:00:00 2001 From: Justin Ferrieu Date: Wed, 21 Aug 2019 12:21:05 +0000 Subject: [PATCH] Add a v2 API endpoint to push DataFrame objects A new endpoint has been made available to admin users on ``POST /v2/dataframes``. This will allow end users to push DataFrames in the form of JSON objects into the CloudKitty storage. Documentation and unit tests are included in this commit. Depends-On: https://review.opendev.org/#/c/668669/ Change-Id: I42641462ecbac89f400a257805fc99f4027903b3 Story: 2005890 Task: 35953 --- cloudkitty/api/v1/controllers/storage.py | 8 +- cloudkitty/api/v2/__init__.py | 1 + cloudkitty/api/v2/dataframes/__init__.py | 26 +++ cloudkitty/api/v2/dataframes/dataframes.py | 42 ++++ cloudkitty/common/policies/__init__.py | 2 + cloudkitty/common/policies/v2/dataframes.py | 31 +++ .../tests/gabbi/gabbits/v2-dataframes.yaml | 186 ++++++++++++++++++ .../_static/cloudkitty.policy.yaml.sample | 4 + .../dataframes/dataframes_post.json | 96 +++++++++ .../v2/dataframes/dataframes.inc | 41 ++++ .../v2/dataframes/dataframes_parameters.yml | 6 + .../v2/dataframes/http_status.yml | 1 + doc/source/api-reference/v2/http_status.yml | 6 + doc/source/api-reference/v2/index.rst | 1 + ...ames-v2-api-endpoint-601825c344ba0e2d.yaml | 6 + 15 files changed, 456 insertions(+), 1 deletion(-) create mode 100644 cloudkitty/api/v2/dataframes/__init__.py create mode 100644 cloudkitty/api/v2/dataframes/dataframes.py create mode 100644 cloudkitty/common/policies/v2/dataframes.py create mode 100644 cloudkitty/tests/gabbi/gabbits/v2-dataframes.yaml create mode 100644 doc/source/api-reference/v2/api_samples/dataframes/dataframes_post.json create mode 100644 doc/source/api-reference/v2/dataframes/dataframes.inc create mode 100644 doc/source/api-reference/v2/dataframes/dataframes_parameters.yml create mode 120000 doc/source/api-reference/v2/dataframes/http_status.yml create mode 100644 releasenotes/notes/add-dataframes-v2-api-endpoint-601825c344ba0e2d.yaml diff --git a/cloudkitty/api/v1/controllers/storage.py b/cloudkitty/api/v1/controllers/storage.py index 41e16428..d1122df2 100644 --- a/cloudkitty/api/v1/controllers/storage.py +++ b/cloudkitty/api/v1/controllers/storage.py @@ -80,7 +80,13 @@ class DataFramesController(rest.RestController): volume=point.qty, rating=point.price) if frame_tenant is None: - frame_tenant = point.desc[scope_key] + # NOTE(jferrieu): Since DataFrame/DataPoint + # implementation patch we cannot guarantee + # anymore that a DataFrame does contain a scope_id + # therefore the __UNDEF__ default value has been + # retained to maintain backward compatibility + # if it would occur being absent + frame_tenant = point.desc.get(scope_key, '__UNDEF__') resources.append(resource) dataframe = storage_models.DataFrame( begin=tzutils.local_to_utc(frame.start, naive=True), diff --git a/cloudkitty/api/v2/__init__.py b/cloudkitty/api/v2/__init__.py index ef785197..1d5b4f0a 100644 --- a/cloudkitty/api/v2/__init__.py +++ b/cloudkitty/api/v2/__init__.py @@ -33,6 +33,7 @@ RESOURCE_SCHEMA = voluptuous.Schema({ API_MODULES = [ 'cloudkitty.api.v2.scope', + 'cloudkitty.api.v2.dataframes', 'cloudkitty.api.v2.summary', ] diff --git a/cloudkitty/api/v2/dataframes/__init__.py b/cloudkitty/api/v2/dataframes/__init__.py new file mode 100644 index 00000000..c05e5e82 --- /dev/null +++ b/cloudkitty/api/v2/dataframes/__init__.py @@ -0,0 +1,26 @@ +# Copyright 2019 Objectif Libre +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +from cloudkitty.api.v2 import utils as api_utils + + +def init(app): + api_utils.do_init(app, 'dataframes', [ + { + 'module': __name__ + '.' + 'dataframes', + 'resource_class': 'DataFrameList', + 'url': '', + }, + ]) + return app diff --git a/cloudkitty/api/v2/dataframes/dataframes.py b/cloudkitty/api/v2/dataframes/dataframes.py new file mode 100644 index 00000000..a3460ca7 --- /dev/null +++ b/cloudkitty/api/v2/dataframes/dataframes.py @@ -0,0 +1,42 @@ +# Copyright 2019 Objectif Libre +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +import flask +import voluptuous +from werkzeug import exceptions as http_exceptions + +from cloudkitty.api.v2 import base +from cloudkitty.api.v2 import utils as api_utils +from cloudkitty.common import policy +from cloudkitty import dataframe + + +class DataFrameList(base.BaseResource): + @api_utils.add_input_schema('body', { + voluptuous.Required('dataframes'): [dataframe.DataFrame.from_dict], + }) + def post(self, dataframes=[]): + policy.authorize( + flask.request.context, + 'dataframes:add', + {}, + ) + + if not dataframes: + raise http_exceptions.BadRequest( + "Parameter dataframes must not be empty.") + + self._storage.push(dataframes) + + return {}, 204 diff --git a/cloudkitty/common/policies/__init__.py b/cloudkitty/common/policies/__init__.py index 5f4c112a..c4759211 100644 --- a/cloudkitty/common/policies/__init__.py +++ b/cloudkitty/common/policies/__init__.py @@ -21,6 +21,7 @@ from cloudkitty.common.policies.v1 import info as v1_info from cloudkitty.common.policies.v1 import rating as v1_rating from cloudkitty.common.policies.v1 import report as v1_report from cloudkitty.common.policies.v1 import storage as v1_storage +from cloudkitty.common.policies.v2 import dataframes as v2_dataframes from cloudkitty.common.policies.v2 import scope as v2_scope from cloudkitty.common.policies.v2 import summary as v2_summary @@ -33,6 +34,7 @@ def list_rules(): v1_rating.list_rules(), v1_report.list_rules(), v1_storage.list_rules(), + v2_dataframes.list_rules(), v2_scope.list_rules(), v2_summary.list_rules(), ) diff --git a/cloudkitty/common/policies/v2/dataframes.py b/cloudkitty/common/policies/v2/dataframes.py new file mode 100644 index 00000000..b99ececb --- /dev/null +++ b/cloudkitty/common/policies/v2/dataframes.py @@ -0,0 +1,31 @@ +# Copyright 2019 Objectif Libre +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +from oslo_policy import policy + +from cloudkitty.common.policies import base + + +dataframes_policies = [ + policy.DocumentedRuleDefault( + name='dataframes:add', + check_str=base.ROLE_ADMIN, + description='Add one or several DataFrames', + operations=[{'path': '/v2/dataframes', + 'method': 'POST'}]), +] + + +def list_rules(): + return dataframes_policies diff --git a/cloudkitty/tests/gabbi/gabbits/v2-dataframes.yaml b/cloudkitty/tests/gabbi/gabbits/v2-dataframes.yaml new file mode 100644 index 00000000..7b8389d7 --- /dev/null +++ b/cloudkitty/tests/gabbi/gabbits/v2-dataframes.yaml @@ -0,0 +1,186 @@ +fixtures: + - ConfigFixtureStorageV2 + - InfluxStorageDataFixture + +tests: + - name: Push dataframes + url: /v2/dataframes + method: POST + status: 204 + request_headers: + content-type: application/json + data: + dataframes: + - period: + begin: 20190723T122810Z + end: 20190723T132810Z + usage: + metric_one: + - vol: + unit: GiB + qty: 1.2 + rating: + price: 0.04 + groupby: + group_one: one + group_two: two + metadata: + attr_one: one + attr_two: two + metric_two: + - vol: + unit: GiB + qty: 1.2 + rating: + price: 0.04 + groupby: + group_one: one + group_two: two + metadata: + attr_one: one + attr_two: two + - period: + begin: 20190723T122810Z + end: 20190723T132810Z + usage: + metric_one: + - vol: + unit: GiB + qty: 1.2 + rating: + price: 0.04 + groupby: + group_one: one + group_two: two + metadata: + attr_one: one + attr_two: two + metric_two: + - vol: + unit: GiB + qty: 1.2 + rating: + price: 0.04 + groupby: + group_one: one + group_two: two + metadata: + attr_one: one + attr_two: two + + - name: Push dataframes with empty dataframes + url: /v2/dataframes + method: POST + status: 400 + request_headers: + content-type: application/json + data: + dataframes: [] + response_strings: + - "Parameter dataframes must not be empty." + + - name: Push dataframes with missing key + url: /v2/dataframes + method: POST + status: 400 + request_headers: + content-type: application/json + data: + dataframes: + - period: + begin: 20190723T122810Z + end: 20190723T132810Z + usage: + metric_one: + - vol: + unit: GiB + qty: 1.2 + rating: + price: 0.04 + groupby: + group_one: one + group_two: two + metadata: + attr_one: one + attr_two: two + metric_two: + - vol: + unit: GiB + qty: 1.2 + rating: + price: 0.04 + groupby: + group_one: one + group_two: two + metadata: + attr_one: one + attr_two: two + - period: + begin: 20190723T122810Z + end: 20190723T132810Z + + - name: Push dataframe with malformed datapoint + url: /v2/dataframes + method: POST + status: 400 + request_headers: + content-type: application/json + data: + dataframes: + - period: + begin: 20190723T122810Z + end: 20190723T132810Z + usage: + metric_one: + - vol: + unit: GiB + qty: 1.2 + metric_two: + - vol: + unit: GiB + qty: 1.2 + rating: + price: 0.04 + groupby: + group_one: one + group_two: two + metadata: + attr_one: one + attr_two: two + + - name: Push dataframe with malformed datetimes + url: /v2/dataframes + method: POST + status: 400 + request_headers: + content-type: application/json + data: + dataframes: + - period: + begin: 20190723TZ + end: 20190723TZ + usage: + metric_one: + - vol: + unit: GiB + qty: 1.2 + rating: + price: 0.04 + groupby: + group_one: one + group_two: two + metadata: + attr_one: one + attr_two: two + metric_two: + - vol: + unit: GiB + qty: 1.2 + rating: + price: 0.04 + groupby: + group_one: one + group_two: two + metadata: + attr_one: one + attr_two: two diff --git a/doc/source/_static/cloudkitty.policy.yaml.sample b/doc/source/_static/cloudkitty.policy.yaml.sample index a0f26a85..d6289f06 100644 --- a/doc/source/_static/cloudkitty.policy.yaml.sample +++ b/doc/source/_static/cloudkitty.policy.yaml.sample @@ -84,6 +84,10 @@ # GET /v1/storage/dataframes #"storage:list_data_frames": "rule:admin_or_owner" +# Add one or several DataFrames +# POST /v2/dataframes +#"dataframes:add": "role:admin" + # Get the state of one or several scopes # GET /v2/scope #"scope:get_state": "role:admin" diff --git a/doc/source/api-reference/v2/api_samples/dataframes/dataframes_post.json b/doc/source/api-reference/v2/api_samples/dataframes/dataframes_post.json new file mode 100644 index 00000000..b6c7460d --- /dev/null +++ b/doc/source/api-reference/v2/api_samples/dataframes/dataframes_post.json @@ -0,0 +1,96 @@ +{ + "dataframes": [ + { + "period": { + "begin": "20190723T122810Z", + "end": "20190723T132810Z" + }, + "usage": { + "metric_one": [ + { + "vol": { + "unit": "GiB", + "qty": 1.2 + }, + "rating": { + "price": 0.04 + }, + "groupby": { + "group_one": "one", + "group_two": "two" + }, + "metadata": { + "attr_one": "one", + "attr_two": "two" + } + } + ], + "metric_two": [ + { + "vol": { + "unit": "MB", + "qty": 200.4 + }, + "rating": { + "price": 0.06 + }, + "groupby": { + "group_one": "one", + "group_two": "two" + }, + "metadata": { + "attr_one": "one", + "attr_two": "two" + } + } + ] + } + }, + { + "period": { + "begin": "20190823T122810Z", + "end": "20190823T132810Z" + }, + "usage": { + "metric_one": [ + { + "vol": { + "unit": "GiB", + "qty": 2.4 + }, + "rating": { + "price": 0.08 + }, + "groupby": { + "group_one": "one", + "group_two": "two" + }, + "metadata": { + "attr_one": "one", + "attr_two": "two" + } + } + ], + "metric_two": [ + { + "vol": { + "unit": "MB", + "qty": 400.8 + }, + "rating": { + "price": 0.12 + }, + "groupby": { + "group_one": "one", + "group_two": "two" + }, + "metadata": { + "attr_one": "one", + "attr_two": "two" + } + } + ] + } + } + ] +} diff --git a/doc/source/api-reference/v2/dataframes/dataframes.inc b/doc/source/api-reference/v2/dataframes/dataframes.inc new file mode 100644 index 00000000..45cb1805 --- /dev/null +++ b/doc/source/api-reference/v2/dataframes/dataframes.inc @@ -0,0 +1,41 @@ +=================== +Dataframes endpoint +=================== + +Add dataframes into the storage backend +======================================= + +Add dataframes into the storage backend. + +.. rest_method:: POST /v2/dataframes + +.. rest_parameters:: dataframes/dataframes_parameters.yml + + - dataframes: dataframes_body + +Request Example +--------------- + +In the body: + +.. literalinclude:: ./api_samples/dataframes/dataframes_post.json + :language: javascript + +Status codes +------------ + +.. rest_status_code:: success http_status.yml + + - 204 + +.. rest_status_code:: error http_status.yml + + - 400 + - 401 + - 403 + - 405 + +Response +-------- + +No content is to be returned. diff --git a/doc/source/api-reference/v2/dataframes/dataframes_parameters.yml b/doc/source/api-reference/v2/dataframes/dataframes_parameters.yml new file mode 100644 index 00000000..20f254d2 --- /dev/null +++ b/doc/source/api-reference/v2/dataframes/dataframes_parameters.yml @@ -0,0 +1,6 @@ +dataframes_body: + in: body + description: | + List of dataframes to add + type: list + required: true diff --git a/doc/source/api-reference/v2/dataframes/http_status.yml b/doc/source/api-reference/v2/dataframes/http_status.yml new file mode 120000 index 00000000..81a848d3 --- /dev/null +++ b/doc/source/api-reference/v2/dataframes/http_status.yml @@ -0,0 +1 @@ +../http_status.yml \ No newline at end of file diff --git a/doc/source/api-reference/v2/http_status.yml b/doc/source/api-reference/v2/http_status.yml index 1fd3510d..728ef591 100644 --- a/doc/source/api-reference/v2/http_status.yml +++ b/doc/source/api-reference/v2/http_status.yml @@ -7,9 +7,15 @@ 202: default: Request has been accepted for asynchronous processing. +204: + default: Request was successful even though no content is to be returned. + 400: default: Invalid request. +401: + default: Unauthenticated user. + 403: default: Forbidden operation for the authentified user. diff --git a/doc/source/api-reference/v2/index.rst b/doc/source/api-reference/v2/index.rst index d76f3828..71131421 100644 --- a/doc/source/api-reference/v2/index.rst +++ b/doc/source/api-reference/v2/index.rst @@ -1,4 +1,5 @@ .. rest_expand_all:: +.. include:: dataframes/dataframes.inc .. include:: scope/scope.inc .. include:: summary/summary.inc diff --git a/releasenotes/notes/add-dataframes-v2-api-endpoint-601825c344ba0e2d.yaml b/releasenotes/notes/add-dataframes-v2-api-endpoint-601825c344ba0e2d.yaml new file mode 100644 index 00000000..930571cf --- /dev/null +++ b/releasenotes/notes/add-dataframes-v2-api-endpoint-601825c344ba0e2d.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Added a v2 API endpoint allowing to push dataframes into the CloudKitty + storage. This endpoint is available via a ``POST`` request on + ``/v2/dataframes``. Admin privileges are required to use this endpoint.