openstack/cloudkitty
Code Issues Proposed changes

Add a v2 API endpoint to push DataFrame objects

Browse Source 
A new endpoint has been made available to admin users on
``POST /v2/dataframes``. This will allow end users to push
DataFrames in the form of JSON objects into the CloudKitty
storage.

Documentation and unit tests are included in this commit.

Depends-On: https://review.opendev.org/#/c/668669/
Change-Id: I42641462ecbac89f400a257805fc99f4027903b3
Story: 2005890
Task: 35953
This commit is contained in:
Justin Ferrieu 2019-08-21 12:21:05 +00:00 committed by Luka Peschke
parent 7ea4dda98e
commit 6e8efde432
15 changed files with 456 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
cloudkitty/api/v1/controllers/storage.py
View File

@ -80,7 +80,13 @@ class DataFramesController(rest.RestController):
volume=point.qty,
rating=point.price)
if frame_tenant is None:
frame_tenant = point.desc[scope_key]
# NOTE(jferrieu): Since DataFrame/DataPoint
# implementation patch we cannot guarantee
# anymore that a DataFrame does contain a scope_id
# therefore the __UNDEF__ default value has been
# retained to maintain backward compatibility
# if it would occur being absent
frame_tenant = point.desc.get(scope_key, '__UNDEF__')
resources.append(resource)
dataframe = storage_models.DataFrame(
begin=tzutils.local_to_utc(frame.start, naive=True),

1
cloudkitty/api/v2/__init__.py
View File

@ -33,6 +33,7 @@ RESOURCE_SCHEMA = voluptuous.Schema({
API_MODULES = [
'cloudkitty.api.v2.scope',
'cloudkitty.api.v2.dataframes',
'cloudkitty.api.v2.summary',
]

26
cloudkitty/api/v2/dataframes/__init__.py Normal file
View File

@ -0,0 +1,26 @@
# Copyright 2019 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
from cloudkitty.api.v2 import utils as api_utils
def init(app):
api_utils.do_init(app, 'dataframes', [
{
'module': __name__ + '.' + 'dataframes',
'resource_class': 'DataFrameList',
'url': '',
},
])
return app

42
cloudkitty/api/v2/dataframes/dataframes.py Normal file
View File

@ -0,0 +1,42 @@
# Copyright 2019 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import flask
import voluptuous
from werkzeug import exceptions as http_exceptions
from cloudkitty.api.v2 import base
from cloudkitty.api.v2 import utils as api_utils
from cloudkitty.common import policy
from cloudkitty import dataframe
class DataFrameList(base.BaseResource):
@api_utils.add_input_schema('body', {
voluptuous.Required('dataframes'): [dataframe.DataFrame.from_dict],
})
def post(self, dataframes=[]):
policy.authorize(
flask.request.context,
'dataframes:add',
{},
)
if not dataframes:
raise http_exceptions.BadRequest(
"Parameter dataframes must not be empty.")
self._storage.push(dataframes)
return {}, 204

2
cloudkitty/common/policies/__init__.py
View File

@ -21,6 +21,7 @@ from cloudkitty.common.policies.v1 import info as v1_info
from cloudkitty.common.policies.v1 import rating as v1_rating
from cloudkitty.common.policies.v1 import report as v1_report
from cloudkitty.common.policies.v1 import storage as v1_storage
from cloudkitty.common.policies.v2 import dataframes as v2_dataframes
from cloudkitty.common.policies.v2 import scope as v2_scope
from cloudkitty.common.policies.v2 import summary as v2_summary
@ -33,6 +34,7 @@ def list_rules():
v1_rating.list_rules(),
v1_report.list_rules(),
v1_storage.list_rules(),
v2_dataframes.list_rules(),
v2_scope.list_rules(),
v2_summary.list_rules(),
)

31
cloudkitty/common/policies/v2/dataframes.py Normal file
View File

@ -0,0 +1,31 @@
# Copyright 2019 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
from oslo_policy import policy
from cloudkitty.common.policies import base
dataframes_policies = [
policy.DocumentedRuleDefault(
name='dataframes:add',
check_str=base.ROLE_ADMIN,
description='Add one or several DataFrames',
operations=[{'path': '/v2/dataframes',
'method': 'POST'}]),
]
def list_rules():
return dataframes_policies

186
cloudkitty/tests/gabbi/gabbits/v2-dataframes.yaml Normal file
View File

@ -0,0 +1,186 @@
fixtures:
- ConfigFixtureStorageV2
- InfluxStorageDataFixture
tests:
- name: Push dataframes
url: /v2/dataframes
method: POST
status: 204
request_headers:
content-type: application/json
data:
dataframes:
- period:
begin: 20190723T122810Z
end: 20190723T132810Z
usage:
metric_one:
- vol:
unit: GiB
qty: 1.2
rating:
price: 0.04
groupby:
group_one: one
group_two: two
metadata:
attr_one: one
attr_two: two
metric_two:
- vol:
unit: GiB
qty: 1.2
rating:
price: 0.04
groupby:
group_one: one
group_two: two
metadata:
attr_one: one
attr_two: two
- period:
begin: 20190723T122810Z
end: 20190723T132810Z
usage:
metric_one:
- vol:
unit: GiB
qty: 1.2
rating:
price: 0.04
groupby:
group_one: one
group_two: two
metadata:
attr_one: one
attr_two: two
metric_two:
- vol:
unit: GiB
qty: 1.2
rating:
price: 0.04
groupby:
group_one: one
group_two: two
metadata:
attr_one: one
attr_two: two
- name: Push dataframes with empty dataframes
url: /v2/dataframes
method: POST
status: 400
request_headers:
content-type: application/json
data:
dataframes: []
response_strings:
- "Parameter dataframes must not be empty."
- name: Push dataframes with missing key
url: /v2/dataframes
method: POST
status: 400
request_headers:
content-type: application/json
data:
dataframes:
- period:
begin: 20190723T122810Z
end: 20190723T132810Z
usage:
metric_one:
- vol:
unit: GiB
qty: 1.2
rating:
price: 0.04
groupby:
group_one: one
group_two: two
metadata:
attr_one: one
attr_two: two
metric_two:
- vol:
unit: GiB
qty: 1.2
rating:
price: 0.04
groupby:
group_one: one
group_two: two
metadata:
attr_one: one
attr_two: two
- period:
begin: 20190723T122810Z
end: 20190723T132810Z
- name: Push dataframe with malformed datapoint
url: /v2/dataframes
method: POST
status: 400
request_headers:
content-type: application/json
data:
dataframes:
- period:
begin: 20190723T122810Z
end: 20190723T132810Z
usage:
metric_one:
- vol:
unit: GiB
qty: 1.2
metric_two:
- vol:
unit: GiB
qty: 1.2
rating:
price: 0.04
groupby:
group_one: one
group_two: two
metadata:
attr_one: one
attr_two: two
- name: Push dataframe with malformed datetimes
url: /v2/dataframes
method: POST
status: 400
request_headers:
content-type: application/json
data:
dataframes:
- period:
begin: 20190723TZ
end: 20190723TZ
usage:
metric_one:
- vol:
unit: GiB
qty: 1.2
rating:
price: 0.04
groupby:
group_one: one
group_two: two
metadata:
attr_one: one
attr_two: two
metric_two:
- vol:
unit: GiB
qty: 1.2
rating:
price: 0.04
groupby:
group_one: one
group_two: two
metadata:
attr_one: one
attr_two: two

4
doc/source/_static/cloudkitty.policy.yaml.sample
View File

@ -84,6 +84,10 @@
# GET /v1/storage/dataframes
#"storage:list_data_frames": "rule:admin_or_owner"
# Add one or several DataFrames
# POST /v2/dataframes
#"dataframes:add": "role:admin"
# Get the state of one or several scopes
# GET /v2/scope
#"scope:get_state": "role:admin"

96
doc/source/api-reference/v2/api_samples/dataframes/dataframes_post.json Normal file
View File

@ -0,0 +1,96 @@
{
"dataframes": [
{
"period": {
"begin": "20190723T122810Z",
"end": "20190723T132810Z"
},
"usage": {
"metric_one": [
{
"vol": {
"unit": "GiB",
"qty": 1.2
},
"rating": {
"price": 0.04
},
"groupby": {
"group_one": "one",
"group_two": "two"
},
"metadata": {
"attr_one": "one",
"attr_two": "two"
}
}
],
"metric_two": [
{
"vol": {
"unit": "MB",
"qty": 200.4
},
"rating": {
"price": 0.06
},
"groupby": {
"group_one": "one",
"group_two": "two"
},
"metadata": {
"attr_one": "one",
"attr_two": "two"
}
}
]
}
},
{
"period": {
"begin": "20190823T122810Z",
"end": "20190823T132810Z"
},
"usage": {
"metric_one": [
{
"vol": {
"unit": "GiB",
"qty": 2.4
},
"rating": {
"price": 0.08
},
"groupby": {
"group_one": "one",
"group_two": "two"
},
"metadata": {
"attr_one": "one",
"attr_two": "two"
}
}
],
"metric_two": [
{
"vol": {
"unit": "MB",
"qty": 400.8
},
"rating": {
"price": 0.12
},
"groupby": {
"group_one": "one",
"group_two": "two"
},
"metadata": {
"attr_one": "one",
"attr_two": "two"
}
}
]
}
}
]
}

41
doc/source/api-reference/v2/dataframes/dataframes.inc Normal file
View File

@ -0,0 +1,41 @@
===================
Dataframes endpoint
===================
Add dataframes into the storage backend
=======================================
Add dataframes into the storage backend.
.. rest_method:: POST /v2/dataframes
.. rest_parameters:: dataframes/dataframes_parameters.yml
- dataframes: dataframes_body
Request Example
---------------
In the body:
.. literalinclude:: ./api_samples/dataframes/dataframes_post.json
:language: javascript
Status codes
------------
.. rest_status_code:: success http_status.yml
- 204
.. rest_status_code:: error http_status.yml
- 400
- 401
- 403
- 405
Response
--------
No content is to be returned.

6
doc/source/api-reference/v2/dataframes/dataframes_parameters.yml Normal file
View File

@ -0,0 +1,6 @@
dataframes_body:
in: body
description: |
List of dataframes to add
type: list
required: true

1
doc/source/api-reference/v2/dataframes/http_status.yml Symbolic link
View File

@ -0,0 +1 @@
../http_status.yml

6
doc/source/api-reference/v2/http_status.yml
View File

@ -7,9 +7,15 @@
202:
default: Request has been accepted for asynchronous processing.
204:
default: Request was successful even though no content is to be returned.
400:
default: Invalid request.
401:
default: Unauthenticated user.
403:
default: Forbidden operation for the authentified user.

1
doc/source/api-reference/v2/index.rst
View File

@ -1,4 +1,5 @@
.. rest_expand_all::
.. include:: dataframes/dataframes.inc
.. include:: scope/scope.inc
.. include:: summary/summary.inc

6
releasenotes/notes/add-dataframes-v2-api-endpoint-601825c344ba0e2d.yaml Normal file
View File

@ -0,0 +1,6 @@
---
features:
- |
Added a v2 API endpoint allowing to push dataframes into the CloudKitty
storage. This endpoint is available via a ``POST`` request on
``/v2/dataframes``. Admin privileges are required to use this endpoint.