openstack/cloudkitty
Code Issues Proposed changes

Add a v2 API endpoint to retrieve DataFrame objects

Browse Source 
A new endpoint has been made available to admin users or
scope owners on ``GET /v2/dataframes``. This will allow
end users to retrieve DataFrames in the form of JSON objects
from the CloudKitty storage backend.

Documentation and unit tests are included in this commit.

Change-Id: I784da5565d040f565945fa93d86dcef11faa72f9
This commit is contained in:
Justin Ferrieu 2019-08-27 15:10:34 +00:00 committed by Luka Peschke
parent 8b234a096e
commit a81c01d6e2
10 changed files with 675 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
cloudkitty/api/v2/dataframes/dataframes.py
View File

@ -20,6 +20,7 @@ from cloudkitty.api.v2 import base
from cloudkitty.api.v2 import utils as api_utils
from cloudkitty.common import policy
from cloudkitty import dataframe
from cloudkitty import tzutils
class DataFrameList(base.BaseResource):
@ -40,3 +41,50 @@ class DataFrameList(base.BaseResource):
self._storage.push(dataframes)
return {}, 204
@api_utils.paginated
@api_utils.add_input_schema('query', {
voluptuous.Optional('begin'):
api_utils.SingleQueryParam(tzutils.dt_from_iso),
voluptuous.Optional('end'):
api_utils.SingleQueryParam(tzutils.dt_from_iso),
voluptuous.Optional('filters'):
api_utils.SingleDictQueryParam(str, str),
})
@api_utils.add_output_schema({
voluptuous.Required('total'): int,
voluptuous.Required('dataframes'):
[dataframe.DataFrame.as_dict],
})
def get(self,
offset=0,
limit=100,
begin=None,
end=None,
filters={}):
policy.authorize(
flask.request.context,
'dataframes:get',
{'tenant_id': flask.request.context.project_id},
)
begin = begin or tzutils.get_month_start()
end = end or tzutils.get_next_month()
metric_types = [filters.pop('type')] if 'type' in filters else None
results = self._storage.retrieve(
begin=begin, end=end,
filters=filters,
metric_types=metric_types,
offset=offset, limit=limit,
)
if results['total'] < 1:
raise http_exceptions.NotFound(
"No resource found for provided filters.")
return {
'total': results['total'],
'dataframes': results['dataframes'],
}

6
cloudkitty/common/policies/v2/dataframes.py
View File

@ -24,6 +24,12 @@ dataframes_policies = [
description='Add one or several DataFrames',
operations=[{'path': '/v2/dataframes',
'method': 'POST'}]),
policy.DocumentedRuleDefault(
name='dataframes:get',
check_str=base.RULE_ADMIN_OR_OWNER,
description='Get DataFrames',
operations=[{'path': '/v2/dataframes',
'method': 'GET'}]),
]

45
cloudkitty/tests/gabbi/fixtures.py
View File

@ -447,7 +447,33 @@ class MetricsConfFixture(fixture.GabbiFixture):
ck_utils.load_conf = self._original_function
class InfluxStorageDataFixture(NowStorageDataFixture):
class NowInfluxStorageDataFixture(NowStorageDataFixture):
def start_fixture(self):
cli = influx_utils.FakeInfluxClient()
st = storage.get_storage()
st._conn = cli
self._get_storage_patch = mock.patch(
'cloudkitty.storage.get_storage',
new=lambda **kw: st,
)
self._get_storage_patch.start()
super(NowInfluxStorageDataFixture, self).start_fixture()
def initialize_data(self):
data = test_utils.generate_v2_storage_data(
start=tzutils.get_month_start(),
end=tzutils.localized_now().replace(hour=0),
)
self.storage.push([data])
def stop_fixture(self):
self._get_storage_patch.stop()
class InfluxStorageDataFixture(StorageDataFixture):
def start_fixture(self):
cli = influx_utils.FakeInfluxClient()
@ -462,17 +488,20 @@ class InfluxStorageDataFixture(NowStorageDataFixture):
super(InfluxStorageDataFixture, self).start_fixture()
def initialize_data(self):
data = test_utils.generate_v2_storage_data(
start=tzutils.get_month_start(),
end=tzutils.localized_now().replace(hour=0),
)
self.storage.push([data])
def stop_fixture(self):
self._get_storage_patch.stop()
class UTCFixture(fixture.GabbiFixture):
"""Set the local timezone to UTC"""
def start_fixture(self):
self._tzmock = mock.patch('cloudkitty.tzutils._LOCAL_TZ', tz.UTC)
self._tzmock.start()
def stop_fixture(self):
self._tzmock.stop()
def setup_app():
messaging.setup()
# FIXME(sheeprine): Extension fixtures are interacting with transformers

400
cloudkitty/tests/gabbi/gabbits/v2-dataframes.yaml
View File

@ -1,6 +1,7 @@
fixtures:
- ConfigFixtureStorageV2
- InfluxStorageDataFixture
- UTCFixture
tests:
- name: Push dataframes
@ -184,3 +185,402 @@ tests:
metadata:
attr_one: one
attr_two: two
- name: fetch period with no data
url: /v2/dataframes
query_parameters:
begin: "2014-01-01T00:00:00"
end: "2015-01-04T00:00:00"
status: 404
response_strings:
- "No resource found for provided filters."
- name: fetch period with no data filtering on tenant_id
url: /v2/dataframes
query_parameters:
begin: "2015-01-01T00:00:00"
end: "2015-01-04T00:00:00"
filters: "project_id:8f82cc70-e50c-466e-8624-24bdea811375"
status: 404
response_strings:
- "No resource found for provided filters."
- name: fetch data for the first tenant without begin time
url: /v2/dataframes
query_parameters:
end: "2015-01-04T00:00:00"
filters: "project_id:8f82cc70-e50c-466e-8624-24bdea811375"
status: 404
response_strings:
- "No resource found for provided filters."
- name: fetch data for the first tenant without end time
url: /v2/dataframes
query_parameters:
begin: "2015-01-04T00:00:00"
filters: "project_id:8f82cc70-e50c-466e-8624-24bdea811375"
status: 200
response_json_paths:
$.dataframes.`len`: 56
- name: fetch data for the first tenant without begin and end time
url: /v2/dataframes
query_parameters:
filters: "project_id:3d9a1b33-482f-42fd-aef9-b575a3da9369"
status: 404
response_strings:
- "No resource found for provided filters."
- name: fetch data for the first tenant when begin time bigger than end time
url: /v2/dataframes
query_parameters:
begin: "2015-01-04T14:00:00"
end: "2015-01-04T13:00:00"
filters: "project_id:8f82cc70-e50c-466e-8624-24bdea811375"
status: 404
response_strings:
- "No resource found for provided filters."
- name: fetch data for the first tenant
url: /v2/dataframes
query_parameters:
begin: "2015-01-04T13:00:00"
end: "2015-01-04T14:00:00"
filters: "project_id:8f82cc70-e50c-466e-8624-24bdea811375"
status: 200
response_json_paths:
$:
total: 4
dataframes:
- usage:
image.size:
- vol:
unit: nothing
qty: 1
rating:
price: 0.121
groupby:
project_id: 8f82cc70-e50c-466e-8624-24bdea811375
fake_meta: 1
metadata:
dummy: true
- vol:
unit: nothing
qty: 1
rating:
price: 0.121
groupby:
project_id: 8f82cc70-e50c-466e-8624-24bdea811375
fake_meta: 1
metadata:
dummy: true
cpu:
- vol:
unit: nothing
qty: 1
rating:
price: 1.337
groupby:
project_id: 8f82cc70-e50c-466e-8624-24bdea811375
fake_meta: 1
metadata:
dummy: true
- vol:
unit: nothing
qty: 1
rating:
price: 1.337
groupby:
project_id: 8f82cc70-e50c-466e-8624-24bdea811375
fake_meta: 1
metadata:
dummy: true
period:
begin: '2015-01-04T13:00:00+00:00'
end: '2015-01-04T14:00:00+00:00'
- name: fetch data for the second tenant
url: /v2/dataframes
query_parameters:
begin: "2015-01-04T13:00:00"
end: "2015-01-04T14:00:00"
filters: "project_id:7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
status: 200
response_json_paths:
$:
total: 4
dataframes:
- usage:
image.size:
- vol:
unit: nothing
qty: 1
rating:
price: 0.121
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
- vol:
unit: nothing
qty: 1
rating:
price: 0.121
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
cpu:
- vol:
unit: nothing
qty: 1
rating:
price: 1.337
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
- vol:
unit: nothing
qty: 1
rating:
price: 1.337
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
period:
begin: '2015-01-04T13:00:00+00:00'
end: '2015-01-04T14:00:00+00:00'
- name: fetch data for multiple tenants
url: /v2/dataframes
query_parameters:
begin: "2015-01-04T13:00:00"
end: "2015-01-04T14:00:00"
status: 200
response_json_paths:
$:
total: 8
dataframes:
- usage:
image.size:
- vol:
unit: nothing
qty: 1
rating:
price: 0.121
groupby:
project_id: 8f82cc70-e50c-466e-8624-24bdea811375
fake_meta: 1
metadata:
dummy: true
- vol:
unit: nothing
qty: 1
rating:
price: 0.121
groupby:
project_id: 8f82cc70-e50c-466e-8624-24bdea811375
fake_meta: 1
metadata:
dummy: true
- vol:
unit: nothing
qty: 1
rating:
price: 0.121
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
- vol:
unit: nothing
qty: 1
rating:
price: 0.121
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
cpu:
- vol:
unit: nothing
qty: 1
rating:
price: 1.337
groupby:
project_id: 8f82cc70-e50c-466e-8624-24bdea811375
fake_meta: 1
metadata:
dummy: true
- vol:
unit: nothing
qty: 1
rating:
price: 1.337
groupby:
project_id: 8f82cc70-e50c-466e-8624-24bdea811375
fake_meta: 1
metadata:
dummy: true
- vol:
unit: nothing
qty: 1
rating:
price: 1.337
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
- vol:
unit: nothing
qty: 1
rating:
price: 1.337
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
period:
begin: '2015-01-04T13:00:00+00:00'
end: '2015-01-04T14:00:00+00:00'
- name: fetch data filtering on cpu service and tenant
url: /v2/dataframes
query_parameters:
begin: "2015-01-04T13:00:00"
end: "2015-01-04T14:00:00"
filters: "type:cpu"
filters: "project_id:7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
status: 200
response_json_paths:
$:
total: 4
dataframes:
- usage:
image.size:
- vol:
unit: nothing
qty: 1
rating:
price: 0.121
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
- vol:
unit: nothing
qty: 1
rating:
price: 0.121
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
cpu:
- vol:
unit: nothing
qty: 1
rating:
price: 1.337
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
- vol:
unit: nothing
qty: 1
rating:
price: 1.337
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
period:
begin: '2015-01-04T13:00:00+00:00'
end: '2015-01-04T14:00:00+00:00'
- name: fetch data filtering on image service and tenant
url: /v2/dataframes
query_parameters:
begin: "2015-01-04T13:00:00"
end: "2015-01-04T14:00:00"
filters: "type:image.size"
filters: "project_id:7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
status: 200
response_json_paths:
$:
total: 4
dataframes:
- usage:
image.size:
- vol:
unit: nothing
qty: 1
rating:
price: 0.121
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
- vol:
unit: nothing
qty: 1
rating:
price: 0.121
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
cpu:
- vol:
unit: nothing
qty: 1
rating:
price: 1.337
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
- vol:
unit: nothing
qty: 1
rating:
price: 1.337
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
period:
begin: '2015-01-04T13:00:00+00:00'
end: '2015-01-04T14:00:00+00:00'
- name: fetch data filtering on service with no data and tenant
url: /v2/dataframes
query_parameters:
begin: "2015-01-04T13:00:00"
end: "2015-01-04T14:00:00"
filters: "type:volume"
filters: "project_id:7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
status: 200
response_json_paths:
$.dataframes.`len`: 1

2
cloudkitty/tests/gabbi/gabbits/v2-summary.yaml
View File

@ -1,6 +1,6 @@
fixtures:
- ConfigFixtureStorageV2
- InfluxStorageDataFixture
- NowInfluxStorageDataFixture
tests:
- name: Get a summary

4
doc/source/_static/cloudkitty.policy.yaml.sample
View File

@ -88,6 +88,10 @@
# POST /v2/dataframes
#"dataframes:add": "role:admin"
# Get DataFrames
# GET /v2/dataframes
#"dataframes:get": "rule:admin_or_owner"
# Get the state of one or several scopes
# GET /v2/scope
#"scope:get_state": "role:admin"

78
doc/source/api-reference/v2/api_samples/dataframes/dataframes_get.json Normal file
View File

@ -0,0 +1,78 @@
{
"total": 3,
"dataframes": [
{
"usage": {
"metric_one": [
{
"vol": {
"unit": "GiB",
"qty": 1.2
},
"rating": {
"price": 0.04
},
"groupby": {
"group_one": "one",
"group_two": "two"
},
"metadata": {
"attr_one": "one",
"attr_two": "two"
}
}
],
"metric_two": [
{
"vol": {
"unit": "GiB",
"qty": 1.2
},
"rating": {
"price": 0.04
},
"groupby": {
"group_one": "one",
"group_two": "two"
},
"metadata": {
"attr_one": "one",
"attr_two": "two"
}
}
]
},
"period": {
"begin": "2019-07-23T12:28:10+00:00",
"end": "2019-07-23T13:28:10+00:00"
}
},
{
"usage": {
"volume.size": [
{
"vol": {
"unit": "GiB",
"qty": 1.9
},
"rating": {
"price": 3.8
},
"groupby": {
"project_id": "8ace6f139a1742548e09f1e446bc9737",
"user_id": "b28fd3f448c34c17bf70e32886900eed",
"id": "be966c6d-78a0-42cf-bab9-e833ed996dee"
},
"metadata": {
"volume_type": ""
}
}
]
},
"period": {
"begin": "2019-08-01T01:00:00+00:00",
"end": "2019-08-01T02:00:00+00:00"
}
}
]
}

44
doc/source/api-reference/v2/dataframes/dataframes.inc
View File

@ -39,3 +39,47 @@ Response
--------
No content is to be returned.
Get dataframes from the storage backend
============================================
Get dataframes from the storage backend.
.. rest_method:: GET /v2/dataframes
.. rest_parameters:: dataframes/dataframes_parameters.yml
- limit: limit
- offset: offset
- begin: begin
- end: end
- filters: filters
Status codes
------------
.. rest_status_code:: success http_status.yml
- 200
.. rest_status_code:: error http_status.yml
- 400
- 401
- 403
- 405
Response
--------
.. rest_parameters:: dataframes/dataframes_parameters.yml
- total: total_resp
- dataframes: dataframes_resp
Response Example
----------------
.. literalinclude:: ./api_samples/dataframes/dataframes_get.json
:language: javascript

51
doc/source/api-reference/v2/dataframes/dataframes_parameters.yml
View File

@ -1,6 +1,55 @@
dataframes_body:
in: body
description: |
List of dataframes to add
List of dataframes to add.
type: list
required: true
dataframes_resp:
in: body
description: |
List of dataframes matching the query parameters.
type: list
required: true
total_resp:
in: body
description: |
Total of datapoints matching the query parameters.
type: int
required: true
limit:
in: query
description: |
For pagination. The maximum number of results to return.
type: int
required: false
offset:
in: query
description: |
For pagination. The index of the first element that should be returned.
type: int
required: false
filters:
in: query
description: |
Optional filters.
type: dict
required: false
begin:
in: query
description: |
Begin of the period for which the dataframes are required.
type: iso8601 timestamp
required: false
end:
in: query
description: |
End of the period for which the dataframes are required.
type: iso8601 timestamp
required: false

7
releasenotes/notes/get-dataframes-v2-api-endpoint-3a4625c6008a5fca.yaml Normal file
View File

@ -0,0 +1,7 @@
---
features:
- |
Added a v2 API endpoint allowing to retrieve dataframes from
the CloudKitty storage. This endpoint is available via a ``GET``
request on ``/v2/dataframes``. Being the owner of the scope or
having admin privileges are required to use this endpoint.