Add a v2 API endpoint to retrieve DataFrame objects

A new endpoint has been made available to admin users or
scope owners on ``GET /v2/dataframes``. This will allow
end users to retrieve DataFrames in the form of JSON objects
from the CloudKitty storage backend.

Documentation and unit tests are included in this commit.

Change-Id: I784da5565d040f565945fa93d86dcef11faa72f9
This commit is contained in:
Justin Ferrieu 2019-08-27 15:10:34 +00:00 committed by Luka Peschke
parent 8b234a096e
commit a81c01d6e2
10 changed files with 675 additions and 10 deletions

View File

@ -20,6 +20,7 @@ from cloudkitty.api.v2 import base
from cloudkitty.api.v2 import utils as api_utils
from cloudkitty.common import policy
from cloudkitty import dataframe
from cloudkitty import tzutils
class DataFrameList(base.BaseResource):
@ -40,3 +41,50 @@ class DataFrameList(base.BaseResource):
self._storage.push(dataframes)
return {}, 204
@api_utils.paginated
@api_utils.add_input_schema('query', {
voluptuous.Optional('begin'):
api_utils.SingleQueryParam(tzutils.dt_from_iso),
voluptuous.Optional('end'):
api_utils.SingleQueryParam(tzutils.dt_from_iso),
voluptuous.Optional('filters'):
api_utils.SingleDictQueryParam(str, str),
})
@api_utils.add_output_schema({
voluptuous.Required('total'): int,
voluptuous.Required('dataframes'):
[dataframe.DataFrame.as_dict],
})
def get(self,
offset=0,
limit=100,
begin=None,
end=None,
filters={}):
policy.authorize(
flask.request.context,
'dataframes:get',
{'tenant_id': flask.request.context.project_id},
)
begin = begin or tzutils.get_month_start()
end = end or tzutils.get_next_month()
metric_types = [filters.pop('type')] if 'type' in filters else None
results = self._storage.retrieve(
begin=begin, end=end,
filters=filters,
metric_types=metric_types,
offset=offset, limit=limit,
)
if results['total'] < 1:
raise http_exceptions.NotFound(
"No resource found for provided filters.")
return {
'total': results['total'],
'dataframes': results['dataframes'],
}

View File

@ -24,6 +24,12 @@ dataframes_policies = [
description='Add one or several DataFrames',
operations=[{'path': '/v2/dataframes',
'method': 'POST'}]),
policy.DocumentedRuleDefault(
name='dataframes:get',
check_str=base.RULE_ADMIN_OR_OWNER,
description='Get DataFrames',
operations=[{'path': '/v2/dataframes',
'method': 'GET'}]),
]

View File

@ -447,7 +447,33 @@ class MetricsConfFixture(fixture.GabbiFixture):
ck_utils.load_conf = self._original_function
class InfluxStorageDataFixture(NowStorageDataFixture):
class NowInfluxStorageDataFixture(NowStorageDataFixture):
def start_fixture(self):
cli = influx_utils.FakeInfluxClient()
st = storage.get_storage()
st._conn = cli
self._get_storage_patch = mock.patch(
'cloudkitty.storage.get_storage',
new=lambda **kw: st,
)
self._get_storage_patch.start()
super(NowInfluxStorageDataFixture, self).start_fixture()
def initialize_data(self):
data = test_utils.generate_v2_storage_data(
start=tzutils.get_month_start(),
end=tzutils.localized_now().replace(hour=0),
)
self.storage.push([data])
def stop_fixture(self):
self._get_storage_patch.stop()
class InfluxStorageDataFixture(StorageDataFixture):
def start_fixture(self):
cli = influx_utils.FakeInfluxClient()
@ -462,17 +488,20 @@ class InfluxStorageDataFixture(NowStorageDataFixture):
super(InfluxStorageDataFixture, self).start_fixture()
def initialize_data(self):
data = test_utils.generate_v2_storage_data(
start=tzutils.get_month_start(),
end=tzutils.localized_now().replace(hour=0),
)
self.storage.push([data])
def stop_fixture(self):
self._get_storage_patch.stop()
class UTCFixture(fixture.GabbiFixture):
"""Set the local timezone to UTC"""
def start_fixture(self):
self._tzmock = mock.patch('cloudkitty.tzutils._LOCAL_TZ', tz.UTC)
self._tzmock.start()
def stop_fixture(self):
self._tzmock.stop()
def setup_app():
messaging.setup()
# FIXME(sheeprine): Extension fixtures are interacting with transformers

View File

@ -1,6 +1,7 @@
fixtures:
- ConfigFixtureStorageV2
- InfluxStorageDataFixture
- UTCFixture
tests:
- name: Push dataframes
@ -184,3 +185,402 @@ tests:
metadata:
attr_one: one
attr_two: two
- name: fetch period with no data
url: /v2/dataframes
query_parameters:
begin: "2014-01-01T00:00:00"
end: "2015-01-04T00:00:00"
status: 404
response_strings:
- "No resource found for provided filters."
- name: fetch period with no data filtering on tenant_id
url: /v2/dataframes
query_parameters:
begin: "2015-01-01T00:00:00"
end: "2015-01-04T00:00:00"
filters: "project_id:8f82cc70-e50c-466e-8624-24bdea811375"
status: 404
response_strings:
- "No resource found for provided filters."
- name: fetch data for the first tenant without begin time
url: /v2/dataframes
query_parameters:
end: "2015-01-04T00:00:00"
filters: "project_id:8f82cc70-e50c-466e-8624-24bdea811375"
status: 404
response_strings:
- "No resource found for provided filters."
- name: fetch data for the first tenant without end time
url: /v2/dataframes
query_parameters:
begin: "2015-01-04T00:00:00"
filters: "project_id:8f82cc70-e50c-466e-8624-24bdea811375"
status: 200
response_json_paths:
$.dataframes.`len`: 56
- name: fetch data for the first tenant without begin and end time
url: /v2/dataframes
query_parameters:
filters: "project_id:3d9a1b33-482f-42fd-aef9-b575a3da9369"
status: 404
response_strings:
- "No resource found for provided filters."
- name: fetch data for the first tenant when begin time bigger than end time
url: /v2/dataframes
query_parameters:
begin: "2015-01-04T14:00:00"
end: "2015-01-04T13:00:00"
filters: "project_id:8f82cc70-e50c-466e-8624-24bdea811375"
status: 404
response_strings:
- "No resource found for provided filters."
- name: fetch data for the first tenant
url: /v2/dataframes
query_parameters:
begin: "2015-01-04T13:00:00"
end: "2015-01-04T14:00:00"
filters: "project_id:8f82cc70-e50c-466e-8624-24bdea811375"
status: 200
response_json_paths:
$:
total: 4
dataframes:
- usage:
image.size:
- vol:
unit: nothing
qty: 1
rating:
price: 0.121
groupby:
project_id: 8f82cc70-e50c-466e-8624-24bdea811375
fake_meta: 1
metadata:
dummy: true
- vol:
unit: nothing
qty: 1
rating:
price: 0.121
groupby:
project_id: 8f82cc70-e50c-466e-8624-24bdea811375
fake_meta: 1
metadata:
dummy: true
cpu:
- vol:
unit: nothing
qty: 1
rating:
price: 1.337
groupby:
project_id: 8f82cc70-e50c-466e-8624-24bdea811375
fake_meta: 1
metadata:
dummy: true
- vol:
unit: nothing
qty: 1
rating:
price: 1.337
groupby:
project_id: 8f82cc70-e50c-466e-8624-24bdea811375
fake_meta: 1
metadata:
dummy: true
period:
begin: '2015-01-04T13:00:00+00:00'
end: '2015-01-04T14:00:00+00:00'
- name: fetch data for the second tenant
url: /v2/dataframes
query_parameters:
begin: "2015-01-04T13:00:00"
end: "2015-01-04T14:00:00"
filters: "project_id:7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
status: 200
response_json_paths:
$:
total: 4
dataframes:
- usage:
image.size:
- vol:
unit: nothing
qty: 1
rating:
price: 0.121
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
- vol:
unit: nothing
qty: 1
rating:
price: 0.121
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
cpu:
- vol:
unit: nothing
qty: 1
rating:
price: 1.337
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
- vol:
unit: nothing
qty: 1
rating:
price: 1.337
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
period:
begin: '2015-01-04T13:00:00+00:00'
end: '2015-01-04T14:00:00+00:00'
- name: fetch data for multiple tenants
url: /v2/dataframes
query_parameters:
begin: "2015-01-04T13:00:00"
end: "2015-01-04T14:00:00"
status: 200
response_json_paths:
$:
total: 8
dataframes:
- usage:
image.size:
- vol:
unit: nothing
qty: 1
rating:
price: 0.121
groupby:
project_id: 8f82cc70-e50c-466e-8624-24bdea811375
fake_meta: 1
metadata:
dummy: true
- vol:
unit: nothing
qty: 1
rating:
price: 0.121
groupby:
project_id: 8f82cc70-e50c-466e-8624-24bdea811375
fake_meta: 1
metadata:
dummy: true
- vol:
unit: nothing
qty: 1
rating:
price: 0.121
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
- vol:
unit: nothing
qty: 1
rating:
price: 0.121
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
cpu:
- vol:
unit: nothing
qty: 1
rating:
price: 1.337
groupby:
project_id: 8f82cc70-e50c-466e-8624-24bdea811375
fake_meta: 1
metadata:
dummy: true
- vol:
unit: nothing
qty: 1
rating:
price: 1.337
groupby:
project_id: 8f82cc70-e50c-466e-8624-24bdea811375
fake_meta: 1
metadata:
dummy: true
- vol:
unit: nothing
qty: 1
rating:
price: 1.337
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
- vol:
unit: nothing
qty: 1
rating:
price: 1.337
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
period:
begin: '2015-01-04T13:00:00+00:00'
end: '2015-01-04T14:00:00+00:00'
- name: fetch data filtering on cpu service and tenant
url: /v2/dataframes
query_parameters:
begin: "2015-01-04T13:00:00"
end: "2015-01-04T14:00:00"
filters: "type:cpu"
filters: "project_id:7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
status: 200
response_json_paths:
$:
total: 4
dataframes:
- usage:
image.size:
- vol:
unit: nothing
qty: 1
rating:
price: 0.121
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
- vol:
unit: nothing
qty: 1
rating:
price: 0.121
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
cpu:
- vol:
unit: nothing
qty: 1
rating:
price: 1.337
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
- vol:
unit: nothing
qty: 1
rating:
price: 1.337
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
period:
begin: '2015-01-04T13:00:00+00:00'
end: '2015-01-04T14:00:00+00:00'
- name: fetch data filtering on image service and tenant
url: /v2/dataframes
query_parameters:
begin: "2015-01-04T13:00:00"
end: "2015-01-04T14:00:00"
filters: "type:image.size"
filters: "project_id:7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
status: 200
response_json_paths:
$:
total: 4
dataframes:
- usage:
image.size:
- vol:
unit: nothing
qty: 1
rating:
price: 0.121
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
- vol:
unit: nothing
qty: 1
rating:
price: 0.121
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
cpu:
- vol:
unit: nothing
qty: 1
rating:
price: 1.337
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
- vol:
unit: nothing
qty: 1
rating:
price: 1.337
groupby:
project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
fake_meta: 1
metadata:
dummy: true
period:
begin: '2015-01-04T13:00:00+00:00'
end: '2015-01-04T14:00:00+00:00'
- name: fetch data filtering on service with no data and tenant
url: /v2/dataframes
query_parameters:
begin: "2015-01-04T13:00:00"
end: "2015-01-04T14:00:00"
filters: "type:volume"
filters: "project_id:7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
status: 200
response_json_paths:
$.dataframes.`len`: 1

View File

@ -1,6 +1,6 @@
fixtures:
- ConfigFixtureStorageV2
- InfluxStorageDataFixture
- NowInfluxStorageDataFixture
tests:
- name: Get a summary

View File

@ -88,6 +88,10 @@
# POST /v2/dataframes
#"dataframes:add": "role:admin"
# Get DataFrames
# GET /v2/dataframes
#"dataframes:get": "rule:admin_or_owner"
# Get the state of one or several scopes
# GET /v2/scope
#"scope:get_state": "role:admin"

View File

@ -0,0 +1,78 @@
{
"total": 3,
"dataframes": [
{
"usage": {
"metric_one": [
{
"vol": {
"unit": "GiB",
"qty": 1.2
},
"rating": {
"price": 0.04
},
"groupby": {
"group_one": "one",
"group_two": "two"
},
"metadata": {
"attr_one": "one",
"attr_two": "two"
}
}
],
"metric_two": [
{
"vol": {
"unit": "GiB",
"qty": 1.2
},
"rating": {
"price": 0.04
},
"groupby": {
"group_one": "one",
"group_two": "two"
},
"metadata": {
"attr_one": "one",
"attr_two": "two"
}
}
]
},
"period": {
"begin": "2019-07-23T12:28:10+00:00",
"end": "2019-07-23T13:28:10+00:00"
}
},
{
"usage": {
"volume.size": [
{
"vol": {
"unit": "GiB",
"qty": 1.9
},
"rating": {
"price": 3.8
},
"groupby": {
"project_id": "8ace6f139a1742548e09f1e446bc9737",
"user_id": "b28fd3f448c34c17bf70e32886900eed",
"id": "be966c6d-78a0-42cf-bab9-e833ed996dee"
},
"metadata": {
"volume_type": ""
}
}
]
},
"period": {
"begin": "2019-08-01T01:00:00+00:00",
"end": "2019-08-01T02:00:00+00:00"
}
}
]
}

View File

@ -39,3 +39,47 @@ Response
--------
No content is to be returned.
Get dataframes from the storage backend
============================================
Get dataframes from the storage backend.
.. rest_method:: GET /v2/dataframes
.. rest_parameters:: dataframes/dataframes_parameters.yml
- limit: limit
- offset: offset
- begin: begin
- end: end
- filters: filters
Status codes
------------
.. rest_status_code:: success http_status.yml
- 200
.. rest_status_code:: error http_status.yml
- 400
- 401
- 403
- 405
Response
--------
.. rest_parameters:: dataframes/dataframes_parameters.yml
- total: total_resp
- dataframes: dataframes_resp
Response Example
----------------
.. literalinclude:: ./api_samples/dataframes/dataframes_get.json
:language: javascript

View File

@ -1,6 +1,55 @@
dataframes_body:
in: body
description: |
List of dataframes to add
List of dataframes to add.
type: list
required: true
dataframes_resp:
in: body
description: |
List of dataframes matching the query parameters.
type: list
required: true
total_resp:
in: body
description: |
Total of datapoints matching the query parameters.
type: int
required: true
limit:
in: query
description: |
For pagination. The maximum number of results to return.
type: int
required: false
offset:
in: query
description: |
For pagination. The index of the first element that should be returned.
type: int
required: false
filters:
in: query
description: |
Optional filters.
type: dict
required: false
begin:
in: query
description: |
Begin of the period for which the dataframes are required.
type: iso8601 timestamp
required: false
end:
in: query
description: |
End of the period for which the dataframes are required.
type: iso8601 timestamp
required: false

View File

@ -0,0 +1,7 @@
---
features:
- |
Added a v2 API endpoint allowing to retrieve dataframes from
the CloudKitty storage. This endpoint is available via a ``GET``
request on ``/v2/dataframes``. Being the owner of the scope or
having admin privileges are required to use this endpoint.