From acd2fec6a13a805e1de405efca40795c3d0b0fc6 Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Tue, 14 May 2019 11:30:13 +0200 Subject: [PATCH] Add a v2 summary endpoint This adds an endpoint on /v2/summary, as a replacement for /v1/summary. It allows to retrieve a summary from the cloudkitty, which can be grouped on custom attributes and filters. Change-Id: I99bff44b24d3dcec2da97281f0b491ac333ed395 Story: 2005664 Task: 30961 --- cloudkitty/api/v2/__init__.py | 1 + cloudkitty/api/v2/base.py | 5 ++ cloudkitty/api/v2/summary/__init__.py | 26 ++++++ cloudkitty/api/v2/summary/summary.py | 64 ++++++++++++++ cloudkitty/api/v2/utils.py | 72 ++++++++++++++++ cloudkitty/common/policies/__init__.py | 2 + cloudkitty/common/policies/v2/summary.py | 30 +++++++ cloudkitty/tests/api/v2/test_utils.py | 67 +++++++++++++++ cloudkitty/tests/gabbi/fixtures.py | 27 ++++++ .../tests/gabbi/gabbits/v2-summary.yaml | 60 ++++++++++++++ .../_static/cloudkitty.policy.yaml.sample | 4 + .../v2/api_samples/summary/summary_get.json | 45 ++++++++++ doc/source/api-reference/v2/index.rst | 1 + .../api-reference/v2/summary/summary.inc | 83 +++++++++++++++++++ .../v2/summary/summary_parameters.yml | 79 ++++++++++++++++++ 15 files changed, 566 insertions(+) create mode 100644 cloudkitty/api/v2/summary/__init__.py create mode 100644 cloudkitty/api/v2/summary/summary.py create mode 100644 cloudkitty/common/policies/v2/summary.py create mode 100644 cloudkitty/tests/gabbi/gabbits/v2-summary.yaml create mode 100644 doc/source/api-reference/v2/api_samples/summary/summary_get.json create mode 100644 doc/source/api-reference/v2/summary/summary.inc create mode 100644 doc/source/api-reference/v2/summary/summary_parameters.yml diff --git a/cloudkitty/api/v2/__init__.py b/cloudkitty/api/v2/__init__.py index e5f8a5e5..ef785197 100644 --- a/cloudkitty/api/v2/__init__.py +++ b/cloudkitty/api/v2/__init__.py @@ -33,6 +33,7 @@ RESOURCE_SCHEMA = voluptuous.Schema({ API_MODULES = [ 'cloudkitty.api.v2.scope', + 'cloudkitty.api.v2.summary', ] diff --git a/cloudkitty/api/v2/base.py b/cloudkitty/api/v2/base.py index 97c48a06..d43f92a6 100644 --- a/cloudkitty/api/v2/base.py +++ b/cloudkitty/api/v2/base.py @@ -16,6 +16,7 @@ import flask_restful from werkzeug import exceptions as http_exceptions from cloudkitty.common import policy +from cloudkitty import storage class BaseResource(flask_restful.Resource): @@ -31,3 +32,7 @@ class BaseResource(flask_restful.Resource): except policy.PolicyNotAuthorized: raise http_exceptions.Forbidden( "You are not authorized to perform this action") + + def __init__(self, *args, **kwargs): + super(BaseResource, self).__init__(*args, **kwargs) + self._storage = storage.get_storage() diff --git a/cloudkitty/api/v2/summary/__init__.py b/cloudkitty/api/v2/summary/__init__.py new file mode 100644 index 00000000..685ecaf3 --- /dev/null +++ b/cloudkitty/api/v2/summary/__init__.py @@ -0,0 +1,26 @@ +# Copyright 2019 Objectif Libre +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +from cloudkitty.api.v2 import utils as api_utils + + +def init(app): + api_utils.do_init(app, 'summary', [ + { + 'module': __name__ + '.' + 'summary', + 'resource_class': 'Summary', + 'url': '', + }, + ]) + return app diff --git a/cloudkitty/api/v2/summary/summary.py b/cloudkitty/api/v2/summary/summary.py new file mode 100644 index 00000000..7543586d --- /dev/null +++ b/cloudkitty/api/v2/summary/summary.py @@ -0,0 +1,64 @@ +# Copyright 2019 Objectif Libre +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +import flask +import voluptuous + +from cloudkitty.api.v2 import base +from cloudkitty.api.v2 import utils as api_utils +from cloudkitty.common import policy +from cloudkitty import utils + + +class Summary(base.BaseResource): + """Resource allowing to retrieve a rating summary.""" + + @api_utils.paginated + @api_utils.add_input_schema('query', { + voluptuous.Optional('groupby'): api_utils.MultiQueryParam(str), + voluptuous.Optional('filters'): + api_utils.SingleDictQueryParam(str, str), + voluptuous.Optional('begin'): voluptuous.Coerce(utils.iso2dt), + voluptuous.Optional('end'): voluptuous.Coerce(utils.iso2dt), + }) + def get(self, groupby=None, filters={}, + begin=None, end=None, + offset=0, limit=100): + policy.authorize( + flask.request.context, + 'summary:get_summary', + {'tenant_id': flask.request.context.project_id}) + begin = begin or utils.get_month_start() + end = end or utils.get_next_month() + + if not flask.request.context.is_admin: + filters['project_id'] = flask.request.context.project_id + + total = self._storage.total( + begin=begin, end=end, + groupby=groupby, + filters=filters, + offset=offset, + limit=limit, + paginate=True, + ) + columns = [] + if len(total['results']) > 0: + columns = list(total['results'][0].keys()) + + return { + 'total': total['total'], + 'columns': columns, + 'results': [list(res.values()) for res in total['results']] + } diff --git a/cloudkitty/api/v2/utils.py b/cloudkitty/api/v2/utils.py index e689263e..21d92960 100644 --- a/cloudkitty/api/v2/utils.py +++ b/cloudkitty/api/v2/utils.py @@ -71,6 +71,78 @@ class MultiQueryParam(object): return self._validate(output) +class DictQueryParam(object): + """Voluptuous helper to validate dict query params. + + This validator converts a dict query parameter to a python dict. + + :param key_type: Type of the dict keys + :param val_type: Type of the dict values + :param unique_values: Defaults to True. Set to True if each key should + contain only one value + :type unique_values: bool + """ + def __init__(self, key_type, val_type, unique_values=True): + self._kval = voluptuous.Coerce(key_type) + self._unique_val = unique_values + + if self._unique_val: + self._vval = voluptuous.Coerce(val_type) + else: + def __vval(values): + return [voluptuous.Coerce(val_type)(v) for v in values] + self._vval = __vval + + @staticmethod + def _append(output, key, val): + if key in output.keys(): + output[key].append(val) + else: + output[key] = [val] + return output + + def __call__(self, v): + if not isinstance(v, list): + v = [v] + + tokens = itertools.chain(*[elem.split(',') for elem in v]) + output = {} + for token in tokens: + try: + key, val = token.split(':') + except ValueError: # Not enough or too many values to unpack + raise voluptuous.DictInvalid( + 'invalid key:value association {}'.format(token)) + + if key in output.keys(): + if self._unique_val: + raise voluptuous.DictInvalid( + 'key {} already provided'.format(key)) + + if self._unique_val: + output[key] = val + else: + output = self._append(output, key, val) + + return {self._kval(k): self._vval(v) for k, v in output.items()} + + +class SingleDictQueryParam(DictQueryParam): + + def __init__(self, key_type, val_type): + super(SingleDictQueryParam, self).__init__(key_type=key_type, + val_type=val_type, + unique_values=True) + + +class MultiDictQueryParam(DictQueryParam): + + def __init__(self, key_type, val_type): + super(MultiDictQueryParam, self).__init__(key_type=key_type, + val_type=val_type, + unique_values=False) + + def add_input_schema(location, schema): """Add a voluptuous schema validation on a method's input diff --git a/cloudkitty/common/policies/__init__.py b/cloudkitty/common/policies/__init__.py index df7064b4..5f4c112a 100644 --- a/cloudkitty/common/policies/__init__.py +++ b/cloudkitty/common/policies/__init__.py @@ -22,6 +22,7 @@ from cloudkitty.common.policies.v1 import rating as v1_rating from cloudkitty.common.policies.v1 import report as v1_report from cloudkitty.common.policies.v1 import storage as v1_storage from cloudkitty.common.policies.v2 import scope as v2_scope +from cloudkitty.common.policies.v2 import summary as v2_summary def list_rules(): @@ -33,4 +34,5 @@ def list_rules(): v1_report.list_rules(), v1_storage.list_rules(), v2_scope.list_rules(), + v2_summary.list_rules(), ) diff --git a/cloudkitty/common/policies/v2/summary.py b/cloudkitty/common/policies/v2/summary.py new file mode 100644 index 00000000..74c25166 --- /dev/null +++ b/cloudkitty/common/policies/v2/summary.py @@ -0,0 +1,30 @@ +# Copyright 2018 Objectif Libre +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +from oslo_policy import policy + +from cloudkitty.common.policies import base + +example_policies = [ + policy.DocumentedRuleDefault( + name='summary:get_summary', + check_str=base.RULE_ADMIN_OR_OWNER, + description='Get a rating summary', + operations=[{'path': '/v2/summary', + 'method': 'GET'}]), +] + + +def list_rules(): + return example_policies diff --git a/cloudkitty/tests/api/v2/test_utils.py b/cloudkitty/tests/api/v2/test_utils.py index 4941d3cd..4cfb5fa0 100644 --- a/cloudkitty/tests/api/v2/test_utils.py +++ b/cloudkitty/tests/api/v2/test_utils.py @@ -82,6 +82,73 @@ class SingleQueryParamTest(tests.TestCase): ) +class DictQueryParamTest(tests.TestCase): + + validator_class = api_utils.DictQueryParam + + def test_empty_list_str_str(self): + validator = self.validator_class(str, str) + input_ = [] + self.assertEqual(validator(input_), {}) + + def test_list_invalid_elem_missing_key_str_str(self): + validator = self.validator_class(str, str) + input_ = ['a:b', 'c'] + self.assertRaises(voluptuous.DictInvalid, validator, input_) + + def test_list_invalid_elem_too_many_columns_str_str(self): + validator = self.validator_class(str, str) + input_ = ['a:b', 'c:d:e'] + self.assertRaises(voluptuous.DictInvalid, validator, input_) + + +class SingleDictQueryParamTest(DictQueryParamTest): + + validator_class = api_utils.SingleDictQueryParam + + def test_single_valid_elem_str_int(self): + validator = self.validator_class(str, int) + input_ = 'life:42' + self.assertEqual(validator(input_), {'life': 42}) + + def test_list_one_valid_elem_str_int(self): + validator = self.validator_class(str, int) + input_ = ['life:42'] + self.assertEqual(validator(input_), {'life': 42}) + + def test_list_several_valid_elems_str_int(self): + validator = self.validator_class(str, int) + input_ = ['life:42', 'one:1', 'two:2'] + self.assertEqual(validator(input_), {'life': 42, 'one': 1, 'two': 2}) + + +class MultiDictQueryParamTest(DictQueryParamTest): + + validator_class = api_utils.MultiDictQueryParam + + def test_single_valid_elem_str_int(self): + validator = self.validator_class(str, int) + input_ = 'life:42' + self.assertEqual(validator(input_), {'life': [42]}) + + def test_list_one_valid_elem_str_int(self): + validator = self.validator_class(str, int) + input_ = ['life:42'] + self.assertEqual(validator(input_), {'life': [42]}) + + def test_list_several_valid_elems_str_int(self): + validator = self.validator_class(str, int) + input_ = ['life:42', 'one:1', 'two:2'] + self.assertEqual(validator(input_), + {'life': [42], 'one': [1], 'two': [2]}) + + def test_list_several_valid_elems_shared_keys_str_int(self): + validator = self.validator_class(str, int) + input_ = ['even:0', 'uneven:1', 'even:2', 'uneven:3', 'even:4'] + self.assertEqual(validator(input_), + {'even': [0, 2, 4], 'uneven': [1, 3]}) + + class AddInputSchemaTest(tests.TestCase): def test_paginated(self): diff --git a/cloudkitty/tests/gabbi/fixtures.py b/cloudkitty/tests/gabbi/fixtures.py index ba17d343..419ae3f2 100644 --- a/cloudkitty/tests/gabbi/fixtures.py +++ b/cloudkitty/tests/gabbi/fixtures.py @@ -43,6 +43,7 @@ from cloudkitty import storage from cloudkitty.storage.v1.sqlalchemy import models from cloudkitty import storage_state from cloudkitty import tests +from cloudkitty.tests.storage.v2 import influx_utils from cloudkitty.tests import utils as test_utils from cloudkitty import utils as ck_utils @@ -431,6 +432,32 @@ class MetricsConfFixture(fixture.GabbiFixture): ck_utils.load_conf = self._original_function +class InfluxStorageDataFixture(NowStorageDataFixture): + + def start_fixture(self): + cli = influx_utils.FakeInfluxClient() + st = storage.get_storage() + st._conn = cli + + self._get_storage_patch = mock.patch( + 'cloudkitty.storage.get_storage', + new=lambda **kw: st, + ) + self._get_storage_patch.start() + + super(InfluxStorageDataFixture, self).start_fixture() + + def initialize_data(self): + data = test_utils.generate_v2_storage_data( + start=ck_utils.get_month_start(), + end=ck_utils.utcnow().replace(hour=0), + ) + self.storage.push([data]) + + def stop_fixture(self): + self._get_storage_patch.stop() + + def setup_app(): messaging.setup() # FIXME(sheeprine): Extension fixtures are interacting with transformers diff --git a/cloudkitty/tests/gabbi/gabbits/v2-summary.yaml b/cloudkitty/tests/gabbi/gabbits/v2-summary.yaml new file mode 100644 index 00000000..d2e5ca9c --- /dev/null +++ b/cloudkitty/tests/gabbi/gabbits/v2-summary.yaml @@ -0,0 +1,60 @@ +fixtures: + - ConfigFixtureStorageV2 + - InfluxStorageDataFixture + +tests: + - name: Get a summary + url: /v2/summary + status: 200 + response_json_paths: + $.results.`len`: 1 + $.total: 1 + + - name: Get a summary by project id + url: /v2/summary + status: 200 + query_parameters: + groupby: project_id + response_json_paths: + $.results.`len`: 2 + $.total: 2 + + - name: Get a summary by type + url: /v2/summary + status: 200 + query_parameters: + groupby: type + response_json_paths: + $.results.`len`: 7 + $.total: 7 + + - name: Get a summary by type and project_id + url: /v2/summary + status: 200 + query_parameters: + groupby: [type, project_id] + response_json_paths: + $.results.`len`: 14 + $.total: 14 + + - name: Get a summary by type and project_id limit 5 offset 0 + url: /v2/summary + status: 200 + query_parameters: + groupby: [type, project_id] + limit: 5 + offset: 0 + response_json_paths: + $.results.`len`: 5 + $.total: 14 + + - name: Get a summary by type and project_id limit 5 offset 5 + url: /v2/summary + status: 200 + query_parameters: + groupby: [type, project_id] + limit: 5 + offset: 5 + response_json_paths: + $.results.`len`: 5 + $.total: 14 diff --git a/doc/source/_static/cloudkitty.policy.yaml.sample b/doc/source/_static/cloudkitty.policy.yaml.sample index ce4d1ac4..3258c742 100644 --- a/doc/source/_static/cloudkitty.policy.yaml.sample +++ b/doc/source/_static/cloudkitty.policy.yaml.sample @@ -88,3 +88,7 @@ # GET /v2/scope #"scope:get_state": "role:admin" +# Get a rating summary +# GET /v2/summary +#"summary:get_summary": "rule:admin_or_owner" + diff --git a/doc/source/api-reference/v2/api_samples/summary/summary_get.json b/doc/source/api-reference/v2/api_samples/summary/summary_get.json new file mode 100644 index 00000000..c07d469a --- /dev/null +++ b/doc/source/api-reference/v2/api_samples/summary/summary_get.json @@ -0,0 +1,45 @@ +{ + "columns": [ + "begin", + "end", + "qty", + "rate", + "project_id", + "type" + ], + "results": [ + [ + "2019-06-01T00:00:00Z", + "2019-07-01T00:00:00Z", + 2590.421676635742, + 1295.210838317871, + "fe9c35372db6420089883805b37a34af", + "image.size" + ], + [ + "2019-06-01T00:00:00Z", + "2019-07-01T00:00:00Z", + 1354, + 3625, + "fe9c35372db6420089883805b37a34af", + "instance" + ], + [ + "2019-06-01T00:00:00Z", + "2019-07-01T00:00:00Z", + 502, + 502, + "fe9c35372db6420089883805b37a34af", + "ip.floating" + ], + [ + "2019-06-01T00:00:00Z", + "2019-07-01T00:00:00Z", + 175.9, + 351.8, + "fe9c35372db6420089883805b37a34af", + "volume.size" + ] + ], + "total": 4 +} diff --git a/doc/source/api-reference/v2/index.rst b/doc/source/api-reference/v2/index.rst index afc5e4d5..d76f3828 100644 --- a/doc/source/api-reference/v2/index.rst +++ b/doc/source/api-reference/v2/index.rst @@ -1,3 +1,4 @@ .. rest_expand_all:: .. include:: scope/scope.inc +.. include:: summary/summary.inc diff --git a/doc/source/api-reference/v2/summary/summary.inc b/doc/source/api-reference/v2/summary/summary.inc new file mode 100644 index 00000000..3d897672 --- /dev/null +++ b/doc/source/api-reference/v2/summary/summary.inc @@ -0,0 +1,83 @@ +================ +Summary endpoint +================ + +Get a rating summary +==================== + +Get a rating summary for one or several tenants. + +.. rest_method:: GET /v2/summary + +.. rest_parameters:: summary/summary_parameters.yml + + - limit: limit + - offset: offset + - begin: begin + - end: end + - groupby: groupby + - filters: filters + +Status codes +------------ + +.. rest_status_code:: success http_status.yml + + - 200 + +.. rest_status_code:: error http_status.yml + + - 400 + - 403 + - 405 + +Response +-------- + +The response has the following format: + +.. code-block:: javascript + + { + "columns": [ + "begin", + "end", + "qty", + "rate", + "group1", + "group2", + ], + "results": [ + [ + "2019-06-01T00:00:00Z", + "2019-07-01T00:00:00Z", + 2590.421676635742, + 1295.210838317871, + "group1", + "group2", + ] + ], + "total": 4 + } + +``total`` is the total amount of found elements. ``columns`` contains the name of +the columns for each element of ``results``. The columns are the four mandatory ones +(``begin``, ``end``, ``qty``, ``rate``) along with each attribute the result is +grouped by. + +.. rest_parameters:: summary/summary_parameters.yml + + - begin: begin_resp + - end: end_resp + - qty: qty_resp + - rate: rate_resp + +Response Example +---------------- + +.. code-block:: shell + + curl "http://cloudkitty-api:8889/v2/summary?filters=project_id%3Afe9c35372db6420089883805b37a34af&groupby=type&groupby=project_id" + +.. literalinclude:: ./api_samples/summary/summary_get.json + :language: javascript diff --git a/doc/source/api-reference/v2/summary/summary_parameters.yml b/doc/source/api-reference/v2/summary/summary_parameters.yml new file mode 100644 index 00000000..138fd58e --- /dev/null +++ b/doc/source/api-reference/v2/summary/summary_parameters.yml @@ -0,0 +1,79 @@ +limit: + in: query + description: | + For pagination. The maximum number of results to return. + type: int + required: false + +offset: &offset + in: query + description: | + For pagination. The index of the first element that should be returned. + type: int + required: false + +filters: + in: query + description: | + Optional filters. + type: dict + required: false + +groupby: + in: query + description: | + Optional attributes to group the summary by. + type: list of strings + required: false + +begin: &begin + in: query + description: | + Begin of the period for which the summary is required. + type: iso8601 timestamp + required: false + +end: &end + in: query + description: | + End of the period for which the summary is required. + type: iso8601 timestamp + required: false + +qty: &qty + in: body + description: | + Qty for the item. + type: float + required: true + +rate: &rate + in: body + description: | + Rate for the item. + type: float + required: true + +begin_resp: + <<: *begin + required: true + description: Begin of the period for the item. + in: body + +end_resp: + <<: *end + required: true + description: End of the period for the item. + in: body + +qty_resp: + <<: *qty + required: true + description: Qty for the item in the specified period. + in: body + +rate_resp: + <<: *rate + required: true + description: Rate for the item in the specified period. + in: body