Add a v2 summary endpoint

This adds an endpoint on /v2/summary, as a replacement for /v1/summary.
It allows to retrieve a summary from the cloudkitty, which can be grouped on
custom attributes and filters.

Change-Id: I99bff44b24d3dcec2da97281f0b491ac333ed395
Story: 2005664
Task: 30961

cloudkitty/api/v2/__init__.py

@@ -33,6 +33,7 @@ RESOURCE_SCHEMA = voluptuous.Schema({
 
 API_MODULES = [
     'cloudkitty.api.v2.scope',
+    'cloudkitty.api.v2.summary',
 ]
 
 

cloudkitty/api/v2/base.py

@@ -16,6 +16,7 @@ import flask_restful
 from werkzeug import exceptions as http_exceptions
 
 from cloudkitty.common import policy
+from cloudkitty import storage
 
 
 class BaseResource(flask_restful.Resource):
@@ -31,3 +32,7 @@ class BaseResource(flask_restful.Resource):
         except policy.PolicyNotAuthorized:
             raise http_exceptions.Forbidden(
                 "You are not authorized to perform this action")
+
+    def __init__(self, *args, **kwargs):
+        super(BaseResource, self).__init__(*args, **kwargs)
+        self._storage = storage.get_storage()

cloudkitty/api/v2/summary/__init__.py

@@ -0,0 +1,26 @@
+# Copyright 2019 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+from cloudkitty.api.v2 import utils as api_utils
+
+
+def init(app):
+    api_utils.do_init(app, 'summary', [
+        {
+            'module': __name__ + '.' + 'summary',
+            'resource_class': 'Summary',
+            'url': '',
+        },
+    ])
+    return app

cloudkitty/api/v2/summary/summary.py

@@ -0,0 +1,64 @@
+# Copyright 2019 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+import flask
+import voluptuous
+
+from cloudkitty.api.v2 import base
+from cloudkitty.api.v2 import utils as api_utils
+from cloudkitty.common import policy
+from cloudkitty import utils
+
+
+class Summary(base.BaseResource):
+    """Resource allowing to retrieve a rating summary."""
+
+    @api_utils.paginated
+    @api_utils.add_input_schema('query', {
+        voluptuous.Optional('groupby'): api_utils.MultiQueryParam(str),
+        voluptuous.Optional('filters'):
+            api_utils.SingleDictQueryParam(str, str),
+        voluptuous.Optional('begin'): voluptuous.Coerce(utils.iso2dt),
+        voluptuous.Optional('end'): voluptuous.Coerce(utils.iso2dt),
+    })
+    def get(self, groupby=None, filters={},
+            begin=None, end=None,
+            offset=0, limit=100):
+        policy.authorize(
+            flask.request.context,
+            'summary:get_summary',
+            {'tenant_id': flask.request.context.project_id})
+        begin = begin or utils.get_month_start()
+        end = end or utils.get_next_month()
+
+        if not flask.request.context.is_admin:
+            filters['project_id'] = flask.request.context.project_id
+
+        total = self._storage.total(
+            begin=begin, end=end,
+            groupby=groupby,
+            filters=filters,
+            offset=offset,
+            limit=limit,
+            paginate=True,
+        )
+        columns = []
+        if len(total['results']) > 0:
+            columns = list(total['results'][0].keys())
+
+        return {
+            'total': total['total'],
+            'columns': columns,
+            'results': [list(res.values()) for res in total['results']]
+        }

cloudkitty/api/v2/utils.py

@@ -71,6 +71,78 @@ class MultiQueryParam(object):
         return self._validate(output)
 
 
+class DictQueryParam(object):
+    """Voluptuous helper to validate dict query params.
+
+    This validator converts a dict query parameter to a python dict.
+
+    :param key_type: Type of the dict keys
+    :param val_type: Type of the dict values
+    :param unique_values: Defaults to True. Set to True if each key should
+                          contain only one value
+    :type unique_values: bool
+    """
+    def __init__(self, key_type, val_type, unique_values=True):
+        self._kval = voluptuous.Coerce(key_type)
+        self._unique_val = unique_values
+
+        if self._unique_val:
+            self._vval = voluptuous.Coerce(val_type)
+        else:
+            def __vval(values):
+                return [voluptuous.Coerce(val_type)(v) for v in values]
+            self._vval = __vval
+
+    @staticmethod
+    def _append(output, key, val):
+        if key in output.keys():
+            output[key].append(val)
+        else:
+            output[key] = [val]
+        return output
+
+    def __call__(self, v):
+        if not isinstance(v, list):
+            v = [v]
+
+        tokens = itertools.chain(*[elem.split(',') for elem in v])
+        output = {}
+        for token in tokens:
+            try:
+                key, val = token.split(':')
+            except ValueError:  # Not enough or too many values to unpack
+                raise voluptuous.DictInvalid(
+                    'invalid key:value association {}'.format(token))
+
+            if key in output.keys():
+                if self._unique_val:
+                    raise voluptuous.DictInvalid(
+                        'key {} already provided'.format(key))
+
+            if self._unique_val:
+                output[key] = val
+            else:
+                output = self._append(output, key, val)
+
+        return {self._kval(k): self._vval(v) for k, v in output.items()}
+
+
+class SingleDictQueryParam(DictQueryParam):
+
+    def __init__(self, key_type, val_type):
+        super(SingleDictQueryParam, self).__init__(key_type=key_type,
+                                                   val_type=val_type,
+                                                   unique_values=True)
+
+
+class MultiDictQueryParam(DictQueryParam):
+
+    def __init__(self, key_type, val_type):
+        super(MultiDictQueryParam, self).__init__(key_type=key_type,
+                                                  val_type=val_type,
+                                                  unique_values=False)
+
+
 def add_input_schema(location, schema):
     """Add a voluptuous schema validation on a method's input
 

cloudkitty/common/policies/__init__.py

@@ -22,6 +22,7 @@ from cloudkitty.common.policies.v1 import rating as v1_rating
 from cloudkitty.common.policies.v1 import report as v1_report
 from cloudkitty.common.policies.v1 import storage as v1_storage
 from cloudkitty.common.policies.v2 import scope as v2_scope
+from cloudkitty.common.policies.v2 import summary as v2_summary
 
 
 def list_rules():
@@ -33,4 +34,5 @@ def list_rules():
         v1_report.list_rules(),
         v1_storage.list_rules(),
         v2_scope.list_rules(),
+        v2_summary.list_rules(),
     )

cloudkitty/common/policies/v2/summary.py

@@ -0,0 +1,30 @@
+# Copyright 2018 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+from oslo_policy import policy
+
+from cloudkitty.common.policies import base
+
+example_policies = [
+    policy.DocumentedRuleDefault(
+        name='summary:get_summary',
+        check_str=base.RULE_ADMIN_OR_OWNER,
+        description='Get a rating summary',
+        operations=[{'path': '/v2/summary',
+                     'method': 'GET'}]),
+]
+
+
+def list_rules():
+    return example_policies

cloudkitty/tests/api/v2/test_utils.py

@@ -82,6 +82,73 @@ class SingleQueryParamTest(tests.TestCase):
         )
 
 
+class DictQueryParamTest(tests.TestCase):
+
+    validator_class = api_utils.DictQueryParam
+
+    def test_empty_list_str_str(self):
+        validator = self.validator_class(str, str)
+        input_ = []
+        self.assertEqual(validator(input_), {})
+
+    def test_list_invalid_elem_missing_key_str_str(self):
+        validator = self.validator_class(str, str)
+        input_ = ['a:b', 'c']
+        self.assertRaises(voluptuous.DictInvalid, validator, input_)
+
+    def test_list_invalid_elem_too_many_columns_str_str(self):
+        validator = self.validator_class(str, str)
+        input_ = ['a:b', 'c:d:e']
+        self.assertRaises(voluptuous.DictInvalid, validator, input_)
+
+
+class SingleDictQueryParamTest(DictQueryParamTest):
+
+    validator_class = api_utils.SingleDictQueryParam
+
+    def test_single_valid_elem_str_int(self):
+        validator = self.validator_class(str, int)
+        input_ = 'life:42'
+        self.assertEqual(validator(input_), {'life': 42})
+
+    def test_list_one_valid_elem_str_int(self):
+        validator = self.validator_class(str, int)
+        input_ = ['life:42']
+        self.assertEqual(validator(input_), {'life': 42})
+
+    def test_list_several_valid_elems_str_int(self):
+        validator = self.validator_class(str, int)
+        input_ = ['life:42', 'one:1', 'two:2']
+        self.assertEqual(validator(input_), {'life': 42, 'one': 1, 'two': 2})
+
+
+class MultiDictQueryParamTest(DictQueryParamTest):
+
+    validator_class = api_utils.MultiDictQueryParam
+
+    def test_single_valid_elem_str_int(self):
+        validator = self.validator_class(str, int)
+        input_ = 'life:42'
+        self.assertEqual(validator(input_), {'life': [42]})
+
+    def test_list_one_valid_elem_str_int(self):
+        validator = self.validator_class(str, int)
+        input_ = ['life:42']
+        self.assertEqual(validator(input_), {'life': [42]})
+
+    def test_list_several_valid_elems_str_int(self):
+        validator = self.validator_class(str, int)
+        input_ = ['life:42', 'one:1', 'two:2']
+        self.assertEqual(validator(input_),
+                         {'life': [42], 'one': [1], 'two': [2]})
+
+    def test_list_several_valid_elems_shared_keys_str_int(self):
+        validator = self.validator_class(str, int)
+        input_ = ['even:0', 'uneven:1', 'even:2', 'uneven:3', 'even:4']
+        self.assertEqual(validator(input_),
+                         {'even': [0, 2, 4], 'uneven': [1, 3]})
+
+
 class AddInputSchemaTest(tests.TestCase):
 
     def test_paginated(self):

cloudkitty/tests/gabbi/fixtures.py

@@ -43,6 +43,7 @@ from cloudkitty import storage
 from cloudkitty.storage.v1.sqlalchemy import models
 from cloudkitty import storage_state
 from cloudkitty import tests
+from cloudkitty.tests.storage.v2 import influx_utils
 from cloudkitty.tests import utils as test_utils
 from cloudkitty import utils as ck_utils
 
@@ -431,6 +432,32 @@ class MetricsConfFixture(fixture.GabbiFixture):
         ck_utils.load_conf = self._original_function
 
 
+class InfluxStorageDataFixture(NowStorageDataFixture):
+
+    def start_fixture(self):
+        cli = influx_utils.FakeInfluxClient()
+        st = storage.get_storage()
+        st._conn = cli
+
+        self._get_storage_patch = mock.patch(
+            'cloudkitty.storage.get_storage',
+            new=lambda **kw: st,
+        )
+        self._get_storage_patch.start()
+
+        super(InfluxStorageDataFixture, self).start_fixture()
+
+    def initialize_data(self):
+        data = test_utils.generate_v2_storage_data(
+            start=ck_utils.get_month_start(),
+            end=ck_utils.utcnow().replace(hour=0),
+        )
+        self.storage.push([data])
+
+    def stop_fixture(self):
+        self._get_storage_patch.stop()
+
+
 def setup_app():
     messaging.setup()
     # FIXME(sheeprine): Extension fixtures are interacting with transformers

cloudkitty/tests/gabbi/gabbits/v2-summary.yaml

@@ -0,0 +1,60 @@
+fixtures:
+  - ConfigFixtureStorageV2
+  - InfluxStorageDataFixture
+
+tests:
+  - name: Get a summary
+    url: /v2/summary
+    status: 200
+    response_json_paths:
+      $.results.`len`: 1
+      $.total: 1
+
+  - name: Get a summary by project id
+    url: /v2/summary
+    status: 200
+    query_parameters:
+      groupby: project_id
+    response_json_paths:
+      $.results.`len`: 2
+      $.total: 2
+
+  - name: Get a summary by type
+    url: /v2/summary
+    status: 200
+    query_parameters:
+      groupby: type
+    response_json_paths:
+      $.results.`len`: 7
+      $.total: 7
+
+  - name: Get a summary by type and project_id
+    url: /v2/summary
+    status: 200
+    query_parameters:
+      groupby: [type, project_id]
+    response_json_paths:
+      $.results.`len`: 14
+      $.total: 14
+
+  - name: Get a summary by type and project_id limit 5 offset 0
+    url: /v2/summary
+    status: 200
+    query_parameters:
+      groupby: [type, project_id]
+      limit: 5
+      offset: 0
+    response_json_paths:
+      $.results.`len`: 5
+      $.total: 14
+
+  - name: Get a summary by type and project_id limit 5 offset 5
+    url: /v2/summary
+    status: 200
+    query_parameters:
+      groupby: [type, project_id]
+      limit: 5
+      offset: 5
+    response_json_paths:
+      $.results.`len`: 5
+      $.total: 14

doc/source/_static/cloudkitty.policy.yaml.sample

@@ -88,3 +88,7 @@
 # GET  /v2/scope
 #"scope:get_state": "role:admin"
 
+# Get a rating summary
+# GET  /v2/summary
+#"summary:get_summary": "rule:admin_or_owner"
+

doc/source/api-reference/v2/api_samples/summary/summary_get.json

@@ -0,0 +1,45 @@
+{
+    "columns": [
+        "begin",
+        "end",
+        "qty",
+        "rate",
+        "project_id",
+        "type"
+    ],
+    "results": [
+        [
+            "2019-06-01T00:00:00Z",
+            "2019-07-01T00:00:00Z",
+            2590.421676635742,
+            1295.210838317871,
+            "fe9c35372db6420089883805b37a34af",
+            "image.size"
+        ],
+        [
+            "2019-06-01T00:00:00Z",
+            "2019-07-01T00:00:00Z",
+            1354,
+            3625,
+            "fe9c35372db6420089883805b37a34af",
+            "instance"
+        ],
+        [
+            "2019-06-01T00:00:00Z",
+            "2019-07-01T00:00:00Z",
+            502,
+            502,
+            "fe9c35372db6420089883805b37a34af",
+            "ip.floating"
+        ],
+        [
+            "2019-06-01T00:00:00Z",
+            "2019-07-01T00:00:00Z",
+            175.9,
+            351.8,
+            "fe9c35372db6420089883805b37a34af",
+            "volume.size"
+        ]
+    ],
+    "total": 4
+}

doc/source/api-reference/v2/index.rst

@@ -1,3 +1,4 @@
 .. rest_expand_all::
 
 .. include:: scope/scope.inc
+.. include:: summary/summary.inc

doc/source/api-reference/v2/summary/summary.inc

@@ -0,0 +1,83 @@
+================
+Summary endpoint
+================
+
+Get a rating summary
+====================
+
+Get a rating summary for one or several tenants.
+
+.. rest_method::  GET /v2/summary
+
+.. rest_parameters:: summary/summary_parameters.yml
+
+   - limit: limit
+   - offset: offset
+   - begin: begin
+   - end: end
+   - groupby: groupby
+   - filters: filters
+
+Status codes
+------------
+
+.. rest_status_code:: success http_status.yml
+
+   - 200
+
+.. rest_status_code:: error http_status.yml
+
+   - 400
+   - 403
+   - 405
+
+Response
+--------
+
+The response has the following format:
+
+.. code-block:: javascript
+
+   {
+       "columns": [
+           "begin",
+           "end",
+           "qty",
+           "rate",
+           "group1",
+           "group2",
+       ],
+       "results": [
+           [
+               "2019-06-01T00:00:00Z",
+               "2019-07-01T00:00:00Z",
+               2590.421676635742,
+               1295.210838317871,
+               "group1",
+               "group2",
+           ]
+       ],
+       "total": 4
+   }
+
+``total`` is the total amount of found elements. ``columns`` contains the name of
+the columns for each element of ``results``. The columns are the four mandatory ones
+(``begin``, ``end``, ``qty``, ``rate``) along with each attribute the result is
+grouped by.
+
+.. rest_parameters:: summary/summary_parameters.yml
+
+   - begin: begin_resp
+   - end: end_resp
+   - qty: qty_resp
+   - rate: rate_resp
+
+Response Example
+----------------
+
+.. code-block:: shell
+
+   curl "http://cloudkitty-api:8889/v2/summary?filters=project_id%3Afe9c35372db6420089883805b37a34af&groupby=type&groupby=project_id"
+
+.. literalinclude:: ./api_samples/summary/summary_get.json
+   :language: javascript

doc/source/api-reference/v2/summary/summary_parameters.yml

@@ -0,0 +1,79 @@
+limit:
+  in: query
+  description: |
+    For pagination. The maximum number of results to return.
+  type: int
+  required: false
+
+offset: &offset
+  in: query
+  description: |
+    For pagination. The index of the first element that should be returned.
+  type: int
+  required: false
+
+filters:
+  in: query
+  description: |
+    Optional filters.
+  type: dict
+  required: false
+
+groupby:
+  in: query
+  description: |
+    Optional attributes to group the summary by.
+  type: list of strings
+  required: false
+
+begin: &begin
+  in: query
+  description: |
+    Begin of the period for which the summary is required.
+  type: iso8601 timestamp
+  required: false
+
+end: &end
+  in: query
+  description: |
+    End of the period for which the summary is required.
+  type: iso8601 timestamp
+  required: false
+
+qty: &qty
+  in: body
+  description: |
+    Qty for the item.
+  type: float
+  required: true
+
+rate: &rate
+  in: body
+  description: |
+    Rate for the item.
+  type: float
+  required: true
+
+begin_resp:
+  <<: *begin
+  required: true
+  description: Begin of the period for the item.
+  in: body
+
+end_resp:
+  <<: *end
+  required: true
+  description: End of the period for the item.
+  in: body
+
+qty_resp:
+  <<: *qty
+  required: true
+  description: Qty for the item in the specified period.
+  in: body
+
+rate_resp:
+  <<: *rate
+  required: true
+  description: Rate for the item in the specified period.
+  in: body