Support pagination in v2 storage total()

This adds the required arguments for pagination to the interface of the v2
storage's 'total()' method.

The required changes have also been applied to the gnocchi backend and to the
v1 storage adapter.

Change-Id: I7d123f7968a50906cd9458a359491862a96ffe69
Story: 2001372
Task: 26836

cloudkitty/api/v1/controllers/report.py

@@ -102,16 +102,15 @@ class ReportController(rest.RestController):
         scope_key = CONF.collect.scope_key
         groupby = [scope_key]
         group_filters = {scope_key: tenant_id} if tenant_id else None
-        total_resources = storage.total(
+        result = storage.total(
             groupby=groupby,
             begin=begin, end=end,
             metric_types=service,
             group_filters=group_filters)
 
-        # TODO(Aaron): `get_total` return a list of dict,
+        if result['total'] < 1:
-        # Get value of rate from index[0]
+            return decimal.Decimal('0')
-        total = sum(total['rate'] for total in total_resources)
+        return sum(total['rate'] for total in result['results'])
-        return total if total else decimal.Decimal('0')
 
     @wsme_pecan.wsexpose(report_models.SummaryCollectionModel,
                          datetime.datetime,
@@ -146,14 +145,14 @@ class ReportController(rest.RestController):
         if groupby is not None and 'res_type' in groupby:
             storage_groupby.append('type')
         group_filters = {scope_key: tenant_id} if tenant_id else None
-        results = storage.total(
+        result = storage.total(
             groupby=storage_groupby,
             begin=begin, end=end,
             metric_types=service,
             group_filters=group_filters)
 
         summarymodels = []
-        for res in results:
+        for res in result['results']:
             kwargs = {
                 'res_type': res.get('type') or res.get('res_type'),
                 'tenant_id': res.get(scope_key) or res.get('tenant_id'),

cloudkitty/storage/__init__.py

@@ -97,7 +97,8 @@ class V1StorageAdapter(storage_v2.BaseStorage):
     def total(self, groupby=None,
               begin=None, end=None,
               metric_types=None,
-              filters=None, group_filters=None):
+              filters=None, group_filters=None,
+              offset=0, limit=100, paginate=True):
         tenant_id = group_filters.get('project_id') if group_filters else None
 
         storage_gby = []
@@ -124,7 +125,10 @@ class V1StorageAdapter(storage_v2.BaseStorage):
                 t['type'] = t.get('res_type')
             else:
                 t['type'] = None
-        return total
+        return {
+            'total': len(total),
+            'results': total,
+        }
 
     def get_tenants(self, begin, end):
         tenants = self.storage.get_tenants(begin, end)

cloudkitty/storage/v2/__init__.py

@@ -118,7 +118,7 @@ class BaseStorage(object):
         :type limit: int
         :param paginate: Defaults to True. If False, all found results
                          will be returned.
-        :type limit: int
+        :type paginate: bool
         :rtype: dict
        """
 
@@ -126,7 +126,8 @@ class BaseStorage(object):
     def total(self, groupby=None,
               begin=None, end=None,
               metric_types=None,
-              filters=None, group_filters=None):
+              filters=None, group_filters=None,
+              offset=0, limit=1000, paginate=True):
         """Returns a grouped total for given groupby.
 
         :param groupby: Attributes on which to group by. These attributes must
@@ -144,9 +145,23 @@ class BaseStorage(object):
         :type group_filters: dict
         :param metric_types: Metric type to filter on.
         :type metric_types: str or list
-        :rtype: list of dicts
+        :param offset: Offset for pagination
+        :type offset: int
+        :param limit: Maximum amount of elements to return
+        :type limit: int
+        :param paginate: Defaults to True. If False, all found results
+                         will be returned.
+        :type paginate: bool
+        :rtype: dict
 
-        returns a list of dicts with the following format::
+        Returns a dict with the following format::
+
+            {
+               'total': int, # total amount of results found
+               'results': list of results,
+            }
+
+        Each result has the following format::
 
             {
                 'begin': XXX,

cloudkitty/storage/v2/gnocchi.py

@@ -701,7 +701,8 @@ class GnocchiStorage(BaseStorage):
     def total(self, groupby=None,
               begin=None, end=None,
               metric_types=None,
-              filters=None, group_filters=None):
+              filters=None, group_filters=None,
+              offset=0, limit=1000, paginate=True):
         begin, end = self._check_begin_end(begin, end)
 
         if groupby is None:
@@ -726,6 +727,15 @@ class GnocchiStorage(BaseStorage):
                 if len(resource['measures']['measures']['aggregated']):
                     rated_resources.append(resource)
 
+        result = {'total': len(rated_resources)}
+        if paginate:
+            rated_resources = rated_resources[offset:limit]
+        if len(rated_resources) < 1:
+            return {
+                'total': 0,
+                'results': [],
+            }
+
         # NOTE(lukapeschke): We undo what has been done previously (grouping
         # per type). This is not performant. Should be fixed as soon as
         # previous note is supported in gnocchi
@@ -749,4 +759,5 @@ class GnocchiStorage(BaseStorage):
                 output_elem['type'] = rated_resource['group'].get(
                     'type', '').replace(RESOURCE_TYPE_NAME_ROOT, '') or ''
             output.append(output_elem)
-        return output
+        result['results'] = output
+        return result

cloudkitty/tests/storage/v1/test_storage.py

@@ -143,7 +143,7 @@ class StorageTotalTest(StorageTest):
         self.insert_data()
         total = self.storage.total(
             begin=begin,
-            end=end)
+            end=end)['results']
         self.assertEqual(1, len(total))
         self.assertEqual(total[0]["rate"], 0)
         self.assertEqual(begin, total[0]["begin"])
@@ -155,7 +155,7 @@ class StorageTotalTest(StorageTest):
         self.insert_data()
         total = self.storage.total(
             begin=begin,
-            end=end)
+            end=end)['results']
         # FIXME(sheeprine): floating point error (transition to decimal)
         self.assertEqual(1, len(total))
         self.assertEqual(1.9473999999999998, total[0]["rate"])
@@ -168,7 +168,7 @@ class StorageTotalTest(StorageTest):
         self.insert_data()
         total = self.storage.total(
             begin=begin,
-            end=end)
+            end=end)['results']
         self.assertEqual(1, len(total))
         self.assertEqual(1.1074, total[0]["rate"])
         self.assertEqual(begin, total[0]["begin"])
@@ -182,7 +182,7 @@ class StorageTotalTest(StorageTest):
         total = self.storage.total(
             begin=begin,
             end=end,
-            group_filters=group_filters)
+            group_filters=group_filters)['results']
         self.assertEqual(1, len(total))
         self.assertEqual(0.5537, total[0]["rate"])
         self.assertEqual(self._tenant_id, total[0]["tenant_id"])
@@ -196,7 +196,7 @@ class StorageTotalTest(StorageTest):
         total = self.storage.total(
             begin=begin,
             end=end,
-            metric_types='instance')
+            metric_types='instance')['results']
         self.assertEqual(1, len(total))
         self.assertEqual(0.84, total[0]["rate"])
         self.assertEqual('instance', total[0]["res_type"])
@@ -210,7 +210,7 @@ class StorageTotalTest(StorageTest):
         total = self.storage.total(
             begin=begin,
             end=end,
-            groupby=['project_id'])
+            groupby=['project_id'])['results']
         self.assertEqual(2, len(total))
         self.assertEqual(0.9737, total[0]["rate"])
         self.assertEqual(self._other_tenant_id, total[0]["tenant_id"])
@@ -228,7 +228,7 @@ class StorageTotalTest(StorageTest):
         total = self.storage.total(
             begin=begin,
             end=end,
-            groupby=['type'])
+            groupby=['type'])['results']
         self.assertEqual(2, len(total))
         self.assertEqual(0.2674, total[0]["rate"])
         self.assertEqual('image.size', total[0]["res_type"])
@@ -246,7 +246,7 @@ class StorageTotalTest(StorageTest):
         total = self.storage.total(
             begin=begin,
             end=end,
-            groupby=['project_id', 'type'])
+            groupby=['project_id', 'type'])['results']
         self.assertEqual(4, len(total))
         self.assertEqual(0.1337, total[0]["rate"])
         self.assertEqual(self._other_tenant_id, total[0]["tenant_id"])