openstack/cloudkitty
Code Issues Proposed changes

Support pagination in v2 storage total()

Browse Source 
This adds the required arguments for pagination to the interface of the v2
storage's 'total()' method.

The required changes have also been applied to the gnocchi backend and to the
v1 storage adapter.

Change-Id: I7d123f7968a50906cd9458a359491862a96ffe69
Story: 2001372
Task: 26836
This commit is contained in:
Luka Peschke 2018-10-02 14:50:21 +02:00
parent da54a587c5
commit f55efdfcb5
5 changed files with 52 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
cloudkitty/api/v1/controllers/report.py
View File

@ -102,16 +102,15 @@ class ReportController(rest.RestController):
scope_key = CONF.collect.scope_key
groupby = [scope_key]
group_filters = {scope_key: tenant_id} if tenant_id else None
total_resources = storage.total(
result = storage.total(
groupby=groupby,
begin=begin, end=end,
metric_types=service,
group_filters=group_filters)
# TODO(Aaron): `get_total` return a list of dict,
# Get value of rate from index[0]
total = sum(total['rate'] for total in total_resources)
return total if total else decimal.Decimal('0')
if result['total'] < 1:
return decimal.Decimal('0')
return sum(total['rate'] for total in result['results'])
@wsme_pecan.wsexpose(report_models.SummaryCollectionModel,
datetime.datetime,
@ -146,14 +145,14 @@ class ReportController(rest.RestController):
if groupby is not None and 'res_type' in groupby:
storage_groupby.append('type')
group_filters = {scope_key: tenant_id} if tenant_id else None
results = storage.total(
result = storage.total(
groupby=storage_groupby,
begin=begin, end=end,
metric_types=service,
group_filters=group_filters)
summarymodels = []
for res in results:
for res in result['results']:
kwargs = {
'res_type': res.get('type') or res.get('res_type'),
'tenant_id': res.get(scope_key) or res.get('tenant_id'),

8
cloudkitty/storage/__init__.py
View File

@ -97,7 +97,8 @@ class V1StorageAdapter(storage_v2.BaseStorage):
def total(self, groupby=None,
begin=None, end=None,
metric_types=None,
filters=None, group_filters=None):
filters=None, group_filters=None,
offset=0, limit=100, paginate=True):
tenant_id = group_filters.get('project_id') if group_filters else None
storage_gby = []
@ -124,7 +125,10 @@ class V1StorageAdapter(storage_v2.BaseStorage):
t['type'] = t.get('res_type')
else:
t['type'] = None
return total
return {
'total': len(total),
'results': total,
}
def get_tenants(self, begin, end):
tenants = self.storage.get_tenants(begin, end)

23
cloudkitty/storage/v2/__init__.py
View File

@ -118,7 +118,7 @@ class BaseStorage(object):
:type limit: int
:param paginate: Defaults to True. If False, all found results
will be returned.
:type limit: int
:type paginate: bool
:rtype: dict
"""
@ -126,7 +126,8 @@ class BaseStorage(object):
def total(self, groupby=None,
begin=None, end=None,
metric_types=None,
filters=None, group_filters=None):
filters=None, group_filters=None,
offset=0, limit=1000, paginate=True):
"""Returns a grouped total for given groupby.
:param groupby: Attributes on which to group by. These attributes must
@ -144,9 +145,23 @@ class BaseStorage(object):
:type group_filters: dict
:param metric_types: Metric type to filter on.
:type metric_types: str or list
:rtype: list of dicts
:param offset: Offset for pagination
:type offset: int
:param limit: Maximum amount of elements to return
:type limit: int
:param paginate: Defaults to True. If False, all found results
will be returned.
:type paginate: bool
:rtype: dict
returns a list of dicts with the following format::
Returns a dict with the following format::
{
'total': int, # total amount of results found
'results': list of results,
}
Each result has the following format::
{
'begin': XXX,

15
cloudkitty/storage/v2/gnocchi.py
View File

@ -701,7 +701,8 @@ class GnocchiStorage(BaseStorage):
def total(self, groupby=None,
begin=None, end=None,
metric_types=None,
filters=None, group_filters=None):
filters=None, group_filters=None,
offset=0, limit=1000, paginate=True):
begin, end = self._check_begin_end(begin, end)
if groupby is None:
@ -726,6 +727,15 @@ class GnocchiStorage(BaseStorage):
if len(resource['measures']['measures']['aggregated']):
rated_resources.append(resource)
result = {'total': len(rated_resources)}
if paginate:
rated_resources = rated_resources[offset:limit]
if len(rated_resources) < 1:
return {
'total': 0,
'results': [],
}
# NOTE(lukapeschke): We undo what has been done previously (grouping
# per type). This is not performant. Should be fixed as soon as
# previous note is supported in gnocchi
@ -749,4 +759,5 @@ class GnocchiStorage(BaseStorage):
output_elem['type'] = rated_resource['group'].get(
'type', '').replace(RESOURCE_TYPE_NAME_ROOT, '') or ''
output.append(output_elem)
return output
result['results'] = output
return result

16
cloudkitty/tests/storage/v1/test_storage.py
View File

@ -143,7 +143,7 @@ class StorageTotalTest(StorageTest):
self.insert_data()
total = self.storage.total(
begin=begin,
end=end)
end=end)['results']
self.assertEqual(1, len(total))
self.assertEqual(total[0]["rate"], 0)
self.assertEqual(begin, total[0]["begin"])
@ -155,7 +155,7 @@ class StorageTotalTest(StorageTest):
self.insert_data()
total = self.storage.total(
begin=begin,
end=end)
end=end)['results']
# FIXME(sheeprine): floating point error (transition to decimal)
self.assertEqual(1, len(total))
self.assertEqual(1.9473999999999998, total[0]["rate"])
@ -168,7 +168,7 @@ class StorageTotalTest(StorageTest):
self.insert_data()
total = self.storage.total(
begin=begin,
end=end)
end=end)['results']
self.assertEqual(1, len(total))
self.assertEqual(1.1074, total[0]["rate"])
self.assertEqual(begin, total[0]["begin"])
@ -182,7 +182,7 @@ class StorageTotalTest(StorageTest):
total = self.storage.total(
begin=begin,
end=end,
group_filters=group_filters)
group_filters=group_filters)['results']
self.assertEqual(1, len(total))
self.assertEqual(0.5537, total[0]["rate"])
self.assertEqual(self._tenant_id, total[0]["tenant_id"])
@ -196,7 +196,7 @@ class StorageTotalTest(StorageTest):
total = self.storage.total(
begin=begin,
end=end,
metric_types='instance')
metric_types='instance')['results']
self.assertEqual(1, len(total))
self.assertEqual(0.84, total[0]["rate"])
self.assertEqual('instance', total[0]["res_type"])
@ -210,7 +210,7 @@ class StorageTotalTest(StorageTest):
total = self.storage.total(
begin=begin,
end=end,
groupby=['project_id'])
groupby=['project_id'])['results']
self.assertEqual(2, len(total))
self.assertEqual(0.9737, total[0]["rate"])
self.assertEqual(self._other_tenant_id, total[0]["tenant_id"])
@ -228,7 +228,7 @@ class StorageTotalTest(StorageTest):
total = self.storage.total(
begin=begin,
end=end,
groupby=['type'])
groupby=['type'])['results']
self.assertEqual(2, len(total))
self.assertEqual(0.2674, total[0]["rate"])
self.assertEqual('image.size', total[0]["res_type"])
@ -246,7 +246,7 @@ class StorageTotalTest(StorageTest):
total = self.storage.total(
begin=begin,
end=end,
groupby=['project_id', 'type'])
groupby=['project_id', 'type'])['results']
self.assertEqual(4, len(total))
self.assertEqual(0.1337, total[0]["rate"])
self.assertEqual(self._other_tenant_id, total[0]["tenant_id"])