From 107bd083ef8f84c41112e1d8ce4f371d89d5e451 Mon Sep 17 00:00:00 2001 From: Radomir Dopieralski Date: Tue, 13 Dec 2016 18:41:06 +0100 Subject: [PATCH] Simple tenant usage pagination Stitches back together all the pages for the os-simple-tenant-usage call, so that the usage statistics are complete. Implements blueprint paginate-simple-tenant-usage Depends-on: If99db6933de012b71cf2c982075f08b3e664361e Change-Id: I8c1206807f707ef47dd92e3cb663970804e3ec8e --- openstack_dashboard/api/nova.py | 65 ++++++++++++++++++- .../test/api_tests/nova_tests.py | 38 +++++++++++ .../test/test_data/nova_data.py | 6 +- 3 files changed, 104 insertions(+), 5 deletions(-) diff --git a/openstack_dashboard/api/nova.py b/openstack_dashboard/api/nova.py index 1d8d28fac..88ce1c4b7 100644 --- a/openstack_dashboard/api/nova.py +++ b/openstack_dashboard/api/nova.py @@ -20,6 +20,7 @@ from __future__ import absolute_import +import collections import logging from django.conf import settings @@ -27,6 +28,7 @@ from django.utils.functional import cached_property # noqa from django.utils.translation import ugettext_lazy as _ import six +from novaclient import api_versions from novaclient import client as nova_client from novaclient import exceptions as nova_exceptions from novaclient.v2.contrib import instance_action as nova_instance_action @@ -866,15 +868,72 @@ def default_quota_update(request, **kwargs): novaclient(request).quota_classes.update(DEFAULT_QUOTA_NAME, **kwargs) +def _get_usage_marker(usage): + marker = None + if hasattr(usage, 'server_usages') and usage.server_usages: + marker = usage.server_usages[-1].get('instance_id') + return marker + + +def _get_usage_list_marker(usage_list): + marker = None + if usage_list: + marker = _get_usage_marker(usage_list[-1]) + return marker + + +def _merge_usage(usage, next_usage): + usage.server_usages.extend(next_usage.server_usages) + usage.total_hours += next_usage.total_hours + usage.total_memory_mb_usage += next_usage.total_memory_mb_usage + usage.total_vcpus_usage += next_usage.total_vcpus_usage + usage.total_local_gb_usage += next_usage.total_local_gb_usage + + +def _merge_usage_list(usages, next_usage_list): + for next_usage in next_usage_list: + if next_usage.tenant_id in usages: + _merge_usage(usages[next_usage.tenant_id], next_usage) + else: + usages[next_usage.tenant_id] = next_usage + + @profiler.trace def usage_get(request, tenant_id, start, end): - return NovaUsage(novaclient(request).usage.get(tenant_id, start, end)) + client = novaclient(request) + usage = client.usage.get(tenant_id, start, end) + if client.api_version >= api_versions.APIVersion('2.40'): + # If the number of instances used to calculate the usage is greater + # than max_limit, the usage will be split across multiple requests + # and the responses will need to be merged back together. + marker = _get_usage_marker(usage) + while marker: + next_usage = client.usage.get(tenant_id, start, end, marker=marker) + marker = _get_usage_marker(next_usage) + if marker: + _merge_usage(usage, next_usage) + return NovaUsage(usage) @profiler.trace def usage_list(request, start, end): - return [NovaUsage(u) for u in - novaclient(request).usage.list(start, end, True)] + client = novaclient(request) + usage_list = client.usage.list(start, end, True) + if client.api_version >= api_versions.APIVersion('2.40'): + # If the number of instances used to calculate the usage is greater + # than max_limit, the usage will be split across multiple requests + # and the responses will need to be merged back together. + usages = collections.OrderedDict() + _merge_usage_list(usages, usage_list) + marker = _get_usage_list_marker(usage_list) + while marker: + next_usage_list = client.usage.list(start, end, True, + marker=marker) + marker = _get_usage_list_marker(next_usage_list) + if marker: + _merge_usage_list(usages, next_usage_list) + usage_list = usages.values() + return [NovaUsage(u) for u in usage_list] @profiler.trace diff --git a/openstack_dashboard/test/api_tests/nova_tests.py b/openstack_dashboard/test/api_tests/nova_tests.py index 306a88a65..43103bdda 100644 --- a/openstack_dashboard/test/api_tests/nova_tests.py +++ b/openstack_dashboard/test/api_tests/nova_tests.py @@ -24,6 +24,7 @@ from django import http from django.test.utils import override_settings from mox3.mox import IsA # noqa +from novaclient import api_versions from novaclient import exceptions as nova_exceptions from novaclient.v2 import flavor_access as nova_flavor_access from novaclient.v2 import servers @@ -201,6 +202,24 @@ class ComputeApiTests(test.APITestCase): 'start', 'end') self.assertIsInstance(ret_val, api.nova.NovaUsage) + def test_usage_get_paginated(self): + novaclient = self.stub_novaclient() + novaclient.api_version = api_versions.APIVersion('2.40') + novaclient.usage = self.mox.CreateMockAnything() + novaclient.usage.get(self.tenant.id, 'start', 'end')\ + .AndReturn(self.usages.first()) + novaclient.usage.get( + self.tenant.id, + 'start', + 'end', + marker=u'063cf7f3-ded1-4297-bc4c-31eae876cc93', + ).AndReturn({}) + self.mox.ReplayAll() + + ret_val = api.nova.usage_get(self.request, self.tenant.id, + 'start', 'end') + self.assertIsInstance(ret_val, api.nova.NovaUsage) + def test_usage_list(self): usages = self.usages.list() @@ -213,6 +232,25 @@ class ComputeApiTests(test.APITestCase): for usage in ret_val: self.assertIsInstance(usage, api.nova.NovaUsage) + def test_usage_list_paginated(self): + usages = self.usages.list() + + novaclient = self.stub_novaclient() + novaclient.api_version = api_versions.APIVersion('2.40') + novaclient.usage = self.mox.CreateMockAnything() + novaclient.usage.list('start', 'end', True).AndReturn(usages) + novaclient.usage.list( + 'start', + 'end', + True, + marker=u'063cf7f3-ded1-4297-bc4c-31eae876cc93', + ).AndReturn({}) + self.mox.ReplayAll() + + ret_val = api.nova.usage_list(self.request, 'start', 'end') + for usage in ret_val: + self.assertIsInstance(usage, api.nova.NovaUsage) + def test_server_get(self): server = self.servers.first() diff --git a/openstack_dashboard/test/test_data/nova_data.py b/openstack_dashboard/test/test_data/nova_data.py index f5d8d7b5b..b889ded41 100644 --- a/openstack_dashboard/test/test_data/nova_data.py +++ b/openstack_dashboard/test/test_data/nova_data.py @@ -139,7 +139,8 @@ USAGE_DATA = """ "hours": 122.87361111111112, "vcpus": %(flavor_vcpus)s, "flavor": "%(flavor_name)s", - "local_gb": %(flavor_disk)s + "local_gb": %(flavor_disk)s, + "instance_id": "063cf7f3-ded1-4297-bc4c-31eae876cc92" }, { "memory_mb": %(flavor_ram)s, @@ -152,7 +153,8 @@ USAGE_DATA = """ "hours": 2.608611111111111, "vcpus": %(flavor_vcpus)s, "flavor": "%(flavor_name)s", - "local_gb": %(flavor_disk)s + "local_gb": %(flavor_disk)s, + "instance_id": "063cf7f3-ded1-4297-bc4c-31eae876cc93" } ] }