From d0881e5b7e4784729a1900ab67c24b7c85057818 Mon Sep 17 00:00:00 2001 From: Ana Krivokapic Date: Fri, 30 May 2014 17:26:37 +0200 Subject: [PATCH] Add unit conversion for metering views Until now, the charts displayed by the metering views (Admin -> Resource Usage) were using units as defined by the Ceilometer meters. This means that the values in the charts were sometimes very hard to read for humans (typically we would see a large number in nanoseconds, bytes, etc). This patch introduces support for normalization of these values. The statistics returned by the Ceilometer API are analyzed to determine the appropriate unit for the series. The whole series is then converted to this unit, so that the values in the series are more easily parsed by humans. Implements blueprint: ceilometer-api-statistics-decorators-and-convertors Change-Id: Ia183ca5270ebd1dcbd1fe6ac20ed4dfe4d6ebcc4 --- .../static/horizon/js/horizon.d3linechart.js | 9 +- horizon/test/tests/utils.py | 60 ++++++++ horizon/utils/functions.py | 53 +++++++ horizon/utils/units.py | 145 ++++++++++++++++++ .../dashboards/admin/metering/tabs.py | 2 +- .../dashboards/admin/metering/views.py | 65 ++++++-- requirements.txt | 1 + test-requirements.txt | 1 + 8 files changed, 317 insertions(+), 19 deletions(-) create mode 100644 horizon/utils/units.py diff --git a/horizon/static/horizon/js/horizon.d3linechart.js b/horizon/static/horizon/js/horizon.d3linechart.js index 2a6c80fdd7..ef12a1d9a7 100644 --- a/horizon/static/horizon/js/horizon.d3linechart.js +++ b/horizon/static/horizon/js/horizon.d3linechart.js @@ -477,6 +477,12 @@ horizon.d3_line_chart = { var hoverDetail = new Rickshaw.Graph.HoverDetail({ graph: graph, formatter: function(series, x, y) { + if(y % 1 === 0) { + y = parseInt(y); + } else { + y = parseFloat(y).toFixed(2); + } + var d = new Date(x * 1000); // Convert datetime to YYYY-MM-DD HH:MM:SS GMT var datetime_string = d.getUTCFullYear() + "-" + @@ -488,8 +494,7 @@ horizon.d3_line_chart = { var date = '' + datetime_string + ''; var swatch = ''; - var content = swatch + series.name + ': ' + parseFloat(y).toFixed(2) + ' ' + series.unit + '
' + date; - return content; + return swatch + series.name + ': ' + y + ' ' + series.unit + '
' + date; } }); } diff --git a/horizon/test/tests/utils.py b/horizon/test/tests/utils.py index 130aa79c71..02f766462c 100644 --- a/horizon/test/tests/utils.py +++ b/horizon/test/tests/utils.py @@ -27,6 +27,7 @@ from horizon.utils.filters import parse_isotime # noqa from horizon.utils import functions from horizon.utils import memoized from horizon.utils import secret_key +from horizon.utils import units from horizon.utils import validators @@ -395,3 +396,62 @@ class GetPageSizeTests(test.TestCase): self.assertRaises(ValueError, functions.get_page_size, request, default) + + +class UnitsTests(test.TestCase): + def test_is_supported(self): + self.assertTrue(units.is_supported('MB')) + self.assertTrue(units.is_supported('min')) + self.assertFalse(units.is_supported('KWh')) + self.assertFalse(units.is_supported('unknown_unit')) + + def test_is_larger(self): + self.assertTrue(units.is_larger('KB', 'B')) + self.assertTrue(units.is_larger('MB', 'B')) + self.assertTrue(units.is_larger('GB', 'B')) + self.assertTrue(units.is_larger('TB', 'B')) + self.assertTrue(units.is_larger('GB', 'MB')) + self.assertFalse(units.is_larger('B', 'KB')) + self.assertFalse(units.is_larger('MB', 'GB')) + + self.assertTrue(units.is_larger('min', 's')) + self.assertTrue(units.is_larger('hr', 'min')) + self.assertTrue(units.is_larger('hr', 's')) + self.assertFalse(units.is_larger('s', 'min')) + + def test_convert(self): + self.assertEqual(units.convert(4096, 'MB', 'GB'), (4, 'GB')) + self.assertEqual(units.convert(4, 'GB', 'MB'), (4096, 'MB')) + + self.assertEqual(units.convert(1.5, 'hr', 'min'), (90, 'min')) + self.assertEqual(units.convert(12, 'hr', 'day'), (0.5, 'day')) + + def test_normalize(self): + self.assertEqual(units.normalize(1, 'B'), (1, 'B')) + self.assertEqual(units.normalize(1000, 'B'), (1000, 'B')) + self.assertEqual(units.normalize(1024, 'B'), (1, 'KB')) + self.assertEqual(units.normalize(1024 * 1024, 'B'), (1, 'MB')) + self.assertEqual(units.normalize(10 * 1024 ** 3, 'B'), (10, 'GB')) + self.assertEqual(units.normalize(1000 * 1024 ** 4, 'B'), (1000, 'TB')) + self.assertEqual(units.normalize(1024, 'KB'), (1, 'MB')) + self.assertEqual(units.normalize(1024 ** 2, 'KB'), (1, 'GB')) + self.assertEqual(units.normalize(10 * 1024, 'MB'), (10, 'GB')) + self.assertEqual(units.normalize(0.5, 'KB'), (512, 'B')) + self.assertEqual(units.normalize(0.0001, 'MB'), (104.9, 'B')) + + self.assertEqual(units.normalize(1, 's'), (1, 's')) + self.assertEqual(units.normalize(60, 's'), (1, 'min')) + self.assertEqual(units.normalize(3600, 's'), (1, 'hr')) + self.assertEqual(units.normalize(3600 * 24, 's'), (1, 'day')) + self.assertEqual(units.normalize(10 * 3600 * 24, 's'), (1.4, 'week')) + self.assertEqual(units.normalize(60, 'min'), (1, 'hr')) + self.assertEqual(units.normalize(90, 'min'), (1.5, 'hr')) + self.assertEqual(units.normalize(60 * 24, 'min'), (1, 'day')) + self.assertEqual(units.normalize(0.5, 'day'), (12, 'hr')) + self.assertEqual(units.normalize(10800000000000, 'ns'), (3, 'hr')) + self.assertEqual(units.normalize(7, 'day'), (1, 'week')) + self.assertEqual(units.normalize(31, 'day'), (1, 'month')) + self.assertEqual(units.normalize(12, 'month'), (1, 'year')) + + self.assertEqual(units.normalize(1, 'unknown_unit'), + (1, 'unknown_unit')) diff --git a/horizon/utils/functions.py b/horizon/utils/functions.py index 85802f53ed..3f4b12c4b0 100644 --- a/horizon/utils/functions.py +++ b/horizon/utils/functions.py @@ -10,6 +10,7 @@ # License for the specific language governing permissions and limitations # under the License. +import decimal import math import re @@ -72,3 +73,55 @@ def get_page_size(request, default=20): def natural_sort(attr): return lambda x: [int(s) if s.isdigit() else s for s in re.split(r'(\d+)', getattr(x, attr, x))] + + +def get_keys(tuple_of_tuples): + """Processes a tuple of 2-element tuples and returns a tuple containing + first component of each tuple. + """ + return tuple([t[0] for t in tuple_of_tuples]) + + +def value_for_key(tuple_of_tuples, key): + """Processes a tuple of 2-element tuples and returns the value + corresponding to the given key. If not value is found, the key is returned. + """ + for t in tuple_of_tuples: + if t[0] == key: + return t[1] + else: + return key + + +def next_key(tuple_of_tuples, key): + """Processes a tuple of 2-element tuples and returns the key which comes + after the given key. + """ + for i, t in enumerate(tuple_of_tuples): + if t[0] == key: + try: + return tuple_of_tuples[i + 1][0] + except IndexError: + return None + + +def previous_key(tuple_of_tuples, key): + """Processes a tuple of 2-element tuples and returns the key which comes + before the given key. + """ + for i, t in enumerate(tuple_of_tuples): + if t[0] == key: + try: + return tuple_of_tuples[i - 1][0] + except IndexError: + return None + + +def format_value(value): + """Returns the given value rounded to one decimal place if it is a + decimal, or integer if it is an integer. + """ + value = decimal.Decimal(str(value)) + if int(value) == value: + return int(value) + return round(value, 1) diff --git a/horizon/utils/units.py b/horizon/utils/units.py new file mode 100644 index 0000000000..1bed48f0bc --- /dev/null +++ b/horizon/utils/units.py @@ -0,0 +1,145 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import decimal + +import pint + +from horizon.utils import functions + +# Mapping of units from Ceilometer to Pint +INFORMATION_UNITS = ( + ('B', 'byte'), + ('KB', 'Kibyte'), + ('MB', 'Mibyte'), + ('GB', 'Gibyte'), + ('TB', 'Tibyte'), + ('PB', 'Pibyte'), + ('EB', 'Eibyte'), +) + +TIME_UNITS = ('ns', 's', 'min', 'hr', 'day', 'week', 'month', 'year') + + +ureg = pint.UnitRegistry() + + +def is_supported(unit): + """Returns a bool indicating whether the unit specified is supported by + this module. + """ + return unit in functions.get_keys(INFORMATION_UNITS) + TIME_UNITS + + +def is_larger(unit_1, unit_2): + """Returns a boolean indicating whether unit_1 is larger than unit_2. + + E.g: + + >>> is_larger('KB', 'B') + True + >>> is_larger('min', 'day') + False + """ + unit_1 = functions.value_for_key(INFORMATION_UNITS, unit_1) + unit_2 = functions.value_for_key(INFORMATION_UNITS, unit_2) + + return ureg.parse_expression(unit_1) > ureg.parse_expression(unit_2) + + +def convert(value, source_unit, target_unit, fmt=False): + """Converts value from source_unit to target_unit. Returns a tuple + containing the converted value and target_unit. Having fmt set to True + causes the value to be formatted to 1 decimal digit if it's a decimal or + be formatted as integer if it's an integer. + + E.g: + + >>> convert(2, 'hr', 'min') + (120.0, 'min') + >>> convert(2, 'hr', 'min', fmt=True) + (120, 'min') + >>> convert(30, 'min', 'hr', fmt=True) + (0.5, 'hr') + """ + orig_target_unit = target_unit + source_unit = functions.value_for_key(INFORMATION_UNITS, source_unit) + target_unit = functions.value_for_key(INFORMATION_UNITS, target_unit) + + q = ureg.Quantity(value, source_unit) + q = q.to(ureg.parse_expression(target_unit)) + value = functions.format_value(q.magnitude) if fmt else q.magnitude + return value, orig_target_unit + + +def normalize(value, unit): + """Converts the value so that it belongs to some expected range. + Returns the new value and new unit. + + E.g: + + >>> normalize(1024, 'KB') + (1, 'MB') + >>> normalize(90, 'min') + (1.5, 'hr') + >>> normalize(1.0, 'object') + (1, 'object') + """ + if value < 0: + raise ValueError('Negative value: %s %s.' % (value, unit)) + + if unit in functions.get_keys(INFORMATION_UNITS): + return _normalize_information(value, unit) + elif unit in TIME_UNITS: + return _normalize_time(value, unit) + else: + # Unknown unit, just return it + return functions.format_value(value), unit + + +def _normalize_information(value, unit): + value = decimal.Decimal(str(value)) + + while value < 1: + prev_unit = functions.previous_key(INFORMATION_UNITS, unit) + if prev_unit is None: + break + value, unit = convert(value, unit, prev_unit) + + while value >= 1024: + next_unit = functions.next_key(INFORMATION_UNITS, unit) + if next_unit is None: + break + value, unit = convert(value, unit, next_unit) + + return functions.format_value(value), unit + + +def _normalize_time(value, unit): + value, unit = convert(value, unit, 's') + + if value >= 60: + value, unit = convert(value, 's', 'min') + + if value >= 60: + value, unit = convert(value, 'min', 'hr') + + if value >= 24: + value, unit = convert(value, 'hr', 'day') + + if value >= 365: + value, unit = convert(value, 'day', 'year') + elif value >= 31: + value, unit = convert(value, 'day', 'month') + elif value >= 7: + value, unit = convert(value, 'day', 'week') + + return functions.format_value(value), unit diff --git a/openstack_dashboard/dashboards/admin/metering/tabs.py b/openstack_dashboard/dashboards/admin/metering/tabs.py index 31c9308a12..35e0b7056c 100644 --- a/openstack_dashboard/dashboards/admin/metering/tabs.py +++ b/openstack_dashboard/dashboards/admin/metering/tabs.py @@ -28,7 +28,7 @@ from openstack_dashboard.utils import metering class GlobalStatsTab(tabs.TableTab): name = _("Stats") slug = "stats" - template_name = ("admin/metering/stats.html") + template_name = "admin/metering/stats.html" preload = False table_classes = (metering_tables.UsageTable,) diff --git a/openstack_dashboard/dashboards/admin/metering/views.py b/openstack_dashboard/dashboards/admin/metering/views.py index e97e5aaa32..1187093b31 100644 --- a/openstack_dashboard/dashboards/admin/metering/views.py +++ b/openstack_dashboard/dashboards/admin/metering/views.py @@ -21,6 +21,7 @@ from horizon import exceptions from horizon import forms from horizon import tabs from horizon.utils import csvbase +from horizon.utils import units from openstack_dashboard.api import ceilometer @@ -46,11 +47,11 @@ class SamplesView(django.views.generic.TemplateView): template_name = "admin/metering/samples.csv" @staticmethod - def _series_for_meter(aggregates, - resource_name, - meter_name, - stats_name, - unit): + def series_for_meter(aggregates, + resource_name, + meter_name, + stats_name, + unit): """Construct datapoint series for a meter from resource aggregates.""" series = [] for resource in aggregates: @@ -65,6 +66,39 @@ class SamplesView(django.views.generic.TemplateView): series.append(point) return series + @staticmethod + def normalize_series_by_unit(series): + """Transform series' values into a more human readable form: + 1) Determine the data point with the maximum value + 2) Decide the unit appropriate for this value (normalize it) + 3) Convert other values to this new unit, if necessary + """ + if not series: + return series + + source_unit = target_unit = series[0]['unit'] + + if not units.is_supported(source_unit): + return series + + # Find the data point with the largest value and normalize it to + # determine its unit - that will be the new unit + maximum = max([d['y'] for point in series for d in point['data']]) + unit = units.normalize(maximum, source_unit)[1] + + # If unit needs to be changed, set the new unit for all data points + # and convert all values to that unit + if units.is_larger(unit, target_unit): + target_unit = unit + for i, point in enumerate(series[:]): + if point['unit'] != target_unit: + series[i]['unit'] = target_unit + for j, d in enumerate(point['data'][:]): + series[i]['data'][j]['y'] = units.convert( + d['y'], source_unit, target_unit, fmt=True)[0] + + return series + def get(self, request, *args, **kwargs): meter = request.GET.get('meter', None) if not meter: @@ -94,19 +128,18 @@ class SamplesView(django.views.generic.TemplateView): query = utils_metering.MeterQuery(request, date_from, date_to, 3600 * 24) - resources, unit = query.query(meter_name) + resources, unit = query.query(meter) resource_name = 'id' if group_by == "project" else 'resource_id' - series = self._series_for_meter(resources, - resource_name, - meter_name, - stats_attr, - unit) - ret = {} - ret['series'] = series - ret['settings'] = {} + series = self.series_for_meter(resources, + resource_name, + meter_name, + stats_attr, + unit) - return HttpResponse(json.dumps(ret), - content_type='application/json') + series = self.normalize_series_by_unit(series) + ret = {'series': series, 'settings': {}} + + return HttpResponse(json.dumps(ret), content_type='application/json') class CsvReportView(django.views.generic.View): diff --git a/requirements.txt b/requirements.txt index 8076b5be48..128e84f72c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,7 @@ pbr>=0.6,!=0.7,<1.0 # Horizon Core Requirements Babel>=1.3 Django>=1.4.2,<1.7 +Pint>=0.5 # BSD django_compressor>=1.4 django_openstack_auth>=1.1.7 django-pyscss>=1.0.3 # BSD License (2 clause) diff --git a/test-requirements.txt b/test-requirements.txt index ec9c33b925..73b718b29a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -23,5 +23,6 @@ oslosphinx>=2.2.0 # Apache-2.0 selenium sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3 testtools>=0.9.36,!=1.2.0,!=1.4.0 +unittest2 # This also needs xvfb library installed on your OS xvfbwrapper>=0.1.3 #license: MIT