From 8fd8d4e40a03613c794bb8e86be7b11851a2e63f Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Wed, 6 Feb 2013 11:33:34 +0100 Subject: [PATCH] Use Mongo finalize to compute avg and duration This reduces the amount of things we do in Python and prepare for future usage of more aggregation. It also returns real duration; API is then free to override this with whatever it wants (and it does so). Change-Id: I9850a4ab146612fbf85587f5553881a22acac67b Signed-off-by: Julien Danjou --- ceilometer/storage/impl_mongodb.py | 47 ++++++++++++--------------- ceilometer/storage/impl_sqlalchemy.py | 2 +- tests/storage/base.py | 7 ++++ tools/test-requires | 3 +- tools/test-requires-folsom | 3 +- 5 files changed, 32 insertions(+), 30 deletions(-) diff --git a/ceilometer/storage/impl_mongodb.py b/ceilometer/storage/impl_mongodb.py index 01b97da750..c201dae8f4 100644 --- a/ceilometer/storage/impl_mongodb.py +++ b/ceilometer/storage/impl_mongodb.py @@ -180,7 +180,7 @@ class Connection(base.Connection): function () { emit('statistics', { min : this.counter_volume, max : this.counter_volume, - qty : this.counter_volume, + sum : this.counter_volume, count : 1, timestamp_min : this.timestamp, timestamp_max : this.timestamp } ) @@ -196,7 +196,7 @@ class Connection(base.Connection): if ( values[i].max > res.max ) res.max = values[i].max; res.count += values[i].count; - res.qty += values[i].qty; + res.sum += values[i].sum; if ( values[i].timestamp_min < res.timestamp_min ) res.timestamp_min = values[i].timestamp_min; if ( values[i].timestamp_max > res.timestamp_max ) @@ -206,6 +206,13 @@ class Connection(base.Connection): } """) + FINALIZE_STATS = bson.code.Code(""" + function (key, value) { + value.avg = value.sum / value.count; + value.duration = (value.timestamp_max - value.timestamp_min) / 1000; + return value; + }""") + def __init__(self, conf): opts = self._parse_connection_url(conf.database_connection) LOG.info('connecting to MongoDB on %s:%s', opts['host'], opts['port']) @@ -466,34 +473,20 @@ class Connection(base.Connection): results = self.db.meter.map_reduce(self.MAP_STATS, self.REDUCE_STATS, {'inline': 1}, + finalize=self.FINALIZE_STATS, query=q, ) if results['results']: - r = results['results'][0]['value'] - (start, end) = self._fix_interval_min_max(r['timestamp_min'], - r['timestamp_max']) - else: - start = None - end = None - r = {'count': 0, - 'min': None, - 'max': None, - 'avg': None, - 'qty': None, - 'duration': None, - 'duration_start': None, - 'duration_end': None, - } - count = int(r['count']) - return {'min': r['min'], - 'sum': r['qty'], - 'count': count, - 'avg': (r['qty'] / count) if count > 0 else None, - 'max': r['max'], - 'duration': 0, - 'duration_start': start, - 'duration_end': end, - } + return results['results'][0]['value'] + + return {'count': 0, + 'min': None, + 'max': None, + 'avg': None, + 'sum': None, + 'duration': None, + 'duration_start': None, + 'duration_end': None} def get_volume_sum(self, event_filter): """Return the sum of the volume field for the events diff --git a/ceilometer/storage/impl_sqlalchemy.py b/ceilometer/storage/impl_sqlalchemy.py index cf758a8e89..14b5da9e66 100644 --- a/ceilometer/storage/impl_sqlalchemy.py +++ b/ceilometer/storage/impl_sqlalchemy.py @@ -407,7 +407,7 @@ class Connection(base.Connection): 'max': res[4], 'avg': (res[2] / count) if count > 0 else None, 'sum': res[2], - 'duration': None, + 'duration': (res[1] - res[0]).seconds, 'duration_start': res[0], 'duration_end': res[1], } diff --git a/tests/storage/base.py b/tests/storage/base.py index ad5bff2ca2..769d353458 100644 --- a/tests/storage/base.py +++ b/tests/storage/base.py @@ -769,6 +769,9 @@ class StatisticsTest(DBTestBase): meter='volume.size', ) results = self.conn.get_meter_statistics(f) + self.assertEqual(results['duration'], + (datetime.datetime(2012, 9, 25, 12, 32) + - datetime.datetime(2012, 9, 25, 10, 30)).seconds) assert results['count'] == 3 assert results['min'] == 8 assert results['max'] == 10 @@ -783,6 +786,7 @@ class StatisticsTest(DBTestBase): end='2012-09-25T11:32:00', ) results = self.conn.get_meter_statistics(f) + self.assertEqual(results['duration'], 0) assert results['count'] == 1 assert results['min'] == 6 assert results['max'] == 6 @@ -795,6 +799,9 @@ class StatisticsTest(DBTestBase): meter='volume.size', ) results = self.conn.get_meter_statistics(f) + self.assertEqual(results['duration'], + (datetime.datetime(2012, 9, 25, 12, 32) + - datetime.datetime(2012, 9, 25, 10, 30)).seconds) assert results['count'] == 3 assert results['min'] == 5 assert results['max'] == 7 diff --git a/tools/test-requires b/tools/test-requires index 66b0be3e4e..d768aba1db 100644 --- a/tools/test-requires +++ b/tools/test-requires @@ -5,7 +5,8 @@ mox Babel>=0.9.6 # NOTE(dhellmann): Ming is necessary to provide the Mongo-in-memory # implementation of MongoDB. -Ming>=0.3.2 +# NOTE(jd): the support for finalize in map reduce in MIM is not yet merged +git+http://git.naquadah.org/git/~jd/merciless.git#egg=Ming http://tarballs.openstack.org/nova/nova-master.tar.gz http://tarballs.openstack.org/glance/glance-master.tar.gz setuptools-git>=0.4 diff --git a/tools/test-requires-folsom b/tools/test-requires-folsom index 9ac3453cce..0c52ed0fb7 100644 --- a/tools/test-requires-folsom +++ b/tools/test-requires-folsom @@ -5,7 +5,8 @@ mock mox # NOTE(dhellmann): Ming is necessary to provide the Mongo-in-memory # implementation of MongoDB. -Ming>=0.3.2 +# NOTE(jd): the support for finalize in map reduce in MIM is not yet merged +git+http://git.naquadah.org/git/~jd/merciless.git#egg=Ming http://tarballs.openstack.org/glance/glance-stable-folsom.tar.gz setuptools-git>=0.4 # 1.7.4 is the Folsom release