Use Mongo finalize to compute avg and duration

This reduces the amount of things we do in Python and prepare for future
usage of more aggregation. It also returns real duration; API is then free
to override this with whatever it wants (and it does so).

Change-Id: I9850a4ab146612fbf85587f5553881a22acac67b
Signed-off-by: Julien Danjou <julien@danjou.info>

ceilometer/storage/impl_mongodb.py

@@ -180,7 +180,7 @@ class Connection(base.Connection):
     function () {
         emit('statistics', { min : this.counter_volume,
                              max : this.counter_volume,
-                             qty : this.counter_volume,
+                             sum : this.counter_volume,
                              count : 1,
                              timestamp_min : this.timestamp,
                              timestamp_max : this.timestamp } )
@@ -196,7 +196,7 @@ class Connection(base.Connection):
             if ( values[i].max > res.max )
                res.max = values[i].max;
             res.count += values[i].count;
-            res.qty += values[i].qty;
+            res.sum += values[i].sum;
             if ( values[i].timestamp_min < res.timestamp_min )
                res.timestamp_min = values[i].timestamp_min;
             if ( values[i].timestamp_max > res.timestamp_max )
@@ -206,6 +206,13 @@ class Connection(base.Connection):
     }
     """)
 
+    FINALIZE_STATS = bson.code.Code("""
+    function (key, value) {
+        value.avg = value.sum / value.count;
+        value.duration = (value.timestamp_max - value.timestamp_min) / 1000;
+        return value;
+    }""")
+
     def __init__(self, conf):
         opts = self._parse_connection_url(conf.database_connection)
         LOG.info('connecting to MongoDB on %s:%s', opts['host'], opts['port'])
@@ -466,34 +473,20 @@ class Connection(base.Connection):
         results = self.db.meter.map_reduce(self.MAP_STATS,
                                            self.REDUCE_STATS,
                                            {'inline': 1},
+                                           finalize=self.FINALIZE_STATS,
                                            query=q,
                                            )
         if results['results']:
-            r = results['results'][0]['value']
-            (start, end) = self._fix_interval_min_max(r['timestamp_min'],
-                                                      r['timestamp_max'])
-        else:
-            start = None
-            end = None
-            r = {'count': 0,
-                 'min': None,
-                 'max': None,
-                 'avg': None,
-                 'qty': None,
-                 'duration': None,
-                 'duration_start': None,
-                 'duration_end': None,
-                 }
-        count = int(r['count'])
-        return {'min': r['min'],
-                'sum': r['qty'],
-                'count': count,
-                'avg': (r['qty'] / count) if count > 0 else None,
-                'max': r['max'],
-                'duration': 0,
-                'duration_start': start,
-                'duration_end': end,
-                }
+            return results['results'][0]['value']
+
+        return {'count': 0,
+                'min': None,
+                'max': None,
+                'avg': None,
+                'sum': None,
+                'duration': None,
+                'duration_start': None,
+                'duration_end': None}
 
     def get_volume_sum(self, event_filter):
         """Return the sum of the volume field for the events

ceilometer/storage/impl_sqlalchemy.py

@@ -407,7 +407,7 @@ class Connection(base.Connection):
                 'max': res[4],
                 'avg': (res[2] / count) if count > 0 else None,
                 'sum': res[2],
-                'duration': None,
+                'duration': (res[1] - res[0]).seconds,
                 'duration_start': res[0],
                 'duration_end': res[1],
                 }

tests/storage/base.py

@@ -769,6 +769,9 @@ class StatisticsTest(DBTestBase):
             meter='volume.size',
         )
         results = self.conn.get_meter_statistics(f)
+        self.assertEqual(results['duration'],
+                         (datetime.datetime(2012, 9, 25, 12, 32)
+                          - datetime.datetime(2012, 9, 25, 10, 30)).seconds)
         assert results['count'] == 3
         assert results['min'] == 8
         assert results['max'] == 10
@@ -783,6 +786,7 @@ class StatisticsTest(DBTestBase):
             end='2012-09-25T11:32:00',
         )
         results = self.conn.get_meter_statistics(f)
+        self.assertEqual(results['duration'], 0)
         assert results['count'] == 1
         assert results['min'] == 6
         assert results['max'] == 6
@@ -795,6 +799,9 @@ class StatisticsTest(DBTestBase):
             meter='volume.size',
         )
         results = self.conn.get_meter_statistics(f)
+        self.assertEqual(results['duration'],
+                         (datetime.datetime(2012, 9, 25, 12, 32)
+                          - datetime.datetime(2012, 9, 25, 10, 30)).seconds)
         assert results['count'] == 3
         assert results['min'] == 5
         assert results['max'] == 7

tools/test-requires

@@ -5,7 +5,8 @@ mox
 Babel>=0.9.6
 # NOTE(dhellmann): Ming is necessary to provide the Mongo-in-memory
 # implementation of MongoDB.
-Ming>=0.3.2
+# NOTE(jd): the support for finalize in map reduce in MIM is not yet merged
+git+http://git.naquadah.org/git/~jd/merciless.git#egg=Ming
 http://tarballs.openstack.org/nova/nova-master.tar.gz
 http://tarballs.openstack.org/glance/glance-master.tar.gz
 setuptools-git>=0.4

tools/test-requires-folsom

@@ -5,7 +5,8 @@ mock
 mox
 # NOTE(dhellmann): Ming is necessary to provide the Mongo-in-memory
 # implementation of MongoDB.
-Ming>=0.3.2
+# NOTE(jd): the support for finalize in map reduce in MIM is not yet merged
+git+http://git.naquadah.org/git/~jd/merciless.git#egg=Ming
 http://tarballs.openstack.org/glance/glance-stable-folsom.tar.gz
 setuptools-git>=0.4
 # 1.7.4 is the Folsom release