From 1d73a6f12f123743e51674b5dbc876bf3d600cd8 Mon Sep 17 00:00:00 2001
From: Ilya Tyaptin <ityaptin@mirantis.com>
Date: Thu, 3 Mar 2016 20:12:06 +0300
Subject: [PATCH] Cast Int64 values to int, float in statistics

Currently statistics requests fail with type errors.
It causes by the fact that pymongo returns bson.Int64
instead of float or int for big nums. It affects a `cpu`
meter usually.

Change-Id: I744a9f79a4bbc3b2ecdd73c126b0859f3f8a733c
Closes-bug: #1532661
---
 ceilometer/storage/impl_mongodb.py            |  5 ++-
 ceilometer/storage/mongo/utils.py             | 20 +++++----
 .../api/v2/test_statistics_scenarios.py       | 42 +++++++++++++++++++
 3 files changed, 58 insertions(+), 9 deletions(-)

diff --git a/ceilometer/storage/impl_mongodb.py b/ceilometer/storage/impl_mongodb.py
index 9eae6b94..4867c141 100644
--- a/ceilometer/storage/impl_mongodb.py
+++ b/ceilometer/storage/impl_mongodb.py
@@ -591,9 +591,10 @@ class Connection(pymongo_base.Connection):
 
     def _stats_result_aggregates(self, result, aggregate):
         stats_args = {}
-        for attr in Connection.STANDARD_AGGREGATES.keys():
+        for attr, func in Connection.STANDARD_AGGREGATES.items():
             if attr in result:
-                stats_args[attr] = result[attr]
+                stats_args.update(func.finalize(result,
+                                                version_array=self.version))
 
         if aggregate:
             stats_args['aggregate'] = {}
diff --git a/ceilometer/storage/mongo/utils.py b/ceilometer/storage/mongo/utils.py
index 6f855335..47b06ac7 100644
--- a/ceilometer/storage/mongo/utils.py
+++ b/ceilometer/storage/mongo/utils.py
@@ -43,7 +43,8 @@ OP_SIGN = {'lt': '$lt', 'le': '$lte', 'ne': '$ne', 'gt': '$gt', 'ge': '$gte'}
 MINIMUM_COMPATIBLE_MONGODB_VERSION = [2, 4]
 COMPLETE_AGGREGATE_COMPATIBLE_VERSION = [2, 6]
 
-FINALIZE_AGGREGATION_LAMBDA = lambda result, param=None: float(result)
+FINALIZE_FLOAT_LAMBDA = lambda result, param=None: float(result)
+FINALIZE_INT_LAMBDA = lambda result, param=None: int(result)
 CARDINALITY_VALIDATION = (lambda name, param: param in ['resource_id',
                                                         'user_id',
                                                         'project_id',
@@ -515,7 +516,7 @@ class AggregationFields(object):
                  finalize=None,
                  parametrized=False,
                  validate=None):
-        self._finalize = finalize or FINALIZE_AGGREGATION_LAMBDA
+        self._finalize = finalize or FINALIZE_FLOAT_LAMBDA
         self.group = lambda *args: group(*args) if parametrized else group
         self.project = (lambda *args: project(*args)
                         if parametrized else project)
@@ -566,23 +567,28 @@ class Aggregation(object):
 SUM_AGGREGATION = Aggregation(
     "sum", AggregationFields(MINIMUM_COMPATIBLE_MONGODB_VERSION,
                              {"sum": {"$sum": "$counter_volume"}},
-                             {"sum": "$sum"}))
+                             {"sum": "$sum"},
+                             ))
 AVG_AGGREGATION = Aggregation(
     "avg", AggregationFields(MINIMUM_COMPATIBLE_MONGODB_VERSION,
                              {"avg": {"$avg": "$counter_volume"}},
-                             {"avg": "$avg"}))
+                             {"avg": "$avg"},
+                             ))
 MIN_AGGREGATION = Aggregation(
     "min", AggregationFields(MINIMUM_COMPATIBLE_MONGODB_VERSION,
                              {"min": {"$min": "$counter_volume"}},
-                             {"min": "$min"}))
+                             {"min": "$min"},
+                             ))
 MAX_AGGREGATION = Aggregation(
     "max", AggregationFields(MINIMUM_COMPATIBLE_MONGODB_VERSION,
                              {"max": {"$max": "$counter_volume"}},
-                             {"max": "$max"}))
+                             {"max": "$max"},
+                             ))
 COUNT_AGGREGATION = Aggregation(
     "count", AggregationFields(MINIMUM_COMPATIBLE_MONGODB_VERSION,
                                {"count": {"$sum": 1}},
-                               {"count": "$count"}))
+                               {"count": "$count"},
+                               FINALIZE_INT_LAMBDA))
 STDDEV_AGGREGATION = Aggregation(
     "stddev",
     AggregationFields(MINIMUM_COMPATIBLE_MONGODB_VERSION,
diff --git a/ceilometer/tests/functional/api/v2/test_statistics_scenarios.py b/ceilometer/tests/functional/api/v2/test_statistics_scenarios.py
index efa7f877..b9e41d32 100644
--- a/ceilometer/tests/functional/api/v2/test_statistics_scenarios.py
+++ b/ceilometer/tests/functional/api/v2/test_statistics_scenarios.py
@@ -1649,3 +1649,45 @@ class TestUnparameterizedAggregates(v2.FunctionalTest):
                                            places=4)
                     for a in standard_aggregates:
                         self.assertNotIn(a, r)
+
+
+@tests_db.run_with('mongodb')
+class TestBigValueStatistics(v2.FunctionalTest):
+
+    PATH = '/meters/volume.size/statistics'
+
+    def setUp(self):
+        super(TestBigValueStatistics, self).setUp()
+        for i in range(0, 3):
+            s = sample.Sample(
+                'volume.size',
+                'gauge',
+                'GiB',
+                (i + 1) * (10 ** 12),
+                'user-id',
+                'project1',
+                'resource-id',
+                timestamp=datetime.datetime(2012, 9, 25, 10 + i, 30 + i),
+                resource_metadata={'display_name': 'test-volume',
+                                   'tag': 'self.sample',
+                                   },
+                source='source1',
+            )
+            msg = utils.meter_message_from_counter(
+                s, self.CONF.publisher.telemetry_secret,
+            )
+            self.conn.record_metering_data(msg)
+
+    def test_big_value_statistics(self):
+        data = self.get_json(self.PATH)
+
+        expected_values = {'count': 3,
+                           'min': 10 ** 12,
+                           'max': 3 * 10 ** 12,
+                           'sum': 6 * 10 ** 12,
+                           'avg': 2 * 10 ** 12}
+        self.assertEqual(1, len(data))
+        for d in data:
+            for name, expected_value in expected_values.items():
+                self.assertIn(name, d)
+                self.assertEqual(expected_value, d[name])