add mandatory limit value to meter list

unrestricted listing of meters can require significant memory.
this patch implements mandatory limit on meter-list

Change-Id: I8b38136c2493e75fbc7d9aef75aa055463a26319
Implements: blueprint mandatory-limit

ceilometer/api/controllers/v2/meters.py

@@ -476,8 +476,8 @@ class MetersController(rest.RestController):
     def _lookup(self, meter_name, *remainder):
         return MeterController(meter_name), remainder
 
-    @wsme_pecan.wsexpose([Meter], [base.Query])
+    @wsme_pecan.wsexpose([Meter], [base.Query], int)
-    def get_all(self, q=None):
+    def get_all(self, q=None, limit=None):
         """Return all known meters, based on the data recorded so far.
 
         :param q: Filter rules for the meters to be returned.
@@ -488,7 +488,9 @@ class MetersController(rest.RestController):
         q = q or []
 
         # Timestamp field is not supported for Meter queries
+        limit = v2_utils.enforce_limit(limit)
         kwargs = v2_utils.query_to_kwargs(
             q, pecan.request.storage_conn.get_meters, allow_timestamps=False)
         return [Meter.from_db_model(m)
-                for m in pecan.request.storage_conn.get_meters(**kwargs)]
+                for m in pecan.request.storage_conn.get_meters(limit=limit,
+                                                               **kwargs)]

ceilometer/storage/impl_hbase.py

@@ -242,7 +242,7 @@ class Connection(hbase_base.Connection, base.Connection):
                     metadata=md)
 
     def get_meters(self, user=None, project=None, resource=None, source=None,
-                   metaquery=None):
+                   metaquery=None, limit=None):
         """Return an iterable of models.Meter instances
 
         :param user: Optional ID for user that owns the resource.
@@ -250,7 +250,10 @@ class Connection(hbase_base.Connection, base.Connection):
         :param resource: Optional resource filter.
         :param source: Optional source filter.
         :param metaquery: Optional dict with metadata to match on.
+        :param limit: Maximum number of results to return.
         """
+        if limit == 0:
+            return
 
         metaquery = metaquery or {}
 
@@ -271,6 +274,8 @@ class Connection(hbase_base.Connection, base.Connection):
                 flatten_result, s, meters, md = hbase_utils.deserialize_entry(
                     data)
                 for m in meters:
+                    if limit and len(result) >= limit:
+                        return
                     _m_rts, m_source, name, m_type, unit = m[0]
                     meter_dict = {'name': name,
                                   'type': m_type,

ceilometer/storage/impl_sqlalchemy.py

@@ -464,7 +464,7 @@ class Connection(base.Connection):
             )
 
     def get_meters(self, user=None, project=None, resource=None, source=None,
-                   metaquery=None):
+                   metaquery=None, limit=None):
         """Return an iterable of api_models.Meter instances
 
         :param user: Optional ID for user that owns the resource.
@@ -472,7 +472,10 @@ class Connection(base.Connection):
         :param resource: Optional ID of the resource.
         :param source: Optional source filter.
         :param metaquery: Optional dict with metadata to match on.
+        :param limit: Maximum number of results to return.
         """
+        if limit == 0:
+            return
         s_filter = storage.SampleFilter(user=user,
                                         project=project,
                                         source=source,
@@ -505,6 +508,7 @@ class Connection(base.Connection):
         query_sample = make_query_from_filter(session, query_sample, s_filter,
                                               require_meter=False)
 
+        query_sample = query_sample.limit(limit) if limit else query_sample
         for row in query_sample.all():
             yield api_models.Meter(
                 name=row.name,

ceilometer/storage/pymongo_base.py

@@ -50,7 +50,7 @@ class Connection(base.Connection):
     )
 
     def get_meters(self, user=None, project=None, resource=None, source=None,
-                   metaquery=None):
+                   metaquery=None, limit=None):
         """Return an iterable of models.Meter instances
 
         :param user: Optional ID for user that owns the resource.
@@ -58,7 +58,10 @@ class Connection(base.Connection):
         :param resource: Optional resource filter.
         :param source: Optional source filter.
         :param metaquery: Optional dict with metadata to match on.
+        :param limit: Maximum number of results to return.
         """
+        if limit == 0:
+            return
 
         metaquery = pymongo_utils.improve_keys(metaquery, metaquery=True) or {}
 
@@ -73,8 +76,13 @@ class Connection(base.Connection):
             q['source'] = source
         q.update(metaquery)
 
+        count = 0
         for r in self.db.resource.find(q):
             for r_meter in r['meter']:
+                if limit and count >= limit:
+                    return
+                else:
+                    count += 1
                 yield models.Meter(
                     name=r_meter['counter_name'],
                     type=r_meter['counter_type'],

ceilometer/tests/api/v2/test_list_meters_scenarios.py

@@ -66,44 +66,62 @@ class TestListMetersRestriction(v2.FunctionalTest,
 
     def setUp(self):
         super(TestListMetersRestriction, self).setUp()
-        self.CONF.set_override('default_api_return_limit', 10, group='api')
+        self.CONF.set_override('default_api_return_limit', 3, group='api')
-        for i in range(20):
+        for x in range(5):
-            s = sample.Sample(
+            for i in range(5):
-                'volume.size',
+                s = sample.Sample(
-                'gauge',
+                    'volume.size%s' % x,
-                'GiB',
+                    'gauge',
-                5 + i,
+                    'GiB',
-                'user-id',
+                    5 + i,
-                'project1',
+                    'user-id',
-                'resource-id',
+                    'project1',
-                timestamp=(datetime.datetime(2012, 9, 25, 10, 30) +
+                    'resource-id',
-                           datetime.timedelta(seconds=i)),
+                    timestamp=(datetime.datetime(2012, 9, 25, 10, 30) +
-                resource_metadata={'display_name': 'test-volume',
+                               datetime.timedelta(seconds=i)),
-                                   'tag': 'self.sample',
+                    resource_metadata={'display_name': 'test-volume',
-                                   },
+                                       'tag': 'self.sample',
-                source='source1',
+                                       },
-            )
+                    source='source1',
-            msg = utils.meter_message_from_counter(
+                )
-                s, self.CONF.publisher.telemetry_secret,
+                msg = utils.meter_message_from_counter(
-            )
+                    s, self.CONF.publisher.telemetry_secret,
-            self.conn.record_metering_data(msg)
+                )
+                self.conn.record_metering_data(msg)
 
     def test_meter_limit(self):
-        data = self.get_json('/meters/volume.size?limit=1')
+        data = self.get_json('/meters?limit=1')
         self.assertEqual(1, len(data))
 
     def test_meter_limit_negative(self):
         self.assertRaises(webtest.app.AppError,
                           self.get_json,
-                          '/meters/volume.size?limit=-2')
+                          '/meters?limit=-2')
 
     def test_meter_limit_bigger(self):
-        data = self.get_json('/meters/volume.size?limit=42')
+        data = self.get_json('/meters?limit=42')
-        self.assertEqual(20, len(data))
+        self.assertEqual(5, len(data))
 
     def test_meter_default_limit(self):
-        data = self.get_json('/meters/volume.size')
+        data = self.get_json('/meters')
-        self.assertEqual(10, len(data))
+        self.assertEqual(3, len(data))
+
+    def test_old_sample_limit(self):
+        data = self.get_json('/meters/volume.size0?limit=1')
+        self.assertEqual(1, len(data))
+
+    def test_old_sample_limit_negative(self):
+        self.assertRaises(webtest.app.AppError,
+                          self.get_json,
+                          '/meters/volume.size0?limit=-2')
+
+    def test_old_sample_limit_bigger(self):
+        data = self.get_json('/meters/volume.size0?limit=42')
+        self.assertEqual(5, len(data))
+
+    def test_old_sample_default_limit(self):
+        data = self.get_json('/meters/volume.size0')
+        self.assertEqual(3, len(data))
 
     def test_sample_limit(self):
         data = self.get_json('/samples?limit=1')
@@ -116,11 +134,11 @@ class TestListMetersRestriction(v2.FunctionalTest,
 
     def test_sample_limit_bigger(self):
         data = self.get_json('/samples?limit=42')
-        self.assertEqual(20, len(data))
+        self.assertEqual(25, len(data))
 
     def test_sample_default_limit(self):
         data = self.get_json('/samples')
-        self.assertEqual(10, len(data))
+        self.assertEqual(3, len(data))
 
 
 class TestListMeters(v2.FunctionalTest,