Enhances get_meters to return unique meters

Enhances get_meters in the storage layer
and API to accept a unique flag and return
back a list of unique meters for clients like
Horizon to consume.

DocImpact
Change-Id: Ifdcb907df867ae650eae733bc7b635a283939064
Closes-Bug: 1506959

ceilometer/api/controllers/v2/meters.py

@@ -477,11 +477,12 @@ class MetersController(rest.RestController):
     def _lookup(self, meter_name, *remainder):
         return MeterController(meter_name), remainder
 
-    @wsme_pecan.wsexpose([Meter], [base.Query], int)
+    @wsme_pecan.wsexpose([Meter], [base.Query], int, str)
-    def get_all(self, q=None, limit=None):
+    def get_all(self, q=None, limit=None, unique=''):
         """Return all known meters, based on the data recorded so far.
 
         :param q: Filter rules for the meters to be returned.
+        :param unique: flag to indicate unique meters to be returned.
         """
 
         rbac.enforce('get_meters', pecan.request)
@@ -494,5 +495,6 @@ class MetersController(rest.RestController):
             q, pecan.request.storage_conn.get_meters,
             ['limit'], allow_timestamps=False)
         return [Meter.from_db_model(m)
-                for m in pecan.request.storage_conn.get_meters(limit=limit,
+                for m in pecan.request.storage_conn.get_meters(
-                                                               **kwargs)]
+                limit=limit, unique=strutils.bool_from_string(unique),
+                **kwargs)]

ceilometer/storage/base.py

@@ -194,7 +194,7 @@ class Connection(object):
 
     @staticmethod
     def get_meters(user=None, project=None, resource=None, source=None,
-                   metaquery=None, limit=None):
+                   metaquery=None, limit=None, unique=False):
         """Return an iterable of model.Meter instances.
 
         Iterable items containing meter information.
@@ -204,6 +204,7 @@ class Connection(object):
         :param source: Optional source filter.
         :param metaquery: Optional dict with metadata to match on.
         :param limit: Maximum number of results to return.
+        :param unique: If set to true, return only unique meter information.
         """
         raise ceilometer.NotImplementedError('Meters not implemented')
 

ceilometer/storage/impl_hbase.py

@@ -241,7 +241,7 @@ class Connection(hbase_base.Connection, base.Connection):
                     metadata=md)
 
     def get_meters(self, user=None, project=None, resource=None, source=None,
-                   metaquery=None, limit=None):
+                   metaquery=None, limit=None, unique=False):
         """Return an iterable of models.Meter instances
 
         :param user: Optional ID for user that owns the resource.
@@ -250,6 +250,7 @@ class Connection(hbase_base.Connection, base.Connection):
         :param source: Optional source filter.
         :param metaquery: Optional dict with metadata to match on.
         :param limit: Maximum number of results to return.
+        :param unique: If set to true, return only unique meter information.
         """
         if limit == 0:
             return
@@ -276,18 +277,32 @@ class Connection(hbase_base.Connection, base.Connection):
                     if limit and len(result) >= limit:
                         return
                     _m_rts, m_source, name, m_type, unit = m[0]
-                    meter_dict = {'name': name,
+                    if unique:
-                                  'type': m_type,
+                        meter_dict = {'name': name,
-                                  'unit': unit,
+                                      'type': m_type,
-                                  'resource_id': flatten_result['resource_id'],
+                                      'unit': unit,
-                                  'project_id': flatten_result['project_id'],
+                                      'resource_id': None,
-                                  'user_id': flatten_result['user_id']}
+                                      'project_id': None,
+                                      'user_id': None,
+                                      'source': None}
+                    else:
+                        meter_dict = {'name': name,
+                                      'type': m_type,
+                                      'unit': unit,
+                                      'resource_id':
+                                          flatten_result['resource_id'],
+                                      'project_id':
+                                          flatten_result['project_id'],
+                                      'user_id':
+                                          flatten_result['user_id']}
+
                     frozen_meter = frozenset(meter_dict.items())
                     if frozen_meter in result:
                         continue
                     result.add(frozen_meter)
-                    meter_dict.update({'source': m_source
+                    if not unique:
-                                       if m_source else None})
+                        meter_dict.update({'source': m_source
+                                           if m_source else None})
 
                     yield models.Meter(**meter_dict)
 

ceilometer/storage/impl_log.py

@@ -80,7 +80,7 @@ class Connection(base.Connection):
         return []
 
     def get_meters(self, user=None, project=None, resource=None, source=None,
-                   limit=None, metaquery=None):
+                   limit=None, metaquery=None, unique=False):
         """Return an iterable of dictionaries containing meter information.
 
         { 'name': name of the meter,
@@ -96,6 +96,7 @@ class Connection(base.Connection):
         :param source: Optional source filter.
         :param limit: Maximum number of results to return.
         :param metaquery: Optional dict with metadata to match on.
+        :param unique: If set to true, return only unique meter information.
         """
         return []
 

ceilometer/storage/impl_sqlalchemy.py

@@ -493,7 +493,7 @@ class Connection(base.Connection):
             )
 
     def get_meters(self, user=None, project=None, resource=None, source=None,
-                   metaquery=None, limit=None):
+                   metaquery=None, limit=None, unique=False):
         """Return an iterable of api_models.Meter instances
 
         :param user: Optional ID for user that owns the resource.
@@ -502,6 +502,7 @@ class Connection(base.Connection):
         :param source: Optional source filter.
         :param metaquery: Optional dict with metadata to match on.
         :param limit: Maximum number of results to return.
+        :param unique: If set to true, return only unique meter information.
         """
         if limit == 0:
             return
@@ -514,10 +515,17 @@ class Connection(base.Connection):
         # NOTE(gordc): get latest sample of each meter/resource. we do not
         #              filter here as we want to filter only on latest record.
         session = self._engine_facade.get_session()
+
         subq = session.query(func.max(models.Sample.id).label('id')).join(
             models.Resource,
-            models.Resource.internal_id == models.Sample.resource_id).group_by(
+            models.Resource.internal_id == models.Sample.resource_id)
-            models.Sample.meter_id, models.Resource.resource_id)
+
+        if unique:
+            subq = subq.group_by(models.Sample.meter_id)
+        else:
+            subq = subq.group_by(models.Sample.meter_id,
+                                 models.Resource.resource_id)
+
         if resource:
             subq = subq.filter(models.Resource.resource_id == resource)
         subq = subq.subquery()
@@ -538,15 +546,27 @@ class Connection(base.Connection):
                                               require_meter=False)
 
         query_sample = query_sample.limit(limit) if limit else query_sample
-        for row in query_sample.all():
+
-            yield api_models.Meter(
+        if unique:
-                name=row.name,
+            for row in query_sample.all():
-                type=row.type,
+                yield api_models.Meter(
-                unit=row.unit,
+                    name=row.name,
-                resource_id=row.resource_id,
+                    type=row.type,
-                project_id=row.project_id,
+                    unit=row.unit,
-                source=row.source_id,
+                    resource_id=None,
-                user_id=row.user_id)
+                    project_id=None,
+                    source=None,
+                    user_id=None)
+        else:
+            for row in query_sample.all():
+                yield api_models.Meter(
+                    name=row.name,
+                    type=row.type,
+                    unit=row.unit,
+                    resource_id=row.resource_id,
+                    project_id=row.project_id,
+                    source=row.source_id,
+                    user_id=row.user_id)
 
     @staticmethod
     def _retrieve_samples(query):

ceilometer/storage/pymongo_base.py

@@ -50,7 +50,7 @@ class Connection(base.Connection):
     )
 
     def get_meters(self, user=None, project=None, resource=None, source=None,
-                   metaquery=None, limit=None):
+                   metaquery=None, limit=None, unique=False):
         """Return an iterable of models.Meter instances
 
         :param user: Optional ID for user that owns the resource.
@@ -59,6 +59,7 @@ class Connection(base.Connection):
         :param source: Optional source filter.
         :param metaquery: Optional dict with metadata to match on.
         :param limit: Maximum number of results to return.
+        :param unique: If set to true, return only unique meter information.
         """
         if limit == 0:
             return
@@ -77,23 +78,44 @@ class Connection(base.Connection):
         q.update(metaquery)
 
         count = 0
+        if unique:
+            meter_names = set()
+
         for r in self.db.resource.find(q):
             for r_meter in r['meter']:
+                if unique:
+                    if r_meter['counter_name'] in meter_names:
+                        continue
+                    else:
+                        meter_names.add(r_meter['counter_name'])
+
                 if limit and count >= limit:
                     return
                 else:
                     count += 1
-                yield models.Meter(
+
-                    name=r_meter['counter_name'],
+                if unique:
-                    type=r_meter['counter_type'],
+                    yield models.Meter(
-                    # Return empty string if 'counter_unit' is not valid for
+                        name=r_meter['counter_name'],
-                    # backward compatibility.
+                        type=r_meter['counter_type'],
-                    unit=r_meter.get('counter_unit', ''),
+                        # Return empty string if 'counter_unit' is not valid
-                    resource_id=r['_id'],
+                        # for backward compatibility.
-                    project_id=r['project_id'],
+                        unit=r_meter.get('counter_unit', ''),
-                    source=r['source'],
+                        resource_id=None,
-                    user_id=r['user_id'],
+                        project_id=None,
-                )
+                        source=None,
+                        user_id=None)
+                else:
+                    yield models.Meter(
+                        name=r_meter['counter_name'],
+                        type=r_meter['counter_type'],
+                        # Return empty string if 'counter_unit' is not valid
+                        # for backward compatibility.
+                        unit=r_meter.get('counter_unit', ''),
+                        resource_id=r['_id'],
+                        project_id=r['project_id'],
+                        source=r['source'],
+                        user_id=r['user_id'])
 
     def get_samples(self, sample_filter, limit=None):
         """Return an iterable of model.Sample instances.

ceilometer/tests/functional/api/v2/test_list_meters_scenarios.py

@@ -271,6 +271,13 @@ class TestListMeters(v2.FunctionalTest):
         self.assertEqual(set(['test_source', 'test_source1']),
                          set(r['source'] for r in data))
 
+    def test_list_unique_meters(self):
+        data = self.get_json('/meters?unique=True')
+        self.assertEqual(4, len(data))
+        self.assertEqual(set(['meter.test', 'meter.mine', 'meter.test.new',
+                              u'meter.accent\xe9\u0437']),
+                         set(r['name'] for r in data))
+
     def test_meters_query_with_timestamp(self):
         date_time = datetime.datetime(2012, 7, 2, 10, 41)
         isotime = date_time.isoformat()

ceilometer/tests/unit/api/v2/test_query.py

@@ -379,7 +379,7 @@ class TestQueryToKwArgs(tests_base.BaseTestCase):
         exc = self.assertRaises(
             wsme.exc.UnknownArgument,
             utils.query_to_kwargs,
-            q, storage_base.Connection.get_meters, ['limit'])
+            q, storage_base.Connection.get_meters, ['limit', 'unique'])
         valid_keys = ['project', 'resource', 'source', 'user']
         msg = ("unrecognized field in query: %s, "
                "valid keys: %s") % (q, valid_keys)