From 52f48485eeed50dfa4705fc73e278214eb9a7f00 Mon Sep 17 00:00:00 2001 From: Rohit Jaiswal Date: Fri, 18 Dec 2015 12:56:59 -0800 Subject: [PATCH] Enhances get_meters to return unique meters Enhances get_meters in the storage layer and API to accept a unique flag and return back a list of unique meters for clients like Horizon to consume. DocImpact Change-Id: Ifdcb907df867ae650eae733bc7b635a283939064 Closes-Bug: 1506959 --- ceilometer/api/controllers/v2/meters.py | 10 ++-- ceilometer/storage/base.py | 3 +- ceilometer/storage/impl_hbase.py | 33 +++++++++---- ceilometer/storage/impl_log.py | 3 +- ceilometer/storage/impl_sqlalchemy.py | 44 +++++++++++++----- ceilometer/storage/pymongo_base.py | 46 ++++++++++++++----- .../api/v2/test_list_meters_scenarios.py | 7 +++ ceilometer/tests/unit/api/v2/test_query.py | 2 +- 8 files changed, 108 insertions(+), 40 deletions(-) diff --git a/ceilometer/api/controllers/v2/meters.py b/ceilometer/api/controllers/v2/meters.py index 35d491bb..a214abc7 100644 --- a/ceilometer/api/controllers/v2/meters.py +++ b/ceilometer/api/controllers/v2/meters.py @@ -477,11 +477,12 @@ class MetersController(rest.RestController): def _lookup(self, meter_name, *remainder): return MeterController(meter_name), remainder - @wsme_pecan.wsexpose([Meter], [base.Query], int) - def get_all(self, q=None, limit=None): + @wsme_pecan.wsexpose([Meter], [base.Query], int, str) + def get_all(self, q=None, limit=None, unique=''): """Return all known meters, based on the data recorded so far. :param q: Filter rules for the meters to be returned. + :param unique: flag to indicate unique meters to be returned. """ rbac.enforce('get_meters', pecan.request) @@ -494,5 +495,6 @@ class MetersController(rest.RestController): q, pecan.request.storage_conn.get_meters, ['limit'], allow_timestamps=False) return [Meter.from_db_model(m) - for m in pecan.request.storage_conn.get_meters(limit=limit, - **kwargs)] + for m in pecan.request.storage_conn.get_meters( + limit=limit, unique=strutils.bool_from_string(unique), + **kwargs)] diff --git a/ceilometer/storage/base.py b/ceilometer/storage/base.py index fb038b3f..a25acdb1 100644 --- a/ceilometer/storage/base.py +++ b/ceilometer/storage/base.py @@ -194,7 +194,7 @@ class Connection(object): @staticmethod def get_meters(user=None, project=None, resource=None, source=None, - metaquery=None, limit=None): + metaquery=None, limit=None, unique=False): """Return an iterable of model.Meter instances. Iterable items containing meter information. @@ -204,6 +204,7 @@ class Connection(object): :param source: Optional source filter. :param metaquery: Optional dict with metadata to match on. :param limit: Maximum number of results to return. + :param unique: If set to true, return only unique meter information. """ raise ceilometer.NotImplementedError('Meters not implemented') diff --git a/ceilometer/storage/impl_hbase.py b/ceilometer/storage/impl_hbase.py index bddbd4f6..ec719116 100644 --- a/ceilometer/storage/impl_hbase.py +++ b/ceilometer/storage/impl_hbase.py @@ -241,7 +241,7 @@ class Connection(hbase_base.Connection, base.Connection): metadata=md) def get_meters(self, user=None, project=None, resource=None, source=None, - metaquery=None, limit=None): + metaquery=None, limit=None, unique=False): """Return an iterable of models.Meter instances :param user: Optional ID for user that owns the resource. @@ -250,6 +250,7 @@ class Connection(hbase_base.Connection, base.Connection): :param source: Optional source filter. :param metaquery: Optional dict with metadata to match on. :param limit: Maximum number of results to return. + :param unique: If set to true, return only unique meter information. """ if limit == 0: return @@ -276,18 +277,32 @@ class Connection(hbase_base.Connection, base.Connection): if limit and len(result) >= limit: return _m_rts, m_source, name, m_type, unit = m[0] - meter_dict = {'name': name, - 'type': m_type, - 'unit': unit, - 'resource_id': flatten_result['resource_id'], - 'project_id': flatten_result['project_id'], - 'user_id': flatten_result['user_id']} + if unique: + meter_dict = {'name': name, + 'type': m_type, + 'unit': unit, + 'resource_id': None, + 'project_id': None, + 'user_id': None, + 'source': None} + else: + meter_dict = {'name': name, + 'type': m_type, + 'unit': unit, + 'resource_id': + flatten_result['resource_id'], + 'project_id': + flatten_result['project_id'], + 'user_id': + flatten_result['user_id']} + frozen_meter = frozenset(meter_dict.items()) if frozen_meter in result: continue result.add(frozen_meter) - meter_dict.update({'source': m_source - if m_source else None}) + if not unique: + meter_dict.update({'source': m_source + if m_source else None}) yield models.Meter(**meter_dict) diff --git a/ceilometer/storage/impl_log.py b/ceilometer/storage/impl_log.py index 22422fc0..42d39d4a 100644 --- a/ceilometer/storage/impl_log.py +++ b/ceilometer/storage/impl_log.py @@ -80,7 +80,7 @@ class Connection(base.Connection): return [] def get_meters(self, user=None, project=None, resource=None, source=None, - limit=None, metaquery=None): + limit=None, metaquery=None, unique=False): """Return an iterable of dictionaries containing meter information. { 'name': name of the meter, @@ -96,6 +96,7 @@ class Connection(base.Connection): :param source: Optional source filter. :param limit: Maximum number of results to return. :param metaquery: Optional dict with metadata to match on. + :param unique: If set to true, return only unique meter information. """ return [] diff --git a/ceilometer/storage/impl_sqlalchemy.py b/ceilometer/storage/impl_sqlalchemy.py index ec46e530..7968a58a 100644 --- a/ceilometer/storage/impl_sqlalchemy.py +++ b/ceilometer/storage/impl_sqlalchemy.py @@ -493,7 +493,7 @@ class Connection(base.Connection): ) def get_meters(self, user=None, project=None, resource=None, source=None, - metaquery=None, limit=None): + metaquery=None, limit=None, unique=False): """Return an iterable of api_models.Meter instances :param user: Optional ID for user that owns the resource. @@ -502,6 +502,7 @@ class Connection(base.Connection): :param source: Optional source filter. :param metaquery: Optional dict with metadata to match on. :param limit: Maximum number of results to return. + :param unique: If set to true, return only unique meter information. """ if limit == 0: return @@ -514,10 +515,17 @@ class Connection(base.Connection): # NOTE(gordc): get latest sample of each meter/resource. we do not # filter here as we want to filter only on latest record. session = self._engine_facade.get_session() + subq = session.query(func.max(models.Sample.id).label('id')).join( models.Resource, - models.Resource.internal_id == models.Sample.resource_id).group_by( - models.Sample.meter_id, models.Resource.resource_id) + models.Resource.internal_id == models.Sample.resource_id) + + if unique: + subq = subq.group_by(models.Sample.meter_id) + else: + subq = subq.group_by(models.Sample.meter_id, + models.Resource.resource_id) + if resource: subq = subq.filter(models.Resource.resource_id == resource) subq = subq.subquery() @@ -538,15 +546,27 @@ class Connection(base.Connection): require_meter=False) query_sample = query_sample.limit(limit) if limit else query_sample - for row in query_sample.all(): - yield api_models.Meter( - name=row.name, - type=row.type, - unit=row.unit, - resource_id=row.resource_id, - project_id=row.project_id, - source=row.source_id, - user_id=row.user_id) + + if unique: + for row in query_sample.all(): + yield api_models.Meter( + name=row.name, + type=row.type, + unit=row.unit, + resource_id=None, + project_id=None, + source=None, + user_id=None) + else: + for row in query_sample.all(): + yield api_models.Meter( + name=row.name, + type=row.type, + unit=row.unit, + resource_id=row.resource_id, + project_id=row.project_id, + source=row.source_id, + user_id=row.user_id) @staticmethod def _retrieve_samples(query): diff --git a/ceilometer/storage/pymongo_base.py b/ceilometer/storage/pymongo_base.py index 93c696d5..bdc6afb8 100644 --- a/ceilometer/storage/pymongo_base.py +++ b/ceilometer/storage/pymongo_base.py @@ -50,7 +50,7 @@ class Connection(base.Connection): ) def get_meters(self, user=None, project=None, resource=None, source=None, - metaquery=None, limit=None): + metaquery=None, limit=None, unique=False): """Return an iterable of models.Meter instances :param user: Optional ID for user that owns the resource. @@ -59,6 +59,7 @@ class Connection(base.Connection): :param source: Optional source filter. :param metaquery: Optional dict with metadata to match on. :param limit: Maximum number of results to return. + :param unique: If set to true, return only unique meter information. """ if limit == 0: return @@ -77,23 +78,44 @@ class Connection(base.Connection): q.update(metaquery) count = 0 + if unique: + meter_names = set() + for r in self.db.resource.find(q): for r_meter in r['meter']: + if unique: + if r_meter['counter_name'] in meter_names: + continue + else: + meter_names.add(r_meter['counter_name']) + if limit and count >= limit: return else: count += 1 - yield models.Meter( - name=r_meter['counter_name'], - type=r_meter['counter_type'], - # Return empty string if 'counter_unit' is not valid for - # backward compatibility. - unit=r_meter.get('counter_unit', ''), - resource_id=r['_id'], - project_id=r['project_id'], - source=r['source'], - user_id=r['user_id'], - ) + + if unique: + yield models.Meter( + name=r_meter['counter_name'], + type=r_meter['counter_type'], + # Return empty string if 'counter_unit' is not valid + # for backward compatibility. + unit=r_meter.get('counter_unit', ''), + resource_id=None, + project_id=None, + source=None, + user_id=None) + else: + yield models.Meter( + name=r_meter['counter_name'], + type=r_meter['counter_type'], + # Return empty string if 'counter_unit' is not valid + # for backward compatibility. + unit=r_meter.get('counter_unit', ''), + resource_id=r['_id'], + project_id=r['project_id'], + source=r['source'], + user_id=r['user_id']) def get_samples(self, sample_filter, limit=None): """Return an iterable of model.Sample instances. diff --git a/ceilometer/tests/functional/api/v2/test_list_meters_scenarios.py b/ceilometer/tests/functional/api/v2/test_list_meters_scenarios.py index e9c14938..6ed3bdd9 100644 --- a/ceilometer/tests/functional/api/v2/test_list_meters_scenarios.py +++ b/ceilometer/tests/functional/api/v2/test_list_meters_scenarios.py @@ -271,6 +271,13 @@ class TestListMeters(v2.FunctionalTest): self.assertEqual(set(['test_source', 'test_source1']), set(r['source'] for r in data)) + def test_list_unique_meters(self): + data = self.get_json('/meters?unique=True') + self.assertEqual(4, len(data)) + self.assertEqual(set(['meter.test', 'meter.mine', 'meter.test.new', + u'meter.accent\xe9\u0437']), + set(r['name'] for r in data)) + def test_meters_query_with_timestamp(self): date_time = datetime.datetime(2012, 7, 2, 10, 41) isotime = date_time.isoformat() diff --git a/ceilometer/tests/unit/api/v2/test_query.py b/ceilometer/tests/unit/api/v2/test_query.py index 7d6fdd68..28880e04 100644 --- a/ceilometer/tests/unit/api/v2/test_query.py +++ b/ceilometer/tests/unit/api/v2/test_query.py @@ -379,7 +379,7 @@ class TestQueryToKwArgs(tests_base.BaseTestCase): exc = self.assertRaises( wsme.exc.UnknownArgument, utils.query_to_kwargs, - q, storage_base.Connection.get_meters, ['limit']) + q, storage_base.Connection.get_meters, ['limit', 'unique']) valid_keys = ['project', 'resource', 'source', 'user'] msg = ("unrecognized field in query: %s, " "valid keys: %s") % (q, valid_keys)