Enhances get_meters to return unique meters

Enhances get_meters in the storage layer
and API to accept a unique flag and return
back a list of unique meters for clients like
Horizon to consume.

DocImpact
Change-Id: Ifdcb907df867ae650eae733bc7b635a283939064
Closes-Bug: 1506959
This commit is contained in:
Rohit Jaiswal 2015-12-18 12:56:59 -08:00
parent 39f22e0f3b
commit 52f48485ee
8 changed files with 108 additions and 40 deletions

View File

@ -477,11 +477,12 @@ class MetersController(rest.RestController):
def _lookup(self, meter_name, *remainder): def _lookup(self, meter_name, *remainder):
return MeterController(meter_name), remainder return MeterController(meter_name), remainder
@wsme_pecan.wsexpose([Meter], [base.Query], int) @wsme_pecan.wsexpose([Meter], [base.Query], int, str)
def get_all(self, q=None, limit=None): def get_all(self, q=None, limit=None, unique=''):
"""Return all known meters, based on the data recorded so far. """Return all known meters, based on the data recorded so far.
:param q: Filter rules for the meters to be returned. :param q: Filter rules for the meters to be returned.
:param unique: flag to indicate unique meters to be returned.
""" """
rbac.enforce('get_meters', pecan.request) rbac.enforce('get_meters', pecan.request)
@ -494,5 +495,6 @@ class MetersController(rest.RestController):
q, pecan.request.storage_conn.get_meters, q, pecan.request.storage_conn.get_meters,
['limit'], allow_timestamps=False) ['limit'], allow_timestamps=False)
return [Meter.from_db_model(m) return [Meter.from_db_model(m)
for m in pecan.request.storage_conn.get_meters(limit=limit, for m in pecan.request.storage_conn.get_meters(
**kwargs)] limit=limit, unique=strutils.bool_from_string(unique),
**kwargs)]

View File

@ -194,7 +194,7 @@ class Connection(object):
@staticmethod @staticmethod
def get_meters(user=None, project=None, resource=None, source=None, def get_meters(user=None, project=None, resource=None, source=None,
metaquery=None, limit=None): metaquery=None, limit=None, unique=False):
"""Return an iterable of model.Meter instances. """Return an iterable of model.Meter instances.
Iterable items containing meter information. Iterable items containing meter information.
@ -204,6 +204,7 @@ class Connection(object):
:param source: Optional source filter. :param source: Optional source filter.
:param metaquery: Optional dict with metadata to match on. :param metaquery: Optional dict with metadata to match on.
:param limit: Maximum number of results to return. :param limit: Maximum number of results to return.
:param unique: If set to true, return only unique meter information.
""" """
raise ceilometer.NotImplementedError('Meters not implemented') raise ceilometer.NotImplementedError('Meters not implemented')

View File

@ -241,7 +241,7 @@ class Connection(hbase_base.Connection, base.Connection):
metadata=md) metadata=md)
def get_meters(self, user=None, project=None, resource=None, source=None, def get_meters(self, user=None, project=None, resource=None, source=None,
metaquery=None, limit=None): metaquery=None, limit=None, unique=False):
"""Return an iterable of models.Meter instances """Return an iterable of models.Meter instances
:param user: Optional ID for user that owns the resource. :param user: Optional ID for user that owns the resource.
@ -250,6 +250,7 @@ class Connection(hbase_base.Connection, base.Connection):
:param source: Optional source filter. :param source: Optional source filter.
:param metaquery: Optional dict with metadata to match on. :param metaquery: Optional dict with metadata to match on.
:param limit: Maximum number of results to return. :param limit: Maximum number of results to return.
:param unique: If set to true, return only unique meter information.
""" """
if limit == 0: if limit == 0:
return return
@ -276,18 +277,32 @@ class Connection(hbase_base.Connection, base.Connection):
if limit and len(result) >= limit: if limit and len(result) >= limit:
return return
_m_rts, m_source, name, m_type, unit = m[0] _m_rts, m_source, name, m_type, unit = m[0]
meter_dict = {'name': name, if unique:
'type': m_type, meter_dict = {'name': name,
'unit': unit, 'type': m_type,
'resource_id': flatten_result['resource_id'], 'unit': unit,
'project_id': flatten_result['project_id'], 'resource_id': None,
'user_id': flatten_result['user_id']} 'project_id': None,
'user_id': None,
'source': None}
else:
meter_dict = {'name': name,
'type': m_type,
'unit': unit,
'resource_id':
flatten_result['resource_id'],
'project_id':
flatten_result['project_id'],
'user_id':
flatten_result['user_id']}
frozen_meter = frozenset(meter_dict.items()) frozen_meter = frozenset(meter_dict.items())
if frozen_meter in result: if frozen_meter in result:
continue continue
result.add(frozen_meter) result.add(frozen_meter)
meter_dict.update({'source': m_source if not unique:
if m_source else None}) meter_dict.update({'source': m_source
if m_source else None})
yield models.Meter(**meter_dict) yield models.Meter(**meter_dict)

View File

@ -80,7 +80,7 @@ class Connection(base.Connection):
return [] return []
def get_meters(self, user=None, project=None, resource=None, source=None, def get_meters(self, user=None, project=None, resource=None, source=None,
limit=None, metaquery=None): limit=None, metaquery=None, unique=False):
"""Return an iterable of dictionaries containing meter information. """Return an iterable of dictionaries containing meter information.
{ 'name': name of the meter, { 'name': name of the meter,
@ -96,6 +96,7 @@ class Connection(base.Connection):
:param source: Optional source filter. :param source: Optional source filter.
:param limit: Maximum number of results to return. :param limit: Maximum number of results to return.
:param metaquery: Optional dict with metadata to match on. :param metaquery: Optional dict with metadata to match on.
:param unique: If set to true, return only unique meter information.
""" """
return [] return []

View File

@ -493,7 +493,7 @@ class Connection(base.Connection):
) )
def get_meters(self, user=None, project=None, resource=None, source=None, def get_meters(self, user=None, project=None, resource=None, source=None,
metaquery=None, limit=None): metaquery=None, limit=None, unique=False):
"""Return an iterable of api_models.Meter instances """Return an iterable of api_models.Meter instances
:param user: Optional ID for user that owns the resource. :param user: Optional ID for user that owns the resource.
@ -502,6 +502,7 @@ class Connection(base.Connection):
:param source: Optional source filter. :param source: Optional source filter.
:param metaquery: Optional dict with metadata to match on. :param metaquery: Optional dict with metadata to match on.
:param limit: Maximum number of results to return. :param limit: Maximum number of results to return.
:param unique: If set to true, return only unique meter information.
""" """
if limit == 0: if limit == 0:
return return
@ -514,10 +515,17 @@ class Connection(base.Connection):
# NOTE(gordc): get latest sample of each meter/resource. we do not # NOTE(gordc): get latest sample of each meter/resource. we do not
# filter here as we want to filter only on latest record. # filter here as we want to filter only on latest record.
session = self._engine_facade.get_session() session = self._engine_facade.get_session()
subq = session.query(func.max(models.Sample.id).label('id')).join( subq = session.query(func.max(models.Sample.id).label('id')).join(
models.Resource, models.Resource,
models.Resource.internal_id == models.Sample.resource_id).group_by( models.Resource.internal_id == models.Sample.resource_id)
models.Sample.meter_id, models.Resource.resource_id)
if unique:
subq = subq.group_by(models.Sample.meter_id)
else:
subq = subq.group_by(models.Sample.meter_id,
models.Resource.resource_id)
if resource: if resource:
subq = subq.filter(models.Resource.resource_id == resource) subq = subq.filter(models.Resource.resource_id == resource)
subq = subq.subquery() subq = subq.subquery()
@ -538,15 +546,27 @@ class Connection(base.Connection):
require_meter=False) require_meter=False)
query_sample = query_sample.limit(limit) if limit else query_sample query_sample = query_sample.limit(limit) if limit else query_sample
for row in query_sample.all():
yield api_models.Meter( if unique:
name=row.name, for row in query_sample.all():
type=row.type, yield api_models.Meter(
unit=row.unit, name=row.name,
resource_id=row.resource_id, type=row.type,
project_id=row.project_id, unit=row.unit,
source=row.source_id, resource_id=None,
user_id=row.user_id) project_id=None,
source=None,
user_id=None)
else:
for row in query_sample.all():
yield api_models.Meter(
name=row.name,
type=row.type,
unit=row.unit,
resource_id=row.resource_id,
project_id=row.project_id,
source=row.source_id,
user_id=row.user_id)
@staticmethod @staticmethod
def _retrieve_samples(query): def _retrieve_samples(query):

View File

@ -50,7 +50,7 @@ class Connection(base.Connection):
) )
def get_meters(self, user=None, project=None, resource=None, source=None, def get_meters(self, user=None, project=None, resource=None, source=None,
metaquery=None, limit=None): metaquery=None, limit=None, unique=False):
"""Return an iterable of models.Meter instances """Return an iterable of models.Meter instances
:param user: Optional ID for user that owns the resource. :param user: Optional ID for user that owns the resource.
@ -59,6 +59,7 @@ class Connection(base.Connection):
:param source: Optional source filter. :param source: Optional source filter.
:param metaquery: Optional dict with metadata to match on. :param metaquery: Optional dict with metadata to match on.
:param limit: Maximum number of results to return. :param limit: Maximum number of results to return.
:param unique: If set to true, return only unique meter information.
""" """
if limit == 0: if limit == 0:
return return
@ -77,23 +78,44 @@ class Connection(base.Connection):
q.update(metaquery) q.update(metaquery)
count = 0 count = 0
if unique:
meter_names = set()
for r in self.db.resource.find(q): for r in self.db.resource.find(q):
for r_meter in r['meter']: for r_meter in r['meter']:
if unique:
if r_meter['counter_name'] in meter_names:
continue
else:
meter_names.add(r_meter['counter_name'])
if limit and count >= limit: if limit and count >= limit:
return return
else: else:
count += 1 count += 1
yield models.Meter(
name=r_meter['counter_name'], if unique:
type=r_meter['counter_type'], yield models.Meter(
# Return empty string if 'counter_unit' is not valid for name=r_meter['counter_name'],
# backward compatibility. type=r_meter['counter_type'],
unit=r_meter.get('counter_unit', ''), # Return empty string if 'counter_unit' is not valid
resource_id=r['_id'], # for backward compatibility.
project_id=r['project_id'], unit=r_meter.get('counter_unit', ''),
source=r['source'], resource_id=None,
user_id=r['user_id'], project_id=None,
) source=None,
user_id=None)
else:
yield models.Meter(
name=r_meter['counter_name'],
type=r_meter['counter_type'],
# Return empty string if 'counter_unit' is not valid
# for backward compatibility.
unit=r_meter.get('counter_unit', ''),
resource_id=r['_id'],
project_id=r['project_id'],
source=r['source'],
user_id=r['user_id'])
def get_samples(self, sample_filter, limit=None): def get_samples(self, sample_filter, limit=None):
"""Return an iterable of model.Sample instances. """Return an iterable of model.Sample instances.

View File

@ -271,6 +271,13 @@ class TestListMeters(v2.FunctionalTest):
self.assertEqual(set(['test_source', 'test_source1']), self.assertEqual(set(['test_source', 'test_source1']),
set(r['source'] for r in data)) set(r['source'] for r in data))
def test_list_unique_meters(self):
data = self.get_json('/meters?unique=True')
self.assertEqual(4, len(data))
self.assertEqual(set(['meter.test', 'meter.mine', 'meter.test.new',
u'meter.accent\xe9\u0437']),
set(r['name'] for r in data))
def test_meters_query_with_timestamp(self): def test_meters_query_with_timestamp(self):
date_time = datetime.datetime(2012, 7, 2, 10, 41) date_time = datetime.datetime(2012, 7, 2, 10, 41)
isotime = date_time.isoformat() isotime = date_time.isoformat()

View File

@ -379,7 +379,7 @@ class TestQueryToKwArgs(tests_base.BaseTestCase):
exc = self.assertRaises( exc = self.assertRaises(
wsme.exc.UnknownArgument, wsme.exc.UnknownArgument,
utils.query_to_kwargs, utils.query_to_kwargs,
q, storage_base.Connection.get_meters, ['limit']) q, storage_base.Connection.get_meters, ['limit', 'unique'])
valid_keys = ['project', 'resource', 'source', 'user'] valid_keys = ['project', 'resource', 'source', 'user']
msg = ("unrecognized field in query: %s, " msg = ("unrecognized field in query: %s, "
"valid keys: %s") % (q, valid_keys) "valid keys: %s") % (q, valid_keys)