openstack
/
deb-ceilometer
Code Issues Proposed changes

Merge "Add the ability to filter on metadata."

This commit is contained in:
Jenkins 2012-12-07 09:03:21 +00:00 committed by Gerrit Code Review
parent 8621a2a757 adef6f07aa
commit cc2ab04937
10 changed files with 172 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
ceilometer/api/v1/blueprint.py
View File

@ -100,14 +100,21 @@ def request_wants_html():
flask.request.accept_mimetypes['application/json']
def _get_metaquery(args):
return dict((k, v)
for (k, v) in args.iteritems()
if k.startswith('metadata.'))
## APIs for working with meters.
@blueprint.route('/meters')
def list_meters_all():
"""Return a list of meters.
:param metadata.<key> match on the metadata within the resource. (optional)
"""
meters = flask.request.storage_conn.get_meters()
rq = flask.request
meters = rq.storage_conn.get_meters(metaquery=_get_metaquery(rq.args))
return flask.jsonify(meters=list(meters))
@ -116,8 +123,11 @@ def list_meters_by_resource(resource):
"""Return a list of meters by resource.
:param resource: The ID of the resource.
:param metadata.<key> match on the metadata within the resource. (optional)
"""
meters = flask.request.storage_conn.get_meters(resource=resource)
rq = flask.request
meters = rq.storage_conn.get_meters(resource=resource,
metaquery=_get_metaquery(rq.args))
return flask.jsonify(meters=list(meters))
@ -126,8 +136,11 @@ def list_meters_by_user(user):
"""Return a list of meters by user.
:param user: The ID of the owning user.
:param metadata.<key> match on the metadata within the resource. (optional)
"""
meters = flask.request.storage_conn.get_meters(user=user)
rq = flask.request
meters = rq.storage_conn.get_meters(user=user,
metaquery=_get_metaquery(rq.args))
return flask.jsonify(meters=list(meters))
@ -136,8 +149,11 @@ def list_meters_by_project(project):
"""Return a list of meters by project.
:param project: The ID of the owning project.
:param metadata.<key> match on the metadata within the resource. (optional)
"""
meters = flask.request.storage_conn.get_meters(project=project)
rq = flask.request
meters = rq.storage_conn.get_meters(project=project,
metaquery=_get_metaquery(rq.args))
return flask.jsonify(meters=list(meters))
@ -146,8 +162,11 @@ def list_meters_by_source(source):
"""Return a list of meters by source.
:param source: The ID of the owning source.
:param metadata.<key> match on the metadata within the resource. (optional)
"""
meters = flask.request.storage_conn.get_meters(source=source)
rq = flask.request
meters = rq.storage_conn.get_meters(source=source,
metaquery=_get_metaquery(rq.args))
return flask.jsonify(meters=list(meters))
@ -156,13 +175,15 @@ def list_meters_by_source(source):
def _list_resources(source=None, user=None, project=None):
"""Return a list of resource identifiers.
"""
q_ts = _get_query_timestamps(flask.request.args)
resources = flask.request.storage_conn.get_resources(
rq = flask.request
q_ts = _get_query_timestamps(rq.args)
resources = rq.storage_conn.get_resources(
source=source,
user=user,
project=project,
start_timestamp=q_ts['start_timestamp'],
end_timestamp=q_ts['end_timestamp'],
metaquery=_get_metaquery(rq.args),
)
return flask.jsonify(resources=list(resources))
@ -178,6 +199,7 @@ def list_resources_by_project(project):
:param end_timestamp: Limits resources by last update time < this value.
(optional)
:type end_timestamp: ISO date in UTC
:param metadata.<key> match on the metadata within the resource. (optional)
"""
return _list_resources(project=project)
@ -192,6 +214,7 @@ def list_all_resources():
:param end_timestamp: Limits resources by last update time < this value.
(optional)
:type end_timestamp: ISO date in UTC
:param metadata.<key> match on the metadata within the resource. (optional)
"""
return _list_resources()
@ -217,6 +240,7 @@ def list_resources_by_source(source):
:param end_timestamp: Limits resources by last update time < this value.
(optional)
:type end_timestamp: ISO date in UTC
:param metadata.<key> match on the metadata within the resource. (optional)
"""
return _list_resources(source=source)
@ -232,6 +256,7 @@ def list_resources_by_user(user):
:param end_timestamp: Limits resources by last update time < this value.
(optional)
:type end_timestamp: ISO date in UTC
:param metadata.<key> match on the metadata within the resource. (optional)
"""
return _list_resources(user=user)
@ -308,6 +333,7 @@ def _list_events(meter,
resource=resource,
start=q_ts['start_timestamp'],
end=q_ts['end_timestamp'],
metaquery=_get_metaquery(flask.request.args),
)
events = list(flask.request.storage_conn.get_raw_events(f))
jsonified = flask.jsonify(events=events)

4
ceilometer/storage/__init__.py
View File

@ -80,9 +80,10 @@ class EventFilter(object):
:param resource: Optional filter for resource id.
:param meter: Optional filter for meter type using the meter name.
:param source: Optional source filter.
:param metaquery: Optional filter on the metadata
"""
def __init__(self, user=None, project=None, start=None, end=None,
resource=None, meter=None, source=None):
resource=None, meter=None, source=None, metaquery={}):
self.user = user
self.project = project
self.start = self._sanitize_timestamp(start)
@ -90,6 +91,7 @@ class EventFilter(object):
self.resource = resource
self.meter = meter
self.source = source
self.metaquery = metaquery
def _sanitize_timestamp(self, timestamp):
"""Return a naive utc datetime object"""

34
ceilometer/storage/base.py
View File

@ -78,7 +78,8 @@ class Connection(object):
@abc.abstractmethod
def get_resources(self, user=None, project=None, source=None,
start_timestamp=None, end_timestamp=None):
start_timestamp=None, end_timestamp=None,
metaquery={}):
"""Return an iterable of dictionaries containing resource information.
{ 'resource_id': UUID of the resource,
@ -94,6 +95,26 @@ class Connection(object):
:param source: Optional source filter.
:param start_timestamp: Optional modified timestamp start range.
:param end_timestamp: Optional modified timestamp end range.
:param metaquery: Optional dict with metadata to match on..
"""
@abc.abstractmethod
def get_meters(self, user=None, project=None, resource=None, source=None,
metaquery={}):
"""Return an iterable of dictionaries containing meter information.
{ 'name': name of the meter,
'type': type of the meter (guage, counter),
'resource_id': UUID of the resource,
'project_id': UUID of project owning the resource,
'user_id': UUID of user owning the resource,
}
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param resource: Optional resource filter.
:param source: Optional source filter.
:param metaquery: Optional dict with metadata to match on.
"""
@abc.abstractmethod
@ -133,14 +154,3 @@ class Connection(object):
( datetime.datetime(), datetime.datetime() )
"""
@abc.abstractmethod
def get_meters(self, user=None, project=None, resource=None, source=None):
"""Return a list of meters.
{'resource_id': UUID string for the resource,
'project_id': UUID of project owning the resource,
'user_id': UUID of user owning the resource,
'name': The name of the meter,
'type': The meter type (gauge, counter, diff),
}
"""

21
ceilometer/storage/impl_log.py
View File

@ -69,18 +69,28 @@ class Connection(base.Connection):
"""
def get_resources(self, user=None, project=None, source=None,
start_timestamp=None, end_timestamp=None):
"""Return an iterable of tuples containing resource ids and
the most recent version of the metadata for the resource.
start_timestamp=None, end_timestamp=None,
metaquery={}):
"""Return an iterable of dictionaries containing resource information.
{ 'resource_id': UUID of the resource,
'project_id': UUID of project owning the resource,
'user_id': UUID of user owning the resource,
'timestamp': UTC datetime of last update to the resource,
'metadata': most current metadata for the resource,
'meter': list of the meters reporting data for the resource,
}
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param source: Optional source filter.
:param start_timestamp: Optional modified timestamp start range.
:param end_timestamp: Optional modified timestamp end range.
:param metaquery: Optional dict with metadata to match on.
"""
def get_meters(self, user=None, project=None, resource=None, source=None):
def get_meters(self, user=None, project=None, resource=None, source=None,
metaquery={}):
"""Return an iterable of dictionaries containing meter information.
{ 'name': name of the meter,
@ -92,8 +102,9 @@ class Connection(base.Connection):
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param resource: Optional ID of the resource.
:param resource: Optional resource filter.
:param source: Optional source filter.
:param metaquery: Optional dict with metadata to match on.
"""
def get_raw_events(self, event_filter):

18
ceilometer/storage/impl_mongodb.py
View File

@ -114,6 +114,10 @@ def make_query_from_filter(event_filter, require_meter=True):
if event_filter.source:
q['source'] = event_filter.source
# so the events call metadata resource_metadata, so we convert
# to that.
q.update(dict(('resource_%s' % k, v)
for (k, v) in event_filter.metaquery.iteritems()))
return q
@ -299,7 +303,8 @@ class Connection(base.Connection):
return sorted(self.db.project.find(q).distinct('_id'))
def get_resources(self, user=None, project=None, source=None,
start_timestamp=None, end_timestamp=None):
start_timestamp=None, end_timestamp=None,
metaquery={}):
"""Return an iterable of dictionaries containing resource information.
{ 'resource_id': UUID of the resource,
@ -315,6 +320,7 @@ class Connection(base.Connection):
:param source: Optional source filter.
:param start_timestamp: Optional modified timestamp start range.
:param end_timestamp: Optional modified timestamp end range.
:param metaquery: Optional dict with metadata to match on.
"""
q = {}
if user is not None:
@ -323,6 +329,8 @@ class Connection(base.Connection):
q['project_id'] = project
if source is not None:
q['source'] = source
q.update(metaquery)
# FIXME(dhellmann): This may not perform very well,
# but doing any better will require changing the database
# schema and that will need more thought than I have time
@ -347,7 +355,8 @@ class Connection(base.Connection):
del r['_id']
yield r
def get_meters(self, user=None, project=None, resource=None, source=None):
def get_meters(self, user=None, project=None, resource=None, source=None,
metaquery={}):
"""Return an iterable of dictionaries containing meter information.
{ 'name': name of the meter,
@ -359,8 +368,9 @@ class Connection(base.Connection):
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param resource: Optional ID of the resource.
:param resource: Optional resource filter.
:param source: Optional source filter.
:param metaquery: Optional dict with metadata to match on.
"""
q = {}
if user is not None:
@ -371,6 +381,8 @@ class Connection(base.Connection):
q['_id'] = resource
if source is not None:
q['source'] = source
q.update(metaquery)
for r in self.db.resource.find(q):
for r_meter in r['meter']:
m = {}

16
ceilometer/storage/impl_sqlalchemy.py
View File

@ -113,6 +113,9 @@ def make_query_from_filter(query, event_filter, require_meter=True):
if event_filter.resource:
query = query.filter_by(resource_id=event_filter.resource)
if event_filter.metaquery is not None and len(event_filter.metaquery) > 0:
raise NotImplementedError('metaquery not implemented')
return query
@ -212,7 +215,8 @@ class Connection(base.Connection):
return (x[0] for x in query.all())
def get_resources(self, user=None, project=None, source=None,
start_timestamp=None, end_timestamp=None):
start_timestamp=None, end_timestamp=None,
metaquery=None):
"""Return an iterable of dictionaries containing resource information.
{ 'resource_id': UUID of the resource,
@ -228,6 +232,7 @@ class Connection(base.Connection):
:param source: Optional source filter.
:param start_timestamp: Optional modified timestamp start range.
:param end_timestamp: Optional modified timestamp end range.
:param metaquery: Optional dict with metadata to match on.
"""
query = model_query(Resource, session=self.session)
if user is not None:
@ -242,6 +247,8 @@ class Connection(base.Connection):
query = query.filter(Resource.project_id == project)
query = query.options(
sqlalchemy_session.sqlalchemy.orm.joinedload('meters'))
if metaquery is not None:
raise NotImplementedError('metaquery not implemented')
for resource in query.all():
r = row2dict(resource)
@ -257,8 +264,8 @@ class Connection(base.Connection):
del r['meters']
yield r
def get_meters(self, user=None, project=None, source=None,
resource=None):
def get_meters(self, user=None, project=None, resource=None, source=None,
metaquery={}):
"""Return an iterable of dictionaries containing meter information.
{ 'name': name of the meter,
@ -272,6 +279,7 @@ class Connection(base.Connection):
:param project: Optional ID for project that owns the resource.
:param resource: Optional ID of the resource.
:param source: Optional source filter.
:param metaquery: Optional dict with metadata to match on.
"""
query = model_query(Resource, session=self.session)
if user is not None:
@ -284,6 +292,8 @@ class Connection(base.Connection):
query = query.filter(Resource.project_id == project)
query = query.options(
sqlalchemy_session.sqlalchemy.orm.joinedload('meters'))
if len(metaquery) > 0:
raise NotImplementedError('metaquery not implemented')
for resource in query.all():
meter_names = set()

15
tests/api/v1/test_list_events.py
View File

@ -124,3 +124,18 @@ class TestListEvents(tests_api.TestBase):
start_timestamp=datetime.datetime(2012, 7, 2, 10, 41),
end_timestamp=datetime.datetime(2012, 7, 2, 10, 42))
self.assertEquals(1, len(data['events']))
def test_metaquery1(self):
q = '/sources/source1/meters/instance'
data = self.get('%s?metadata.tag=self.counter2' % q)
self.assertEquals(1, len(data['events']))
def test_metaquery2(self):
q = '/sources/source1/meters/instance'
data = self.get('%s?metadata.tag=self.counter' % q)
self.assertEquals(2, len(data['events']))
def test_metaquery3(self):
q = '/sources/source1/meters/instance'
data = self.get('%s?metadata.display_name=test-server' % q)
self.assertEquals(3, len(data['events']))

14
tests/api/v1/test_list_meters.py
View File

@ -74,7 +74,7 @@ class TestListMeters(tests_api.TestBase):
'resource-id2',
timestamp=datetime.datetime(2012, 7, 2, 10, 41),
resource_metadata={'display_name': 'test-server',
'tag': 'self.counter2',
'tag': 'two.counter',
}),
counter.Counter(
'meter.test',
@ -85,7 +85,7 @@ class TestListMeters(tests_api.TestBase):
'resource-id3',
timestamp=datetime.datetime(2012, 7, 2, 10, 42),
resource_metadata={'display_name': 'test-server',
'tag': 'self.counter3',
'tag': 'three.counter',
}),
counter.Counter(
'meter.mine',
@ -96,7 +96,7 @@ class TestListMeters(tests_api.TestBase):
'resource-id4',
timestamp=datetime.datetime(2012, 7, 2, 10, 43),
resource_metadata={'display_name': 'test-server',
'tag': 'self.counter4',
'tag': 'four.counter',
})]:
msg = meter.meter_message_from_counter(cnt,
cfg.CONF.metering_secret,
@ -153,3 +153,11 @@ class TestListMeters(tests_api.TestBase):
def test_with_project_non_existent(self):
data = self.get('/projects/jd-was-here/meters')
self.assertEquals(data['meters'], [])
def test_metaquery1(self):
data = self.get('/meters?metadata.tag=self.counter')
self.assertEquals(1, len(data['meters']))
def test_metaquery2(self):
data = self.get('/meters?metadata.tag=four.counter')
self.assertEquals(1, len(data['meters']))

10
tests/api/v1/test_list_resources.py
View File

@ -166,3 +166,13 @@ class TestListResources(tests_api.TestBase):
def test_with_project_non_existent(self):
data = self.get('/projects/jd-was-here/resources')
self.assertEquals(data['resources'], [])
def test_metaquery1(self):
q = '/sources/test_list_resources/resources'
data = self.get('%s?metadata.display_name=test-server' % q)
self.assertEquals(3, len(data['resources']))
def test_metaquery2(self):
q = '/sources/test_list_resources/resources'
data = self.get('%s?metadata.tag=self.counter4' % q)
self.assertEquals(1, len(data['resources']))

36
tests/storage/test_impl_sqlalchemy.py
View File

@ -299,6 +299,21 @@ class ResourceTest(SQLAlchemyEngineTestBase):
ids = set(r['resource_id'] for r in resources)
assert ids == set(['resource-id', 'resource-id-alternate'])
def test_get_resources_by_metaquery(self):
q = {'metadata.display_name': 'test-server'}
got_not_imp = False
try:
list(self.conn.get_resources(metaquery=q))
except NotImplementedError:
got_not_imp = True
self.assertTrue(got_not_imp)
#this should work, but it doesn't.
#actually unless I wrap get_resources in list()
#it doesn't get called - weird
#self.assertRaises(NotImplementedError,
# self.conn.get_resources,
# metaquery=q)
class MeterTest(SQLAlchemyEngineTestBase):
@ -336,16 +351,33 @@ class MeterTest(SQLAlchemyEngineTestBase):
def test_get_meters_by_project(self):
results = list(self.conn.get_meters(project='project-id'))
for r in results:
print r
assert len(results) == 2
def test_get_meters_by_metaquery(self):
q = {'metadata.display_name': 'test-server'}
got_not_imp = False
try:
list(self.conn.get_meters(metaquery=q))
except NotImplementedError:
got_not_imp = True
self.assertTrue(got_not_imp)
def test_get_raw_events_by_user(self):
f = storage.EventFilter(user='user-id')
results = list(self.conn.get_raw_events(f))
assert len(results) == 2
self._iterate_msgs(results)
def test_get_events_by_metaquery(self):
q = {'metadata.display_name': 'test-server'}
f = storage.EventFilter(metaquery=q)
got_not_imp = False
try:
list(self.conn.get_raw_events(f))
except NotImplementedError:
got_not_imp = True
self.assertTrue(got_not_imp)
def test_get_raw_events_by_project(self):
f = storage.EventFilter(project='project-id')
results = list(self.conn.get_raw_events(f))