Merge "Add the ability to filter on metadata."

ceilometer/api/v1/blueprint.py

@@ -100,14 +100,21 @@ def request_wants_html():
         flask.request.accept_mimetypes['application/json']
 
 
+def _get_metaquery(args):
+    return dict((k, v)
+                for (k, v) in args.iteritems()
+                    if k.startswith('metadata.'))
+
 ## APIs for working with meters.
 
 
 @blueprint.route('/meters')
 def list_meters_all():
     """Return a list of meters.
+    :param metadata.<key> match on the metadata within the resource. (optional)
     """
-    meters = flask.request.storage_conn.get_meters()
+    rq = flask.request
+    meters = rq.storage_conn.get_meters(metaquery=_get_metaquery(rq.args))
     return flask.jsonify(meters=list(meters))
 
 
@@ -116,8 +123,11 @@ def list_meters_by_resource(resource):
     """Return a list of meters by resource.
 
     :param resource: The ID of the resource.
+    :param metadata.<key> match on the metadata within the resource. (optional)
     """
-    meters = flask.request.storage_conn.get_meters(resource=resource)
+    rq = flask.request
+    meters = rq.storage_conn.get_meters(resource=resource,
+                                        metaquery=_get_metaquery(rq.args))
     return flask.jsonify(meters=list(meters))
 
 
@@ -126,8 +136,11 @@ def list_meters_by_user(user):
     """Return a list of meters by user.
 
     :param user: The ID of the owning user.
+    :param metadata.<key> match on the metadata within the resource. (optional)
     """
-    meters = flask.request.storage_conn.get_meters(user=user)
+    rq = flask.request
+    meters = rq.storage_conn.get_meters(user=user,
+                                        metaquery=_get_metaquery(rq.args))
     return flask.jsonify(meters=list(meters))
 
 
@@ -136,8 +149,11 @@ def list_meters_by_project(project):
     """Return a list of meters by project.
 
     :param project: The ID of the owning project.
+    :param metadata.<key> match on the metadata within the resource. (optional)
     """
-    meters = flask.request.storage_conn.get_meters(project=project)
+    rq = flask.request
+    meters = rq.storage_conn.get_meters(project=project,
+                                        metaquery=_get_metaquery(rq.args))
     return flask.jsonify(meters=list(meters))
 
 
@@ -146,8 +162,11 @@ def list_meters_by_source(source):
     """Return a list of meters by source.
 
     :param source: The ID of the owning source.
+    :param metadata.<key> match on the metadata within the resource. (optional)
     """
-    meters = flask.request.storage_conn.get_meters(source=source)
+    rq = flask.request
+    meters = rq.storage_conn.get_meters(source=source,
+                                        metaquery=_get_metaquery(rq.args))
     return flask.jsonify(meters=list(meters))
 
 
@@ -156,13 +175,15 @@ def list_meters_by_source(source):
 def _list_resources(source=None, user=None, project=None):
     """Return a list of resource identifiers.
     """
-    q_ts = _get_query_timestamps(flask.request.args)
+    rq = flask.request
-    resources = flask.request.storage_conn.get_resources(
+    q_ts = _get_query_timestamps(rq.args)
+    resources = rq.storage_conn.get_resources(
         source=source,
         user=user,
         project=project,
         start_timestamp=q_ts['start_timestamp'],
         end_timestamp=q_ts['end_timestamp'],
+        metaquery=_get_metaquery(rq.args),
         )
     return flask.jsonify(resources=list(resources))
 
@@ -178,6 +199,7 @@ def list_resources_by_project(project):
     :param end_timestamp: Limits resources by last update time < this value.
         (optional)
     :type end_timestamp: ISO date in UTC
+    :param metadata.<key> match on the metadata within the resource. (optional)
     """
     return _list_resources(project=project)
 
@@ -192,6 +214,7 @@ def list_all_resources():
     :param end_timestamp: Limits resources by last update time < this value.
         (optional)
     :type end_timestamp: ISO date in UTC
+    :param metadata.<key> match on the metadata within the resource. (optional)
     """
     return _list_resources()
 
@@ -217,6 +240,7 @@ def list_resources_by_source(source):
     :param end_timestamp: Limits resources by last update time < this value.
         (optional)
     :type end_timestamp: ISO date in UTC
+    :param metadata.<key> match on the metadata within the resource. (optional)
     """
     return _list_resources(source=source)
 
@@ -232,6 +256,7 @@ def list_resources_by_user(user):
     :param end_timestamp: Limits resources by last update time < this value.
         (optional)
     :type end_timestamp: ISO date in UTC
+    :param metadata.<key> match on the metadata within the resource. (optional)
     """
     return _list_resources(user=user)
 
@@ -308,6 +333,7 @@ def _list_events(meter,
                             resource=resource,
                             start=q_ts['start_timestamp'],
                             end=q_ts['end_timestamp'],
+                            metaquery=_get_metaquery(flask.request.args),
                             )
     events = list(flask.request.storage_conn.get_raw_events(f))
     jsonified = flask.jsonify(events=events)

ceilometer/storage/__init__.py

@@ -80,9 +80,10 @@ class EventFilter(object):
     :param resource: Optional filter for resource id.
     :param meter: Optional filter for meter type using the meter name.
     :param source: Optional source filter.
+    :param metaquery: Optional filter on the metadata
     """
     def __init__(self, user=None, project=None, start=None, end=None,
-                 resource=None, meter=None, source=None):
+                 resource=None, meter=None, source=None, metaquery={}):
         self.user = user
         self.project = project
         self.start = self._sanitize_timestamp(start)
@@ -90,6 +91,7 @@ class EventFilter(object):
         self.resource = resource
         self.meter = meter
         self.source = source
+        self.metaquery = metaquery
 
     def _sanitize_timestamp(self, timestamp):
         """Return a naive utc datetime object"""

ceilometer/storage/base.py

@@ -78,7 +78,8 @@ class Connection(object):
 
     @abc.abstractmethod
     def get_resources(self, user=None, project=None, source=None,
-                      start_timestamp=None, end_timestamp=None):
+                      start_timestamp=None, end_timestamp=None,
+                      metaquery={}):
         """Return an iterable of dictionaries containing resource information.
 
         { 'resource_id': UUID of the resource,
@@ -94,6 +95,26 @@ class Connection(object):
         :param source: Optional source filter.
         :param start_timestamp: Optional modified timestamp start range.
         :param end_timestamp: Optional modified timestamp end range.
+        :param metaquery: Optional dict with metadata to match on..
+        """
+
+    @abc.abstractmethod
+    def get_meters(self, user=None, project=None, resource=None, source=None,
+                   metaquery={}):
+        """Return an iterable of dictionaries containing meter information.
+
+        { 'name': name of the meter,
+          'type': type of the meter (guage, counter),
+          'resource_id': UUID of the resource,
+          'project_id': UUID of project owning the resource,
+          'user_id': UUID of user owning the resource,
+          }
+
+        :param user: Optional ID for user that owns the resource.
+        :param project: Optional ID for project that owns the resource.
+        :param resource: Optional resource filter.
+        :param source: Optional source filter.
+        :param metaquery: Optional dict with metadata to match on.
         """
 
     @abc.abstractmethod
@@ -133,14 +154,3 @@ class Connection(object):
 
         ( datetime.datetime(), datetime.datetime() )
         """
-
-    @abc.abstractmethod
-    def get_meters(self, user=None, project=None, resource=None, source=None):
-        """Return a list of meters.
-        {'resource_id': UUID string for the resource,
-         'project_id': UUID of project owning the resource,
-         'user_id': UUID of user owning the resource,
-         'name': The name of the meter,
-         'type': The meter type (gauge, counter, diff),
-        }
-        """

ceilometer/storage/impl_log.py

@@ -69,18 +69,28 @@ class Connection(base.Connection):
         """
 
     def get_resources(self, user=None, project=None, source=None,
-                      start_timestamp=None, end_timestamp=None):
+                      start_timestamp=None, end_timestamp=None,
-        """Return an iterable of tuples containing resource ids and
+                      metaquery={}):
-        the most recent version of the metadata for the resource.
+        """Return an iterable of dictionaries containing resource information.
+
+        { 'resource_id': UUID of the resource,
+          'project_id': UUID of project owning the resource,
+          'user_id': UUID of user owning the resource,
+          'timestamp': UTC datetime of last update to the resource,
+          'metadata': most current metadata for the resource,
+          'meter': list of the meters reporting data for the resource,
+          }
 
         :param user: Optional ID for user that owns the resource.
         :param project: Optional ID for project that owns the resource.
         :param source: Optional source filter.
         :param start_timestamp: Optional modified timestamp start range.
         :param end_timestamp: Optional modified timestamp end range.
+        :param metaquery: Optional dict with metadata to match on.
         """
 
-    def get_meters(self, user=None, project=None, resource=None, source=None):
+    def get_meters(self, user=None, project=None, resource=None, source=None,
+                   metaquery={}):
         """Return an iterable of dictionaries containing meter information.
 
         { 'name': name of the meter,
@@ -92,8 +102,9 @@ class Connection(base.Connection):
 
         :param user: Optional ID for user that owns the resource.
         :param project: Optional ID for project that owns the resource.
-        :param resource: Optional ID of the resource.
+        :param resource: Optional resource filter.
         :param source: Optional source filter.
+        :param metaquery: Optional dict with metadata to match on.
         """
 
     def get_raw_events(self, event_filter):

ceilometer/storage/impl_mongodb.py

@@ -114,6 +114,10 @@ def make_query_from_filter(event_filter, require_meter=True):
     if event_filter.source:
         q['source'] = event_filter.source
 
+    # so the events call metadata resource_metadata, so we convert
+    # to that.
+    q.update(dict(('resource_%s' % k, v)
+                  for (k, v) in event_filter.metaquery.iteritems()))
     return q
 
 
@@ -299,7 +303,8 @@ class Connection(base.Connection):
         return sorted(self.db.project.find(q).distinct('_id'))
 
     def get_resources(self, user=None, project=None, source=None,
-                      start_timestamp=None, end_timestamp=None):
+                      start_timestamp=None, end_timestamp=None,
+                      metaquery={}):
         """Return an iterable of dictionaries containing resource information.
 
         { 'resource_id': UUID of the resource,
@@ -315,6 +320,7 @@ class Connection(base.Connection):
         :param source: Optional source filter.
         :param start_timestamp: Optional modified timestamp start range.
         :param end_timestamp: Optional modified timestamp end range.
+        :param metaquery: Optional dict with metadata to match on.
         """
         q = {}
         if user is not None:
@@ -323,6 +329,8 @@ class Connection(base.Connection):
             q['project_id'] = project
         if source is not None:
             q['source'] = source
+        q.update(metaquery)
+
         # FIXME(dhellmann): This may not perform very well,
         # but doing any better will require changing the database
         # schema and that will need more thought than I have time
@@ -347,7 +355,8 @@ class Connection(base.Connection):
             del r['_id']
             yield r
 
-    def get_meters(self, user=None, project=None, resource=None, source=None):
+    def get_meters(self, user=None, project=None, resource=None, source=None,
+                   metaquery={}):
         """Return an iterable of dictionaries containing meter information.
 
         { 'name': name of the meter,
@@ -359,8 +368,9 @@ class Connection(base.Connection):
 
         :param user: Optional ID for user that owns the resource.
         :param project: Optional ID for project that owns the resource.
-        :param resource: Optional ID of the resource.
+        :param resource: Optional resource filter.
         :param source: Optional source filter.
+        :param metaquery: Optional dict with metadata to match on.
         """
         q = {}
         if user is not None:
@@ -371,6 +381,8 @@ class Connection(base.Connection):
             q['_id'] = resource
         if source is not None:
             q['source'] = source
+        q.update(metaquery)
+
         for r in self.db.resource.find(q):
             for r_meter in r['meter']:
                 m = {}

ceilometer/storage/impl_sqlalchemy.py

@@ -113,6 +113,9 @@ def make_query_from_filter(query, event_filter, require_meter=True):
     if event_filter.resource:
         query = query.filter_by(resource_id=event_filter.resource)
 
+    if event_filter.metaquery is not None and len(event_filter.metaquery) > 0:
+        raise NotImplementedError('metaquery not implemented')
+
     return query
 
 
@@ -212,7 +215,8 @@ class Connection(base.Connection):
         return (x[0] for x in query.all())
 
     def get_resources(self, user=None, project=None, source=None,
-                      start_timestamp=None, end_timestamp=None):
+                      start_timestamp=None, end_timestamp=None,
+                      metaquery=None):
         """Return an iterable of dictionaries containing resource information.
 
         { 'resource_id': UUID of the resource,
@@ -228,6 +232,7 @@ class Connection(base.Connection):
         :param source: Optional source filter.
         :param start_timestamp: Optional modified timestamp start range.
         :param end_timestamp: Optional modified timestamp end range.
+        :param metaquery: Optional dict with metadata to match on.
         """
         query = model_query(Resource, session=self.session)
         if user is not None:
@@ -242,6 +247,8 @@ class Connection(base.Connection):
             query = query.filter(Resource.project_id == project)
         query = query.options(
                     sqlalchemy_session.sqlalchemy.orm.joinedload('meters'))
+        if metaquery is not None:
+            raise NotImplementedError('metaquery not implemented')
 
         for resource in query.all():
             r = row2dict(resource)
@@ -257,8 +264,8 @@ class Connection(base.Connection):
             del r['meters']
             yield r
 
-    def get_meters(self, user=None, project=None, source=None,
+    def get_meters(self, user=None, project=None, resource=None, source=None,
-                   resource=None):
+                   metaquery={}):
         """Return an iterable of dictionaries containing meter information.
 
         { 'name': name of the meter,
@@ -272,6 +279,7 @@ class Connection(base.Connection):
         :param project: Optional ID for project that owns the resource.
         :param resource: Optional ID of the resource.
         :param source: Optional source filter.
+        :param metaquery: Optional dict with metadata to match on.
         """
         query = model_query(Resource, session=self.session)
         if user is not None:
@@ -284,6 +292,8 @@ class Connection(base.Connection):
             query = query.filter(Resource.project_id == project)
         query = query.options(
                     sqlalchemy_session.sqlalchemy.orm.joinedload('meters'))
+        if len(metaquery) > 0:
+            raise NotImplementedError('metaquery not implemented')
 
         for resource in query.all():
             meter_names = set()

tests/api/v1/test_list_events.py

@@ -124,3 +124,18 @@ class TestListEvents(tests_api.TestBase):
                         start_timestamp=datetime.datetime(2012, 7, 2, 10, 41),
                           end_timestamp=datetime.datetime(2012, 7, 2, 10, 42))
         self.assertEquals(1, len(data['events']))
+
+    def test_metaquery1(self):
+        q = '/sources/source1/meters/instance'
+        data = self.get('%s?metadata.tag=self.counter2' % q)
+        self.assertEquals(1, len(data['events']))
+
+    def test_metaquery2(self):
+        q = '/sources/source1/meters/instance'
+        data = self.get('%s?metadata.tag=self.counter' % q)
+        self.assertEquals(2, len(data['events']))
+
+    def test_metaquery3(self):
+        q = '/sources/source1/meters/instance'
+        data = self.get('%s?metadata.display_name=test-server' % q)
+        self.assertEquals(3, len(data['events']))

tests/api/v1/test_list_meters.py

@@ -74,7 +74,7 @@ class TestListMeters(tests_api.TestBase):
                     'resource-id2',
                     timestamp=datetime.datetime(2012, 7, 2, 10, 41),
                     resource_metadata={'display_name': 'test-server',
-                                       'tag': 'self.counter2',
+                                       'tag': 'two.counter',
                                    }),
                 counter.Counter(
                     'meter.test',
@@ -85,7 +85,7 @@ class TestListMeters(tests_api.TestBase):
                     'resource-id3',
                     timestamp=datetime.datetime(2012, 7, 2, 10, 42),
                     resource_metadata={'display_name': 'test-server',
-                                       'tag': 'self.counter3',
+                                       'tag': 'three.counter',
                                    }),
                 counter.Counter(
                     'meter.mine',
@@ -96,7 +96,7 @@ class TestListMeters(tests_api.TestBase):
                     'resource-id4',
                     timestamp=datetime.datetime(2012, 7, 2, 10, 43),
                     resource_metadata={'display_name': 'test-server',
-                                       'tag': 'self.counter4',
+                                       'tag': 'four.counter',
                                    })]:
             msg = meter.meter_message_from_counter(cnt,
                                                    cfg.CONF.metering_secret,
@@ -153,3 +153,11 @@ class TestListMeters(tests_api.TestBase):
     def test_with_project_non_existent(self):
         data = self.get('/projects/jd-was-here/meters')
         self.assertEquals(data['meters'], [])
+
+    def test_metaquery1(self):
+        data = self.get('/meters?metadata.tag=self.counter')
+        self.assertEquals(1, len(data['meters']))
+
+    def test_metaquery2(self):
+        data = self.get('/meters?metadata.tag=four.counter')
+        self.assertEquals(1, len(data['meters']))

tests/api/v1/test_list_resources.py

@@ -166,3 +166,13 @@ class TestListResources(tests_api.TestBase):
     def test_with_project_non_existent(self):
         data = self.get('/projects/jd-was-here/resources')
         self.assertEquals(data['resources'], [])
+
+    def test_metaquery1(self):
+        q = '/sources/test_list_resources/resources'
+        data = self.get('%s?metadata.display_name=test-server' % q)
+        self.assertEquals(3, len(data['resources']))
+
+    def test_metaquery2(self):
+        q = '/sources/test_list_resources/resources'
+        data = self.get('%s?metadata.tag=self.counter4' % q)
+        self.assertEquals(1, len(data['resources']))

tests/storage/test_impl_sqlalchemy.py

@@ -299,6 +299,21 @@ class ResourceTest(SQLAlchemyEngineTestBase):
         ids = set(r['resource_id'] for r in resources)
         assert ids == set(['resource-id', 'resource-id-alternate'])
 
+    def test_get_resources_by_metaquery(self):
+        q = {'metadata.display_name': 'test-server'}
+        got_not_imp = False
+        try:
+            list(self.conn.get_resources(metaquery=q))
+        except NotImplementedError:
+            got_not_imp = True
+        self.assertTrue(got_not_imp)
+        #this should work, but it doesn't.
+        #actually unless I wrap get_resources in list()
+        #it doesn't get called - weird
+        #self.assertRaises(NotImplementedError,
+        #                  self.conn.get_resources,
+        #                  metaquery=q)
+
 
 class MeterTest(SQLAlchemyEngineTestBase):
 
@@ -336,16 +351,33 @@ class MeterTest(SQLAlchemyEngineTestBase):
 
     def test_get_meters_by_project(self):
         results = list(self.conn.get_meters(project='project-id'))
-        for r in results:
-            print r
         assert len(results) == 2
 
+    def test_get_meters_by_metaquery(self):
+        q = {'metadata.display_name': 'test-server'}
+        got_not_imp = False
+        try:
+            list(self.conn.get_meters(metaquery=q))
+        except NotImplementedError:
+            got_not_imp = True
+        self.assertTrue(got_not_imp)
+
     def test_get_raw_events_by_user(self):
         f = storage.EventFilter(user='user-id')
         results = list(self.conn.get_raw_events(f))
         assert len(results) == 2
         self._iterate_msgs(results)
 
+    def test_get_events_by_metaquery(self):
+        q = {'metadata.display_name': 'test-server'}
+        f = storage.EventFilter(metaquery=q)
+        got_not_imp = False
+        try:
+            list(self.conn.get_raw_events(f))
+        except NotImplementedError:
+            got_not_imp = True
+        self.assertTrue(got_not_imp)
+
     def test_get_raw_events_by_project(self):
         f = storage.EventFilter(project='project-id')
         results = list(self.conn.get_raw_events(f))