Add API endpoint for listing raw event data

This change adds some of the endpoints for listing raw
event data from the database. It does not yet support
listing events by project id.

It also fixes a problem with the MongoDB driver returning
Mongo's ObjectId instances in the results of the event
query, which makes them impossible to serialize via JSON.

Change-Id: I08d122ecd2f726fb1b2880bc22e28113f6a3aeb1
Signed-off-by: Doug Hellmann <doug.hellmann@dreamhost.com>

ceilometer/api/v1.py

@@ -20,17 +20,25 @@
 
 import flask
 
+from ceilometer.openstack.common import log
+from ceilometer import storage
+
+
+LOG = log.getLogger(__name__)
+
 
 blueprint = flask.Blueprint('v1', __name__)
 
+
 ## APIs for working with resources.
 
 
 @blueprint.route('/resources', defaults={'source': None})
 @blueprint.route('/sources/<source>/resources')
 def list_resources(source):
-    resources = list(flask.request.storage_conn.get_resources(source=source))
-    return flask.jsonify(resources=resources)
+    resources = flask.request.storage_conn.get_resources(source=source)
+    return flask.jsonify(resources=list(resources))
+
 
 ## APIs for working with users.
 
@@ -38,5 +46,28 @@ def list_resources(source):
 @blueprint.route('/users', defaults={'source': None})
 @blueprint.route('/sources/<source>/users')
 def list_users(source):
-    users = list(flask.request.storage_conn.get_users(source=source))
-    return flask.jsonify(users=users)
+    users = flask.request.storage_conn.get_users(source=source)
+    return flask.jsonify(users=list(users))
+
+
+## APIs for working with events.
+
+
+@blueprint.route('/users/<user>')
+@blueprint.route('/users/<user>/meters/<meter>')
+@blueprint.route('/users/<user>/resources/<resource>')
+@blueprint.route('/users/<user>/resources/<resource>/meter/<meter>')
+@blueprint.route('/sources/<source>/users/<user>')
+@blueprint.route('/sources/<source>/users/<user>/meters/<meter>')
+@blueprint.route('/sources/<source>/users/<user>/resources/<resource>')
+@blueprint.route(
+    '/sources/<source>/users/<user>/resources/<resource>/meter/<meter>'
+    )
+def list_events(user, meter=None, resource=None, source=None):
+    f = storage.EventFilter(user=user,
+                            source=source,
+                            meter=meter,
+                            resource=resource,
+                            )
+    events = flask.request.storage_conn.get_raw_events(f)
+    return flask.jsonify(events=list(events))

ceilometer/storage/impl_mongodb.py

@@ -18,6 +18,7 @@
 """MongoDB storage backend
 """
 
+import copy
 import datetime
 
 from ceilometer.openstack.common import log
@@ -240,8 +241,11 @@ class Connection(base.Connection):
             upsert=True,
             )
 
-        # Record the raw data for the event
-        self.db.meter.insert(data)
+        # Record the raw data for the event. Use a copy so we do not
+        # modify a data structure owned by our caller (the driver adds
+        # a new key '_id').
+        record = copy.copy(data)
+        self.db.meter.insert(record)
         return
 
     def get_users(self, source=None):
@@ -293,6 +297,8 @@ class Connection(base.Connection):
         for resource in self.db.resource.find(q):
             r = {}
             r.update(resource)
+            # Replace the '_id' key with 'resource_id' to meet the
+            # caller's expectations.
             r['resource_id'] = r['_id']
             del r['_id']
             yield r
@@ -302,7 +308,12 @@ class Connection(base.Connection):
         """
         q = make_query_from_filter(event_filter, require_meter=False)
         events = self.db.meter.find(q)
-        return events
+        for e in events:
+            # Remove the ObjectId generated by the database when
+            # the event was inserted. It is an implementation
+            # detail that should not leak outside of the driver.
+            del e['_id']
+            yield e
 
     def get_volume_sum(self, event_filter):
         """Return the sum of the volume field for the events

ceilometer/tests/api.py

@@ -52,6 +52,8 @@ class Connection(impl_mongodb.Connection):
 
 class TestBase(unittest.TestCase):
 
+    DBNAME = 'testdb'
+
     def setUp(self):
         super(TestBase, self).setUp()
         self.app = flask.Flask('test')
@@ -61,17 +63,24 @@ class TestBase(unittest.TestCase):
         self.conf.metering_storage_engine = 'mongodb'
         self.conf.mongodb_host = 'localhost'
         self.conf.mongodb_port = 27017
-        self.conf.mongodb_dbname = 'testdb'
+        self.conf.mongodb_dbname = self.DBNAME
         self.conn = Connection(self.conf)
-        self.conn.conn.drop_database('testdb')
-        self.conn.conn['testdb']
+        self.conn.conn.drop_database(self.DBNAME)
+        self.conn.conn[self.DBNAME]
 
         @self.app.before_request
         def attach_storage_connection():
             flask.request.storage_conn = self.conn
         return
 
+    def tearDown(self):
+        self.conn.conn.drop_database(self.DBNAME)
+
     def get(self, path):
         rv = self.test_app.get(path)
-        data = json.loads(rv.data)
+        try:
+            data = json.loads(rv.data)
+        except ValueError:
+            print 'RAW DATA:', rv
+            raise
         return data

tests/api/v1/test_list_events.py

@@ -0,0 +1,86 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright © 2012 New Dream Network, LLC (DreamHost)
+#
+# Author: Doug Hellmann <doug.hellmann@dreamhost.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+"""Test listing raw events.
+"""
+
+import datetime
+import logging
+
+from ceilometer import counter
+from ceilometer import meter
+
+from ceilometer.tests import api as tests_api
+
+LOG = logging.getLogger(__name__)
+
+
+class TestListEvents(tests_api.TestBase):
+
+    def setUp(self):
+        super(TestListEvents, self).setUp()
+        self.counter1 = counter.Counter(
+            'source1',
+            'instance',
+            'cumulative',
+            1,
+            'user-id',
+            'project-id',
+            'resource-id',
+            timestamp=datetime.datetime(2012, 7, 2, 10, 40),
+            duration=0,
+            resource_metadata={'display_name': 'test-server',
+                               'tag': 'self.counter',
+                               }
+            )
+        msg = meter.meter_message_from_counter(self.counter1)
+        self.conn.record_metering_data(msg)
+
+        self.counter2 = counter.Counter(
+            'source2',
+            'instance',
+            'cumulative',
+            1,
+            'user-id',
+            'project-id',
+            'resource-id-alternate',
+            timestamp=datetime.datetime(2012, 7, 2, 10, 41),
+            duration=0,
+            resource_metadata={'display_name': 'test-server',
+                               'tag': 'self.counter2',
+                               }
+            )
+        msg2 = meter.meter_message_from_counter(self.counter2)
+        self.conn.record_metering_data(msg2)
+
+    def test_empty(self):
+        data = self.get('/users/no-such-user')
+        self.assertEquals({'events': []}, data)
+
+    def test_with_user(self):
+        data = self.get('/users/user-id')
+        self.assertEquals(2, len(data['events']))
+
+    def test_with_source_and_user(self):
+        data = self.get('/sources/source1/users/user-id')
+        ids = [r['resource_id'] for r in data['events']]
+        self.assertEquals(['resource-id'], ids)
+
+    def test_with_resource(self):
+        data = self.get('/users/user-id/resources/resource-id')
+        ids = [r['resource_id'] for r in data['events']]
+        self.assertEquals(['resource-id'], ids)