From 74e381fc9d0adb17034f9eb8467b8ca4934020c3 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 30 Jul 2012 17:49:33 -0400 Subject: [PATCH] Add API endpoint for listing raw event data This change adds some of the endpoints for listing raw event data from the database. It does not yet support listing events by project id. It also fixes a problem with the MongoDB driver returning Mongo's ObjectId instances in the results of the event query, which makes them impossible to serialize via JSON. Change-Id: I08d122ecd2f726fb1b2880bc22e28113f6a3aeb1 Signed-off-by: Doug Hellmann --- ceilometer/api/v1.py | 39 ++++++++++++-- ceilometer/storage/impl_mongodb.py | 17 ++++-- ceilometer/tests/api.py | 17 ++++-- tests/api/v1/test_list_events.py | 86 ++++++++++++++++++++++++++++++ 4 files changed, 148 insertions(+), 11 deletions(-) create mode 100644 tests/api/v1/test_list_events.py diff --git a/ceilometer/api/v1.py b/ceilometer/api/v1.py index 1e4ab372..bb9b2212 100644 --- a/ceilometer/api/v1.py +++ b/ceilometer/api/v1.py @@ -20,17 +20,25 @@ import flask +from ceilometer.openstack.common import log +from ceilometer import storage + + +LOG = log.getLogger(__name__) + blueprint = flask.Blueprint('v1', __name__) + ## APIs for working with resources. @blueprint.route('/resources', defaults={'source': None}) @blueprint.route('/sources//resources') def list_resources(source): - resources = list(flask.request.storage_conn.get_resources(source=source)) - return flask.jsonify(resources=resources) + resources = flask.request.storage_conn.get_resources(source=source) + return flask.jsonify(resources=list(resources)) + ## APIs for working with users. @@ -38,5 +46,28 @@ def list_resources(source): @blueprint.route('/users', defaults={'source': None}) @blueprint.route('/sources//users') def list_users(source): - users = list(flask.request.storage_conn.get_users(source=source)) - return flask.jsonify(users=users) + users = flask.request.storage_conn.get_users(source=source) + return flask.jsonify(users=list(users)) + + +## APIs for working with events. + + +@blueprint.route('/users/') +@blueprint.route('/users//meters/') +@blueprint.route('/users//resources/') +@blueprint.route('/users//resources//meter/') +@blueprint.route('/sources//users/') +@blueprint.route('/sources//users//meters/') +@blueprint.route('/sources//users//resources/') +@blueprint.route( + '/sources//users//resources//meter/' + ) +def list_events(user, meter=None, resource=None, source=None): + f = storage.EventFilter(user=user, + source=source, + meter=meter, + resource=resource, + ) + events = flask.request.storage_conn.get_raw_events(f) + return flask.jsonify(events=list(events)) diff --git a/ceilometer/storage/impl_mongodb.py b/ceilometer/storage/impl_mongodb.py index b67467f3..7216d82d 100644 --- a/ceilometer/storage/impl_mongodb.py +++ b/ceilometer/storage/impl_mongodb.py @@ -18,6 +18,7 @@ """MongoDB storage backend """ +import copy import datetime from ceilometer.openstack.common import log @@ -240,8 +241,11 @@ class Connection(base.Connection): upsert=True, ) - # Record the raw data for the event - self.db.meter.insert(data) + # Record the raw data for the event. Use a copy so we do not + # modify a data structure owned by our caller (the driver adds + # a new key '_id'). + record = copy.copy(data) + self.db.meter.insert(record) return def get_users(self, source=None): @@ -293,6 +297,8 @@ class Connection(base.Connection): for resource in self.db.resource.find(q): r = {} r.update(resource) + # Replace the '_id' key with 'resource_id' to meet the + # caller's expectations. r['resource_id'] = r['_id'] del r['_id'] yield r @@ -302,7 +308,12 @@ class Connection(base.Connection): """ q = make_query_from_filter(event_filter, require_meter=False) events = self.db.meter.find(q) - return events + for e in events: + # Remove the ObjectId generated by the database when + # the event was inserted. It is an implementation + # detail that should not leak outside of the driver. + del e['_id'] + yield e def get_volume_sum(self, event_filter): """Return the sum of the volume field for the events diff --git a/ceilometer/tests/api.py b/ceilometer/tests/api.py index 13d738dc..4348a6c4 100644 --- a/ceilometer/tests/api.py +++ b/ceilometer/tests/api.py @@ -52,6 +52,8 @@ class Connection(impl_mongodb.Connection): class TestBase(unittest.TestCase): + DBNAME = 'testdb' + def setUp(self): super(TestBase, self).setUp() self.app = flask.Flask('test') @@ -61,17 +63,24 @@ class TestBase(unittest.TestCase): self.conf.metering_storage_engine = 'mongodb' self.conf.mongodb_host = 'localhost' self.conf.mongodb_port = 27017 - self.conf.mongodb_dbname = 'testdb' + self.conf.mongodb_dbname = self.DBNAME self.conn = Connection(self.conf) - self.conn.conn.drop_database('testdb') - self.conn.conn['testdb'] + self.conn.conn.drop_database(self.DBNAME) + self.conn.conn[self.DBNAME] @self.app.before_request def attach_storage_connection(): flask.request.storage_conn = self.conn return + def tearDown(self): + self.conn.conn.drop_database(self.DBNAME) + def get(self, path): rv = self.test_app.get(path) - data = json.loads(rv.data) + try: + data = json.loads(rv.data) + except ValueError: + print 'RAW DATA:', rv + raise return data diff --git a/tests/api/v1/test_list_events.py b/tests/api/v1/test_list_events.py new file mode 100644 index 00000000..31fd3929 --- /dev/null +++ b/tests/api/v1/test_list_events.py @@ -0,0 +1,86 @@ +# -*- encoding: utf-8 -*- +# +# Copyright © 2012 New Dream Network, LLC (DreamHost) +# +# Author: Doug Hellmann +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +"""Test listing raw events. +""" + +import datetime +import logging + +from ceilometer import counter +from ceilometer import meter + +from ceilometer.tests import api as tests_api + +LOG = logging.getLogger(__name__) + + +class TestListEvents(tests_api.TestBase): + + def setUp(self): + super(TestListEvents, self).setUp() + self.counter1 = counter.Counter( + 'source1', + 'instance', + 'cumulative', + 1, + 'user-id', + 'project-id', + 'resource-id', + timestamp=datetime.datetime(2012, 7, 2, 10, 40), + duration=0, + resource_metadata={'display_name': 'test-server', + 'tag': 'self.counter', + } + ) + msg = meter.meter_message_from_counter(self.counter1) + self.conn.record_metering_data(msg) + + self.counter2 = counter.Counter( + 'source2', + 'instance', + 'cumulative', + 1, + 'user-id', + 'project-id', + 'resource-id-alternate', + timestamp=datetime.datetime(2012, 7, 2, 10, 41), + duration=0, + resource_metadata={'display_name': 'test-server', + 'tag': 'self.counter2', + } + ) + msg2 = meter.meter_message_from_counter(self.counter2) + self.conn.record_metering_data(msg2) + + def test_empty(self): + data = self.get('/users/no-such-user') + self.assertEquals({'events': []}, data) + + def test_with_user(self): + data = self.get('/users/user-id') + self.assertEquals(2, len(data['events'])) + + def test_with_source_and_user(self): + data = self.get('/sources/source1/users/user-id') + ids = [r['resource_id'] for r in data['events']] + self.assertEquals(['resource-id'], ids) + + def test_with_resource(self): + data = self.get('/users/user-id/resources/resource-id') + ids = [r['resource_id'] for r in data['events']] + self.assertEquals(['resource-id'], ids)