openstack/deb-ceilometer
Code Issues Proposed changes
f3bc7d0109
deb-ceilometer/tests/storage/test_impl_mongodb.py
Angus Salkeld f3bc7d0109 Flatten the v2 API 
This is a squashed commit of work done by Doug and myself.
Thanks Doug!

Author: Angus Salkeld <asalkeld@redhat.com>

    Add a Statistics class
    Note this is a bit different to the spec
    (http://wiki.openstack.org/Ceilometer/blueprints/APIv2)
    As wsme doen't really like different types been returned from
    the same method.

    I have:
    GET /v2/meters/<meter> - raw samples
    GET /v2/meters/<meter>/statistics - for the stats

    Make the error reporting better for invalid fields
    Try and protect from passing in the wrong arguments into the db api
    Also get_resources() takes start/stop_timestamp not start/stop.
    Fix most of the duration test cases (overlapping ones are still broken)
    Add some log messages to warn of unimplemented queries
    Fix the start/end timestamp passed into calc_duration()
    Make the query op default to 'eq'
    Fix v2 event list paths
    Remove v2 list projects tests
    Re-Add the duration
    Implement get_meter_statistics() for sqlalchemy.
    Add tests for get_meter_statistics()
    Fix the latest pep8 1.4 issues

Author: Doug Hellmann <doug.hellmann@dreamhost.com>
    fixme comment
    Fix duration calculation
    fix event listing tests
    remove obsolete list tests
    update resource listing tests
    remove obsolete list tests
    fix max statistics tests for projects
    fix max tests for resource queries
    fix tests for stats using project filter
    Fix sum tests for resource queries
    Fix the statistics calculation in the mongo driver to handle getting
    no data back from the query.
    Update the queries in the test code.
    enable logging for wsme in the tests to help with debugging
    always include all query fields to keep values aligned
    only include the start and end timestamp keywords wanted by the EventFilter
    update url used in acl tests
    update tests for listing meters
    convert prints to logging calls and add a few todo/fixme notes
    add some debugging and error checking to _query_to_kwargs
    add q argument to get_json() to make it easier to pass queries to the service
    do not stub out controller we have deleted
    fix whitespace issues to make pep8 happy

Change-Id: I1b9a4c26fb8cc74ae1a002f93b84db05d0b20192
Blueprint: api-aggregate-average
Blueprint: api-server-pecan-wsme
2013-01-19 16:57:57 +11:00

615 lines
22 KiB
Python
Raw Blame History

# -*- encoding: utf-8 -*-
#
# Copyright © 2012 New Dream Network, LLC (DreamHost)
#
# Author: Doug Hellmann <doug.hellmann@dreamhost.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Tests for ceilometer/storage/impl_mongodb.py
.. note::
(dhellmann) These tests have some dependencies which cannot be
installed in the CI environment right now.
Ming is necessary to provide the Mongo-in-memory implementation for
of MongoDB. The original source for Ming is at
http://sourceforge.net/project/merciless but there does not seem to
be a way to point to a "zipball" of the latest HEAD there, and we
need features present only in that version. I forked the project to
github to make it easier to install, and put the URL into the
test-requires file. Then I ended up making some changes to it so it
would be compatible with PyMongo's API.
https://github.com/dreamhost/Ming/zipball/master#egg=Ming
In order to run the tests that use map-reduce with MIM, some
additional system-level packages are required::
apt-get install nspr-config
apt-get install libnspr4-dev
apt-get install pkg-config
pip install python-spidermonkey
To run the tests *without* mim, set the environment variable
CEILOMETER_TEST_LIVE=1 before running tox.
"""
import datetime
import unittest
import mox
from ceilometer.collector import meter
from ceilometer import counter
from ceilometer import storage
from ceilometer.tests.db import TestConnection, require_map_reduce
class MongoDBEngineTestBase(unittest.TestCase):
DBNAME = 'testdb'
def setUp(self):
super(MongoDBEngineTestBase, self).setUp()
conf = mox.Mox().CreateMockAnything()
conf.database_connection = 'mongodb://localhost/%s' % self.DBNAME
self.conn = TestConnection(conf)
self.conn.drop_database(self.DBNAME)
self.db = self.conn.conn[self.DBNAME]
self.counter = counter.Counter(
'instance',
counter.TYPE_CUMULATIVE,
'',
1,
'user-id',
'project-id',
'resource-id',
timestamp=datetime.datetime(2012, 7, 2, 10, 40),
resource_metadata={'display_name': 'test-server',
'tag': 'self.counter',
}
)
self.msg = meter.meter_message_from_counter(self.counter,
'not-so-secret',
'test-1')
self.conn.record_metering_data(self.msg)
self.counter2 = counter.Counter(
'instance',
counter.TYPE_CUMULATIVE,
'',
1,
'user-id',
'project-id',
'resource-id-alternate',
timestamp=datetime.datetime(2012, 7, 2, 10, 41),
resource_metadata={'display_name': 'test-server',
'tag': 'self.counter2',
}
)
self.msg2 = meter.meter_message_from_counter(self.counter2,
'not-so-secret',
'test-2')
self.conn.record_metering_data(self.msg2)
self.counter3 = counter.Counter(
'instance',
counter.TYPE_CUMULATIVE,
'',
1,
'user-id-alternate',
'project-id',
'resource-id-alternate',
timestamp=datetime.datetime(2012, 7, 2, 10, 41),
resource_metadata={'display_name': 'test-server',
'tag': 'self.counter3',
}
)
self.msg3 = meter.meter_message_from_counter(self.counter3,
'not-so-secret',
'test-3')
self.conn.record_metering_data(self.msg3)
for i in range(2, 4):
c = counter.Counter(
'instance',
counter.TYPE_CUMULATIVE,
'',
1,
'user-id-%s' % i,
'project-id-%s' % i,
'resource-id-%s' % i,
timestamp=datetime.datetime(2012, 7, 2, 10, 40 + i),
resource_metadata={'display_name': 'test-server',
'tag': 'counter-%s' % i,
}
)
msg = meter.meter_message_from_counter(c, 'not-so-secret', 'test')
self.conn.record_metering_data(msg)
def tearDown(self):
self.conn.drop_database(self.DBNAME)
class IndexTest(MongoDBEngineTestBase):
def test_indexes_exist(self):
# ensure_index returns none if index already exists
assert not self.db.resource.ensure_index('foo', name='resource_idx')
assert not self.db.meter.ensure_index('foo', name='meter_idx')
class UserTest(MongoDBEngineTestBase):
def test_new_user(self):
user = self.db.user.find_one({'_id': 'user-id'})
assert user is not None
def test_new_user_source(self):
user = self.db.user.find_one({'_id': 'user-id'})
assert 'source' in user
assert user['source'] == ['test-1', 'test-2']
def test_get_users(self):
users = self.conn.get_users()
assert set(users) == set(['user-id',
'user-id-alternate',
'user-id-2',
'user-id-3',
])
def test_get_users_by_source(self):
users = list(self.conn.get_users(source='test-1'))
assert len(users) == 1
assert users == ['user-id']
class ProjectTest(MongoDBEngineTestBase):
def test_new_project(self):
project = self.db.project.find_one({'_id': 'project-id'})
assert project is not None
def test_new_project_source(self):
project = self.db.project.find_one({'_id': 'project-id'})
assert 'source' in project
assert project['source'] == ['test-1', 'test-2', 'test-3']
def test_get_projects(self):
projects = self.conn.get_projects()
expected = set(['project-id', 'project-id-2', 'project-id-3'])
assert set(projects) == expected
def test_get_projects_by_source(self):
projects = self.conn.get_projects(source='test-1')
expected = ['project-id']
assert projects == expected
class ResourceTest(MongoDBEngineTestBase):
def test_new_resource(self):
resource = self.db.resource.find_one({'_id': 'resource-id'})
assert resource is not None
def test_new_resource_project(self):
resource = self.db.resource.find_one({'_id': 'resource-id'})
assert 'project_id' in resource
assert resource['project_id'] == 'project-id'
def test_new_resource_user(self):
resource = self.db.resource.find_one({'_id': 'resource-id'})
assert 'user_id' in resource
assert resource['user_id'] == 'user-id'
def test_new_resource_meter(self):
resource = self.db.resource.find_one({'_id': 'resource-id'})
assert 'meter' in resource
assert resource['meter'] == [{'counter_name': 'instance',
'counter_type': 'cumulative',
'counter_unit': '',
}]
def test_new_resource_metadata(self):
resource = self.db.resource.find_one({'_id': 'resource-id'})
assert 'metadata' in resource
def test_get_resources(self):
resources = list(self.conn.get_resources())
assert len(resources) == 4
for resource in resources:
if resource['resource_id'] != 'resource-id':
continue
assert resource['resource_id'] == 'resource-id'
assert resource['project_id'] == 'project-id'
assert resource['user_id'] == 'user-id'
assert 'metadata' in resource
assert resource['meter'] == [{'counter_name': 'instance',
'counter_type': 'cumulative',
'counter_unit': '',
}]
break
else:
assert False, 'Never found resource-id'
def test_get_resources_start_timestamp(self):
timestamp = datetime.datetime(2012, 7, 2, 10, 42)
resources = list(self.conn.get_resources(start_timestamp=timestamp))
resource_ids = [r['resource_id'] for r in resources]
expected = set(['resource-id-2', 'resource-id-3'])
assert set(resource_ids) == expected
def test_get_resources_end_timestamp(self):
timestamp = datetime.datetime(2012, 7, 2, 10, 42)
resources = list(self.conn.get_resources(end_timestamp=timestamp))
resource_ids = [r['resource_id'] for r in resources]
expected = set(['resource-id', 'resource-id-alternate'])
assert set(resource_ids) == expected
def test_get_resources_both_timestamps(self):
start_ts = datetime.datetime(2012, 7, 2, 10, 42)
end_ts = datetime.datetime(2012, 7, 2, 10, 43)
resources = list(self.conn.get_resources(start_timestamp=start_ts,
end_timestamp=end_ts))
resource_ids = [r['resource_id'] for r in resources]
expected = set(['resource-id-2'])
assert set(resource_ids) == expected
def test_get_resources_by_source(self):
resources = list(self.conn.get_resources(source='test-1'))
assert len(resources) == 1
ids = set(r['resource_id'] for r in resources)
assert ids == set(['resource-id'])
def test_get_resources_by_user(self):
resources = list(self.conn.get_resources(user='user-id'))
num_resources = len(resources)
assert num_resources == 1
ids = set(r['resource_id'] for r in resources)
assert ids == set(['resource-id'])
def test_get_resources_by_project(self):
resources = list(self.conn.get_resources(project='project-id'))
assert len(resources) == 2
ids = set(r['resource_id'] for r in resources)
assert ids == set(['resource-id', 'resource-id-alternate'])
class MeterTest(MongoDBEngineTestBase):
def test_new_meter(self):
meter = self.db.meter.find_one()
assert meter is not None
def test_get_meters(self):
results = list(self.conn.get_meters())
assert len(results) == 4
def test_get_meters_by_user(self):
results = list(self.conn.get_meters(user='user-id'))
assert len(results) == 1
def test_get_meters_by_project(self):
results = list(self.conn.get_meters(project='project-id'))
assert len(results) == 2
def test_get_raw_events_by_user(self):
f = storage.EventFilter(user='user-id')
results = list(self.conn.get_raw_events(f))
assert len(results) == 2
for meter in results:
assert meter in [self.msg, self.msg2]
def test_get_raw_events_by_project(self):
f = storage.EventFilter(project='project-id')
results = list(self.conn.get_raw_events(f))
assert results
for meter in results:
assert meter in [self.msg, self.msg2, self.msg3]
def test_get_raw_events_by_resource(self):
f = storage.EventFilter(user='user-id', resource='resource-id')
results = list(self.conn.get_raw_events(f))
assert results
meter = results[0]
assert meter is not None
assert meter == self.msg
def test_get_raw_events_by_start_time(self):
f = storage.EventFilter(
user='user-id',
start=datetime.datetime(2012, 7, 2, 10, 41),
)
results = list(self.conn.get_raw_events(f))
length = len(results)
assert length == 1
assert results[0]['timestamp'] == datetime.datetime(2012, 7, 2, 10, 41)
def test_get_raw_events_by_end_time(self):
f = storage.EventFilter(
user='user-id',
end=datetime.datetime(2012, 7, 2, 10, 41),
)
results = list(self.conn.get_raw_events(f))
length = len(results)
assert length == 1
assert results[0]['timestamp'] == datetime.datetime(2012, 7, 2, 10, 40)
def test_get_raw_events_by_both_times(self):
f = storage.EventFilter(
start=datetime.datetime(2012, 7, 2, 10, 42),
end=datetime.datetime(2012, 7, 2, 10, 43),
)
results = list(self.conn.get_raw_events(f))
length = len(results)
assert length == 1
assert results[0]['timestamp'] == datetime.datetime(2012, 7, 2, 10, 42)
def test_get_raw_events_by_meter(self):
f = storage.EventFilter(
user='user-id',
meter='no-such-meter',
)
results = list(self.conn.get_raw_events(f))
assert not results
def test_get_raw_events_by_meter2(self):
f = storage.EventFilter(
user='user-id',
meter='instance',
)
results = list(self.conn.get_raw_events(f))
assert results
def test_get_raw_events_by_source(self):
f = storage.EventFilter(source='test-1')
results = list(self.conn.get_raw_events(f))
assert results
assert len(results) == 1
class SumTest(MongoDBEngineTestBase):
def setUp(self):
super(SumTest, self).setUp()
require_map_reduce(self.conn)
def test_by_user(self):
f = storage.EventFilter(
user='user-id',
meter='instance',
)
results = list(self.conn.get_volume_sum(f))
assert results
counts = dict((r['resource_id'], r['value'])
for r in results)
assert counts['resource-id'] == 1
assert counts['resource-id-alternate'] == 1
assert set(counts.keys()) == set(['resource-id',
'resource-id-alternate'])
def test_by_project(self):
f = storage.EventFilter(
project='project-id',
meter='instance',
)
results = list(self.conn.get_volume_sum(f))
assert results
counts = dict((r['resource_id'], r['value'])
for r in results)
assert counts['resource-id'] == 1
assert counts['resource-id-alternate'] == 2
assert set(counts.keys()) == set(['resource-id',
'resource-id-alternate'])
def test_one_resource(self):
f = storage.EventFilter(
user='user-id',
meter='instance',
resource='resource-id',
)
results = list(self.conn.get_volume_sum(f))
assert results
counts = dict((r['resource_id'], r['value'])
for r in results)
assert counts['resource-id'] == 1
assert set(counts.keys()) == set(['resource-id'])
class TestGetEventInterval(MongoDBEngineTestBase):
def setUp(self):
super(TestGetEventInterval, self).setUp()
require_map_reduce(self.conn)
# Create events relative to the range and pretend
# that the intervening events exist.
self.start = datetime.datetime(2012, 8, 28, 0, 0)
self.end = datetime.datetime(2012, 8, 29, 0, 0)
self.early1 = self.start - datetime.timedelta(minutes=20)
self.early2 = self.start - datetime.timedelta(minutes=10)
self.middle1 = self.start + datetime.timedelta(minutes=10)
self.middle2 = self.end - datetime.timedelta(minutes=10)
self.late1 = self.end + datetime.timedelta(minutes=10)
self.late2 = self.end + datetime.timedelta(minutes=20)
self._filter = storage.EventFilter(
resource='resource-id',
meter='instance',
start=self.start,
end=self.end,
)
def _make_events(self, *timestamps):
for t in timestamps:
c = counter.Counter(
name='instance',
type=counter.TYPE_CUMULATIVE,
unit='',
volume=1,
user_id='user-id',
project_id='project-id',
resource_id='resource-id',
timestamp=t,
resource_metadata={'display_name': 'test-server',
}
)
msg = meter.meter_message_from_counter(counter=c,
secret='not-so-secret',
source='test',
)
self.conn.record_metering_data(msg)
def test_before_range(self):
self._make_events(self.early1, self.early2)
s, e = self.conn.get_event_interval(self._filter)
assert s is None
assert e is None
def test_overlap_range_start(self):
self._make_events(self.early1, self.start, self.middle1)
s, e = self.conn.get_event_interval(self._filter)
assert s == self.start
assert e == self.middle1
def test_within_range(self):
self._make_events(self.middle1, self.middle2)
s, e = self.conn.get_event_interval(self._filter)
assert s == self.middle1
assert e == self.middle2
def test_within_range_zero_duration(self):
self._make_events(self.middle1)
s, e = self.conn.get_event_interval(self._filter)
assert s == self.middle1
assert e == self.middle1
def test_within_range_zero_duration_two_events(self):
self._make_events(self.middle1, self.middle1)
s, e = self.conn.get_event_interval(self._filter)
assert s == self.middle1
assert e == self.middle1
def test_overlap_range_end(self):
self._make_events(self.middle2, self.end, self.late1)
s, e = self.conn.get_event_interval(self._filter)
assert s == self.middle2
assert e == self.middle2
def test_overlap_range_end_with_offset(self):
self._make_events(self.middle2, self.end, self.late1)
self._filter.end = self.late1
s, e = self.conn.get_event_interval(self._filter)
assert s == self.middle2
assert e == self.end
def test_after_range(self):
self._make_events(self.late1, self.late2)
s, e = self.conn.get_event_interval(self._filter)
assert s is None
assert e is None
class StatisticsTest(MongoDBEngineTestBase):
def setUp(self):
super(StatisticsTest, self).setUp()
require_map_reduce(self.conn)
self.counters = []
for i in range(3):
c = counter.Counter(
'volume.size',
'gauge',
'GiB',
5 + i,
'user-id',
'project1',
'resource-id',
timestamp=datetime.datetime(2012, 9, 25, 10 + i, 30 + i),
resource_metadata={'display_name': 'test-volume',
'tag': 'self.counter',
}
)
self.counters.append(c)
msg = meter.meter_message_from_counter(c,
secret='not-so-secret',
source='test',
)
self.conn.record_metering_data(msg)
for i in range(3):
c = counter.Counter(
'volume.size',
'gauge',
'GiB',
8 + i,
'user-5',
'project2',
'resource-6',
timestamp=datetime.datetime(2012, 9, 25, 10 + i, 30 + i),
resource_metadata={'display_name': 'test-volume',
'tag': 'self.counter',
}
)
self.counters.append(c)
msg = meter.meter_message_from_counter(c,
secret='not-so-secret',
source='test',
)
self.conn.record_metering_data(msg)
def test_by_user(self):
f = storage.EventFilter(
user='user-5',
meter='volume.size',
)
results = self.conn.get_meter_statistics(f)
assert results['count'] == 3
assert results['min'] == 8
assert results['max'] == 10
assert results['sum'] == 27
assert results['avg'] == 9
def test_by_project(self):
f = storage.EventFilter(
meter='volume.size',
resource='resource-id',
start='2012-09-25T11:30:00',
end='2012-09-25T11:32:00',
)
results = self.conn.get_meter_statistics(f)
assert results['count'] == 1
assert results['min'] == 6
assert results['max'] == 6
assert results['sum'] == 6
assert results['avg'] == 6
def test_one_resource(self):
f = storage.EventFilter(
user='user-id',
meter='volume.size',
)
results = self.conn.get_meter_statistics(f)
assert results['count'] == 3
assert results['min'] == 5
assert results['max'] == 7
assert results['sum'] == 18
assert results['avg'] == 6
View Git Blame Copy Permalink