From 8d23f431ab0bd638edbf2197e56bea68d7b06a21 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Mon, 12 Dec 2016 19:06:30 +0100 Subject: [PATCH] Remove events storage and API This now has been moved to Panko. Change-Id: I179eb0d436752e3bb8abaed714664cf74f5615e6 --- ceilometer/api/controllers/v2/capabilities.py | 12 +- ceilometer/api/controllers/v2/events.py | 327 -------- ceilometer/api/controllers/v2/root.py | 13 +- ceilometer/api/hooks.py | 21 +- ceilometer/cmd/storage.py | 148 ++-- ceilometer/dispatcher/database.py | 44 +- ceilometer/event/storage/__init__.py | 57 -- ceilometer/event/storage/base.py | 99 --- .../event/storage/impl_elasticsearch.py | 288 ------- ceilometer/event/storage/impl_hbase.py | 221 ------ ceilometer/event/storage/impl_log.py | 33 - ceilometer/event/storage/impl_mongodb.py | 85 --- ceilometer/event/storage/impl_sqlalchemy.py | 456 ------------ ceilometer/event/storage/pymongo_base.py | 147 ---- ceilometer/storage/__init__.py | 18 +- ceilometer/storage/hbase/migration.py | 31 +- ceilometer/storage/hbase/utils.py | 48 +- ceilometer/storage/mongo/utils.py | 52 -- ceilometer/storage/sqlalchemy/models.py | 95 --- ceilometer/tests/db.py | 43 +- .../functional/api/v2/test_acl_scenarios.py | 104 --- .../functional/api/v2/test_event_scenarios.py | 703 ------------------ ceilometer/tests/functional/gabbi/fixtures.py | 29 - .../gabbi/gabbits/api-events-no-data.yaml | 210 ------ .../gabbi/gabbits/api-events-with-data.yaml | 242 ------ .../gabbi/gabbits/capabilities.yaml | 1 - .../tests/functional/publisher/test_direct.py | 19 - .../functional/storage/test_impl_hbase.py | 9 - .../functional/storage/test_impl_mongodb.py | 16 - .../storage/test_impl_sqlalchemy.py | 71 -- .../storage/test_storage_scenarios.py | 437 ----------- ceilometer/tests/functional/test_bin.py | 3 - ceilometer/tests/unit/api/v2/test_query.py | 14 - ceilometer/tests/unit/dispatcher/test_db.py | 14 - .../tests/unit/dispatcher/test_dispatcher.py | 9 - .../tests/unit/storage/test_get_connection.py | 42 +- ceilometer/tests/unit/storage/test_models.py | 94 --- ceilometer/tests/unit/test_collector.py | 8 +- devstack/plugin.sh | 18 +- doc/source/architecture.rst | 2 +- doc/source/webapi/v2.rst | 32 - etc/ceilometer/policy.json | 3 - ...er-event-api-removed-49c57835e307b997.yaml | 6 + setup.cfg | 10 - test-requirements.txt | 1 - tools/ceilometer-test-event.py | 83 --- tools/make_test_event_data.py | 116 --- tools/test_hbase_table_utils.py | 3 - tox.ini | 5 +- 49 files changed, 111 insertions(+), 4431 deletions(-) delete mode 100644 ceilometer/api/controllers/v2/events.py delete mode 100644 ceilometer/event/storage/base.py delete mode 100644 ceilometer/event/storage/impl_elasticsearch.py delete mode 100644 ceilometer/event/storage/impl_hbase.py delete mode 100644 ceilometer/event/storage/impl_log.py delete mode 100644 ceilometer/event/storage/impl_mongodb.py delete mode 100644 ceilometer/event/storage/impl_sqlalchemy.py delete mode 100644 ceilometer/event/storage/pymongo_base.py delete mode 100644 ceilometer/tests/functional/api/v2/test_event_scenarios.py delete mode 100644 ceilometer/tests/functional/gabbi/gabbits/api-events-no-data.yaml delete mode 100644 ceilometer/tests/functional/gabbi/gabbits/api-events-with-data.yaml delete mode 100644 ceilometer/tests/unit/storage/test_models.py create mode 100644 releasenotes/notes/ceilometer-event-api-removed-49c57835e307b997.yaml delete mode 100755 tools/ceilometer-test-event.py delete mode 100755 tools/make_test_event_data.py diff --git a/ceilometer/api/controllers/v2/capabilities.py b/ceilometer/api/controllers/v2/capabilities.py index 2a8c70d4b4..0e2e41338a 100644 --- a/ceilometer/api/controllers/v2/capabilities.py +++ b/ceilometer/api/controllers/v2/capabilities.py @@ -41,8 +41,6 @@ class Capabilities(base.Base): "A flattened dictionary of API capabilities" storage = {wtypes.text: bool} "A flattened dictionary of storage capabilities" - event_storage = {wtypes.text: bool} - "A flattened dictionary of event storage capabilities" @classmethod def sample(cls): @@ -68,12 +66,9 @@ class Capabilities(base.Base): 'stddev': True, 'cardinality': True, 'quartile': False}}}, - 'events': {'query': {'simple': True}}, }), storage=_flatten_capabilities( {'storage': {'production_ready': True}}), - event_storage=_flatten_capabilities( - {'storage': {'production_ready': True}}), ) @@ -89,12 +84,7 @@ class CapabilitiesController(rest.RestController): # variation in API capabilities is effectively determined by # the lack of strict feature parity across storage drivers conn = pecan.request.storage_conn - event_conn = pecan.request.event_storage_conn driver_capabilities = conn.get_capabilities().copy() - driver_capabilities['events'] = event_conn.get_capabilities()['events'] driver_perf = conn.get_storage_capabilities() - event_driver_perf = event_conn.get_storage_capabilities() return Capabilities(api=_flatten_capabilities(driver_capabilities), - storage=_flatten_capabilities(driver_perf), - event_storage=_flatten_capabilities( - event_driver_perf)) + storage=_flatten_capabilities(driver_perf)) diff --git a/ceilometer/api/controllers/v2/events.py b/ceilometer/api/controllers/v2/events.py deleted file mode 100644 index 58db241abe..0000000000 --- a/ceilometer/api/controllers/v2/events.py +++ /dev/null @@ -1,327 +0,0 @@ -# -# Copyright 2012 New Dream Network, LLC (DreamHost) -# Copyright 2013 IBM Corp. -# Copyright 2013 eNovance -# Copyright Ericsson AB 2013. All rights reserved -# Copyright 2014 Hewlett-Packard Company -# Copyright 2015 Huawei Technologies Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import datetime - -from oslo_log import log -import pecan -from pecan import rest -import six -import wsme -from wsme import types as wtypes -import wsmeext.pecan as wsme_pecan - -from ceilometer.api.controllers.v2 import base -from ceilometer.api.controllers.v2 import utils as v2_utils -from ceilometer.api import rbac -from ceilometer.event import storage -from ceilometer.event.storage import models as event_models -from ceilometer.i18n import _, _LE - -LOG = log.getLogger(__name__) - - -class TraitDescription(base.Base): - """A description of a trait, with no associated value.""" - - type = wtypes.text - "the data type, defaults to string" - - name = wtypes.text - "the name of the trait" - - @classmethod - def sample(cls): - return cls(name='service', - type='string' - ) - - -class EventQuery(base.Query): - """Query arguments for Event Queries.""" - - _supported_types = ['integer', 'float', 'string', 'datetime'] - - type = wsme.wsattr(wtypes.text, default='string') - "the type of the trait filter, defaults to string" - - def __repr__(self): - # for logging calls - return '' % (self.field, - self.op, - self._get_value_as_type(), - self.type) - - @classmethod - def sample(cls): - return cls(field="event_type", - type="string", - op="eq", - value="compute.instance.create.start") - - -class Trait(base.Base): - """A Trait associated with an event.""" - - name = wtypes.text - "The name of the trait" - - value = wtypes.text - "the value of the trait" - - type = wtypes.text - "the type of the trait (string, integer, float or datetime)" - - @staticmethod - def _convert_storage_trait(trait): - """Helper method to convert a storage model into an API trait instance. - - If an API trait instance is passed in, just return it. - """ - if isinstance(trait, Trait): - return trait - value = (six.text_type(trait.value) - if not trait.dtype == event_models.Trait.DATETIME_TYPE - else trait.value.isoformat()) - trait_type = event_models.Trait.get_name_by_type(trait.dtype) - return Trait(name=trait.name, type=trait_type, value=value) - - @classmethod - def sample(cls): - return cls(name='service', - type='string', - value='compute.hostname' - ) - - -class Event(base.Base): - """A System event.""" - - message_id = wtypes.text - "The message ID for the notification" - - event_type = wtypes.text - "The type of the event" - - _traits = None - - def get_traits(self): - return self._traits - - def set_traits(self, traits): - self._traits = map(Trait._convert_storage_trait, traits) - - traits = wsme.wsproperty(wtypes.ArrayType(Trait), - get_traits, - set_traits) - "Event specific properties" - - generated = datetime.datetime - "The time the event occurred" - - raw = base.JsonType() - "The raw copy of notification" - - @classmethod - def sample(cls): - return cls( - event_type='compute.instance.update', - generated=datetime.datetime(2015, 1, 1, 12, 0, 0, 0), - message_id='94834db1-8f1b-404d-b2ec-c35901f1b7f0', - traits={ - Trait(name='request_id', - value='req-4e2d67b8-31a4-48af-bb2f-9df72a353a72'), - Trait(name='service', - value='conductor.tem-devstack-01'), - Trait(name='tenant_id', - value='7f13f2b17917463b9ee21aa92c4b36d6') - }, - raw={'status': {'nested': 'started'}} - ) - - -def _build_rbac_query_filters(): - filters = {'t_filter': [], 'admin_proj': None} - # Returns user_id, proj_id for non-admins - user_id, proj_id = rbac.get_limited_to(pecan.request.headers) - # If non-admin, filter events by user and project - if user_id and proj_id: - filters['t_filter'].append({"key": "project_id", "string": proj_id, - "op": "eq"}) - filters['t_filter'].append({"key": "user_id", "string": user_id, - "op": "eq"}) - elif not user_id and not proj_id: - filters['admin_proj'] = pecan.request.headers.get('X-Project-Id') - return filters - - -def _event_query_to_event_filter(q): - evt_model_filter = { - 'event_type': None, - 'message_id': None, - 'start_timestamp': None, - 'end_timestamp': None - } - filters = _build_rbac_query_filters() - traits_filter = filters['t_filter'] - admin_proj = filters['admin_proj'] - - for i in q: - if not i.op: - i.op = 'eq' - elif i.op not in base.operation_kind: - error = (_('Operator %(operator)s is not supported. The supported' - ' operators are: %(supported)s') % - {'operator': i.op, 'supported': base.operation_kind}) - raise base.ClientSideError(error) - if i.field in evt_model_filter: - if i.op != 'eq' and i.field in ('event_type', 'message_id'): - error = (_('Operator %(operator)s is not supported. Only' - ' `eq\' operator is available for field' - ' %(field)s') % - {'operator': i.op, 'field': i.field}) - raise base.ClientSideError(error) - if i.op != 'ge' and i.field == 'start_timestamp': - error = (_('Operator %(operator)s is not supported. Only' - ' `ge\' operator is available for field' - ' %(field)s') % - {'operator': i.op, 'field': i.field}) - raise base.ClientSideError(error) - if i.op != 'le' and i.field == 'end_timestamp': - error = (_('Operator %(operator)s is not supported. Only' - ' `le\' operator is available for field' - ' %(field)s') % - {'operator': i.op, 'field': i.field}) - raise base.ClientSideError(error) - evt_model_filter[i.field] = i.value - else: - trait_type = i.type or 'string' - traits_filter.append({"key": i.field, - trait_type: i._get_value_as_type(), - "op": i.op}) - return storage.EventFilter(traits_filter=traits_filter, - admin_proj=admin_proj, **evt_model_filter) - - -class TraitsController(rest.RestController): - """Works on Event Traits.""" - - @v2_utils.requires_admin - @wsme_pecan.wsexpose([Trait], wtypes.text, wtypes.text) - def get_one(self, event_type, trait_name): - """Return all instances of a trait for an event type. - - :param event_type: Event type to filter traits by - :param trait_name: Trait to return values for - """ - LOG.debug("Getting traits for %s", event_type) - return [Trait._convert_storage_trait(t) - for t in pecan.request.event_storage_conn - .get_traits(event_type, trait_name)] - - @v2_utils.requires_admin - @wsme_pecan.wsexpose([TraitDescription], wtypes.text) - def get_all(self, event_type): - """Return all trait names for an event type. - - :param event_type: Event type to filter traits by - """ - get_trait_name = event_models.Trait.get_name_by_type - return [TraitDescription(name=t['name'], - type=get_trait_name(t['data_type'])) - for t in pecan.request.event_storage_conn - .get_trait_types(event_type)] - - -class EventTypesController(rest.RestController): - """Works on Event Types in the system.""" - - traits = TraitsController() - - @v2_utils.requires_admin - @wsme_pecan.wsexpose(None, wtypes.text) - def get_one(self, event_type): - """Unused API, will always return 404. - - :param event_type: A event type - """ - pecan.abort(404) - - @v2_utils.requires_admin - @wsme_pecan.wsexpose([six.text_type]) - def get_all(self): - """Get all event types.""" - return list(pecan.request.event_storage_conn.get_event_types()) - - -class EventsController(rest.RestController): - """Works on Events.""" - - @v2_utils.requires_context - @wsme_pecan.wsexpose([Event], [EventQuery], int) - def get_all(self, q=None, limit=None): - """Return all events matching the query filters. - - :param q: Filter arguments for which Events to return - :param limit: Maximum number of samples to be returned. - """ - rbac.enforce("events:index", pecan.request) - q = q or [] - limit = v2_utils.enforce_limit(limit) - event_filter = _event_query_to_event_filter(q) - return [Event(message_id=event.message_id, - event_type=event.event_type, - generated=event.generated, - traits=event.traits, - raw=event.raw) - for event in - pecan.request.event_storage_conn.get_events(event_filter, - limit)] - - @v2_utils.requires_context - @wsme_pecan.wsexpose(Event, wtypes.text) - def get_one(self, message_id): - """Return a single event with the given message id. - - :param message_id: Message ID of the Event to be returned - """ - rbac.enforce("events:show", pecan.request) - filters = _build_rbac_query_filters() - t_filter = filters['t_filter'] - admin_proj = filters['admin_proj'] - event_filter = storage.EventFilter(traits_filter=t_filter, - admin_proj=admin_proj, - message_id=message_id) - events = [event for event - in pecan.request.event_storage_conn.get_events(event_filter)] - if not events: - raise base.EntityNotFound(_("Event"), message_id) - - if len(events) > 1: - LOG.error(_LE("More than one event with " - "id %s returned from storage driver") % message_id) - - event = events[0] - - return Event(message_id=event.message_id, - event_type=event.event_type, - generated=event.generated, - traits=event.traits, - raw=event.raw) diff --git a/ceilometer/api/controllers/v2/root.py b/ceilometer/api/controllers/v2/root.py index f106ad90e1..8add14d2d3 100644 --- a/ceilometer/api/controllers/v2/root.py +++ b/ceilometer/api/controllers/v2/root.py @@ -25,7 +25,6 @@ from oslo_utils import strutils import pecan from ceilometer.api.controllers.v2 import capabilities -from ceilometer.api.controllers.v2 import events from ceilometer.api.controllers.v2 import meters from ceilometer.api.controllers.v2 import query from ceilometer.api.controllers.v2 import resources @@ -209,14 +208,10 @@ class V2Controller(object): aodh_abort() elif kind == 'alarms' and self.aodh_url: _redirect(self.aodh_url) - elif kind == 'events': - if self.panko_url: - return _redirect(self.panko_url) - return events.EventsController(), remainder - elif kind == 'event_types': - if self.panko_url: - return _redirect(self.panko_url) - return events.EventTypesController(), remainder + elif kind == 'events' and self.panko_url: + return _redirect(self.panko_url) + elif kind == 'event_types' and self.panko_url: + return _redirect(self.panko_url) else: pecan.abort(404) diff --git a/ceilometer/api/hooks.py b/ceilometer/api/hooks.py index 7dd52203ce..c460310b5f 100644 --- a/ceilometer/api/hooks.py +++ b/ceilometer/api/hooks.py @@ -45,27 +45,22 @@ class ConfigHook(hooks.PecanHook): class DBHook(hooks.PecanHook): def __init__(self, conf): - self.storage_connection = self.get_connection(conf, 'metering') - self.event_storage_connection = self.get_connection(conf, 'event') + self.storage_connection = self.get_connection(conf) - if (not self.storage_connection - and not self.event_storage_connection): - raise Exception("Api failed to start. Failed to connect to " - "databases, purpose: %s" % - ', '.join(['metering', 'event'])) + if not self.storage_connection: + raise Exception( + "API failed to start. Failed to connect to database") def before(self, state): state.request.storage_conn = self.storage_connection - state.request.event_storage_conn = self.event_storage_connection @staticmethod - def get_connection(conf, purpose): + def get_connection(conf): try: - return storage.get_connection_from_config(conf, purpose) + return storage.get_connection_from_config(conf) except Exception as err: - params = {"purpose": purpose, "err": err} - LOG.exception(_LE("Failed to connect to db, purpose %(purpose)s " - "retry later: %(err)s") % params) + LOG.exception(_LE("Failed to connect to db" "retry later: %s"), + err) class NotifierHook(hooks.PecanHook): diff --git a/ceilometer/cmd/storage.py b/ceilometer/cmd/storage.py index 01c83095a3..517be7c16f 100644 --- a/ceilometer/cmd/storage.py +++ b/ceilometer/cmd/storage.py @@ -33,9 +33,6 @@ def upgrade(): cfg.BoolOpt('skip-metering-database', help='Skip metering database upgrade.', default=False), - cfg.BoolOpt('skip-event-database', - help='Skip event database upgrade.', - default=False), cfg.BoolOpt('skip-gnocchi-resource-types', help='Skip gnocchi resource-types upgrade.', default=False), @@ -46,13 +43,7 @@ def upgrade(): LOG.info("Skipping metering database upgrade") else: LOG.debug("Upgrading metering database") - storage.get_connection_from_config(conf, 'metering').upgrade() - - if conf.skip_event_database: - LOG.info("Skipping event database upgrade") - else: - LOG.debug("Upgrading event database") - storage.get_connection_from_config(conf, 'event').upgrade() + storage.get_connection_from_config(conf).upgrade() if conf.skip_gnocchi_resource_types: LOG.info("Skipping Gnocchi resource types upgrade") @@ -67,94 +58,89 @@ def expirer(): if conf.database.metering_time_to_live > 0: LOG.debug("Clearing expired metering data") - storage_conn = storage.get_connection_from_config(conf, 'metering') + storage_conn = storage.get_connection_from_config(conf) storage_conn.clear_expired_metering_data( conf.database.metering_time_to_live) else: LOG.info(_LI("Nothing to clean, database metering time to live " "is disabled")) - if conf.database.event_time_to_live > 0: - LOG.debug("Clearing expired event data") - event_conn = storage.get_connection_from_config(conf, 'event') - event_conn.clear_expired_event_data( - conf.database.event_time_to_live) - else: - LOG.info(_LI("Nothing to clean, database event time to live " - "is disabled")) - def db_clean_legacy(): conf = cfg.ConfigOpts() conf.register_cli_opts([ - cfg.strOpt('confirm-drop-alarm-table', + cfg.strOpt('confirm-drop-table', short='n', - help='confirm to drop the legacy alarm tables')]) - if not conf.confirm_drop_alarm_table: - confirm = moves.input("Do you really want to drop the legacy alarm " - "tables? This will destroy data definitely " - "if it exist. Please type 'YES' to confirm: ") + help='confirm to drop the legacy tables')]) + if not conf.confirm_drop_table: + confirm = moves.input("Do you really want to drop the legacy " + "alarm and event tables? This will destroy " + "data definitively if it exist. Please type " + "'YES' to confirm: ") if confirm != 'YES': print("DB legacy cleanup aborted!") return service.prepare_service(conf=conf) - for purpose in ['metering', 'event']: - url = (getattr(conf.database, '%s_connection' % purpose) or - conf.database.connection) - parsed = urlparse.urlparse(url) - if parsed.password: - masked_netloc = '****'.join(parsed.netloc.rsplit(parsed.password)) - masked_url = parsed._replace(netloc=masked_netloc) - masked_url = urlparse.urlunparse(masked_url) - else: - masked_url = url - LOG.info(_LI('Starting to drop alarm and alarm history tables in ' - '%(purpose)s backend: %(url)s'), { - 'purpose': purpose, 'url': masked_url}) + url = (getattr(conf.database, "metering_connection") or + conf.database.connection) + parsed = urlparse.urlparse(url) - connection_scheme = parsed.scheme - conn = storage.get_connection_from_config(conf, purpose) - if connection_scheme in ('mysql', 'mysql+pymysql', 'postgresql', - 'sqlite'): - engine = conn._engine_facade.get_engine() - meta = sa.MetaData(bind=engine) - for table_name in ['alarm', 'alarm_history']: - if engine.has_table(table_name): - alarm = sa.Table(table_name, meta, autoload=True) - alarm.drop() - LOG.info(_LI("Legacy %s table of SQL backend has been " - "dropped."), table_name) - else: - LOG.info(_LI('%s table does not exist.'), table_name) + if parsed.password: + masked_netloc = '****'.join(parsed.netloc.rsplit(parsed.password)) + masked_url = parsed._replace(netloc=masked_netloc) + masked_url = urlparse.urlunparse(masked_url) + else: + masked_url = url + LOG.info(_LI('Starting to drop event, alarm and alarm history tables in ' + 'backend: %s'), masked_url) - elif connection_scheme == 'hbase': - with conn.conn_pool.connection() as h_conn: - tables = h_conn.tables() - table_name_mapping = {'alarm': 'alarm', - 'alarm_h': 'alarm history'} - for table_name in ['alarm', 'alarm_h']: - try: - if table_name in tables: - h_conn.disable_table(table_name) - h_conn.delete_table(table_name) - LOG.info(_LI("Legacy %s table of Hbase backend " - "has been dropped."), - table_name_mapping[table_name]) - else: - LOG.info(_LI('%s table does not exist.'), - table_name_mapping[table_name]) - except Exception as e: - LOG.error(_LE('Error occurred while dropping alarm ' - 'tables of Hbase, %s'), e) + connection_scheme = parsed.scheme + conn = storage.get_connection_from_config(conf) + if connection_scheme in ('mysql', 'mysql+pymysql', 'postgresql', + 'sqlite'): + engine = conn._engine_facade.get_engine() + meta = sa.MetaData(bind=engine) + for table_name in ('alarm', 'alarm_history', + 'trait_text', 'trait_int', + 'trait_float', 'trait_datetime', + 'event', 'event_type'): + if engine.has_table(table_name): + table = sa.Table(table_name, meta, autoload=True) + table.drop() + LOG.info(_LI("Legacy %s table of SQL backend has been " + "dropped."), table_name) + else: + LOG.info(_LI('%s table does not exist.'), table_name) - elif connection_scheme == 'mongodb': - for table_name in ['alarm', 'alarm_history']: - if table_name in conn.db.conn.collection_names(): - conn.db.conn.drop_collection(table_name) - LOG.info(_LI("Legacy %s table of Mongodb backend has been " - "dropped."), table_name) - else: - LOG.info(_LI('%s table does not exist.'), table_name) - LOG.info('Legacy alarm tables cleanup done.') + elif connection_scheme == 'hbase': + with conn.conn_pool.connection() as h_conn: + tables = h_conn.tables() + table_name_mapping = {'alarm': 'alarm', + 'alarm_h': 'alarm history', + 'event': 'event'} + for table_name in ('alarm', 'alarm_h', 'event'): + try: + if table_name in tables: + h_conn.disable_table(table_name) + h_conn.delete_table(table_name) + LOG.info(_LI("Legacy %s table of Hbase backend " + "has been dropped."), + table_name_mapping[table_name]) + else: + LOG.info(_LI('%s table does not exist.'), + table_name_mapping[table_name]) + except Exception as e: + LOG.error(_LE('Error occurred while dropping alarm ' + 'tables of Hbase, %s'), e) + + elif connection_scheme == 'mongodb': + for table_name in ('alarm', 'alarm_history', 'event'): + if table_name in conn.db.conn.collection_names(): + conn.db.conn.drop_collection(table_name) + LOG.info(_LI("Legacy %s table of Mongodb backend has been " + "dropped."), table_name) + else: + LOG.info(_LI('%s table does not exist.'), table_name) + LOG.info('Legacy alarm and event tables cleanup done.') diff --git a/ceilometer/dispatcher/database.py b/ceilometer/dispatcher/database.py index 9ee881b549..9be30199b7 100644 --- a/ceilometer/dispatcher/database.py +++ b/ceilometer/dispatcher/database.py @@ -13,19 +13,17 @@ # License for the specific language governing permissions and limitations # under the License. -from debtcollector import removals from oslo_log import log from oslo_utils import timeutils from ceilometer import dispatcher -from ceilometer.event.storage import models from ceilometer.i18n import _LE from ceilometer import storage LOG = log.getLogger(__name__) -class DatabaseDispatcher(dispatcher.Base): +class MeterDatabaseDispatcher(dispatcher.MeterDispatcherBase): """Dispatcher class for recording metering data into database. The dispatcher class which records each meter into a database configured @@ -36,21 +34,15 @@ class DatabaseDispatcher(dispatcher.Base): [DEFAULT] meter_dispatchers = database - event_dispatchers = database """ @property def conn(self): if not hasattr(self, "_conn"): self._conn = storage.get_connection_from_config( - self.conf, self.CONNECTION_TYPE) + self.conf) return self._conn - -class MeterDatabaseDispatcher(dispatcher.MeterDispatcherBase, - DatabaseDispatcher): - CONNECTION_TYPE = 'metering' - def record_metering_data(self, data): # We may have receive only one counter on the wire if not data: @@ -78,35 +70,3 @@ class MeterDatabaseDispatcher(dispatcher.MeterDispatcherBase, LOG.error(_LE('Failed to record %(len)s: %(err)s.'), {'len': len(data), 'err': err}) raise - - -@removals.removed_class("EventDatabaseDispatcher", - message="Use panko instead", - removal_version="8.0.0") -class EventDatabaseDispatcher(dispatcher.EventDispatcherBase, - DatabaseDispatcher): - CONNECTION_TYPE = 'event' - - def record_events(self, events): - if not isinstance(events, list): - events = [events] - - event_list = [] - for ev in events: - try: - event_list.append( - models.Event( - message_id=ev['message_id'], - event_type=ev['event_type'], - generated=timeutils.normalize_time( - timeutils.parse_isotime(ev['generated'])), - traits=[models.Trait( - name, dtype, - models.Trait.convert_value(dtype, value)) - for name, dtype, value in ev['traits']], - raw=ev.get('raw', {})) - ) - except Exception: - LOG.exception(_LE("Error processing event and it will be " - "dropped: %s"), ev) - self.conn.record_events(event_list) diff --git a/ceilometer/event/storage/__init__.py b/ceilometer/event/storage/__init__.py index fcf6a9049e..e69de29bb2 100644 --- a/ceilometer/event/storage/__init__.py +++ b/ceilometer/event/storage/__init__.py @@ -1,57 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import six - -from ceilometer import utils - - -class EventFilter(object): - """Properties for building an Event query. - - :param start_timestamp: UTC start datetime (mandatory) - :param end_timestamp: UTC end datetime (mandatory) - :param event_type: the name of the event. None for all. - :param message_id: the message_id of the event. None for all. - :param admin_proj: the project_id of admin role. None if non-admin user. - :param traits_filter: the trait filter dicts, all of which are optional. - This parameter is a list of dictionaries that specify trait values: - - .. code-block:: python - - {'key': , - 'string': , - 'integer': , - 'datetime': , - 'float': , - 'op': } - """ - - def __init__(self, start_timestamp=None, end_timestamp=None, - event_type=None, message_id=None, traits_filter=None, - admin_proj=None): - self.start_timestamp = utils.sanitize_timestamp(start_timestamp) - self.end_timestamp = utils.sanitize_timestamp(end_timestamp) - self.message_id = message_id - self.event_type = event_type - self.traits_filter = traits_filter or [] - self.admin_proj = admin_proj - - def __repr__(self): - return ("" % - (self.start_timestamp, - self.end_timestamp, - self.event_type, - six.text_type(self.traits_filter))) diff --git a/ceilometer/event/storage/base.py b/ceilometer/event/storage/base.py deleted file mode 100644 index e107c69473..0000000000 --- a/ceilometer/event/storage/base.py +++ /dev/null @@ -1,99 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -import ceilometer - - -class Connection(object): - """Base class for event storage system connections.""" - - # A dictionary representing the capabilities of this driver. - CAPABILITIES = { - 'events': {'query': {'simple': False}}, - } - - STORAGE_CAPABILITIES = { - 'storage': {'production_ready': False}, - } - - def __init__(self, conf, url): - self.conf = conf - - @staticmethod - def upgrade(): - """Migrate the database to `version` or the most recent version.""" - - @staticmethod - def clear(): - """Clear database.""" - - @staticmethod - def record_events(events): - """Write the events to the backend storage system. - - :param events: a list of model.Event objects. - """ - raise ceilometer.NotImplementedError('Events not implemented.') - - @staticmethod - def get_events(event_filter, limit=None): - """Return an iterable of model.Event objects.""" - raise ceilometer.NotImplementedError('Events not implemented.') - - @staticmethod - def get_event_types(): - """Return all event types as an iterable of strings.""" - raise ceilometer.NotImplementedError('Events not implemented.') - - @staticmethod - def get_trait_types(event_type): - """Return a dictionary containing the name and data type of the trait. - - Only trait types for the provided event_type are - returned. - :param event_type: the type of the Event - """ - raise ceilometer.NotImplementedError('Events not implemented.') - - @staticmethod - def get_traits(event_type, trait_type=None): - """Return all trait instances associated with an event_type. - - If trait_type is specified, only return instances of that trait type. - :param event_type: the type of the Event to filter by - :param trait_type: the name of the Trait to filter by - """ - - raise ceilometer.NotImplementedError('Events not implemented.') - - @classmethod - def get_capabilities(cls): - """Return an dictionary with the capabilities of each driver.""" - return cls.CAPABILITIES - - @classmethod - def get_storage_capabilities(cls): - """Return a dictionary representing the performance capabilities. - - This is needed to evaluate the performance of each driver. - """ - return cls.STORAGE_CAPABILITIES - - @staticmethod - def clear_expired_event_data(ttl): - """Clear expired data from the backend storage system. - - Clearing occurs according to the time-to-live. - - :param ttl: Number of seconds to keep records for. - """ - raise ceilometer.NotImplementedError('Clearing events not implemented') diff --git a/ceilometer/event/storage/impl_elasticsearch.py b/ceilometer/event/storage/impl_elasticsearch.py deleted file mode 100644 index 55e48583a4..0000000000 --- a/ceilometer/event/storage/impl_elasticsearch.py +++ /dev/null @@ -1,288 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import datetime -import operator - -import elasticsearch as es -from elasticsearch import helpers -from oslo_log import log -from oslo_utils import netutils -from oslo_utils import timeutils -import six - -from ceilometer.event.storage import base -from ceilometer.event.storage import models -from ceilometer.i18n import _LE, _LI -from ceilometer import storage -from ceilometer import utils - -LOG = log.getLogger(__name__) - - -AVAILABLE_CAPABILITIES = { - 'events': {'query': {'simple': True}}, -} - - -AVAILABLE_STORAGE_CAPABILITIES = { - 'storage': {'production_ready': True}, -} - - -class Connection(base.Connection): - """Put the event data into an ElasticSearch db. - - Events in ElasticSearch are indexed by day and stored by event_type. - An example document:: - - {"_index":"events_2014-10-21", - "_type":"event_type0", - "_id":"dc90e464-65ab-4a5d-bf66-ecb956b5d779", - "_score":1.0, - "_source":{"timestamp": "2014-10-21T20:02:09.274797" - "traits": {"id4_0": "2014-10-21T20:02:09.274797", - "id3_0": 0.7510790937279408, - "id2_0": 5, - "id1_0": "18c97ba1-3b74-441a-b948-a702a30cbce2"} - } - } - """ - - CAPABILITIES = utils.update_nested(base.Connection.CAPABILITIES, - AVAILABLE_CAPABILITIES) - STORAGE_CAPABILITIES = utils.update_nested( - base.Connection.STORAGE_CAPABILITIES, - AVAILABLE_STORAGE_CAPABILITIES, - ) - index_name = 'events' - # NOTE(gordc): mainly for testing, data is not searchable after write, - # it is only searchable after periodic refreshes. - _refresh_on_write = False - - def __init__(self, conf, url): - super(Connection, self).__init__(conf, url) - url_split = netutils.urlsplit(url) - self.conn = es.Elasticsearch(url_split.netloc) - - def upgrade(self): - iclient = es.client.IndicesClient(self.conn) - ts_template = { - 'template': '*', - 'mappings': {'_default_': - {'_timestamp': {'enabled': True, - 'store': True}, - 'properties': {'traits': {'type': 'nested'}}}}} - iclient.put_template(name='enable_timestamp', body=ts_template) - - def record_events(self, events): - - def _build_bulk_index(event_list): - for ev in event_list: - traits = {t.name: t.value for t in ev.traits} - yield {'_op_type': 'create', - '_index': '%s_%s' % (self.index_name, - ev.generated.date().isoformat()), - '_type': ev.event_type, - '_id': ev.message_id, - '_source': {'timestamp': ev.generated.isoformat(), - 'traits': traits, - 'raw': ev.raw}} - - error = None - for ok, result in helpers.streaming_bulk( - self.conn, _build_bulk_index(events)): - if not ok: - __, result = result.popitem() - if result['status'] == 409: - LOG.info(_LI('Duplicate event detected, skipping it: %s'), - result) - else: - LOG.exception(_LE('Failed to record event: %s'), result) - error = storage.StorageUnknownWriteError(result) - - if self._refresh_on_write: - self.conn.indices.refresh(index='%s_*' % self.index_name) - while self.conn.cluster.pending_tasks(local=True)['tasks']: - pass - if error: - raise error - - def _make_dsl_from_filter(self, indices, ev_filter): - q_args = {} - filters = [] - - if ev_filter.start_timestamp: - filters.append({'range': {'timestamp': - {'ge': ev_filter.start_timestamp.isoformat()}}}) - while indices[0] < ( - '%s_%s' % (self.index_name, - ev_filter.start_timestamp.date().isoformat())): - del indices[0] - if ev_filter.end_timestamp: - filters.append({'range': {'timestamp': - {'le': ev_filter.end_timestamp.isoformat()}}}) - while indices[-1] > ( - '%s_%s' % (self.index_name, - ev_filter.end_timestamp.date().isoformat())): - del indices[-1] - q_args['index'] = indices - - if ev_filter.event_type: - q_args['doc_type'] = ev_filter.event_type - if ev_filter.message_id: - filters.append({'term': {'_id': ev_filter.message_id}}) - if ev_filter.traits_filter or ev_filter.admin_proj: - trait_filters = [] - or_cond = [] - for t_filter in ev_filter.traits_filter or []: - value = None - for val_type in ['integer', 'string', 'float', 'datetime']: - if t_filter.get(val_type): - value = t_filter.get(val_type) - if isinstance(value, six.string_types): - value = value.lower() - elif isinstance(value, datetime.datetime): - value = value.isoformat() - break - if t_filter.get('op') in ['gt', 'ge', 'lt', 'le']: - op = (t_filter.get('op').replace('ge', 'gte') - .replace('le', 'lte')) - trait_filters.append( - {'range': {t_filter['key']: {op: value}}}) - else: - tf = {"query": {"query_string": { - "query": "%s: \"%s\"" % (t_filter['key'], value)}}} - if t_filter.get('op') == 'ne': - tf = {"not": tf} - trait_filters.append(tf) - if ev_filter.admin_proj: - or_cond = [{'missing': {'field': 'project_id'}}, - {'term': {'project_id': ev_filter.admin_proj}}] - filters.append( - {'nested': {'path': 'traits', 'query': {'filtered': { - 'filter': {'bool': {'must': trait_filters, - 'should': or_cond}}}}}}) - - q_args['body'] = {'query': {'filtered': - {'filter': {'bool': {'must': filters}}}}} - return q_args - - def get_events(self, event_filter, limit=None): - if limit == 0: - return - iclient = es.client.IndicesClient(self.conn) - indices = iclient.get_mapping('%s_*' % self.index_name).keys() - if indices: - filter_args = self._make_dsl_from_filter(indices, event_filter) - if limit is not None: - filter_args['size'] = limit - results = self.conn.search(fields=['_id', 'timestamp', - '_type', '_source'], - sort='timestamp:asc', - **filter_args) - trait_mappings = {} - for record in results['hits']['hits']: - trait_list = [] - if not record['_type'] in trait_mappings: - trait_mappings[record['_type']] = list( - self.get_trait_types(record['_type'])) - for key in record['_source']['traits'].keys(): - value = record['_source']['traits'][key] - for t_map in trait_mappings[record['_type']]: - if t_map['name'] == key: - dtype = t_map['data_type'] - break - else: - dtype = models.Trait.TEXT_TYPE - trait_list.append(models.Trait( - name=key, dtype=dtype, - value=models.Trait.convert_value(dtype, value))) - gen_ts = timeutils.normalize_time(timeutils.parse_isotime( - record['_source']['timestamp'])) - yield models.Event(message_id=record['_id'], - event_type=record['_type'], - generated=gen_ts, - traits=sorted( - trait_list, - key=operator.attrgetter('dtype')), - raw=record['_source']['raw']) - - def get_event_types(self): - iclient = es.client.IndicesClient(self.conn) - es_mappings = iclient.get_mapping('%s_*' % self.index_name) - seen_types = set() - for index in es_mappings.keys(): - for ev_type in es_mappings[index]['mappings'].keys(): - seen_types.add(ev_type) - # TODO(gordc): tests assume sorted ordering but backends are not - # explicitly ordered. - # NOTE: _default_ is a type that appears in all mappings but is not - # real 'type' - seen_types.discard('_default_') - return sorted(list(seen_types)) - - @staticmethod - def _remap_es_types(d_type): - if d_type == 'string': - d_type = 'text' - elif d_type == 'long': - d_type = 'int' - elif d_type == 'double': - d_type = 'float' - elif d_type == 'date' or d_type == 'date_time': - d_type = 'datetime' - return d_type - - def get_trait_types(self, event_type): - iclient = es.client.IndicesClient(self.conn) - es_mappings = iclient.get_mapping('%s_*' % self.index_name) - seen_types = [] - for index in es_mappings.keys(): - # if event_type exists in index and has traits - if (es_mappings[index]['mappings'].get(event_type) and - es_mappings[index]['mappings'][event_type]['properties'] - ['traits'].get('properties')): - for t_type in (es_mappings[index]['mappings'][event_type] - ['properties']['traits']['properties'].keys()): - d_type = (es_mappings[index]['mappings'][event_type] - ['properties']['traits']['properties'] - [t_type]['type']) - d_type = models.Trait.get_type_by_name( - self._remap_es_types(d_type)) - if (t_type, d_type) not in seen_types: - yield {'name': t_type, 'data_type': d_type} - seen_types.append((t_type, d_type)) - - def get_traits(self, event_type, trait_type=None): - t_types = dict((res['name'], res['data_type']) - for res in self.get_trait_types(event_type)) - if not t_types or (trait_type and trait_type not in t_types.keys()): - return - result = self.conn.search('%s_*' % self.index_name, event_type) - for ev in result['hits']['hits']: - if trait_type and ev['_source']['traits'].get(trait_type): - yield models.Trait( - name=trait_type, - dtype=t_types[trait_type], - value=models.Trait.convert_value( - t_types[trait_type], - ev['_source']['traits'][trait_type])) - else: - for trait in ev['_source']['traits'].keys(): - yield models.Trait( - name=trait, - dtype=t_types[trait], - value=models.Trait.convert_value( - t_types[trait], - ev['_source']['traits'][trait])) diff --git a/ceilometer/event/storage/impl_hbase.py b/ceilometer/event/storage/impl_hbase.py deleted file mode 100644 index 6dbc0bd50b..0000000000 --- a/ceilometer/event/storage/impl_hbase.py +++ /dev/null @@ -1,221 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import operator - -from oslo_log import log - -from ceilometer.event.storage import base -from ceilometer.event.storage import models -from ceilometer.i18n import _LE -from ceilometer.storage.hbase import base as hbase_base -from ceilometer.storage.hbase import utils as hbase_utils -from ceilometer import utils - -LOG = log.getLogger(__name__) - - -AVAILABLE_CAPABILITIES = { - 'events': {'query': {'simple': True}}, -} - - -AVAILABLE_STORAGE_CAPABILITIES = { - 'storage': {'production_ready': True}, -} - - -class Connection(hbase_base.Connection, base.Connection): - """Put the event data into a HBase database - - Collections: - - - events: - - - row_key: timestamp of event's generation + uuid of event - in format: "%s:%s" % (ts, Event.message_id) - - Column Families: - - f: contains the following qualifiers: - - - event_type: description of event's type - - timestamp: time stamp of event generation - - all traits for this event in format: - - .. code-block:: python - - "%s:%s" % (trait_name, trait_type) - """ - - CAPABILITIES = utils.update_nested(base.Connection.CAPABILITIES, - AVAILABLE_CAPABILITIES) - STORAGE_CAPABILITIES = utils.update_nested( - base.Connection.STORAGE_CAPABILITIES, - AVAILABLE_STORAGE_CAPABILITIES, - ) - _memory_instance = None - - EVENT_TABLE = "event" - - def upgrade(self): - tables = [self.EVENT_TABLE] - column_families = {'f': dict(max_versions=1)} - with self.conn_pool.connection() as conn: - hbase_utils.create_tables(conn, tables, column_families) - - def clear(self): - LOG.debug('Dropping HBase schema...') - with self.conn_pool.connection() as conn: - for table in [self.EVENT_TABLE]: - try: - conn.disable_table(table) - except Exception: - LOG.debug('Cannot disable table but ignoring error') - try: - conn.delete_table(table) - except Exception: - LOG.debug('Cannot delete table but ignoring error') - - def record_events(self, event_models): - """Write the events to Hbase. - - :param event_models: a list of models.Event objects. - """ - error = None - with self.conn_pool.connection() as conn: - events_table = conn.table(self.EVENT_TABLE) - for event_model in event_models: - # Row key consists of timestamp and message_id from - # models.Event or purposes of storage event sorted by - # timestamp in the database. - ts = event_model.generated - row = hbase_utils.prepare_key( - hbase_utils.timestamp(ts, reverse=False), - event_model.message_id) - event_type = event_model.event_type - traits = {} - if event_model.traits: - for trait in event_model.traits: - key = hbase_utils.prepare_key(trait.name, trait.dtype) - traits[key] = trait.value - record = hbase_utils.serialize_entry(traits, - event_type=event_type, - timestamp=ts, - raw=event_model.raw) - try: - events_table.put(row, record) - except Exception as ex: - LOG.exception(_LE("Failed to record event: %s") % ex) - error = ex - if error: - raise error - - def get_events(self, event_filter, limit=None): - """Return an iter of models.Event objects. - - :param event_filter: storage.EventFilter object, consists of filters - for events that are stored in database. - """ - if limit == 0: - return - q, start, stop = hbase_utils.make_events_query_from_filter( - event_filter) - with self.conn_pool.connection() as conn: - events_table = conn.table(self.EVENT_TABLE) - - gen = events_table.scan(filter=q, row_start=start, row_stop=stop, - limit=limit) - - for event_id, data in gen: - traits = [] - events_dict = hbase_utils.deserialize_entry(data)[0] - for key, value in events_dict.items(): - if isinstance(key, tuple): - trait_name, trait_dtype = key - traits.append(models.Trait(name=trait_name, - dtype=int(trait_dtype), - value=value)) - ts, mess = event_id.split(':') - - yield models.Event( - message_id=hbase_utils.unquote(mess), - event_type=events_dict['event_type'], - generated=events_dict['timestamp'], - traits=sorted(traits, - key=operator.attrgetter('dtype')), - raw=events_dict['raw'] - ) - - def get_event_types(self): - """Return all event types as an iterable of strings.""" - with self.conn_pool.connection() as conn: - events_table = conn.table(self.EVENT_TABLE) - gen = events_table.scan() - - event_types = set() - for event_id, data in gen: - events_dict = hbase_utils.deserialize_entry(data)[0] - for key, value in events_dict.items(): - if not isinstance(key, tuple) and key.startswith('event_type'): - if value not in event_types: - event_types.add(value) - yield value - - def get_trait_types(self, event_type): - """Return a dictionary containing the name and data type of the trait. - - Only trait types for the provided event_type are returned. - - :param event_type: the type of the Event - """ - - q = hbase_utils.make_query(event_type=event_type) - trait_names = set() - with self.conn_pool.connection() as conn: - events_table = conn.table(self.EVENT_TABLE) - gen = events_table.scan(filter=q) - for event_id, data in gen: - events_dict = hbase_utils.deserialize_entry(data)[0] - for key, value in events_dict.items(): - if isinstance(key, tuple): - trait_name, trait_type = key - if trait_name not in trait_names: - # Here we check that our method return only unique - # trait types, for ex. if it is found the same trait - # types in different events with equal event_type, - # method will return only one trait type. It is - # proposed that certain trait name could have only one - # trait type. - trait_names.add(trait_name) - data_type = models.Trait.type_names[int(trait_type)] - yield {'name': trait_name, 'data_type': data_type} - - def get_traits(self, event_type, trait_type=None): - """Return all trait instances associated with an event_type. - - If trait_type is specified, only return instances of that trait type. - :param event_type: the type of the Event to filter by - :param trait_type: the name of the Trait to filter by - """ - q = hbase_utils.make_query(event_type=event_type, - trait_type=trait_type) - with self.conn_pool.connection() as conn: - events_table = conn.table(self.EVENT_TABLE) - gen = events_table.scan(filter=q) - for event_id, data in gen: - events_dict = hbase_utils.deserialize_entry(data)[0] - for key, value in events_dict.items(): - if isinstance(key, tuple): - trait_name, trait_type = key - yield models.Trait(name=trait_name, - dtype=int(trait_type), value=value) diff --git a/ceilometer/event/storage/impl_log.py b/ceilometer/event/storage/impl_log.py deleted file mode 100644 index 50c51908a1..0000000000 --- a/ceilometer/event/storage/impl_log.py +++ /dev/null @@ -1,33 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslo_log import log - -from ceilometer.event.storage import base -from ceilometer.i18n import _LI - -LOG = log.getLogger(__name__) - - -class Connection(base.Connection): - """Log event data.""" - - @staticmethod - def clear_expired_event_data(ttl): - """Clear expired data from the backend storage system. - - Clearing occurs according to the time-to-live. - - :param ttl: Number of seconds to keep records for. - """ - LOG.info(_LI("Dropping event data with TTL %d"), ttl) diff --git a/ceilometer/event/storage/impl_mongodb.py b/ceilometer/event/storage/impl_mongodb.py deleted file mode 100644 index ae1b63c543..0000000000 --- a/ceilometer/event/storage/impl_mongodb.py +++ /dev/null @@ -1,85 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""MongoDB storage backend""" - -from oslo_log import log -import pymongo - -from ceilometer.event.storage import pymongo_base -from ceilometer import storage -from ceilometer.storage import impl_mongodb -from ceilometer.storage.mongo import utils as pymongo_utils - -LOG = log.getLogger(__name__) - - -class Connection(pymongo_base.Connection): - """Put the event data into a MongoDB database.""" - - CONNECTION_POOL = pymongo_utils.ConnectionPool() - - def __init__(self, conf, url): - super(Connection, self).__init__(conf, url) - - # NOTE(jd) Use our own connection pooling on top of the Pymongo one. - # We need that otherwise we overflow the MongoDB instance with new - # connection since we instantiate a Pymongo client each time someone - # requires a new storage connection. - self.conn = self.CONNECTION_POOL.connect(conf, url) - - # Require MongoDB 2.4 to use $setOnInsert - if self.conn.server_info()['versionArray'] < [2, 4]: - raise storage.StorageBadVersion("Need at least MongoDB 2.4") - - connection_options = pymongo.uri_parser.parse_uri(url) - self.db = getattr(self.conn, connection_options['database']) - if connection_options.get('username'): - self.db.authenticate(connection_options['username'], - connection_options['password']) - - # NOTE(jd) Upgrading is just about creating index, so let's do this - # on connection to be sure at least the TTL is correctly updated if - # needed. - self.upgrade() - - def upgrade(self): - # create collection if not present - if 'event' not in self.db.conn.collection_names(): - self.db.conn.create_collection('event') - # Establish indexes - # NOTE(idegtiarov): This indexes cover get_events, get_event_types, and - # get_trait_types requests based on event_type and timestamp fields. - self.db.event.create_index( - [('event_type', pymongo.ASCENDING), - ('timestamp', pymongo.ASCENDING)], - name='event_type_idx' - ) - ttl = self.conf.database.event_time_to_live - impl_mongodb.Connection.update_ttl(ttl, 'event_ttl', 'timestamp', - self.db.event) - - def clear(self): - self.conn.drop_database(self.db.name) - # Connection will be reopened automatically if needed - self.conn.close() - - @staticmethod - def clear_expired_event_data(ttl): - """Clear expired data from the backend storage system. - - Clearing occurs according to the time-to-live. - - :param ttl: Number of seconds to keep records for. - """ - LOG.debug("Clearing expired event data is based on native " - "MongoDB time to live feature and going in background.") diff --git a/ceilometer/event/storage/impl_sqlalchemy.py b/ceilometer/event/storage/impl_sqlalchemy.py deleted file mode 100644 index bb190625c4..0000000000 --- a/ceilometer/event/storage/impl_sqlalchemy.py +++ /dev/null @@ -1,456 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""SQLAlchemy storage backend.""" - -from __future__ import absolute_import -import datetime -import os - -from oslo_db import exception as dbexc -from oslo_db.sqlalchemy import session as db_session -from oslo_log import log -from oslo_utils import timeutils -import sqlalchemy as sa - -from ceilometer.event.storage import base -from ceilometer.event.storage import models as api_models -from ceilometer.i18n import _LE, _LI -from ceilometer import storage -from ceilometer.storage.sqlalchemy import models -from ceilometer import utils - -LOG = log.getLogger(__name__) - - -AVAILABLE_CAPABILITIES = { - 'events': {'query': {'simple': True}}, -} - - -AVAILABLE_STORAGE_CAPABILITIES = { - 'storage': {'production_ready': True}, -} - - -TRAIT_MAPLIST = [(api_models.Trait.NONE_TYPE, models.TraitText), - (api_models.Trait.TEXT_TYPE, models.TraitText), - (api_models.Trait.INT_TYPE, models.TraitInt), - (api_models.Trait.FLOAT_TYPE, models.TraitFloat), - (api_models.Trait.DATETIME_TYPE, models.TraitDatetime)] - - -TRAIT_ID_TO_MODEL = dict((x, y) for x, y in TRAIT_MAPLIST) -TRAIT_MODEL_TO_ID = dict((y, x) for x, y in TRAIT_MAPLIST) - - -trait_models_dict = {'string': models.TraitText, - 'integer': models.TraitInt, - 'datetime': models.TraitDatetime, - 'float': models.TraitFloat} - - -def _build_trait_query(session, trait_type, key, value, op='eq'): - trait_model = trait_models_dict[trait_type] - op_dict = {'eq': (trait_model.value == value), - 'lt': (trait_model.value < value), - 'le': (trait_model.value <= value), - 'gt': (trait_model.value > value), - 'ge': (trait_model.value >= value), - 'ne': (trait_model.value != value)} - conditions = [trait_model.key == key, op_dict[op]] - return (session.query(trait_model.event_id.label('ev_id')) - .filter(*conditions)) - - -class Connection(base.Connection): - """Put the event data into a SQLAlchemy database. - - Tables:: - - - EventType - - event definition - - { id: event type id - desc: description of event - } - - Event - - event data - - { id: event id - message_id: message id - generated = timestamp of event - event_type_id = event type -> eventtype.id - } - - TraitInt - - int trait value - - { event_id: event -> event.id - key: trait name - value: integer value - } - - TraitDatetime - - datetime trait value - - { event_id: event -> event.id - key: trait name - value: datetime value - } - - TraitText - - text trait value - - { event_id: event -> event.id - key: trait name - value: text value - } - - TraitFloat - - float trait value - - { event_id: event -> event.id - key: trait name - value: float value - } - - """ - CAPABILITIES = utils.update_nested(base.Connection.CAPABILITIES, - AVAILABLE_CAPABILITIES) - STORAGE_CAPABILITIES = utils.update_nested( - base.Connection.STORAGE_CAPABILITIES, - AVAILABLE_STORAGE_CAPABILITIES, - ) - - def __init__(self, conf, url): - super(Connection, self).__init__(conf, url) - # Set max_retries to 0, since oslo.db in certain cases may attempt - # to retry making the db connection retried max_retries ^ 2 times - # in failure case and db reconnection has already been implemented - # in storage.__init__.get_connection_from_config function - options = dict(self.conf.database.items()) - options['max_retries'] = 0 - # oslo.db doesn't support options defined by Ceilometer - for opt in storage.OPTS: - options.pop(opt.name, None) - self._engine_facade = db_session.EngineFacade(url, **options) - - def upgrade(self): - # NOTE(gordc): to minimise memory, only import migration when needed - from oslo_db.sqlalchemy import migration - path = os.path.join(os.path.abspath(os.path.dirname(__file__)), - '..', '..', 'storage', 'sqlalchemy', - 'migrate_repo') - engine = self._engine_facade.get_engine() - - from migrate import exceptions as migrate_exc - from migrate.versioning import api - from migrate.versioning import repository - - repo = repository.Repository(path) - try: - api.db_version(engine, repo) - except migrate_exc.DatabaseNotControlledError: - models.Base.metadata.create_all(engine) - api.version_control(engine, repo, repo.latest) - else: - migration.db_sync(engine, path) - - def clear(self): - engine = self._engine_facade.get_engine() - for table in reversed(models.Base.metadata.sorted_tables): - engine.execute(table.delete()) - engine.dispose() - - def _get_or_create_event_type(self, event_type, session=None): - """Check if an event type with the supplied name is already exists. - - If not, we create it and return the record. This may result in a flush. - """ - try: - if session is None: - session = self._engine_facade.get_session() - with session.begin(subtransactions=True): - et = session.query(models.EventType).filter( - models.EventType.desc == event_type).first() - if not et: - et = models.EventType(event_type) - session.add(et) - except dbexc.DBDuplicateEntry: - et = self._get_or_create_event_type(event_type, session) - - return et - - def record_events(self, event_models): - """Write the events to SQL database via sqlalchemy. - - :param event_models: a list of model.Event objects. - """ - session = self._engine_facade.get_session() - error = None - for event_model in event_models: - event = None - try: - with session.begin(): - event_type = self._get_or_create_event_type( - event_model.event_type, session=session) - event = models.Event(event_model.message_id, event_type, - event_model.generated, - event_model.raw) - session.add(event) - session.flush() - - if event_model.traits: - trait_map = {} - for trait in event_model.traits: - if trait_map.get(trait.dtype) is None: - trait_map[trait.dtype] = [] - trait_map[trait.dtype].append( - {'event_id': event.id, - 'key': trait.name, - 'value': trait.value}) - for dtype in trait_map.keys(): - model = TRAIT_ID_TO_MODEL[dtype] - session.execute(model.__table__.insert(), - trait_map[dtype]) - except dbexc.DBDuplicateEntry as e: - LOG.info(_LI("Duplicate event detected, skipping it: %s"), e) - except KeyError as e: - LOG.exception(_LE('Failed to record event: %s'), e) - except Exception as e: - LOG.exception(_LE('Failed to record event: %s'), e) - error = e - if error: - raise error - - def get_events(self, event_filter, limit=None): - """Return an iterable of model.Event objects. - - :param event_filter: EventFilter instance - """ - if limit == 0: - return - session = self._engine_facade.get_session() - with session.begin(): - # Build up the join conditions - event_join_conditions = [models.EventType.id == - models.Event.event_type_id] - - if event_filter.event_type: - event_join_conditions.append(models.EventType.desc == - event_filter.event_type) - - # Build up the where conditions - event_filter_conditions = [] - if event_filter.message_id: - event_filter_conditions.append( - models.Event.message_id == event_filter.message_id) - if event_filter.start_timestamp: - event_filter_conditions.append( - models.Event.generated >= event_filter.start_timestamp) - if event_filter.end_timestamp: - event_filter_conditions.append( - models.Event.generated <= event_filter.end_timestamp) - - trait_subq = None - # Build trait filter - if event_filter.traits_filter: - filters = list(event_filter.traits_filter) - trait_filter = filters.pop() - key = trait_filter.pop('key') - op = trait_filter.pop('op', 'eq') - trait_type, value = list(trait_filter.items())[0] - trait_subq = _build_trait_query(session, trait_type, - key, value, op) - for trait_filter in filters: - key = trait_filter.pop('key') - op = trait_filter.pop('op', 'eq') - trait_type, value = list(trait_filter.items())[0] - q = _build_trait_query(session, trait_type, - key, value, op) - trait_subq = trait_subq.filter( - trait_subq.subquery().c.ev_id == q.subquery().c.ev_id) - trait_subq = trait_subq.subquery() - - query = (session.query(models.Event.id) - .join(models.EventType, - sa.and_(*event_join_conditions))) - if trait_subq is not None: - query = query.join(trait_subq, - trait_subq.c.ev_id == models.Event.id) - if event_filter.admin_proj: - no_proj_q = session.query(models.TraitText.event_id).filter( - models.TraitText.key == 'project_id') - admin_q = (session.query(models.TraitText.event_id).filter( - ~sa.exists().where(models.TraitText.event_id == - no_proj_q.subquery().c.event_id)).union( - session.query(models.TraitText.event_id).filter(sa.and_( - models.TraitText.key == 'project_id', - models.TraitText.value == event_filter.admin_proj, - models.Event.id == models.TraitText.event_id)))) - query = query.filter(sa.exists().where( - models.Event.id == - admin_q.subquery().c.trait_text_event_id)) - if event_filter_conditions: - query = query.filter(sa.and_(*event_filter_conditions)) - - query = query.order_by(models.Event.generated).limit(limit) - event_list = {} - # get a list of all events that match filters - for (id_, generated, message_id, - desc, raw) in query.add_columns( - models.Event.generated, models.Event.message_id, - models.EventType.desc, models.Event.raw).all(): - event_list[id_] = api_models.Event(message_id, desc, - generated, [], raw) - # Query all traits related to events. - # NOTE (gordc): cast is done because pgsql defaults to TEXT when - # handling unknown values such as null. - trait_q = ( - session.query( - models.TraitDatetime.event_id, - models.TraitDatetime.key, models.TraitDatetime.value, - sa.cast(sa.null(), sa.Integer), - sa.cast(sa.null(), sa.Float(53)), - sa.cast(sa.null(), sa.String(255))) - .filter(sa.exists().where( - models.TraitDatetime.event_id == query.subquery().c.id)) - ).union_all( - session.query( - models.TraitInt.event_id, - models.TraitInt.key, sa.null(), - models.TraitInt.value, sa.null(), sa.null()) - .filter(sa.exists().where( - models.TraitInt.event_id == query.subquery().c.id)), - session.query( - models.TraitFloat.event_id, - models.TraitFloat.key, sa.null(), sa.null(), - models.TraitFloat.value, sa.null()) - .filter(sa.exists().where( - models.TraitFloat.event_id == query.subquery().c.id)), - session.query( - models.TraitText.event_id, - models.TraitText.key, sa.null(), sa.null(), sa.null(), - models.TraitText.value) - .filter(sa.exists().where( - models.TraitText.event_id == query.subquery().c.id))) - - for id_, key, t_date, t_int, t_float, t_text in ( - trait_q.order_by(models.TraitDatetime.key)).all(): - if t_int is not None: - dtype = api_models.Trait.INT_TYPE - val = t_int - elif t_float is not None: - dtype = api_models.Trait.FLOAT_TYPE - val = t_float - elif t_date is not None: - dtype = api_models.Trait.DATETIME_TYPE - val = t_date - else: - dtype = api_models.Trait.TEXT_TYPE - val = t_text - - try: - trait_model = api_models.Trait(key, dtype, val) - event_list[id_].append_trait(trait_model) - except KeyError: - # NOTE(gordc): this is expected as we do not set REPEATABLE - # READ (bug 1506717). if query is run while recording new - # event data, trait query may return more data than event - # query. they can be safely discarded. - pass - - return event_list.values() - - def get_event_types(self): - """Return all event types as an iterable of strings.""" - - session = self._engine_facade.get_session() - with session.begin(): - query = (session.query(models.EventType.desc). - order_by(models.EventType.desc)) - for name in query.all(): - # The query returns a tuple with one element. - yield name[0] - - def get_trait_types(self, event_type): - """Return a dictionary containing the name and data type of the trait. - - Only trait types for the provided event_type are returned. - :param event_type: the type of the Event - """ - session = self._engine_facade.get_session() - - with session.begin(): - for trait_model in [models.TraitText, models.TraitInt, - models.TraitFloat, models.TraitDatetime]: - query = (session.query(trait_model.key) - .join(models.Event, - models.Event.id == trait_model.event_id) - .join(models.EventType, - sa.and_(models.EventType.id == - models.Event.event_type_id, - models.EventType.desc == event_type)) - .distinct()) - - dtype = TRAIT_MODEL_TO_ID.get(trait_model) - for row in query.all(): - yield {'name': row[0], 'data_type': dtype} - - def get_traits(self, event_type, trait_type=None): - """Return all trait instances associated with an event_type. - - If trait_type is specified, only return instances of that trait type. - :param event_type: the type of the Event to filter by - :param trait_type: the name of the Trait to filter by - """ - - session = self._engine_facade.get_session() - with session.begin(): - for trait_model in [models.TraitText, models.TraitInt, - models.TraitFloat, models.TraitDatetime]: - query = (session.query(trait_model.key, trait_model.value) - .join(models.Event, - models.Event.id == trait_model.event_id) - .join(models.EventType, - sa.and_(models.EventType.id == - models.Event.event_type_id, - models.EventType.desc == event_type)) - .order_by(trait_model.key)) - if trait_type: - query = query.filter(trait_model.key == trait_type) - - dtype = TRAIT_MODEL_TO_ID.get(trait_model) - for k, v in query.all(): - yield api_models.Trait(name=k, - dtype=dtype, - value=v) - - def clear_expired_event_data(self, ttl): - """Clear expired data from the backend storage system. - - Clearing occurs according to the time-to-live. - - :param ttl: Number of seconds to keep records for. - """ - session = self._engine_facade.get_session() - with session.begin(): - end = timeutils.utcnow() - datetime.timedelta(seconds=ttl) - event_q = (session.query(models.Event.id) - .filter(models.Event.generated < end)) - - event_subq = event_q.subquery() - for trait_model in [models.TraitText, models.TraitInt, - models.TraitFloat, models.TraitDatetime]: - (session.query(trait_model) - .filter(trait_model.event_id.in_(event_subq)) - .delete(synchronize_session="fetch")) - event_rows = event_q.delete() - - # remove EventType and TraitType with no corresponding - # matching events and traits - (session.query(models.EventType) - .filter(~models.EventType.events.any()) - .delete(synchronize_session="fetch")) - LOG.info(_LI("%d events are removed from database"), event_rows) diff --git a/ceilometer/event/storage/pymongo_base.py b/ceilometer/event/storage/pymongo_base.py deleted file mode 100644 index be53316de4..0000000000 --- a/ceilometer/event/storage/pymongo_base.py +++ /dev/null @@ -1,147 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Common functions for MongoDB backend -""" -from oslo_log import log -import pymongo - -from ceilometer.event.storage import base -from ceilometer.event.storage import models -from ceilometer.i18n import _LE, _LI -from ceilometer.storage.mongo import utils as pymongo_utils -from ceilometer import utils - -LOG = log.getLogger(__name__) - - -COMMON_AVAILABLE_CAPABILITIES = { - 'events': {'query': {'simple': True}}, -} - - -AVAILABLE_STORAGE_CAPABILITIES = { - 'storage': {'production_ready': True}, -} - - -class Connection(base.Connection): - """Base event Connection class for MongoDB driver.""" - CAPABILITIES = utils.update_nested(base.Connection.CAPABILITIES, - COMMON_AVAILABLE_CAPABILITIES) - - STORAGE_CAPABILITIES = utils.update_nested( - base.Connection.STORAGE_CAPABILITIES, - AVAILABLE_STORAGE_CAPABILITIES, - ) - - def record_events(self, event_models): - """Write the events to database. - - :param event_models: a list of models.Event objects. - """ - error = None - for event_model in event_models: - traits = [] - if event_model.traits: - for trait in event_model.traits: - traits.append({'trait_name': trait.name, - 'trait_type': trait.dtype, - 'trait_value': trait.value}) - try: - self.db.event.insert_one( - {'_id': event_model.message_id, - 'event_type': event_model.event_type, - 'timestamp': event_model.generated, - 'traits': traits, 'raw': event_model.raw}) - except pymongo.errors.DuplicateKeyError as ex: - LOG.info(_LI("Duplicate event detected, skipping it: %s") % ex) - except Exception as ex: - LOG.exception(_LE("Failed to record event: %s") % ex) - error = ex - if error: - raise error - - def get_events(self, event_filter, limit=None): - """Return an iter of models.Event objects. - - :param event_filter: storage.EventFilter object, consists of filters - for events that are stored in database. - :param limit: Maximum number of results to return. - """ - if limit == 0: - return - q = pymongo_utils.make_events_query_from_filter(event_filter) - if limit is not None: - results = self.db.event.find(q, limit=limit) - else: - results = self.db.event.find(q) - for event in results: - traits = [] - for trait in event['traits']: - traits.append(models.Trait(name=trait['trait_name'], - dtype=int(trait['trait_type']), - value=trait['trait_value'])) - yield models.Event(message_id=event['_id'], - event_type=event['event_type'], - generated=event['timestamp'], - traits=traits, raw=event.get('raw')) - - def get_event_types(self): - """Return all event types as an iter of strings.""" - return self.db.event.distinct('event_type') - - def get_trait_types(self, event_type): - """Return a dictionary containing the name and data type of the trait. - - Only trait types for the provided event_type are returned. - - :param event_type: the type of the Event. - """ - trait_names = set() - events = self.db.event.find({'event_type': event_type}) - - for event in events: - for trait in event['traits']: - trait_name = trait['trait_name'] - if trait_name not in trait_names: - # Here we check that our method return only unique - # trait types. Method will return only one trait type. It - # is proposed that certain trait name could have only one - # trait type. - trait_names.add(trait_name) - yield {'name': trait_name, - 'data_type': trait['trait_type']} - - def get_traits(self, event_type, trait_name=None): - """Return all trait instances associated with an event_type. - - If trait_type is specified, only return instances of that trait type. - - :param event_type: the type of the Event to filter by - :param trait_name: the name of the Trait to filter by - """ - if not trait_name: - events = self.db.event.find({'event_type': event_type}) - else: - # We choose events that simultaneously have event_type and certain - # trait_name, and retrieve events contains only mentioned traits. - events = self.db.event.find({'$and': [{'event_type': event_type}, - {'traits.trait_name': trait_name}]}, - {'traits': {'$elemMatch': - {'trait_name': trait_name}} - }) - for event in events: - for trait in event['traits']: - yield models.Trait(name=trait['trait_name'], - dtype=trait['trait_type'], - value=trait['trait_value']) diff --git a/ceilometer/storage/__init__.py b/ceilometer/storage/__init__.py index 41a5fea9de..ee52ec5359 100644 --- a/ceilometer/storage/__init__.py +++ b/ceilometer/storage/__init__.py @@ -34,18 +34,10 @@ OPTS = [ "in the database for (<= 0 means forever).", deprecated_opts=[cfg.DeprecatedOpt('time_to_live', 'database')]), - cfg.IntOpt('event_time_to_live', - default=-1, - help=("Number of seconds that events are kept " - "in the database for (<= 0 means forever).")), cfg.StrOpt('metering_connection', secret=True, help='The connection string used to connect to the metering ' 'database. (if unset, connection is used)'), - cfg.StrOpt('event_connection', - secret=True, - help='The connection string used to connect to the event ' - 'database. (if unset, connection is used)'), cfg.BoolOpt('sql_expire_samples_only', default=False, help="Indicates if expirer expires only samples. If set true," @@ -67,7 +59,7 @@ class StorageBadAggregate(Exception): code = 400 -def get_connection_from_config(conf, purpose='metering'): +def get_connection_from_config(conf): retries = conf.database.max_retries @tenacity.retry( @@ -76,20 +68,20 @@ def get_connection_from_config(conf, purpose='metering'): else tenacity.stop_never), reraise=True) def _inner(): - namespace = 'ceilometer.%s.storage' % purpose - url = (getattr(conf.database, '%s_connection' % purpose) or + url = (getattr(conf.database, 'metering_connection') or conf.database.connection) - return get_connection(conf, url, namespace) + return get_connection(conf, url) return _inner() -def get_connection(conf, url, namespace): +def get_connection(conf, url): """Return an open connection to the database.""" connection_scheme = urlparse.urlparse(url).scheme # SqlAlchemy connections specify may specify a 'dialect' or # 'dialect+driver'. Handle the case where driver is specified. engine_name = connection_scheme.split('+')[0] + namespace = 'ceilometer.metering.storage' # NOTE: translation not applied bug #1446983 LOG.debug('looking for %(name)r driver in %(namespace)r', {'name': engine_name, 'namespace': namespace}) diff --git a/ceilometer/storage/hbase/migration.py b/ceilometer/storage/hbase/migration.py index 9cc3df9304..86a5253fcb 100644 --- a/ceilometer/storage/hbase/migration.py +++ b/ceilometer/storage/hbase/migration.py @@ -13,8 +13,6 @@ """HBase storage backend migrations """ -import re - from ceilometer.storage.hbase import utils as hbase_utils @@ -64,35 +62,8 @@ def migrate_meter_table(conn, table): meter_table.delete(row) -def migrate_event_table(conn, table): - """Migrate table 'event' in HBase. - - Change row format from ""%d_%s" % timestamp, event_id, - to new separator format "%s:%s" % timestamp, event_id - Also change trait columns from %s+%s % trait.name, trait.dtype - to %s:%s % trait.name, trait.dtype - """ - event_table = conn.table(table) - event_filter = "RowFilter(=, 'regexstring:\\d*_\\w*')" - gen = event_table.scan(filter=event_filter) - trait_pattern = re.compile("f:[\w\-_]*\+\w") - column_prefix = "f:" - for row, data in gen: - row_parts = row.split("_", 1) - update_data = {} - for column, value in data.items(): - if trait_pattern.match(column): - trait_parts = column[2:].rsplit('+', 1) - column = hbase_utils.prepare_key(*trait_parts) - update_data[column_prefix + column] = value - new_row = hbase_utils.prepare_key(*row_parts) - event_table.put(new_row, update_data) - event_table.delete(row) - - TABLE_MIGRATION_FUNCS = {'resource': migrate_resource_table, - 'meter': migrate_meter_table, - 'event': migrate_event_table} + 'meter': migrate_meter_table} def migrate_tables(conn, tables): diff --git a/ceilometer/storage/hbase/utils.py b/ceilometer/storage/hbase/utils.py index 1c5519ca41..dc050f4b05 100644 --- a/ceilometer/storage/hbase/utils.py +++ b/ceilometer/storage/hbase/utils.py @@ -25,8 +25,6 @@ from ceilometer import utils LOG = log.getLogger(__name__) -EVENT_TRAIT_TYPES = {'none': 0, 'string': 1, 'integer': 2, 'float': 3, - 'datetime': 4} OP_SIGN = {'eq': '=', 'lt': '<', 'le': '<=', 'ne': '!=', 'gt': '>', 'ge': '>='} # We need this additional dictionary because we have reverted timestamp in # row-keys for stored metrics @@ -58,31 +56,6 @@ def timestamp(dt, reverse=True): return 0x7fffffffffffffff - ts if reverse else ts -def make_events_query_from_filter(event_filter): - """Return start and stop row for filtering and a query. - - Query is based on the selected parameter. - :param event_filter: storage.EventFilter object. - """ - start = "%s" % (timestamp(event_filter.start_timestamp, reverse=False) - if event_filter.start_timestamp else "") - stop = "%s" % (timestamp(event_filter.end_timestamp, reverse=False) - if event_filter.end_timestamp else "") - kwargs = {'event_type': event_filter.event_type, - 'event_id': event_filter.message_id} - res_q = make_query(**kwargs) - - if event_filter.traits_filter: - for trait_filter in event_filter.traits_filter: - q_trait = make_query(trait_query=True, **trait_filter) - if q_trait: - if res_q: - res_q += " AND " + q_trait - else: - res_q = q_trait - return res_q, start, stop - - def make_timestamp_query(func, start=None, start_op=None, end=None, end_op=None, bounds_only=False, **kwargs): """Return a filter start and stop row for filtering and a query. @@ -127,31 +100,16 @@ def get_start_end_rts(start, end): return rts_start, rts_end -def make_query(metaquery=None, trait_query=None, **kwargs): +def make_query(metaquery=None, **kwargs): """Return a filter query string based on the selected parameters. :param metaquery: optional metaquery dict - :param trait_query: optional boolean, for trait_query from kwargs :param kwargs: key-value pairs to filter on. Key should be a real column name in db """ q = [] res_q = None - # Query for traits differs from others. It is constructed with - # SingleColumnValueFilter with the possibility to choose comparison - # operator - if trait_query: - trait_name = kwargs.pop('key') - op = kwargs.pop('op', 'eq') - for k, v in kwargs.items(): - if v is not None: - res_q = ("SingleColumnValueFilter " - "('f', '%s', %s, 'binary:%s', true, true)" % - (prepare_key(trait_name, EVENT_TRAIT_TYPES[k]), - OP_SIGN[op], dump(v))) - return res_q - # Note: we use extended constructor for SingleColumnValueFilter here. # It is explicitly specified that entry should not be returned if CF is not # found in table. @@ -161,10 +119,6 @@ def make_query(metaquery=None, trait_query=None, **kwargs): q.append("SingleColumnValueFilter " "('f', 's_%s', =, 'binary:%s', true, true)" % (value, dump('1'))) - elif key == 'trait_type': - q.append("ColumnPrefixFilter('%s')" % value) - elif key == 'event_id': - q.append("RowFilter ( = , 'regexstring:\d*:%s')" % value) else: q.append("SingleColumnValueFilter " "('f', '%s', =, 'binary:%s', true, true)" % diff --git a/ceilometer/storage/mongo/utils.py b/ceilometer/storage/mongo/utils.py index c123245a7a..d84a0a6a24 100644 --- a/ceilometer/storage/mongo/utils.py +++ b/ceilometer/storage/mongo/utils.py @@ -32,10 +32,6 @@ ERROR_INDEX_WITH_DIFFERENT_SPEC_ALREADY_EXISTS = 86 LOG = log.getLogger(__name__) -EVENT_TRAIT_TYPES = {'none': 0, 'string': 1, 'integer': 2, 'float': 3, - 'datetime': 4} -OP_SIGN = {'lt': '$lt', 'le': '$lte', 'ne': '$ne', 'gt': '$gt', 'ge': '$gte'} - MINIMUM_COMPATIBLE_MONGODB_VERSION = [2, 4] COMPLETE_AGGREGATE_COMPATIBLE_VERSION = [2, 6] @@ -73,54 +69,6 @@ def make_timestamp_range(start, end, return ts_range -def make_events_query_from_filter(event_filter): - """Return start and stop row for filtering and a query. - - Query is based on the selected parameter. - - :param event_filter: storage.EventFilter object. - """ - query = {} - q_list = [] - ts_range = make_timestamp_range(event_filter.start_timestamp, - event_filter.end_timestamp) - if ts_range: - q_list.append({'timestamp': ts_range}) - if event_filter.event_type: - q_list.append({'event_type': event_filter.event_type}) - if event_filter.message_id: - q_list.append({'_id': event_filter.message_id}) - - if event_filter.traits_filter: - for trait_filter in event_filter.traits_filter: - op = trait_filter.pop('op', 'eq') - dict_query = {} - for k, v in six.iteritems(trait_filter): - if v is not None: - # All parameters in EventFilter['traits'] are optional, so - # we need to check if they are in the query or no. - if k == 'key': - dict_query.setdefault('trait_name', v) - elif k in ['string', 'integer', 'datetime', 'float']: - dict_query.setdefault('trait_type', - EVENT_TRAIT_TYPES[k]) - dict_query.setdefault('trait_value', - v if op == 'eq' - else {OP_SIGN[op]: v}) - dict_query = {'$elemMatch': dict_query} - q_list.append({'traits': dict_query}) - if event_filter.admin_proj: - q_list.append({'$or': [ - {'traits': {'$not': {'$elemMatch': {'trait_name': 'project_id'}}}}, - {'traits': { - '$elemMatch': {'trait_name': 'project_id', - 'trait_value': event_filter.admin_proj}}}]}) - if q_list: - query = {'$and': q_list} - - return query - - def make_query_from_filter(sample_filter, require_meter=True): """Return a query dictionary based on the settings in the filter. diff --git a/ceilometer/storage/sqlalchemy/models.py b/ceilometer/storage/sqlalchemy/models.py index 6d8de90643..386d736189 100644 --- a/ceilometer/storage/sqlalchemy/models.py +++ b/ceilometer/storage/sqlalchemy/models.py @@ -248,98 +248,3 @@ class FullSample(object): project_id = Resource.project_id resource_metadata = Resource.resource_metadata internal_id = Resource.internal_id - - -class EventType(Base): - """Types of event records.""" - __tablename__ = 'event_type' - - id = Column(Integer, primary_key=True) - desc = Column(String(255), unique=True) - - def __init__(self, event_type): - self.desc = event_type - - def __repr__(self): - return "" % self.desc - - -class Event(Base): - __tablename__ = 'event' - __table_args__ = ( - Index('ix_event_message_id', 'message_id'), - Index('ix_event_type_id', 'event_type_id'), - Index('ix_event_generated', 'generated'), - _COMMON_TABLE_ARGS, - ) - id = Column(Integer, primary_key=True) - message_id = Column(String(50), unique=True) - generated = Column(PreciseTimestamp()) - raw = deferred(Column(JSONEncodedDict())) - - event_type_id = Column(Integer, ForeignKey('event_type.id')) - event_type = relationship("EventType", backref='events') - - def __init__(self, message_id, event_type, generated, raw): - self.message_id = message_id - self.event_type = event_type - self.generated = generated - self.raw = raw - - def __repr__(self): - return "" % (self.id, - self.message_id, - self.event_type, - self.generated) - - -class TraitText(Base): - """Event text traits.""" - - __tablename__ = 'trait_text' - __table_args__ = ( - Index('ix_trait_text_event_id_key', 'event_id', 'key'), - _COMMON_TABLE_ARGS, - ) - event_id = Column(Integer, ForeignKey('event.id'), primary_key=True) - key = Column(String(255), primary_key=True) - value = Column(String(255)) - - -class TraitInt(Base): - """Event integer traits.""" - - __tablename__ = 'trait_int' - __table_args__ = ( - Index('ix_trait_int_event_id_key', 'event_id', 'key'), - _COMMON_TABLE_ARGS, - ) - event_id = Column(Integer, ForeignKey('event.id'), primary_key=True) - key = Column(String(255), primary_key=True) - value = Column(Integer) - - -class TraitFloat(Base): - """Event float traits.""" - - __tablename__ = 'trait_float' - __table_args__ = ( - Index('ix_trait_float_event_id_key', 'event_id', 'key'), - _COMMON_TABLE_ARGS, - ) - event_id = Column(Integer, ForeignKey('event.id'), primary_key=True) - key = Column(String(255), primary_key=True) - value = Column(Float(53)) - - -class TraitDatetime(Base): - """Event datetime traits.""" - - __tablename__ = 'trait_datetime' - __table_args__ = ( - Index('ix_trait_datetime_event_id_key', 'event_id', 'key'), - _COMMON_TABLE_ARGS, - ) - event_id = Column(Integer, ForeignKey('event.id'), primary_key=True) - key = Column(String(255), primary_key=True) - value = Column(PreciseTimestamp()) diff --git a/ceilometer/tests/db.py b/ceilometer/tests/db.py index 65348b8bb8..462eff1ab6 100644 --- a/ceilometer/tests/db.py +++ b/ceilometer/tests/db.py @@ -52,10 +52,7 @@ class MongoDbManager(DBManager): action='ignore', message='.*you must provide a username and password.*') try: - self.connection = storage.get_connection( - self._conf, self.url, 'ceilometer.metering.storage') - self.event_connection = storage.get_connection( - self._conf, self.url, 'ceilometer.event.storage') + self.connection = storage.get_connection(self._conf, self.url) except storage.StorageBadVersion as e: raise testcase.TestSkipped(six.text_type(e)) @@ -82,10 +79,7 @@ class SQLManager(DBManager): def setUp(self): super(SQLManager, self).setUp() - self.connection = storage.get_connection( - self._conf, self.url, 'ceilometer.metering.storage') - self.event_connection = storage.get_connection( - self._conf, self.url, 'ceilometer.event.storage') + self.connection = storage.get_connection(self._conf, self.url) class PgSQLManager(SQLManager): @@ -102,26 +96,10 @@ class MySQLManager(SQLManager): conn.execute('CREATE DATABASE %s;' % db_name) -class ElasticSearchManager(DBManager): - def setUp(self): - super(ElasticSearchManager, self).setUp() - self.connection = storage.get_connection( - self._conf, 'sqlite://', 'ceilometer.metering.storage') - self.event_connection = storage.get_connection( - self._conf, self.url, 'ceilometer.event.storage') - # prefix each test with unique index name - self.event_connection.index_name = 'events_%s' % uuid.uuid4().hex - # force index on write so data is queryable right away - self.event_connection._refresh_on_write = True - - class HBaseManager(DBManager): def setUp(self): super(HBaseManager, self).setUp() - self.connection = storage.get_connection( - self._conf, self.url, 'ceilometer.metering.storage') - self.event_connection = storage.get_connection( - self._conn, self.url, 'ceilometer.event.storage') + self.connection = storage.get_connection(self._conf, self.url) # Unique prefix for each test to keep data is distinguished because # all test data is stored in one table data_prefix = str(uuid.uuid4().hex) @@ -155,10 +133,7 @@ class SQLiteManager(DBManager): def setUp(self): super(SQLiteManager, self).setUp() self.url = self._url - self.connection = storage.get_connection( - self._conf, self._url, 'ceilometer.metering.storage') - self.event_connection = storage.get_connection( - self._conf, self._url, 'ceilometer.event.storage') + self.connection = storage.get_connection(self._conf, self._url) @six.add_metaclass(test_base.SkipNotImplementedMeta) @@ -169,7 +144,6 @@ class TestBase(test_base.BaseTestCase): 'mysql': MySQLManager, 'postgresql': PgSQLManager, 'sqlite': SQLiteManager, - 'es': ElasticSearchManager, } if mocks is not None: DRIVER_MANAGERS['hbase'] = HBaseManager @@ -205,9 +179,6 @@ class TestBase(test_base.BaseTestCase): self.conn = self.db_manager.connection self.conn.upgrade() - self.event_conn = self.db_manager.event_connection - self.event_conn.upgrade() - self.useFixture(mockpatch.Patch('ceilometer.storage.get_connection', side_effect=self._get_connection)) @@ -221,15 +192,11 @@ class TestBase(test_base.BaseTestCase): ) def tearDown(self): - self.event_conn.clear() - self.event_conn = None self.conn.clear() self.conn = None super(TestBase, self).tearDown() - def _get_connection(self, conf, url, namespace): - if namespace == "ceilometer.event.storage": - return self.event_conn + def _get_connection(self, conf, url): return self.conn diff --git a/ceilometer/tests/functional/api/v2/test_acl_scenarios.py b/ceilometer/tests/functional/api/v2/test_acl_scenarios.py index 49a4463f2b..d8001f6c3a 100644 --- a/ceilometer/tests/functional/api/v2/test_acl_scenarios.py +++ b/ceilometer/tests/functional/api/v2/test_acl_scenarios.py @@ -15,16 +15,12 @@ """Test ACL.""" import datetime -import os import uuid from keystonemiddleware import fixture as ksm_fixture -from oslo_utils import fileutils -import six import webtest from ceilometer.api import app -from ceilometer.event.storage import models as ev_model from ceilometer.publisher import utils from ceilometer import sample from ceilometer.tests.functional.api import v2 @@ -182,103 +178,3 @@ class TestAPIACL(v2.FunctionalTest): 'value': 'project-naughty', }]) self.assertEqual(401, data.status_int) - - -class TestAPIEventACL(TestAPIACL): - - PATH = '/events' - - def test_non_admin_get_event_types(self): - data = self.get_json('/event_types', expect_errors=True, - headers={"X-Roles": "Member", - "X-Auth-Token": VALID_TOKEN2, - "X-Project-Id": "project-good"}) - self.assertEqual(401, data.status_int) - - -class TestBaseApiEventRBAC(v2.FunctionalTest): - - PATH = '/events' - - def setUp(self): - super(TestBaseApiEventRBAC, self).setUp() - traits = [ev_model.Trait('project_id', 1, 'project-good'), - ev_model.Trait('user_id', 1, 'user-good')] - self.message_id = str(uuid.uuid4()) - ev = ev_model.Event(self.message_id, 'event_type', - datetime.datetime.now(), traits, {}) - self.event_conn.record_events([ev]) - - def test_get_events_without_project(self): - headers_no_proj = {"X-Roles": "admin", "X-User-Id": "user-good"} - resp = self.get_json(self.PATH, expect_errors=True, - headers=headers_no_proj, status=403) - self.assertEqual(403, resp.status_int) - - def test_get_events_without_user(self): - headers_no_user = {"X-Roles": "admin", "X-Project-Id": "project-good"} - resp = self.get_json(self.PATH, expect_errors=True, - headers=headers_no_user, status=403) - self.assertEqual(403, resp.status_int) - - def test_get_events_without_scope(self): - headers_no_user_proj = {"X-Roles": "admin"} - resp = self.get_json(self.PATH, - expect_errors=True, - headers=headers_no_user_proj, - status=403) - self.assertEqual(403, resp.status_int) - - def test_get_events(self): - headers = {"X-Roles": "Member", "X-User-Id": "user-good", - "X-Project-Id": "project-good"} - self.get_json(self.PATH, headers=headers, status=200) - - def test_get_event(self): - headers = {"X-Roles": "Member", "X-User-Id": "user-good", - "X-Project-Id": "project-good"} - self.get_json(self.PATH + "/" + self.message_id, headers=headers, - status=200) - - -class TestApiEventAdminRBAC(TestBaseApiEventRBAC): - - def _make_app(self, enable_acl=False): - content = ('{"context_is_admin": "role:admin",' - '"telemetry:events:index": "rule:context_is_admin",' - '"telemetry:events:show": "rule:context_is_admin"}') - if six.PY3: - content = content.encode('utf-8') - self.tempfile = fileutils.write_to_tempfile(content=content, - prefix='policy', - suffix='.json') - - self.CONF.set_override("policy_file", self.tempfile, - group='oslo_policy') - return super(TestApiEventAdminRBAC, self)._make_app() - - def tearDown(self): - os.remove(self.tempfile) - super(TestApiEventAdminRBAC, self).tearDown() - - def test_get_events(self): - headers_rbac = {"X-Roles": "admin", "X-User-Id": "user-good", - "X-Project-Id": "project-good"} - self.get_json(self.PATH, headers=headers_rbac, status=200) - - def test_get_events_bad(self): - headers_rbac = {"X-Roles": "Member", "X-User-Id": "user-good", - "X-Project-Id": "project-good"} - self.get_json(self.PATH, headers=headers_rbac, status=403) - - def test_get_event(self): - headers = {"X-Roles": "admin", "X-User-Id": "user-good", - "X-Project-Id": "project-good"} - self.get_json(self.PATH + "/" + self.message_id, headers=headers, - status=200) - - def test_get_event_bad(self): - headers = {"X-Roles": "Member", "X-User-Id": "user-good", - "X-Project-Id": "project-good"} - self.get_json(self.PATH + "/" + self.message_id, headers=headers, - status=403) diff --git a/ceilometer/tests/functional/api/v2/test_event_scenarios.py b/ceilometer/tests/functional/api/v2/test_event_scenarios.py deleted file mode 100644 index b5a2df8169..0000000000 --- a/ceilometer/tests/functional/api/v2/test_event_scenarios.py +++ /dev/null @@ -1,703 +0,0 @@ -# -# Copyright 2013 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Test event, event_type and trait retrieval.""" - -import datetime -import uuid - -import webtest.app - -from ceilometer.event.storage import models -from ceilometer.tests import db as tests_db -from ceilometer.tests.functional.api import v2 - -USER_ID = uuid.uuid4().hex -PROJ_ID = uuid.uuid4().hex -HEADERS = {"X-Roles": "admin", - "X-User-Id": USER_ID, - "X-Project-Id": PROJ_ID} - - -class EventTestBase(v2.FunctionalTest): - - def setUp(self): - super(EventTestBase, self).setUp() - self._generate_models() - - def _generate_models(self): - event_models = [] - base = 0 - self.s_time = datetime.datetime(2013, 12, 31, 5, 0) - self.trait_time = datetime.datetime(2013, 12, 31, 5, 0) - for event_type in ['Foo', 'Bar', 'Zoo']: - trait_models = [models.Trait(name, type, value) - for name, type, value in [ - ('trait_A', models.Trait.TEXT_TYPE, - "my_%s_text" % event_type), - ('trait_B', models.Trait.INT_TYPE, - base + 1), - ('trait_C', models.Trait.FLOAT_TYPE, - float(base) + 0.123456), - ('trait_D', models.Trait.DATETIME_TYPE, - self.trait_time)]] - - # Message ID for test will be 'base'. So, message ID for the first - # event will be '0', the second '100', and so on. - # trait_time in first event will be equal to self.trait_time - # (datetime.datetime(2013, 12, 31, 5, 0)), next will add 1 day, so - # second will be (datetime.datetime(2014, 01, 01, 5, 0)) and so on. - event_models.append( - models.Event(message_id=str(base), - event_type=event_type, - generated=self.trait_time, - traits=trait_models, - raw={'status': {'nested': 'started'}})) - base += 100 - self.trait_time += datetime.timedelta(days=1) - self.event_conn.record_events(event_models) - - -class TestEventTypeAPI(EventTestBase): - - PATH = '/event_types' - - def test_event_types(self): - data = self.get_json(self.PATH, headers=HEADERS) - for event_type in ['Foo', 'Bar', 'Zoo']: - self.assertIn(event_type, data) - - -class TestTraitAPI(EventTestBase): - - PATH = '/event_types/%s/traits' - - def test_get_traits_for_event(self): - path = self.PATH % "Foo" - data = self.get_json(path, headers=HEADERS) - - self.assertEqual(4, len(data)) - - def test_get_event_invalid_path(self): - data = self.get_json('/event_types/trait_A/', headers=HEADERS, - expect_errors=True) - self.assertEqual(404, data.status_int) - - def test_get_traits_for_non_existent_event(self): - path = self.PATH % "NO_SUCH_EVENT_TYPE" - data = self.get_json(path, headers=HEADERS) - - self.assertEqual([], data) - - def test_get_trait_data_for_event(self): - path = (self.PATH % "Foo") + "/trait_A" - data = self.get_json(path, headers=HEADERS) - self.assertEqual(1, len(data)) - self.assertEqual("trait_A", data[0]['name']) - - path = (self.PATH % "Foo") + "/trait_B" - data = self.get_json(path, headers=HEADERS) - self.assertEqual(1, len(data)) - self.assertEqual("trait_B", data[0]['name']) - self.assertEqual("1", data[0]['value']) - - path = (self.PATH % "Foo") + "/trait_D" - data = self.get_json(path, headers=HEADERS) - self.assertEqual(1, len(data)) - self.assertEqual("trait_D", data[0]['name']) - self.assertEqual((self.trait_time - datetime.timedelta(days=3)). - isoformat(), data[0]['value']) - - def test_get_trait_data_for_non_existent_event(self): - path = (self.PATH % "NO_SUCH_EVENT") + "/trait_A" - data = self.get_json(path, headers=HEADERS) - - self.assertEqual([], data) - - def test_get_trait_data_for_non_existent_trait(self): - path = (self.PATH % "Foo") + "/no_such_trait" - data = self.get_json(path, headers=HEADERS) - - self.assertEqual([], data) - - -class TestEventAPI(EventTestBase): - - PATH = '/events' - - def test_get_events(self): - data = self.get_json(self.PATH, headers=HEADERS) - self.assertEqual(3, len(data)) - # We expect to get native UTC generated time back - trait_time = self.s_time - for event in data: - expected_generated = trait_time.isoformat() - self.assertIn(event['event_type'], ['Foo', 'Bar', 'Zoo']) - self.assertEqual(4, len(event['traits'])) - self.assertEqual({'status': {'nested': 'started'}}, event['raw']), - self.assertEqual(expected_generated, event['generated']) - for trait_name in ['trait_A', 'trait_B', - 'trait_C', 'trait_D']: - self.assertIn(trait_name, map(lambda x: x['name'], - event['traits'])) - trait_time += datetime.timedelta(days=1) - - def test_get_event_by_message_id(self): - event = self.get_json(self.PATH + "/100", headers=HEADERS) - expected_traits = [{'name': 'trait_A', - 'type': 'string', - 'value': 'my_Bar_text'}, - {'name': 'trait_B', - 'type': 'integer', - 'value': '101'}, - {'name': 'trait_C', - 'type': 'float', - 'value': '100.123456'}, - {'name': 'trait_D', - 'type': 'datetime', - 'value': '2014-01-01T05:00:00'}] - self.assertEqual('100', event['message_id']) - self.assertEqual('Bar', event['event_type']) - self.assertEqual('2014-01-01T05:00:00', event['generated']) - self.assertEqual(expected_traits, event['traits']) - - def test_get_event_by_message_id_no_such_id(self): - data = self.get_json(self.PATH + "/DNE", headers=HEADERS, - expect_errors=True) - self.assertEqual(404, data.status_int) - - def test_get_events_filter_event_type(self): - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'event_type', - 'value': 'Foo'}]) - self.assertEqual(1, len(data)) - - def test_get_events_filter_trait_no_type(self): - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_A', - 'value': 'my_Foo_text'}]) - self.assertEqual(1, len(data)) - self.assertEqual('Foo', data[0]['event_type']) - - def test_get_events_filter_trait_empty_type(self): - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_A', - 'value': 'my_Foo_text', - 'type': ''}]) - self.assertEqual(1, len(data)) - self.assertEqual('Foo', data[0]['event_type']) - - def test_get_events_filter_trait_invalid_type(self): - resp = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_A', - 'value': 'my_Foo_text', - 'type': 'whats-up'}], - expect_errors=True) - self.assertEqual(400, resp.status_code) - self.assertEqual("The data type whats-up is not supported. The " - "supported data type list is: [\'integer\', " - "\'float\', \'string\', \'datetime\']", - resp.json['error_message']['faultstring']) - - def test_get_events_filter_operator_invalid_type(self): - resp = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_A', - 'value': 'my_Foo_text', - 'op': 'whats-up'}], - expect_errors=True) - self.assertEqual(400, resp.status_code) - self.assertEqual("Operator whats-up is not supported. The " - "supported operators are: (\'lt\', \'le\', " - "\'eq\', \'ne\', \'ge\', \'gt\')", - resp.json['error_message']['faultstring']) - - def test_get_events_filter_start_timestamp(self): - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'start_timestamp', - 'op': 'ge', - 'value': '2014-01-01T00:00:00'}]) - self.assertEqual(2, len(data)) - sorted_types = sorted([d['event_type'] for d in data]) - event_types = ['Foo', 'Bar', 'Zoo'] - self.assertEqual(sorted_types, sorted(event_types[1:])) - - def test_get_events_filter_start_timestamp_invalid_op(self): - resp = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'start_timestamp', - 'op': 'gt', - 'value': '2014-01-01T00:00:00'}], - expect_errors=True) - self.assertEqual(400, resp.status_code) - self.assertEqual(u'Operator gt is not supported. Only' - ' `ge\' operator is available for field' - ' start_timestamp', - resp.json['error_message']['faultstring']) - - def test_get_events_filter_end_timestamp(self): - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'end_timestamp', - 'op': 'le', - 'value': '2014-01-03T00:00:00'}]) - self.assertEqual(3, len(data)) - event_types = ['Foo', 'Bar', 'Zoo'] - sorted_types = sorted([d['event_type'] for d in data]) - self.assertEqual(sorted_types, sorted(event_types[:3])) - - def test_get_events_filter_end_timestamp_invalid_op(self): - resp = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'end_timestamp', - 'op': 'gt', - 'value': '2014-01-03T00:00:00'}], - expect_errors=True) - self.assertEqual(400, resp.status_code) - self.assertEqual(u'Operator gt is not supported. Only' - ' `le\' operator is available for field' - ' end_timestamp', - resp.json['error_message']['faultstring']) - - def test_get_events_filter_start_end_timestamp(self): - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'start_timestamp', - 'op': 'ge', - 'value': '2014-01-02T00:00:00'}, - {'field': 'end_timestamp', - 'op': 'le', - 'value': '2014-01-03T10:00:00'}]) - self.assertEqual(1, len(data)) - sorted_types = sorted([d['event_type'] for d in data]) - event_types = ['Foo', 'Bar', 'Zoo'] - self.assertEqual(sorted_types, sorted(event_types[2:3])) - - def test_get_events_filter_text_trait(self): - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_A', - 'value': 'my_Foo_text', - 'type': 'string'}]) - self.assertEqual(1, len(data)) - self.assertEqual('Foo', data[0]['event_type']) - - def test_get_events_filter_int_trait(self): - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_B', - 'value': '101', - 'type': 'integer'}]) - self.assertEqual(1, len(data)) - self.assertEqual('Bar', data[0]['event_type']) - - traits = [x for x in data[0]['traits'] if x['name'] == 'trait_B'] - self.assertEqual(1, len(traits)) - self.assertEqual('integer', traits[0]['type']) - self.assertEqual('101', traits[0]['value']) - - def test_get_events_filter_float_trait(self): - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_C', - 'value': '200.123456', - 'type': 'float'}]) - self.assertEqual(1, len(data)) - self.assertEqual('Zoo', data[0]['event_type']) - - traits = [x for x in data[0]['traits'] if x['name'] == 'trait_C'] - self.assertEqual(1, len(traits)) - self.assertEqual('float', traits[0]['type']) - self.assertEqual('200.123456', traits[0]['value']) - - def test_get_events_filter_datetime_trait(self): - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_D', - 'value': '2014-01-01T05:00:00', - 'type': 'datetime'}]) - self.assertEqual(1, len(data)) - traits = [x for x in data[0]['traits'] if x['name'] == 'trait_D'] - self.assertEqual(1, len(traits)) - self.assertEqual('datetime', traits[0]['type']) - self.assertEqual('2014-01-01T05:00:00', traits[0]['value']) - - def test_get_events_multiple_filters(self): - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_B', - 'value': '1', - 'type': 'integer'}, - {'field': 'trait_A', - 'value': 'my_Foo_text', - 'type': 'string'}]) - self.assertEqual(1, len(data)) - self.assertEqual('Foo', data[0]['event_type']) - - def test_get_events_multiple_filters_no_matches(self): - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_B', - 'value': '101', - 'type': 'integer'}, - {'field': 'trait_A', - 'value': 'my_Foo_text', - 'type': 'string'}]) - - self.assertEqual(0, len(data)) - - def test_get_events_multiple_filters_same_field_different_values(self): - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_A', - 'value': 'my_Foo_text', - 'type': 'string'}, - {'field': 'trait_A', - 'value': 'my_Bar_text', - 'type': 'string'}]) - self.assertEqual(0, len(data)) - - def test_get_events_not_filters(self): - data = self.get_json(self.PATH, headers=HEADERS, - q=[]) - self.assertEqual(3, len(data)) - - def test_get_events_filter_op_string(self): - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_A', - 'value': 'my_Foo_text', - 'type': 'string', - 'op': 'eq'}]) - self.assertEqual(1, len(data)) - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_A', - 'value': 'my_Bar_text', - 'type': 'string', - 'op': 'lt'}]) - self.assertEqual(0, len(data)) - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_A', - 'value': 'my_Zoo_text', - 'type': 'string', - 'op': 'le'}]) - self.assertEqual(3, len(data)) - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_A', - 'value': 'my_Foo_text', - 'type': 'string', - 'op': 'ne'}]) - self.assertEqual(2, len(data)) - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_A', - 'value': 'my_Bar_text', - 'type': 'string', - 'op': 'gt'}]) - self.assertEqual(2, len(data)) - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_A', - 'value': 'my_Zoo_text', - 'type': 'string', - 'op': 'ge'}]) - self.assertEqual(1, len(data)) - - def test_get_events_filter_op_integer(self): - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_B', - 'value': '101', - 'type': 'integer', - 'op': 'eq'}]) - self.assertEqual(1, len(data)) - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_B', - 'value': '201', - 'type': 'integer', - 'op': 'lt'}]) - self.assertEqual(2, len(data)) - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_B', - 'value': '1', - 'type': 'integer', - 'op': 'le'}]) - self.assertEqual(1, len(data)) - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_B', - 'value': '101', - 'type': 'integer', - 'op': 'ne'}]) - self.assertEqual(2, len(data)) - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_B', - 'value': '201', - 'type': 'integer', - 'op': 'gt'}]) - self.assertEqual(0, len(data)) - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_B', - 'value': '1', - 'type': 'integer', - 'op': 'ge'}]) - self.assertEqual(3, len(data)) - - def test_get_events_filter_op_float(self): - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_C', - 'value': '100.123456', - 'type': 'float', - 'op': 'eq'}]) - self.assertEqual(1, len(data)) - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_C', - 'value': '200.123456', - 'type': 'float', - 'op': 'lt'}]) - self.assertEqual(2, len(data)) - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_C', - 'value': '0.123456', - 'type': 'float', - 'op': 'le'}]) - self.assertEqual(1, len(data)) - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_C', - 'value': '100.123456', - 'type': 'float', - 'op': 'ne'}]) - self.assertEqual(2, len(data)) - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_C', - 'value': '200.123456', - 'type': 'float', - 'op': 'gt'}]) - self.assertEqual(0, len(data)) - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_C', - 'value': '0.123456', - 'type': 'float', - 'op': 'ge'}]) - self.assertEqual(3, len(data)) - - def test_get_events_filter_op_datatime(self): - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_D', - 'value': '2014-01-01T05:00:00', - 'type': 'datetime', - 'op': 'eq'}]) - self.assertEqual(1, len(data)) - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_D', - 'value': '2014-01-02T05:00:00', - 'type': 'datetime', - 'op': 'lt'}]) - self.assertEqual(2, len(data)) - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_D', - 'value': '2013-12-31T05:00:00', - 'type': 'datetime', - 'op': 'le'}]) - self.assertEqual(1, len(data)) - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_D', - 'value': '2014-01-01T05:00:00', - 'type': 'datetime', - 'op': 'ne'}]) - self.assertEqual(2, len(data)) - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_D', - 'value': '2014-01-02T05:00:00', - 'type': 'datetime', - 'op': 'gt'}]) - self.assertEqual(0, len(data)) - data = self.get_json(self.PATH, headers=HEADERS, - q=[{'field': 'trait_D', - 'value': '2013-12-31T05:00:00', - 'type': 'datetime', - 'op': 'ge'}]) - self.assertEqual(3, len(data)) - - def test_get_events_filter_wrong_op(self): - self.assertRaises(webtest.app.AppError, - self.get_json, self.PATH, headers=HEADERS, - q=[{'field': 'trait_B', - 'value': '1', - 'type': 'integer', - 'op': 'el'}]) - - -class AclRestrictedEventTestBase(v2.FunctionalTest): - - def setUp(self): - super(AclRestrictedEventTestBase, self).setUp() - self.admin_user_id = uuid.uuid4().hex - self.admin_proj_id = uuid.uuid4().hex - self.user_id = uuid.uuid4().hex - self.proj_id = uuid.uuid4().hex - self._generate_models() - - def _generate_models(self): - event_models = [] - self.s_time = datetime.datetime(2013, 12, 31, 5, 0) - event_models.append( - models.Event(message_id='1', - event_type='empty_ev', - generated=self.s_time, - traits=[models.Trait('random', - models.Trait.TEXT_TYPE, - 'blah')], - raw={})) - event_models.append( - models.Event(message_id='2', - event_type='admin_ev', - generated=self.s_time, - traits=[models.Trait('project_id', - models.Trait.TEXT_TYPE, - self.admin_proj_id), - models.Trait('user_id', - models.Trait.TEXT_TYPE, - self.admin_user_id)], - raw={})) - event_models.append( - models.Event(message_id='3', - event_type='user_ev', - generated=self.s_time, - traits=[models.Trait('project_id', - models.Trait.TEXT_TYPE, - self.proj_id), - models.Trait('user_id', - models.Trait.TEXT_TYPE, - self.user_id)], - raw={})) - self.event_conn.record_events(event_models) - - def test_non_admin_access(self): - a_headers = {"X-Roles": "member", - "X-User-Id": self.user_id, - "X-Project-Id": self.proj_id} - data = self.get_json('/events', headers=a_headers) - self.assertEqual(1, len(data)) - self.assertEqual('user_ev', data[0]['event_type']) - - def test_non_admin_access_single(self): - a_headers = {"X-Roles": "member", - "X-User-Id": self.user_id, - "X-Project-Id": self.proj_id} - data = self.get_json('/events/3', headers=a_headers) - self.assertEqual('user_ev', data['event_type']) - - def test_non_admin_access_incorrect_user(self): - a_headers = {"X-Roles": "member", - "X-User-Id": 'blah', - "X-Project-Id": self.proj_id} - data = self.get_json('/events', headers=a_headers) - self.assertEqual(0, len(data)) - - def test_non_admin_access_incorrect_proj(self): - a_headers = {"X-Roles": "member", - "X-User-Id": self.user_id, - "X-Project-Id": 'blah'} - data = self.get_json('/events', headers=a_headers) - self.assertEqual(0, len(data)) - - def test_non_admin_access_single_invalid(self): - a_headers = {"X-Roles": "member", - "X-User-Id": self.user_id, - "X-Project-Id": self.proj_id} - data = self.get_json('/events/1', headers=a_headers, - expect_errors=True) - self.assertEqual(404, data.status_int) - - @tests_db.run_with('sqlite', 'mysql', 'pgsql', 'mongodb', 'es') - def test_admin_access(self): - a_headers = {"X-Roles": "admin", - "X-User-Id": self.admin_user_id, - "X-Project-Id": self.admin_proj_id} - data = self.get_json('/events', headers=a_headers) - self.assertEqual(2, len(data)) - self.assertEqual(set(['empty_ev', 'admin_ev']), - set(ev['event_type'] for ev in data)) - - @tests_db.run_with('sqlite', 'mysql', 'pgsql', 'mongodb', 'es') - def test_admin_access_trait_filter(self): - a_headers = {"X-Roles": "admin", - "X-User-Id": self.admin_user_id, - "X-Project-Id": self.admin_proj_id} - data = self.get_json('/events', headers=a_headers, - q=[{'field': 'random', - 'value': 'blah', - 'type': 'string', - 'op': 'eq'}]) - self.assertEqual(1, len(data)) - self.assertEqual('empty_ev', data[0]['event_type']) - - @tests_db.run_with('sqlite', 'mysql', 'pgsql', 'mongodb', 'es') - def test_admin_access_single(self): - a_headers = {"X-Roles": "admin", - "X-User-Id": self.admin_user_id, - "X-Project-Id": self.admin_proj_id} - data = self.get_json('/events/1', headers=a_headers) - self.assertEqual('empty_ev', data['event_type']) - data = self.get_json('/events/2', headers=a_headers) - self.assertEqual('admin_ev', data['event_type']) - - @tests_db.run_with('sqlite', 'mysql', 'pgsql', 'mongodb', 'es') - def test_admin_access_trait_filter_no_access(self): - a_headers = {"X-Roles": "admin", - "X-User-Id": self.admin_user_id, - "X-Project-Id": self.admin_proj_id} - data = self.get_json('/events', headers=a_headers, - q=[{'field': 'user_id', - 'value': self.user_id, - 'type': 'string', - 'op': 'eq'}]) - self.assertEqual(0, len(data)) - - -class EventRestrictionTestBase(v2.FunctionalTest): - - def setUp(self): - super(EventRestrictionTestBase, self).setUp() - self.CONF.set_override('default_api_return_limit', 10, group='api') - self._generate_models() - - def _generate_models(self): - event_models = [] - base = 0 - self.s_time = datetime.datetime(2013, 12, 31, 5, 0) - self.trait_time = datetime.datetime(2013, 12, 31, 5, 0) - for i in range(20): - trait_models = [models.Trait(name, type, value) - for name, type, value in [ - ('trait_A', models.Trait.TEXT_TYPE, - "my_text"), - ('trait_B', models.Trait.INT_TYPE, - base + 1), - ('trait_C', models.Trait.FLOAT_TYPE, - float(base) + 0.123456), - ('trait_D', models.Trait.DATETIME_TYPE, - self.trait_time)]] - - event_models.append( - models.Event(message_id=str(uuid.uuid4()), - event_type='foo.bar', - generated=self.trait_time, - traits=trait_models, - raw={'status': {'nested': 'started'}})) - self.trait_time += datetime.timedelta(seconds=1) - self.event_conn.record_events(event_models) - - -class TestEventRestriction(EventRestrictionTestBase): - - def test_get_limit(self): - data = self.get_json('/events?limit=1', headers=HEADERS) - self.assertEqual(1, len(data)) - - def test_get_limit_negative(self): - self.assertRaises(webtest.app.AppError, - self.get_json, '/events?limit=-2', headers=HEADERS) - - def test_get_limit_bigger(self): - data = self.get_json('/events?limit=100', headers=HEADERS) - self.assertEqual(20, len(data)) - - def test_get_default_limit(self): - data = self.get_json('/events', headers=HEADERS) - self.assertEqual(10, len(data)) diff --git a/ceilometer/tests/functional/gabbi/fixtures.py b/ceilometer/tests/functional/gabbi/fixtures.py index 5a0823b620..eeb2f12587 100644 --- a/ceilometer/tests/functional/gabbi/fixtures.py +++ b/ceilometer/tests/functional/gabbi/fixtures.py @@ -29,7 +29,6 @@ import six from six.moves.urllib import parse as urlparse from ceilometer.api import app -from ceilometer.event.storage import models from ceilometer.publisher import utils from ceilometer import sample from ceilometer import service @@ -98,7 +97,6 @@ class ConfigFixture(fixture.GabbiFixture): database_name = '%s-%s' % (db_url, str(uuid.uuid4())) conf.set_override('connection', database_name, group='database') conf.set_override('metering_connection', '', group='database') - conf.set_override('event_connection', '', group='database') conf.set_override('gnocchi_is_enabled', False, group='api') conf.set_override('aodh_is_enabled', False, group='api') @@ -153,33 +151,6 @@ class SampleDataFixture(fixture.GabbiFixture): print('meter', self.conn.db.meter.remove({'source': self.source})) -class EventDataFixture(fixture.GabbiFixture): - """Instantiate some sample event data for use in testing.""" - - def start_fixture(self): - """Create some events.""" - global LOAD_APP_KWARGS - conf = LOAD_APP_KWARGS['conf'] - self.conn = storage.get_connection_from_config(conf, 'event') - events = [] - name_list = ['chocolate.chip', 'peanut.butter', 'sugar'] - for ix, name in enumerate(name_list): - timestamp = datetime.datetime.utcnow() - message_id = 'fea1b15a-1d47-4175-85a5-a4bb2c72924{}'.format(ix) - traits = [models.Trait('type', 1, name), - models.Trait('ate', 2, ix)] - event = models.Event(message_id, - 'cookies_{}'.format(name), - timestamp, - traits, {'nested': {'inside': 'value'}}) - events.append(event) - self.conn.record_events(events) - - def stop_fixture(self): - """Destroy the events.""" - self.conn.db.event.remove({'event_type': '/^cookies_/'}) - - class CORSConfigFixture(fixture.GabbiFixture): """Inject mock configuration for the CORS middleware.""" diff --git a/ceilometer/tests/functional/gabbi/gabbits/api-events-no-data.yaml b/ceilometer/tests/functional/gabbi/gabbits/api-events-no-data.yaml deleted file mode 100644 index 742a38f05a..0000000000 --- a/ceilometer/tests/functional/gabbi/gabbits/api-events-no-data.yaml +++ /dev/null @@ -1,210 +0,0 @@ -# These test run against the Events API with no data preloaded into the -# datastore. This allows us to verify that requests are still processed -# normally even if data is missing for that endpoint. -fixtures: -- ConfigFixture - -tests: - -# this attempts to get all the events and expects an empty list back -- name: get all events - url: /v2/events - request_headers: - X-Roles: admin - X-User-Id: user1 - X-Project-Id: project1 - response_headers: - content-type: application/json - response_strings: - - "[]" - -# this attempts to get all the events with no role/user/project -# info in header and expects a 403 -- name: get events with bad headers - url: /v2/events - status: 403 - -# this attempts to get all the events with no user/project -# info in header and expects a 403 -- name: get events with admin only header - url: /v2/events - request_headers: - X-Roles: admin - status: 403 - -# this attempts to get all the events with no project -# info in header and expects a 403 -- name: get events with no project header - url: /v2/events - request_headers: - X-Roles: admin - X-User-Id: user1 - status: 403 - -# this attempts to get all the events with no user -# info in header and expects a 403 -- name: get events with no user header - url: /v2/events - request_headers: - X-Roles: admin - X-Project-Id: project1 - status: 403 - -# this attempts to get all the events with invalid parameters and expects a 400 -- name: get events with bad params - url: /v2/events?bad_Stuff_here - request_headers: - X-Roles: admin - X-User-Id: user1 - X-Project-Id: project1 - status: 400 - -# this attempts to query the events with the correct parameterized query syntax -# and expects an empty list -- name: get events that match query - url: /v2/events?q.field=event_type&q.op=eq&q.type=string&q.value=cookies_chocolate.chip - request_headers: - X-Roles: admin - X-User-Id: user1 - X-Project-Id: project1 - response_headers: - content-type: application/json - response_strings: - - "[]" - -# this attempts to query the events with the correct data query syntax and -# expects an empty list -- name: get events that match query via request data - url: /v2/events - request_headers: - content-type: application/json - X-Roles: admin - X-User-Id: user1 - X-Project-Id: project1 - data: - q: - - field: event_type - op: eq - type: string - value: cookies_chocolate.chip - response_headers: - content-type: application/json - response_strings: - - "[]" - -# this attempts to query the events with the correct parameterized query syntax -# but a bad field name and expects an empty list -- name: get events that match bad query - url: /v2/events?q.field=bad_field&q.op=eq&q.type=string&q.value=cookies_chocolate.chip - request_headers: - X-Roles: admin - X-User-Id: user1 - X-Project-Id: project1 - response_headers: - content-type: application/json - response_strings: - - "[]" - -# this attempts to query the events with the correct data query syntax and -# a bad field name and expects an empty list -- name: get events that match bad query via request data - url: /v2/events - request_headers: - content-type: application/json - X-Roles: admin - X-User-Id: user1 - X-Project-Id: project1 - data: - q: - - field: bad_field - op: eq - type: string - value: cookies_chocolate.chip - response_headers: - content-type: application/json - response_strings: - - "[]" - -# this attempts to query the events with the wrong data query syntax missing the -# q object but supplying the field list and a bad field name and expects a 400 -- name: get events that match bad query via request data malformed list - url: /v2/events - request_headers: - content-type: application/json - X-Roles: admin - X-User-Id: user1 - X-Project-Id: project1 - data: - - field: bad_field - op: eq - type: string - value: cookies_chocolate.chip - status: 400 - -# this attempts to query the events with the wrong data query syntax missing the -# q object but supplying the field list along with a bad content-type. Should -# return a 400 -- name: get events that match bad query via request data wrong type - url: /v2/events - request_headers: - content-type: text/plain - X-Roles: admin - X-User-Id: user1 - X-Project-Id: project1 - data: - "field: bad_field op: eq type: string value: cookies_chocolate.chip xfail: True" - status: 415 - -# Get a single event by message_id no data is present so should return a 404 -- name: get a single event - url: /v2/events/fea1b15a-1d47-4175-85a5-a4bb2c729240 - request_headers: - X-Roles: admin - X-User-Id: user1 - X-Project-Id: project1 - status: 404 - -# Get all the event types should return an empty list -- name: get all event types - url: /v2/event_types - request_headers: - X-Roles: admin - X-User-Id: user1 - X-Project-Id: project1 - response_headers: - content-type: application/json - response_strings: - - "[]" - -# Get a single event type by name, this API is unused and should return a 404 -- name: get event types for good event_type unused api - url: /v2/event_types/cookies_chocolate.chip - request_headers: - X-Roles: admin - X-User-Id: user1 - X-Project-Id: project1 - status: 404 - -# Get all traits for an event type should return an empty list -- name: get all traits for event type - url: /v2/event_types/cookies_chocolate.chip/traits - request_headers: - X-Roles: admin - X-User-Id: user1 - X-Project-Id: project1 - response_headers: - content-type: application/json - response_strings: - - "[]" - -# Get all traits named ate for an event type should return an empty list -- name: get all traits named ate for event type - url: /v2/event_types/cookies_chocolate.chip/traits/ate - request_headers: - X-Roles: admin - X-User-Id: user1 - X-Project-Id: project1 - response_headers: - content-type: application/json - response_strings: - - "[]" diff --git a/ceilometer/tests/functional/gabbi/gabbits/api-events-with-data.yaml b/ceilometer/tests/functional/gabbi/gabbits/api-events-with-data.yaml deleted file mode 100644 index 8ae32a0622..0000000000 --- a/ceilometer/tests/functional/gabbi/gabbits/api-events-with-data.yaml +++ /dev/null @@ -1,242 +0,0 @@ -# These test run against the Events API with data preloaded into the datastore. -fixtures: -- ConfigFixture -- EventDataFixture - -tests: - -# this attempts to get all the events and checks to make sure they are valid -- name: get all events - url: /v2/events - request_headers: - X-Roles: admin - X-User-Id: user1 - X-Project-Id: project1 - response_headers: - content-type: application/json - response_json_paths: - $.[0].event_type: cookies_chocolate.chip - $.[0].traits.[0].value: chocolate.chip - $.[0].traits.[1].value: '0' - $.[0].raw.nested.inside: value - $.[1].event_type: cookies_peanut.butter - $.[1].traits.[0].name: type - $.[1].traits.[1].name: ate - $.[1].raw.nested.inside: value - $.[2].event_type: cookies_sugar - $.[2].traits.[0].type: string - $.[2].traits.[1].type: integer - $.[2].raw.nested.inside: value - -# this attempts to get all the events with invalid parameters and expects a 400 -- name: get events with bad params - url: /v2/events?bad_Stuff_here - request_headers: - X-Roles: admin - X-User-Id: user1 - X-Project-Id: project1 - status: 400 - -# this attempts to query the events with the correct parameterized query syntax -# and expects a matching event -- name: get events that match query - url: /v2/events?q.field=event_type&q.op=eq&q.type=string&q.value=cookies_chocolate.chip - request_headers: - X-Roles: admin - X-User-Id: user1 - X-Project-Id: project1 - response_headers: - content-type: application/json - response_json_paths: - $.[0].event_type: cookies_chocolate.chip - $.[0].traits.[0].value: chocolate.chip - -# this attempts to query the events with the correct data query syntax and -# expects a matching event -- name: get events that match query via data - url: /v2/events - request_headers: - content-type: application/json - X-Roles: admin - X-User-Id: user1 - X-Project-Id: project1 - data: - q: - - field: event_type - op: eq - type: string - value: cookies_chocolate.chip - response_headers: - content-type: application/json - response_json_paths: - $.[0].event_type: cookies_chocolate.chip - $.[0].traits.[0].value: chocolate.chip - -# this attempts to query the events with the correct parameterized query syntax -# but a bad field name and expects an empty list -- name: get events that match bad query - url: /v2/events?q.field=bad_field&q.op=eq&q.type=string&q.value=cookies_chocolate.chip - request_headers: - X-Roles: admin - X-User-Id: user1 - X-Project-Id: project1 - response_headers: - content-type: application/json - response_strings: - - "[]" - -# this attempts to query the events with the correct data query syntax and -# a bad field name and expects an empty list -- name: get events that match bad query via data - url: /v2/events - request_headers: - content-type: application/json - X-Roles: admin - X-User-Id: user1 - X-Project-Id: project1 - data: - q: - - field: bad_field - op: eq - type: string - value: cookies_chocolate.chip - response_headers: - content-type: application/json - response_strings: - - "[]" - -# this attempts to query the events with the wrong data query syntax missing the -# q object but supplying the field list and a bad field name and expects a 400 -- name: get events that match bad query via data list - url: /v2/events - request_headers: - content-type: application/json - X-Roles: admin - X-User-Id: user1 - X-Project-Id: project1 - data: - - field: bad_field - op: eq - type: string - value: cookies_chocolate.chip - status: 400 - -# Get a single event by message_id should return an event -- name: get a single event - url: /v2/events/fea1b15a-1d47-4175-85a5-a4bb2c729240 - request_headers: - X-Roles: admin - X-User-Id: user1 - X-Project-Id: project1 - response_headers: - content-type: application/json - response_json_paths: - $.event_type: cookies_chocolate.chip - $.traits.[0].value: chocolate.chip - $.traits.[1].value: '0' - -# Get a single event by message_id no data is present so should return a 404 -- name: get a single event that does not exist - url: /v2/events/bad-id - request_headers: - X-Roles: admin - X-User-Id: user1 - X-Project-Id: project1 - status: 404 - -# Get all the event types should return a list of event types -- name: get all event types - url: /v2/event_types - request_headers: - X-Roles: admin - X-User-Id: user1 - X-Project-Id: project1 - response_headers: - content-type: application/json - response_strings: - - cookies_chocolate.chip - - cookies_peanut.butter - - cookies_sugar - -# Get a single event type by valid name, this API is unused and should return a 404 -- name: get event types for good event_type unused api - url: /v2/event_types/cookies_chocolate.chip - request_headers: - X-Roles: admin - X-User-Id: user1 - X-Project-Id: project1 - status: 404 - -# Get a single event type by invalid name, this API is unused and should return a 404 -- name: get event types for bad event_type unused api - url: /v2/event_types/bad_event_type - request_headers: - X-Roles: admin - X-User-Id: user1 - X-Project-Id: project1 - status: 404 - -# Get all traits for a valid event type should return an list of traits -- name: get all traits for event type - url: /v2/event_types/cookies_chocolate.chip/traits - request_headers: - X-Roles: admin - X-User-Id: user1 - X-Project-Id: project1 - response_headers: - content-type: application/json - response_json_paths: - $.[0].type: string - $.[1].name: ate - -# Get all traits for an invalid event type should return an empty list -- name: get all traits names for event type bad event type - url: /v2/event_types/bad_event_type/traits - request_headers: - X-Roles: admin - X-User-Id: user1 - X-Project-Id: project1 - response_headers: - content-type: application/json - response_strings: - - "[]" - -# Get all traits of type ate for a valid event type should return an list of -# traits -- name: get all traits of type ate for event type - url: /v2/event_types/cookies_chocolate.chip/traits/ate - request_headers: - X-Roles: admin - X-User-Id: user1 - X-Project-Id: project1 - response_headers: - content-type: application/json - response_json_paths: - $.[0].name: ate - $.[0].value: '0' - -# Get all traits of type ate for an invalid event type should return an empty -# list -- name: get all traits of type for event type bad event type - url: /v2/event_types/bad_event_type/traits/ate - request_headers: - X-Roles: admin - X-User-Id: user1 - X-Project-Id: project1 - response_headers: - content-type: application/json - response_strings: - - "[]" - -# Get all traits of type bad_trait_name for a valid event type should return an -# empty list -- name: get all traits of type instances for event type bad trait name - url: /v2/event_types/cookies_chocolate.chip/traits/bad_trait_name - request_headers: - X-Roles: admin - X-User-Id: user1 - X-Project-Id: project1 - response_headers: - content-type: application/json - response_strings: - - "[]" diff --git a/ceilometer/tests/functional/gabbi/gabbits/capabilities.yaml b/ceilometer/tests/functional/gabbi/gabbits/capabilities.yaml index 5b9c91644a..512d296e8e 100644 --- a/ceilometer/tests/functional/gabbi/gabbits/capabilities.yaml +++ b/ceilometer/tests/functional/gabbi/gabbits/capabilities.yaml @@ -10,5 +10,4 @@ tests: desc: retrieve capabilities for the mongo store url: /v2/capabilities response_json_paths: - $.event_storage.['storage:production_ready']: true $.storage.['storage:production_ready']: true diff --git a/ceilometer/tests/functional/publisher/test_direct.py b/ceilometer/tests/functional/publisher/test_direct.py index 3200185cd9..ce1cbffbcd 100644 --- a/ceilometer/tests/functional/publisher/test_direct.py +++ b/ceilometer/tests/functional/publisher/test_direct.py @@ -20,7 +20,6 @@ import uuid from oslo_utils import netutils -from ceilometer.event.storage import models as event from ceilometer.publisher import direct from ceilometer import sample from ceilometer.tests import db as tests_db @@ -79,21 +78,3 @@ class TestDirectPublisher(tests_db.TestBase): self.assertEqual(3, len(meters), 'There should be 3 samples') self.assertEqual(['alpha', 'beta', 'gamma'], names) - - -class TestEventDirectPublisher(tests_db.TestBase): - test_data = [event.Event(message_id=str(uuid.uuid4()), - event_type='event_%d' % i, - generated=datetime.datetime.utcnow(), - traits=[], raw={}) - for i in range(0, 5)] - - def test_direct_publisher(self): - parsed_url = netutils.urlsplit('direct://dispatcher=database') - publisher = direct.DirectPublisher(self.CONF, parsed_url) - publisher.publish_events(self.test_data) - - e_types = list(self.event_conn.get_event_types()) - self.assertEqual(5, len(e_types)) - self.assertEqual(['event_%d' % i for i in range(0, 5)], - sorted(e_types)) diff --git a/ceilometer/tests/functional/storage/test_impl_hbase.py b/ceilometer/tests/functional/storage/test_impl_hbase.py index 1805ef2d43..09a307375f 100644 --- a/ceilometer/tests/functional/storage/test_impl_hbase.py +++ b/ceilometer/tests/functional/storage/test_impl_hbase.py @@ -29,7 +29,6 @@ except ImportError: import testtools.testcase raise testtools.testcase.TestSkipped("happybase is needed") -from ceilometer.event.storage import impl_hbase as hbase_event from ceilometer.storage import impl_hbase as hbase from ceilometer.tests import base as test_base from ceilometer.tests import db as tests_db @@ -87,14 +86,6 @@ class CapabilitiesTest(test_base.BaseTestCase): actual_capabilities = hbase.Connection.get_capabilities() self.assertEqual(expected_capabilities, actual_capabilities) - def test_event_capabilities(self): - expected_capabilities = { - 'events': {'query': {'simple': True}}, - } - - actual_capabilities = hbase_event.Connection.get_capabilities() - self.assertEqual(expected_capabilities, actual_capabilities) - def test_storage_capabilities(self): expected_capabilities = { 'storage': {'production_ready': True}, diff --git a/ceilometer/tests/functional/storage/test_impl_mongodb.py b/ceilometer/tests/functional/storage/test_impl_mongodb.py index 109ad74c3e..d96257fd91 100644 --- a/ceilometer/tests/functional/storage/test_impl_mongodb.py +++ b/ceilometer/tests/functional/storage/test_impl_mongodb.py @@ -21,7 +21,6 @@ """ -from ceilometer.event.storage import impl_mongodb as impl_mongodb_event from ceilometer.storage import impl_mongodb from ceilometer.tests import base as test_base from ceilometer.tests import db as tests_db @@ -60,10 +59,6 @@ class IndexTest(tests_db.TestBase): self._test_ttl_index_absent(self.conn, 'meter', 'metering_time_to_live') - def test_event_ttl_index_absent(self): - self._test_ttl_index_absent(self.event_conn, 'event', - 'event_time_to_live') - def _test_ttl_index_present(self, conn, coll_name, ttl_opt): coll = getattr(conn.db, coll_name) self.CONF.set_override(ttl_opt, 456789, group='database') @@ -81,10 +76,6 @@ class IndexTest(tests_db.TestBase): self._test_ttl_index_present(self.conn, 'meter', 'metering_time_to_live') - def test_event_ttl_index_present(self): - self._test_ttl_index_present(self.event_conn, 'event', - 'event_time_to_live') - class CapabilitiesTest(test_base.BaseTestCase): # Check the returned capabilities list, which is specific to each DB @@ -117,13 +108,6 @@ class CapabilitiesTest(test_base.BaseTestCase): actual_capabilities = impl_mongodb.Connection.get_capabilities() self.assertEqual(expected_capabilities, actual_capabilities) - def test_event_capabilities(self): - expected_capabilities = { - 'events': {'query': {'simple': True}}, - } - actual_capabilities = impl_mongodb_event.Connection.get_capabilities() - self.assertEqual(expected_capabilities, actual_capabilities) - def test_storage_capabilities(self): expected_capabilities = { 'storage': {'production_ready': True}, diff --git a/ceilometer/tests/functional/storage/test_impl_sqlalchemy.py b/ceilometer/tests/functional/storage/test_impl_sqlalchemy.py index c02c2eac99..026345f82a 100644 --- a/ceilometer/tests/functional/storage/test_impl_sqlalchemy.py +++ b/ceilometer/tests/functional/storage/test_impl_sqlalchemy.py @@ -24,10 +24,7 @@ import warnings import mock from oslo_db import exception from oslo_utils import timeutils -from six.moves import reprlib -from ceilometer.event.storage import impl_sqlalchemy as impl_sqla_event -from ceilometer.event.storage import models from ceilometer.publisher import utils from ceilometer import sample from ceilometer.storage import impl_sqlalchemy @@ -53,71 +50,10 @@ class EngineFacadeTest(tests_db.TestBase): @mock.patch.object(warnings, 'warn') def test_no_not_supported_warning(self, mocked): impl_sqlalchemy.Connection(self.CONF, 'sqlite://') - impl_sqla_event.Connection(self.CONF, 'sqlite://') self.assertNotIn(mock.call(mock.ANY, exception.NotSupportedWarning), mocked.call_args_list) -@tests_db.run_with('sqlite', 'mysql', 'pgsql') -class EventTypeTest(tests_db.TestBase): - # EventType is a construct specific to sqlalchemy - # Not applicable to other drivers. - - def test_event_type_exists(self): - et1 = self.event_conn._get_or_create_event_type("foo") - self.assertTrue(et1.id >= 0) - et2 = self.event_conn._get_or_create_event_type("foo") - self.assertEqual(et2.id, et1.id) - self.assertEqual(et2.desc, et1.desc) - - def test_event_type_unique(self): - et1 = self.event_conn._get_or_create_event_type("foo") - self.assertTrue(et1.id >= 0) - et2 = self.event_conn._get_or_create_event_type("blah") - self.assertNotEqual(et1.id, et2.id) - self.assertNotEqual(et1.desc, et2.desc) - # Test the method __repr__ returns a string - self.assertTrue(reprlib.repr(et2)) - - -@tests_db.run_with('sqlite', 'mysql', 'pgsql') -class EventTest(tests_db.TestBase): - def _verify_data(self, trait, trait_table): - now = datetime.datetime.utcnow() - ev = models.Event('1', 'name', now, [trait], {}) - self.event_conn.record_events([ev]) - session = self.event_conn._engine_facade.get_session() - t_tables = [sql_models.TraitText, sql_models.TraitFloat, - sql_models.TraitInt, sql_models.TraitDatetime] - for table in t_tables: - if table == trait_table: - self.assertEqual(1, session.query(table).count()) - else: - self.assertEqual(0, session.query(table).count()) - - def test_string_traits(self): - model = models.Trait("Foo", models.Trait.TEXT_TYPE, "my_text") - self._verify_data(model, sql_models.TraitText) - - def test_int_traits(self): - model = models.Trait("Foo", models.Trait.INT_TYPE, 100) - self._verify_data(model, sql_models.TraitInt) - - def test_float_traits(self): - model = models.Trait("Foo", models.Trait.FLOAT_TYPE, 123.456) - self._verify_data(model, sql_models.TraitFloat) - - def test_datetime_traits(self): - now = datetime.datetime.utcnow() - model = models.Trait("Foo", models.Trait.DATETIME_TYPE, now) - self._verify_data(model, sql_models.TraitDatetime) - - def test_event_repr(self): - ev = sql_models.Event('msg_id', None, False, {}) - ev.id = 100 - self.assertTrue(reprlib.repr(ev)) - - @tests_db.run_with('sqlite', 'mysql', 'pgsql') class RelationshipTest(scenarios.DBTestBase): # Note: Do not derive from SQLAlchemyEngineTestBase, since we @@ -175,13 +111,6 @@ class CapabilitiesTest(test_base.BaseTestCase): actual_capabilities = impl_sqlalchemy.Connection.get_capabilities() self.assertEqual(expected_capabilities, actual_capabilities) - def test_event_capabilities(self): - expected_capabilities = { - 'events': {'query': {'simple': True}}, - } - actual_capabilities = impl_sqla_event.Connection.get_capabilities() - self.assertEqual(expected_capabilities, actual_capabilities) - def test_storage_capabilities(self): expected_capabilities = { 'storage': {'production_ready': True}, diff --git a/ceilometer/tests/functional/storage/test_storage_scenarios.py b/ceilometer/tests/functional/storage/test_storage_scenarios.py index 7307bcf8a2..1273cc9c3f 100644 --- a/ceilometer/tests/functional/storage/test_storage_scenarios.py +++ b/ceilometer/tests/functional/storage/test_storage_scenarios.py @@ -15,7 +15,6 @@ """Base classes for DB backend implementation test""" import datetime -import operator import mock from oslo_db import api @@ -24,8 +23,6 @@ from oslo_utils import timeutils import pymongo import ceilometer -from ceilometer.event import storage as event_storage -from ceilometer.event.storage import models as event_models from ceilometer.publisher import utils from ceilometer import sample from ceilometer import storage @@ -2633,440 +2630,6 @@ class CounterDataTypeTest(DBTestBase): self.assertEqual(1938495037.53697, results[0].counter_volume) -class EventTestBase(tests_db.TestBase): - """Separate test base class. - - We don't want to inherit all the Meter stuff. - """ - - def setUp(self): - super(EventTestBase, self).setUp() - self.prepare_data() - - def prepare_data(self): - self.event_models = [] - base = 0 - self.start = datetime.datetime(2013, 12, 31, 5, 0) - now = self.start - for event_type in ['Foo', 'Bar', 'Zoo', 'Foo', 'Bar', 'Zoo']: - trait_models = [event_models.Trait(name, dtype, value) - for name, dtype, value in [ - ('trait_A', event_models.Trait.TEXT_TYPE, - "my_%s_text" % event_type), - ('trait_B', event_models.Trait.INT_TYPE, - base + 1), - ('trait_C', event_models.Trait.FLOAT_TYPE, - float(base) + 0.123456), - ('trait_D', event_models.Trait.DATETIME_TYPE, - now)]] - self.event_models.append( - event_models.Event("id_%s_%d" % (event_type, base), - event_type, now, trait_models, - {'status': {'nested': 'started'}})) - base += 100 - now = now + datetime.timedelta(hours=1) - self.end = now - - self.event_conn.record_events(self.event_models) - - -@tests_db.run_with('sqlite', 'mysql', 'pgsql') -class EventTTLTest(EventTestBase): - - @mock.patch.object(timeutils, 'utcnow') - def test_clear_expired_event_data(self, mock_utcnow): - mock_utcnow.return_value = datetime.datetime(2013, 12, 31, 10, 0) - self.event_conn.clear_expired_event_data(3600) - - events = list(self.event_conn.get_events(event_storage.EventFilter())) - self.assertEqual(2, len(events)) - event_types = list(self.event_conn.get_event_types()) - self.assertEqual(['Bar', 'Zoo'], event_types) - for event_type in event_types: - trait_types = list(self.event_conn.get_trait_types(event_type)) - self.assertEqual(4, len(trait_types)) - traits = list(self.event_conn.get_traits(event_type)) - self.assertEqual(4, len(traits)) - - -@tests_db.run_with('sqlite', 'mysql', 'pgsql', 'mongodb') -class EventTest(EventTestBase): - def test_duplicate_message_id(self): - now = datetime.datetime.utcnow() - m = [event_models.Event("1", "Foo", now, None, {}), - event_models.Event("1", "Zoo", now, [], {})] - with mock.patch('%s.LOG' % - self.event_conn.record_events.__module__) as log: - self.event_conn.record_events(m) - self.assertEqual(1, log.info.call_count) - - def test_bad_event(self): - now = datetime.datetime.utcnow() - broken_event = event_models.Event("1", "Foo", now, None, {}) - del(broken_event.__dict__['raw']) - m = [broken_event, broken_event] - with mock.patch('%s.LOG' % - self.event_conn.record_events.__module__) as log: - self.assertRaises(AttributeError, self.event_conn.record_events, m) - # ensure that record_events does not break on first error but - # delays exception and tries to record each event. - self.assertEqual(2, log.exception.call_count) - - -class GetEventTest(EventTestBase): - - def test_generated_is_datetime(self): - event_filter = event_storage.EventFilter(self.start, self.end) - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(6, len(events)) - for i, event in enumerate(events): - self.assertIsInstance(event.generated, datetime.datetime) - self.assertEqual(event.generated, - self.event_models[i].generated) - model_traits = self.event_models[i].traits - for j, trait in enumerate(event.traits): - if trait.dtype == event_models.Trait.DATETIME_TYPE: - self.assertIsInstance(trait.value, datetime.datetime) - self.assertEqual(trait.value, model_traits[j].value) - - def test_simple_get(self): - event_filter = event_storage.EventFilter(self.start, self.end) - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(6, len(events)) - start_time = None - for i, type in enumerate(['Foo', 'Bar', 'Zoo']): - self.assertEqual(type, events[i].event_type) - self.assertEqual(4, len(events[i].traits)) - # Ensure sorted results ... - if start_time is not None: - # Python 2.6 has no assertLess :( - self.assertTrue(start_time < events[i].generated) - start_time = events[i].generated - - def test_simple_get_event_type(self): - expected_trait_values = { - 'id_Bar_100': { - 'trait_A': 'my_Bar_text', - 'trait_B': 101, - 'trait_C': 100.123456, - 'trait_D': self.start + datetime.timedelta(hours=1) - }, - 'id_Bar_400': { - 'trait_A': 'my_Bar_text', - 'trait_B': 401, - 'trait_C': 400.123456, - 'trait_D': self.start + datetime.timedelta(hours=4) - } - } - - event_filter = event_storage.EventFilter(self.start, self.end, "Bar") - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(2, len(events)) - self.assertEqual("Bar", events[0].event_type) - self.assertEqual("Bar", events[1].event_type) - self.assertEqual(4, len(events[0].traits)) - self.assertEqual(4, len(events[1].traits)) - for event in events: - trait_values = expected_trait_values.get(event.message_id, - None) - if not trait_values: - self.fail("Unexpected event ID returned:" % event.message_id) - - for trait in event.traits: - expected_val = trait_values.get(trait.name) - if not expected_val: - self.fail("Unexpected trait type: %s" % trait.dtype) - self.assertEqual(expected_val, trait.value) - - def test_get_event_trait_filter(self): - trait_filters = [{'key': 'trait_B', 'integer': 101}] - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(1, len(events)) - self.assertEqual("Bar", events[0].event_type) - self.assertEqual(4, len(events[0].traits)) - - def test_get_event_trait_filter_op_string(self): - trait_filters = [{'key': 'trait_A', 'string': 'my_Foo_text', - 'op': 'eq'}] - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(2, len(events)) - self.assertEqual("Foo", events[0].event_type) - self.assertEqual(4, len(events[0].traits)) - trait_filters[0].update({'key': 'trait_A', 'op': 'lt'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(2, len(events)) - self.assertEqual("Bar", events[0].event_type) - trait_filters[0].update({'key': 'trait_A', 'op': 'le'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(4, len(events)) - self.assertEqual("Bar", events[1].event_type) - trait_filters[0].update({'key': 'trait_A', 'op': 'ne'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(4, len(events)) - self.assertEqual("Zoo", events[3].event_type) - trait_filters[0].update({'key': 'trait_A', 'op': 'gt'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(2, len(events)) - self.assertEqual("Zoo", events[0].event_type) - trait_filters[0].update({'key': 'trait_A', 'op': 'ge'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(4, len(events)) - self.assertEqual("Foo", events[2].event_type) - - def test_get_event_trait_filter_op_integer(self): - trait_filters = [{'key': 'trait_B', 'integer': 101, 'op': 'eq'}] - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(1, len(events)) - self.assertEqual("Bar", events[0].event_type) - self.assertEqual(4, len(events[0].traits)) - trait_filters[0].update({'key': 'trait_B', 'op': 'lt'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(1, len(events)) - self.assertEqual("Foo", events[0].event_type) - trait_filters[0].update({'key': 'trait_B', 'op': 'le'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(2, len(events)) - self.assertEqual("Bar", events[1].event_type) - trait_filters[0].update({'key': 'trait_B', 'op': 'ne'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(5, len(events)) - self.assertEqual("Zoo", events[4].event_type) - trait_filters[0].update({'key': 'trait_B', 'op': 'gt'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(4, len(events)) - self.assertEqual("Zoo", events[0].event_type) - trait_filters[0].update({'key': 'trait_B', 'op': 'ge'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(5, len(events)) - self.assertEqual("Foo", events[2].event_type) - - def test_get_event_trait_filter_op_float(self): - trait_filters = [{'key': 'trait_C', 'float': 300.123456, 'op': 'eq'}] - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(1, len(events)) - self.assertEqual("Foo", events[0].event_type) - self.assertEqual(4, len(events[0].traits)) - trait_filters[0].update({'key': 'trait_C', 'op': 'lt'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(3, len(events)) - self.assertEqual("Zoo", events[2].event_type) - trait_filters[0].update({'key': 'trait_C', 'op': 'le'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(4, len(events)) - self.assertEqual("Bar", events[1].event_type) - trait_filters[0].update({'key': 'trait_C', 'op': 'ne'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(5, len(events)) - self.assertEqual("Zoo", events[2].event_type) - trait_filters[0].update({'key': 'trait_C', 'op': 'gt'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(2, len(events)) - self.assertEqual("Bar", events[0].event_type) - trait_filters[0].update({'key': 'trait_C', 'op': 'ge'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(3, len(events)) - self.assertEqual("Zoo", events[2].event_type) - - def test_get_event_trait_filter_op_datetime(self): - trait_filters = [{'key': 'trait_D', - 'datetime': self.start + datetime.timedelta(hours=2), - 'op': 'eq'}] - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(1, len(events)) - self.assertEqual("Zoo", events[0].event_type) - self.assertEqual(4, len(events[0].traits)) - trait_filters[0].update({'key': 'trait_D', 'op': 'lt'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(2, len(events)) - trait_filters[0].update({'key': 'trait_D', 'op': 'le'}) - self.assertEqual("Bar", events[1].event_type) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(3, len(events)) - self.assertEqual("Bar", events[1].event_type) - trait_filters[0].update({'key': 'trait_D', 'op': 'ne'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(5, len(events)) - self.assertEqual("Foo", events[2].event_type) - trait_filters[0].update({'key': 'trait_D', 'op': 'gt'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(3, len(events)) - self.assertEqual("Zoo", events[2].event_type) - trait_filters[0].update({'key': 'trait_D', 'op': 'ge'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(4, len(events)) - self.assertEqual("Bar", events[2].event_type) - - def test_get_event_multiple_trait_filter(self): - trait_filters = [{'key': 'trait_B', 'integer': 1}, - {'key': 'trait_A', 'string': 'my_Foo_text'}, - {'key': 'trait_C', 'float': 0.123456}] - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(1, len(events)) - self.assertEqual("Foo", events[0].event_type) - self.assertEqual(4, len(events[0].traits)) - - def test_get_event_multiple_trait_filter_expect_none(self): - trait_filters = [{'key': 'trait_B', 'integer': 1}, - {'key': 'trait_A', 'string': 'my_Zoo_text'}] - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(0, len(events)) - - def test_get_event_types(self): - event_types = [e for e in - self.event_conn.get_event_types()] - - self.assertEqual(3, len(event_types)) - self.assertIn("Bar", event_types) - self.assertIn("Foo", event_types) - self.assertIn("Zoo", event_types) - - def test_get_trait_types(self): - trait_types = [tt for tt in - self.event_conn.get_trait_types("Foo")] - self.assertEqual(4, len(trait_types)) - trait_type_names = map(lambda x: x['name'], trait_types) - self.assertIn("trait_A", trait_type_names) - self.assertIn("trait_B", trait_type_names) - self.assertIn("trait_C", trait_type_names) - self.assertIn("trait_D", trait_type_names) - - def test_get_trait_types_unknown_event(self): - trait_types = [tt for tt in - self.event_conn.get_trait_types("Moo")] - self.assertEqual(0, len(trait_types)) - - def test_get_traits(self): - traits = self.event_conn.get_traits("Bar") - # format results in a way that makes them easier to work with - trait_dict = {} - for trait in traits: - trait_dict[trait.name] = trait.dtype - - self.assertIn("trait_A", trait_dict) - self.assertEqual(event_models.Trait.TEXT_TYPE, trait_dict["trait_A"]) - self.assertIn("trait_B", trait_dict) - self.assertEqual(event_models.Trait.INT_TYPE, trait_dict["trait_B"]) - self.assertIn("trait_C", trait_dict) - self.assertEqual(event_models.Trait.FLOAT_TYPE, trait_dict["trait_C"]) - self.assertIn("trait_D", trait_dict) - self.assertEqual(event_models.Trait.DATETIME_TYPE, - trait_dict["trait_D"]) - - def test_get_all_traits(self): - traits = self.event_conn.get_traits("Foo") - traits = sorted([t for t in traits], key=operator.attrgetter('dtype')) - self.assertEqual(8, len(traits)) - trait = traits[0] - self.assertEqual("trait_A", trait.name) - self.assertEqual(event_models.Trait.TEXT_TYPE, trait.dtype) - - def test_simple_get_event_no_traits(self): - new_events = [event_models.Event("id_notraits", "NoTraits", - self.start, [], {})] - self.event_conn.record_events(new_events) - event_filter = event_storage.EventFilter( - self.start, self.end, "NoTraits") - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(1, len(events)) - self.assertEqual("id_notraits", events[0].message_id) - self.assertEqual("NoTraits", events[0].event_type) - self.assertEqual(0, len(events[0].traits)) - - def test_simple_get_no_filters(self): - event_filter = event_storage.EventFilter(None, None, None) - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(6, len(events)) - - def test_get_by_message_id(self): - new_events = [event_models.Event("id_testid", - "MessageIDTest", - self.start, - [], {})] - - self.event_conn.record_events(new_events) - event_filter = event_storage.EventFilter(message_id="id_testid") - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertEqual(1, len(events)) - event = events[0] - self.assertEqual("id_testid", event.message_id) - - def test_simple_get_raw(self): - event_filter = event_storage.EventFilter() - events = [event for event in self.event_conn.get_events(event_filter)] - self.assertTrue(events) - self.assertEqual({'status': {'nested': 'started'}}, events[0].raw) - - def test_trait_type_enforced_on_none(self): - new_events = [event_models.Event( - "id_testid", "MessageIDTest", self.start, - [event_models.Trait('text', event_models.Trait.TEXT_TYPE, ''), - event_models.Trait('int', event_models.Trait.INT_TYPE, 0), - event_models.Trait('float', event_models.Trait.FLOAT_TYPE, 0.0)], - {})] - self.event_conn.record_events(new_events) - event_filter = event_storage.EventFilter(message_id="id_testid") - events = [event for event in self.event_conn.get_events(event_filter)] - options = [(event_models.Trait.TEXT_TYPE, ''), - (event_models.Trait.INT_TYPE, 0.0), - (event_models.Trait.FLOAT_TYPE, 0.0)] - for trait in events[0].traits: - options.remove((trait.dtype, trait.value)) - - class BigIntegerTest(tests_db.TestBase): def test_metadata_bigint(self): metadata = {'bigint': 99999999999999} diff --git a/ceilometer/tests/functional/test_bin.py b/ceilometer/tests/functional/test_bin.py index e1117520e2..3949946b53 100644 --- a/ceilometer/tests/functional/test_bin.py +++ b/ceilometer/tests/functional/test_bin.py @@ -54,8 +54,6 @@ class BinTestCase(base.BaseTestCase): self.assertEqual(0, subp.poll()) self.assertIn(b"Nothing to clean, database metering " b"time to live is disabled", stdout) - self.assertIn(b"Nothing to clean, database event " - b"time to live is disabled", stdout) def _test_run_expirer_ttl_enabled(self, ttl_name, data_name): content = ("[DEFAULT]\n" @@ -83,7 +81,6 @@ class BinTestCase(base.BaseTestCase): self._test_run_expirer_ttl_enabled('metering_time_to_live', 'metering') self._test_run_expirer_ttl_enabled('time_to_live', 'metering') - self._test_run_expirer_ttl_enabled('event_time_to_live', 'event') class BinSendSampleTestCase(base.BaseTestCase): diff --git a/ceilometer/tests/unit/api/v2/test_query.py b/ceilometer/tests/unit/api/v2/test_query.py index 5ed3d12cc0..99960fb73d 100644 --- a/ceilometer/tests/unit/api/v2/test_query.py +++ b/ceilometer/tests/unit/api/v2/test_query.py @@ -24,7 +24,6 @@ from oslotest import mockpatch import wsme from ceilometer.api.controllers.v2 import base as v2_base -from ceilometer.api.controllers.v2 import events from ceilometer.api.controllers.v2 import meters from ceilometer.api.controllers.v2 import utils from ceilometer import storage @@ -37,10 +36,6 @@ class TestQuery(base.BaseTestCase): super(TestQuery, self).setUp() self.useFixture(fixtures.MonkeyPatch( 'pecan.response', mock.MagicMock())) - self.useFixture(mockpatch.Patch('ceilometer.api.controllers.v2.events' - '._build_rbac_query_filters', - return_value={'t_filter': [], - 'admin_proj': None})) def test_get_value_as_type_with_integer(self): query = v2_base.Query(field='metadata.size', @@ -158,15 +153,6 @@ class TestQuery(base.BaseTestCase): expected = value self.assertEqual(expected, query._get_value_as_type()) - def test_event_query_to_event_filter_with_bad_op(self): - # bug 1511592 - query = v2_base.Query(field='event_type', - op='ne', - value='compute.instance.create.end', - type='string') - self.assertRaises(v2_base.ClientSideError, - events._event_query_to_event_filter, [query]) - class TestValidateGroupByFields(base.BaseTestCase): diff --git a/ceilometer/tests/unit/dispatcher/test_db.py b/ceilometer/tests/unit/dispatcher/test_db.py index 78d5d6a2f5..d1d83f5b46 100644 --- a/ceilometer/tests/unit/dispatcher/test_db.py +++ b/ceilometer/tests/unit/dispatcher/test_db.py @@ -13,14 +13,12 @@ # License for the specific language governing permissions and limitations # under the License. import datetime -import uuid import mock from oslo_config import fixture as fixture_config from oslotest import base from ceilometer.dispatcher import database -from ceilometer.event.storage import models as event_models from ceilometer.publisher import utils @@ -31,18 +29,6 @@ class TestDispatcherDB(base.BaseTestCase): self.CONF = self.useFixture(fixture_config.Config()).conf self.CONF.set_override('connection', 'sqlite://', group='database') self.meter_dispatcher = database.MeterDatabaseDispatcher(self.CONF) - self.event_dispatcher = database.EventDatabaseDispatcher(self.CONF) - - def test_event_conn(self): - event = event_models.Event(uuid.uuid4(), 'test', - datetime.datetime(2012, 7, 2, 13, 53, 40), - [], {}) - event = utils.message_from_event(event, - self.CONF.publisher.telemetry_secret) - with mock.patch.object(self.event_dispatcher.conn, - 'record_events') as record_events: - self.event_dispatcher.record_events(event) - self.assertEqual(1, len(record_events.call_args_list[0][0][0])) def test_valid_message(self): msg = {'counter_name': 'test', diff --git a/ceilometer/tests/unit/dispatcher/test_dispatcher.py b/ceilometer/tests/unit/dispatcher/test_dispatcher.py index f0fbab7d06..a6af08ce7f 100644 --- a/ceilometer/tests/unit/dispatcher/test_dispatcher.py +++ b/ceilometer/tests/unit/dispatcher/test_dispatcher.py @@ -24,11 +24,6 @@ class FakeMeterDispatcher(dispatcher.MeterDispatcherBase): pass -class FakeEventDispatcher(dispatcher.EventDispatcherBase): - def record_events(self, events): - pass - - class TestDispatchManager(base.BaseTestCase): def setUp(self): super(TestDispatchManager, self).setUp() @@ -42,11 +37,7 @@ class TestDispatchManager(base.BaseTestCase): self.useFixture(mockpatch.Patch( 'ceilometer.dispatcher.database.MeterDatabaseDispatcher', new=FakeMeterDispatcher)) - self.useFixture(mockpatch.Patch( - 'ceilometer.dispatcher.database.EventDatabaseDispatcher', - new=FakeEventDispatcher)) def test_load(self): sample_mg, event_mg = dispatcher.load_dispatcher_manager(self.CONF) self.assertEqual(2, len(list(sample_mg))) - self.assertEqual(1, len(list(event_mg))) diff --git a/ceilometer/tests/unit/storage/test_get_connection.py b/ceilometer/tests/unit/storage/test_get_connection.py index 85b9100476..20cfb6981c 100644 --- a/ceilometer/tests/unit/storage/test_get_connection.py +++ b/ceilometer/tests/unit/storage/test_get_connection.py @@ -14,16 +14,10 @@ # under the License. """Tests for ceilometer/storage/ """ -import unittest - import mock from oslo_config import fixture as fixture_config from oslotest import base -try: - from ceilometer.event.storage import impl_hbase as impl_hbase_event -except ImportError: - impl_hbase_event = None from ceilometer import storage from ceilometer.storage import impl_log from ceilometer.storage import impl_sqlalchemy @@ -38,15 +32,13 @@ class EngineTest(base.BaseTestCase): def test_get_connection(self): engine = storage.get_connection(self.CONF, - 'log://localhost', - 'ceilometer.metering.storage') + 'log://localhost') self.assertIsInstance(engine, impl_log.Connection) def test_get_connection_no_such_engine(self): try: storage.get_connection(self.CONF, - 'no-such-engine://localhost', - 'ceilometer.metering.storage') + 'no-such-engine://localhost') except RuntimeError as err: self.assertIn('no-such-engine', six.text_type(err)) @@ -77,44 +69,14 @@ class ConnectionConfigTest(base.BaseTestCase): self.CONF.set_override("connection", "log://", group="database") conn = storage.get_connection_from_config(self.CONF) self.assertIsInstance(conn, impl_log.Connection) - conn = storage.get_connection_from_config(self.CONF, 'metering') - self.assertIsInstance(conn, impl_log.Connection) def test_two_urls(self): self.CONF.set_override("connection", "log://", group="database") conn = storage.get_connection_from_config(self.CONF) self.assertIsInstance(conn, impl_log.Connection) - conn = storage.get_connection_from_config(self.CONF, 'metering') - self.assertIsInstance(conn, impl_log.Connection) - - @unittest.skipUnless(impl_hbase_event, 'need hbase implementation') - def test_three_urls(self): - self.CONF.set_override("connection", "log://", group="database") - self.CONF.set_override("event_connection", "hbase://__test__", - group="database") - conn = storage.get_connection_from_config(self.CONF) - self.assertIsInstance(conn, impl_log.Connection) - conn = storage.get_connection_from_config(self.CONF, 'metering') - self.assertIsInstance(conn, impl_log.Connection) - conn = storage.get_connection_from_config(self.CONF, 'event') - self.assertIsInstance(conn, impl_hbase_event.Connection) - - @unittest.skipUnless(impl_hbase_event, 'need hbase implementation') - def test_three_urls_no_default(self): - self.CONF.set_override("connection", None, group="database") - self.CONF.set_override("metering_connection", "log://", - group="database") - self.CONF.set_override("event_connection", "hbase://__test__", - group="database") - conn = storage.get_connection_from_config(self.CONF) - self.assertIsInstance(conn, impl_log.Connection) - conn = storage.get_connection_from_config(self.CONF, 'event') - self.assertIsInstance(conn, impl_hbase_event.Connection) def test_sqlalchemy_driver(self): self.CONF.set_override("connection", "sqlite+pysqlite://", group="database") conn = storage.get_connection_from_config(self.CONF) self.assertIsInstance(conn, impl_sqlalchemy.Connection) - conn = storage.get_connection_from_config(self.CONF, 'metering') - self.assertIsInstance(conn, impl_sqlalchemy.Connection) diff --git a/ceilometer/tests/unit/storage/test_models.py b/ceilometer/tests/unit/storage/test_models.py deleted file mode 100644 index 9790d2414f..0000000000 --- a/ceilometer/tests/unit/storage/test_models.py +++ /dev/null @@ -1,94 +0,0 @@ -# -# Copyright 2013 New Dream Network, LLC (DreamHost) -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import datetime - -from oslotest import base as testbase -import six - -from ceilometer.event.storage import models as event_models -from ceilometer.storage import base -from ceilometer.storage import models - - -class FakeModel(base.Model): - def __init__(self, arg1, arg2): - base.Model.__init__(self, arg1=arg1, arg2=arg2) - - -class ModelTest(testbase.BaseTestCase): - - def test_create_attributes(self): - m = FakeModel(1, 2) - self.assertEqual(1, m.arg1) - self.assertEqual(2, m.arg2) - - def test_as_dict(self): - m = FakeModel(1, 2) - d = m.as_dict() - self.assertEqual({'arg1': 1, 'arg2': 2}, d) - - def test_as_dict_recursive(self): - m = FakeModel(1, FakeModel('a', 'b')) - d = m.as_dict() - self.assertEqual({'arg1': 1, - 'arg2': {'arg1': 'a', - 'arg2': 'b'}}, - d) - - def test_as_dict_recursive_list(self): - m = FakeModel(1, [FakeModel('a', 'b')]) - d = m.as_dict() - self.assertEqual({'arg1': 1, - 'arg2': [{'arg1': 'a', - 'arg2': 'b'}]}, - d) - - def test_event_repr_no_traits(self): - x = event_models.Event("1", "name", "now", None, {}) - self.assertEqual("", repr(x)) - - def test_get_field_names_of_sample(self): - sample_fields = ["source", "counter_name", "counter_type", - "counter_unit", "counter_volume", "user_id", - "project_id", "resource_id", "timestamp", - "resource_metadata", "message_id", - "message_signature", "recorded_at"] - - self.assertEqual(set(sample_fields), - set(models.Sample.get_field_names())) - - -class TestTraitModel(testbase.BaseTestCase): - - def test_convert_value(self): - v = event_models.Trait.convert_value( - event_models.Trait.INT_TYPE, '10') - self.assertEqual(10, v) - self.assertIsInstance(v, int) - v = event_models.Trait.convert_value( - event_models.Trait.FLOAT_TYPE, '10') - self.assertEqual(10.0, v) - self.assertIsInstance(v, float) - - v = event_models.Trait.convert_value( - event_models.Trait.DATETIME_TYPE, '2013-08-08 21:05:37.123456') - self.assertEqual(datetime.datetime(2013, 8, 8, 21, 5, 37, 123456), v) - self.assertIsInstance(v, datetime.datetime) - - v = event_models.Trait.convert_value( - event_models.Trait.TEXT_TYPE, 10) - self.assertEqual("10", v) - self.assertIsInstance(v, six.text_type) diff --git a/ceilometer/tests/unit/test_collector.py b/ceilometer/tests/unit/test_collector.py index a8a89f66e3..6bf24381aa 100644 --- a/ceilometer/tests/unit/test_collector.py +++ b/ceilometer/tests/unit/test_collector.py @@ -36,9 +36,9 @@ class TestEventDispatcherVerifier(base.BaseTestCase): self.conf.import_opt('telemetry_secret', 'ceilometer.publisher.utils', 'publisher') - self.conf.set_override("event_dispatchers", ['database']) + self.conf.set_override("event_dispatchers", ['file']) self.useFixture(mockpatch.Patch( - 'ceilometer.dispatcher.database.EventDatabaseDispatcher', + 'ceilometer.dispatcher.file.FileDispatcher', new=FakeDispatcher)) @mock.patch('ceilometer.publisher.utils.verify_signature') @@ -50,9 +50,9 @@ class TestEventDispatcherVerifier(base.BaseTestCase): manager = dispatcher.load_dispatcher_manager(self.conf)[1] v = collector.EventEndpoint("secret", manager) v.sample([sample]) - self.assertEqual([], manager['database'].obj.events) + self.assertEqual([], manager['file'].obj.events) del sample['payload'][0]['message_signature'] sample['payload'][0]['message_signature'] = utils.compute_signature( sample['payload'][0], "secret") v.sample([sample]) - self.assertEqual(sample['payload'], manager['database'].obj.events) + self.assertEqual(sample['payload'], manager['file'].obj.events) diff --git a/devstack/plugin.sh b/devstack/plugin.sh index f8c2ab9cc5..3d96155117 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -34,7 +34,7 @@ # of Ceilometer (see within for additional settings): # # CEILOMETER_PIPELINE_INTERVAL: Seconds between pipeline processing runs. Default 600. -# CEILOMETER_BACKEND: Database backend (e.g. 'mysql', 'mongodb', 'es', 'gnocchi', 'none') +# CEILOMETER_BACKEND: Database backend (e.g. 'mysql', 'mongodb', 'gnocchi', 'none') # CEILOMETER_COORDINATION_URL: URL for group membership service provided by tooz. # CEILOMETER_EVENT_ALARM: Set to True to enable publisher for event alarming @@ -138,11 +138,6 @@ function _ceilometer_prepare_storage_backend { pip_install_gr pymongo _ceilometer_install_mongodb fi - - if [ "$CEILOMETER_BACKEND" = 'es' ] ; then - ${TOP_DIR}/pkg/elasticsearch.sh download - ${TOP_DIR}/pkg/elasticsearch.sh install - fi } @@ -205,8 +200,6 @@ function _ceilometer_drop_database { if is_service_enabled ceilometer-collector ceilometer-api ; then if [ "$CEILOMETER_BACKEND" = 'mongodb' ] ; then mongo ceilometer --eval "db.dropDatabase();" - elif [ "$CEILOMETER_BACKEND" = 'es' ] ; then - curl -XDELETE "localhost:9200/events_*" fi fi } @@ -245,16 +238,9 @@ function _ceilometer_configure_storage_backend { fi elif [ "$CEILOMETER_BACKEND" = 'mysql' ] || [ "$CEILOMETER_BACKEND" = 'postgresql' ] ; then iniset $CEILOMETER_CONF DEFAULT meter_dispatchers database - iniset $CEILOMETER_CONF database event_connection $(database_connection_url ceilometer) iniset $CEILOMETER_CONF database metering_connection $(database_connection_url ceilometer) - elif [ "$CEILOMETER_BACKEND" = 'es' ] ; then - # es is only supported for events. we will use sql for metering. - iniset $CEILOMETER_CONF database event_connection es://localhost:9200 - iniset $CEILOMETER_CONF database metering_connection $(database_connection_url ceilometer) - ${TOP_DIR}/pkg/elasticsearch.sh start elif [ "$CEILOMETER_BACKEND" = 'mongodb' ] ; then iniset $CEILOMETER_CONF DEFAULT meter_dispatchers database - iniset $CEILOMETER_CONF database event_connection mongodb://localhost:27017/ceilometer iniset $CEILOMETER_CONF database metering_connection mongodb://localhost:27017/ceilometer elif [ "$CEILOMETER_BACKEND" = 'gnocchi' ] ; then iniset $CEILOMETER_CONF DEFAULT meter_dispatchers gnocchi @@ -371,7 +357,7 @@ function init_ceilometer { if is_service_enabled gnocchi ; then if [ "$CEILOMETER_BACKEND" = 'gnocchi' ]; then set -e - $CEILOMETER_BIN_DIR/ceilometer-upgrade --skip-metering-database --skip-event-database + $CEILOMETER_BIN_DIR/ceilometer-upgrade --skip-metering-database set +e fi fi diff --git a/doc/source/architecture.rst b/doc/source/architecture.rst index 848b80a22d..45283ed754 100644 --- a/doc/source/architecture.rst +++ b/doc/source/architecture.rst @@ -194,7 +194,7 @@ Currently, processed data can be published using 7 different transports: 5. kafka, which publishes data to a Kafka message queue to be consumed by any system that supports Kafka. 6. file, which publishes samples to a file with specified name and location; -7. database, which stores samples/events to the legacy ceilometer database system. +7. database, which stores samples to the legacy ceilometer database system. Storing/Accessing the data diff --git a/doc/source/webapi/v2.rst b/doc/source/webapi/v2.rst index 43e171b34d..58cfb7be3a 100644 --- a/doc/source/webapi/v2.rst +++ b/doc/source/webapi/v2.rst @@ -102,26 +102,6 @@ available in the backend. .. autotype:: ceilometer.api.controllers.v2.capabilities.Capabilities :members: -Events and Traits -================= - -.. rest-controller:: ceilometer.api.controllers.v2.events:EventTypesController - :webprefix: /v2/event_types - -.. rest-controller:: ceilometer.api.controllers.v2.events:TraitsController - :webprefix: /v2/event_types/(event_type)/traits - -.. rest-controller:: ceilometer.api.controllers.v2.events:EventsController - :webprefix: /v2/events - -.. autotype:: ceilometer.api.controllers.v2.events.Event - :members: - -.. autotype:: ceilometer.api.controllers.v2.events.Trait - :members: - -.. autotype:: ceilometer.api.controllers.v2.events.TraitDescription - :members: Filtering Queries ================= @@ -158,18 +138,6 @@ field of *Sample*). See :ref:`api-queries` for how to query the API. .. autotype:: ceilometer.api.controllers.v2.base.Query :members: -Event Query -+++++++++++ - -Event query is similar to simple query, its type EventQuery is actually -a subclass of Query, so EventQuery has every attribute Query has. -But there are some differences. If a field is one of the following: -event_type, message_id, start_timestamp, end_timestamp, then this field -will be applied on event, otherwise it will be treated as trait name and -applied on trait. See :ref:`api-queries` for how to query the API. - -.. autotype:: ceilometer.api.controllers.v2.events.EventQuery - :members: Complex Query +++++++++++++ diff --git a/etc/ceilometer/policy.json b/etc/ceilometer/policy.json index a5e836af4a..0aa0a3d17a 100644 --- a/etc/ceilometer/policy.json +++ b/etc/ceilometer/policy.json @@ -12,7 +12,4 @@ "telemetry:get_resource": "", "telemetry:get_resources": "", - - "telemetry:events:index": "", - "telemetry:events:show": "" } diff --git a/releasenotes/notes/ceilometer-event-api-removed-49c57835e307b997.yaml b/releasenotes/notes/ceilometer-event-api-removed-49c57835e307b997.yaml new file mode 100644 index 0000000000..0fa279eabb --- /dev/null +++ b/releasenotes/notes/ceilometer-event-api-removed-49c57835e307b997.yaml @@ -0,0 +1,6 @@ +--- +other: + - >- + The Events API (exposed at /v2/events) which was deprecated has been + removed. The Panko project is now responsible for providing this API and can + be installed separately. diff --git a/setup.cfg b/setup.cfg index e603c7b735..f06becc1d1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -202,15 +202,6 @@ ceilometer.poll.central = ceilometer.builder.poll.central = hardware.snmp = ceilometer.hardware.pollsters.generic:GenericHardwareDeclarativePollster -ceilometer.event.storage = - es = ceilometer.event.storage.impl_elasticsearch:Connection - log = ceilometer.event.storage.impl_log:Connection - mongodb = ceilometer.event.storage.impl_mongodb:Connection - mysql = ceilometer.event.storage.impl_sqlalchemy:Connection - postgresql = ceilometer.event.storage.impl_sqlalchemy:Connection - sqlite = ceilometer.event.storage.impl_sqlalchemy:Connection - hbase = ceilometer.event.storage.impl_hbase:Connection - ceilometer.metering.storage = log = ceilometer.storage.impl_log:Connection mongodb = ceilometer.storage.impl_mongodb:Connection @@ -285,7 +276,6 @@ ceilometer.dispatcher.meter = gnocchi = ceilometer.dispatcher.gnocchi:GnocchiDispatcher ceilometer.dispatcher.event = - database = ceilometer.dispatcher.database:EventDatabaseDispatcher file = ceilometer.dispatcher.file:FileDispatcher http = ceilometer.dispatcher.http:HttpDispatcher gnocchi = ceilometer.dispatcher.gnocchi:GnocchiDispatcher diff --git a/test-requirements.txt b/test-requirements.txt index c3380bc165..d392569f2f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,7 +4,6 @@ contextlib2>=0.4.0 # PSF License coverage>=3.6 # Apache-2.0 -elasticsearch>=1.3.0 # Apache-2.0 fixtures<2.0,>=1.3.1 # Apache-2.0/BSD happybase!=0.7,>=0.5,<1.0.0;python_version=='2.7' # MIT mock>=1.2 # BSD diff --git a/tools/ceilometer-test-event.py b/tools/ceilometer-test-event.py deleted file mode 100755 index dedc7ac09d..0000000000 --- a/tools/ceilometer-test-event.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2013 Rackspace Hosting. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Command line tool help you debug your event definitions. - -Feed it a list of test notifications in json format, and it will show -you what events will be generated. -""" - -import json -import sys - -from oslo_config import cfg -from stevedore import extension - -from ceilometer.event import converter -from ceilometer import service - - -cfg.CONF.register_cli_opts([ - cfg.StrOpt('input-file', - short='i', - help='File to read test notifications from.' - ' (Containing a json list of notifications.)' - ' defaults to stdin.'), - cfg.StrOpt('output-file', - short='o', - help='File to write results to. Defaults to stdout.'), -]) - -TYPES = {1: 'text', - 2: 'int', - 3: 'float', - 4: 'datetime'} - - -service.prepare_service() - -output_file = cfg.CONF.output_file -input_file = cfg.CONF.input_file - -if output_file is None: - out = sys.stdout -else: - out = open(output_file, 'w') - -if input_file is None: - notifications = json.load(sys.stdin) -else: - with open(input_file, 'r') as f: - notifications = json.load(f) - -out.write("Definitions file: %s\n" % cfg.CONF.event.definitions_cfg_file) -out.write("Notifications tested: %s\n" % len(notifications)) - -event_converter = converter.setup_events( - extension.ExtensionManager( - namespace='ceilometer.event.trait_plugin')) - -for notification in notifications: - event = event_converter.to_event(notification) - if event is None: - out.write("Dropped notification: %s\n" % - notification['message_id']) - continue - out.write("Event: %s at %s\n" % (event.event_type, event.generated)) - for trait in event.traits: - dtype = TYPES[trait.dtype] - out.write(" Trait: name: %s, type: %s, value: %s\n" % ( - trait.name, dtype, trait.value)) diff --git a/tools/make_test_event_data.py b/tools/make_test_event_data.py deleted file mode 100755 index 1df6db4f35..0000000000 --- a/tools/make_test_event_data.py +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env python -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Command line tool for creating event test data for Ceilometer. - -Usage: - -Generate testing data for e.g. for default time span - -source .tox/py27/bin/activate -./tools/make_test_event_data.py --event_types 3 -""" -import argparse -import datetime -import random -import uuid - -from oslo_config import cfg -from oslo_utils import timeutils - -from ceilometer.event.storage import models -from ceilometer import storage - - -def make_test_data(conn, start, end, interval, event_types): - - # Compute start and end timestamps for the new data. - if isinstance(start, datetime.datetime): - timestamp = start - else: - timestamp = timeutils.parse_strtime(start) - - if not isinstance(end, datetime.datetime): - end = timeutils.parse_strtime(end) - - increment = datetime.timedelta(minutes=interval) - - print('Adding new events') - n = 0 - while timestamp <= end: - data = [] - for i in range(event_types): - traits = [models.Trait('id1_%d' % i, 1, str(uuid.uuid4())), - models.Trait('id2_%d' % i, 2, random.randint(1, 10)), - models.Trait('id3_%d' % i, 3, random.random()), - models.Trait('id4_%d' % i, 4, timestamp)] - data.append(models.Event(str(uuid.uuid4()), - 'event_type%d' % i, - timestamp, - traits, - {})) - n += 1 - conn.record_events(data) - timestamp = timestamp + increment - print('Added %d new events' % n) - - -def main(): - cfg.CONF([], project='ceilometer') - - parser = argparse.ArgumentParser( - description='generate event data', - ) - parser.add_argument( - '--interval', - default=10, - type=int, - help='The period between events, in minutes.', - ) - parser.add_argument( - '--start', - default=31, - type=int, - help='The number of days in the past to start timestamps.', - ) - parser.add_argument( - '--end', - default=2, - type=int, - help='The number of days into the future to continue timestamps.', - ) - parser.add_argument( - '--event_types', - default=3, - type=int, - help='The number of unique event_types.', - ) - args = parser.parse_args() - - # Connect to the event database - conn = storage.get_connection_from_config(cfg.CONF, 'event') - - # Compute the correct time span - start = datetime.datetime.utcnow() - datetime.timedelta(days=args.start) - end = datetime.datetime.utcnow() + datetime.timedelta(days=args.end) - - make_test_data(conn=conn, - start=start, - end=end, - interval=args.interval, - event_types=args.event_types) - - -if __name__ == '__main__': - main() diff --git a/tools/test_hbase_table_utils.py b/tools/test_hbase_table_utils.py index 10294e31cf..3a8b08fa3e 100755 --- a/tools/test_hbase_table_utils.py +++ b/tools/test_hbase_table_utils.py @@ -27,14 +27,11 @@ def main(argv): (os.getenv("CEILOMETER_TEST_STORAGE_URL"), os.getenv("CEILOMETER_TEST_HBASE_TABLE_PREFIX", "test"))) conn = storage.get_connection(url, 'ceilometer.metering.storage') - event_conn = storage.get_connection(url, 'ceilometer.event.storage') for arg in argv: if arg == "--upgrade": conn.upgrade() - event_conn.upgrade() if arg == "--clear": conn.clear() - event_conn.clear() if __name__ == '__main__': diff --git a/tox.ini b/tox.ini index bbf85cfb52..afa63c6df0 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] minversion = 1.8 skipsdist = True -envlist = py{34,27,35},{debug,py,py34,py27,py35}-{mongodb,mysql,postgresql,elastic,functional},pep8 +envlist = py{34,27,35},{debug,py,py34,py27,py35}-{mongodb,mysql,postgresql,functional},pep8 [testenv] deps = .[mongo,mysql,postgresql,gnocchi] @@ -14,11 +14,10 @@ setenv = VIRTUAL_ENV={envdir} CEILOMETER_TEST_BACKEND={env:CEILOMETER_TEST_BACKEND:none} CEILOMETER_TEST_DEBUG={env:CEILOMETER_TEST_DEBUG:} debug: CEILOMETER_TEST_DEBUG=True - {mongodb,mysql,postgresql,elastic,functional}: OS_TEST_PATH=ceilometer/tests/functional/ + {mongodb,mysql,postgresql,functional}: OS_TEST_PATH=ceilometer/tests/functional/ mongodb: CEILOMETER_TEST_BACKEND=mongodb mysql: CEILOMETER_TEST_BACKEND=mysql postgresql: CEILOMETER_TEST_BACKEND=postgresql - elastic: CEILOMETER_TEST_BACKEND=elasticsearch functional: CEILOMETER_TEST_BACKEND={env:CEILOMETER_TEST_BACKEND:mongodb} passenv = OS_TEST_TIMEOUT OS_STDOUT_CAPTURE OS_STDERR_CAPTURE OS_LOG_CAPTURE CEILOMETER_* commands =