From 41f345f55254932dda0645a0ec63251fd7de6a22 Mon Sep 17 00:00:00 2001 From: gord chung Date: Thu, 12 Jan 2017 19:19:25 +0000 Subject: [PATCH] remove event redundancy - we don't need to prefix 'event/storage' because that's all panko does. cleanup inline with aodh. - test_hbase_table_utils is not used anywhere and last reference was from icehouse. - drop a few places that use 'event_*' prefix since it wastes line space Change-Id: I348f5a3dfb59682589b4912eaed4e4b8abb15ebb --- panko/api/controllers/v2/events.py | 4 +- panko/dispatcher/database.py | 2 +- panko/event/__init__.py | 0 panko/event/storage/__init__.py | 57 ----- panko/event/storage/base.py | 100 --------- panko/storage/__init__.py | 48 ++++- panko/storage/base.py | 87 ++++++++ .../{event => }/storage/impl_elasticsearch.py | 4 +- panko/{event => }/storage/impl_hbase.py | 4 +- panko/{event => }/storage/impl_log.py | 2 +- panko/{event => }/storage/impl_mongodb.py | 2 +- panko/{event => }/storage/impl_sqlalchemy.py | 7 +- panko/{event => }/storage/models.py | 0 panko/{event => }/storage/pymongo_base.py | 4 +- .../functional/api/v2/test_acl_scenarios.py | 10 +- .../functional/api/v2/test_event_scenarios.py | 2 +- panko/tests/functional/gabbi/fixtures.py | 2 +- .../functional/storage/test_impl_mongodb.py | 2 +- .../storage/test_impl_sqlalchemy.py | 4 +- .../storage/test_storage_scenarios.py | 196 +++++++++--------- panko/tests/unit/dispatcher/test_db.py | 8 +- .../tests/unit/storage/test_get_connection.py | 7 +- setup.cfg | 16 +- tools/make_test_event_data.py | 2 +- tools/test_hbase_table_utils.py | 36 ---- 25 files changed, 270 insertions(+), 336 deletions(-) delete mode 100644 panko/event/__init__.py delete mode 100644 panko/event/storage/__init__.py delete mode 100644 panko/event/storage/base.py rename panko/{event => }/storage/impl_elasticsearch.py (99%) rename panko/{event => }/storage/impl_hbase.py (99%) rename panko/{event => }/storage/impl_log.py (96%) rename panko/{event => }/storage/impl_mongodb.py (98%) rename panko/{event => }/storage/impl_sqlalchemy.py (99%) rename panko/{event => }/storage/models.py (100%) rename panko/{event => }/storage/pymongo_base.py (98%) delete mode 100755 tools/test_hbase_table_utils.py diff --git a/panko/api/controllers/v2/events.py b/panko/api/controllers/v2/events.py index 5d990512..399f4f4c 100644 --- a/panko/api/controllers/v2/events.py +++ b/panko/api/controllers/v2/events.py @@ -31,8 +31,8 @@ import wsmeext.pecan as wsme_pecan from panko.api.controllers.v2 import base from panko.api.controllers.v2 import utils as v2_utils from panko.api import rbac -from panko.event import storage -from panko.event.storage import models as event_models +from panko import storage +from panko.storage import models as event_models from panko.i18n import _, _LE LOG = log.getLogger(__name__) diff --git a/panko/dispatcher/database.py b/panko/dispatcher/database.py index 102a1c9b..b665a078 100644 --- a/panko/dispatcher/database.py +++ b/panko/dispatcher/database.py @@ -16,10 +16,10 @@ from oslo_log import log from oslo_utils import timeutils -from panko.event.storage import models from panko.i18n import _LE from panko import service from panko import storage +from panko.storage import models LOG = log.getLogger(__name__) diff --git a/panko/event/__init__.py b/panko/event/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/panko/event/storage/__init__.py b/panko/event/storage/__init__.py deleted file mode 100644 index 8ba13a09..00000000 --- a/panko/event/storage/__init__.py +++ /dev/null @@ -1,57 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import six - -from panko import utils - - -class EventFilter(object): - """Properties for building an Event query. - - :param start_timestamp: UTC start datetime (mandatory) - :param end_timestamp: UTC end datetime (mandatory) - :param event_type: the name of the event. None for all. - :param message_id: the message_id of the event. None for all. - :param admin_proj: the project_id of admin role. None if non-admin user. - :param traits_filter: the trait filter dicts, all of which are optional. - This parameter is a list of dictionaries that specify trait values: - - .. code-block:: python - - {'key': , - 'string': , - 'integer': , - 'datetime': , - 'float': , - 'op': } - """ - - def __init__(self, start_timestamp=None, end_timestamp=None, - event_type=None, message_id=None, traits_filter=None, - admin_proj=None): - self.start_timestamp = utils.sanitize_timestamp(start_timestamp) - self.end_timestamp = utils.sanitize_timestamp(end_timestamp) - self.message_id = message_id - self.event_type = event_type - self.traits_filter = traits_filter or [] - self.admin_proj = admin_proj - - def __repr__(self): - return ("" % - (self.start_timestamp, - self.end_timestamp, - self.event_type, - six.text_type(self.traits_filter))) diff --git a/panko/event/storage/base.py b/panko/event/storage/base.py deleted file mode 100644 index bf5b45f4..00000000 --- a/panko/event/storage/base.py +++ /dev/null @@ -1,100 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -import panko - - -class Connection(object): - """Base class for event storage system connections.""" - - # A dictionary representing the capabilities of this driver. - CAPABILITIES = { - 'events': {'query': {'simple': False}}, - } - - STORAGE_CAPABILITIES = { - 'storage': {'production_ready': False}, - } - - @staticmethod - def __init__(url, conf): - pass - - @staticmethod - def upgrade(): - """Migrate the database to `version` or the most recent version.""" - - @staticmethod - def clear(): - """Clear database.""" - - @staticmethod - def record_events(events): - """Write the events to the backend storage system. - - :param events: a list of model.Event objects. - """ - raise panko.NotImplementedError('Events not implemented.') - - @staticmethod - def get_events(event_filter, pagination=None): - """Return an iterable of model.Event objects.""" - raise panko.NotImplementedError('Events not implemented.') - - @staticmethod - def get_event_types(): - """Return all event types as an iterable of strings.""" - raise panko.NotImplementedError('Events not implemented.') - - @staticmethod - def get_trait_types(event_type): - """Return a dictionary containing the name and data type of the trait. - - Only trait types for the provided event_type are - returned. - :param event_type: the type of the Event - """ - raise panko.NotImplementedError('Events not implemented.') - - @staticmethod - def get_traits(event_type, trait_type=None): - """Return all trait instances associated with an event_type. - - If trait_type is specified, only return instances of that trait type. - :param event_type: the type of the Event to filter by - :param trait_type: the name of the Trait to filter by - """ - - raise panko.NotImplementedError('Events not implemented.') - - @classmethod - def get_capabilities(cls): - """Return an dictionary with the capabilities of each driver.""" - return cls.CAPABILITIES - - @classmethod - def get_storage_capabilities(cls): - """Return a dictionary representing the performance capabilities. - - This is needed to evaluate the performance of each driver. - """ - return cls.STORAGE_CAPABILITIES - - @staticmethod - def clear_expired_event_data(ttl): - """Clear expired data from the backend storage system. - - Clearing occurs according to the time-to-live. - - :param ttl: Number of seconds to keep records for. - """ - raise panko.NotImplementedError('Clearing events not implemented') diff --git a/panko/storage/__init__.py b/panko/storage/__init__.py index dac22042..55f37b97 100644 --- a/panko/storage/__init__.py +++ b/panko/storage/__init__.py @@ -17,10 +17,12 @@ from oslo_config import cfg from oslo_log import log +import six import six.moves.urllib.parse as urlparse from stevedore import driver import tenacity +from panko import utils LOG = log.getLogger(__name__) @@ -77,7 +79,49 @@ def get_connection(url, conf): # 'dialect+driver'. Handle the case where driver is specified. engine_name = connection_scheme.split('+')[0] # NOTE: translation not applied bug #1446983 - LOG.debug('looking for %(name)r driver in panko.event.storage', + LOG.debug('looking for %(name)r driver in panko.storage', {'name': engine_name}) - mgr = driver.DriverManager('panko.event.storage', engine_name) + mgr = driver.DriverManager('panko.storage', engine_name) return mgr.driver(url, conf) + + +class EventFilter(object): + """Properties for building an Event query. + + :param start_timestamp: UTC start datetime (mandatory) + :param end_timestamp: UTC end datetime (mandatory) + :param event_type: the name of the event. None for all. + :param message_id: the message_id of the event. None for all. + :param admin_proj: the project_id of admin role. None if non-admin user. + :param traits_filter: the trait filter dicts, all of which are optional. + This parameter is a list of dictionaries that specify trait values: + + .. code-block:: python + + {'key': , + 'string': , + 'integer': , + 'datetime': , + 'float': , + 'op': } + """ + + def __init__(self, start_timestamp=None, end_timestamp=None, + event_type=None, message_id=None, traits_filter=None, + admin_proj=None): + self.start_timestamp = utils.sanitize_timestamp(start_timestamp) + self.end_timestamp = utils.sanitize_timestamp(end_timestamp) + self.message_id = message_id + self.event_type = event_type + self.traits_filter = traits_filter or [] + self.admin_proj = admin_proj + + def __repr__(self): + return ("" % + (self.start_timestamp, + self.end_timestamp, + self.event_type, + six.text_type(self.traits_filter))) diff --git a/panko/storage/base.py b/panko/storage/base.py index 4b075bfa..a9270bb1 100644 --- a/panko/storage/base.py +++ b/panko/storage/base.py @@ -17,6 +17,8 @@ import six +import panko + class Model(object): """Base class for storage API models.""" @@ -42,3 +44,88 @@ class Model(object): def __ne__(self, other): return not self.__eq__(other) + + +class Connection(object): + """Base class for event storage system connections.""" + + # A dictionary representing the capabilities of this driver. + CAPABILITIES = { + 'events': {'query': {'simple': False}}, + } + + STORAGE_CAPABILITIES = { + 'storage': {'production_ready': False}, + } + + @staticmethod + def __init__(url, conf): + pass + + @staticmethod + def upgrade(): + """Migrate the database to `version` or the most recent version.""" + + @staticmethod + def clear(): + """Clear database.""" + + @staticmethod + def record_events(events): + """Write the events to the backend storage system. + + :param events: a list of model.Event objects. + """ + raise panko.NotImplementedError('Events not implemented.') + + @staticmethod + def get_events(event_filter, pagination=None): + """Return an iterable of model.Event objects.""" + + @staticmethod + def get_event_types(): + """Return all event types as an iterable of strings.""" + raise panko.NotImplementedError('Events not implemented.') + + @staticmethod + def get_trait_types(event_type): + """Return a dictionary containing the name and data type of the trait. + + Only trait types for the provided event_type are + returned. + :param event_type: the type of the Event + """ + raise panko.NotImplementedError('Events not implemented.') + + @staticmethod + def get_traits(event_type, trait_type=None): + """Return all trait instances associated with an event_type. + + If trait_type is specified, only return instances of that trait type. + :param event_type: the type of the Event to filter by + :param trait_type: the name of the Trait to filter by + """ + + raise panko.NotImplementedError('Events not implemented.') + + @classmethod + def get_capabilities(cls): + """Return an dictionary with the capabilities of each driver.""" + return cls.CAPABILITIES + + @classmethod + def get_storage_capabilities(cls): + """Return a dictionary representing the performance capabilities. + + This is needed to evaluate the performance of each driver. + """ + return cls.STORAGE_CAPABILITIES + + @staticmethod + def clear_expired_event_data(ttl): + """Clear expired data from the backend storage system. + + Clearing occurs according to the time-to-live. + :param ttl: Number of seconds to keep records for. + """ + raise panko.NotImplementedError('Clearing events not implemented') diff --git a/panko/event/storage/impl_elasticsearch.py b/panko/storage/impl_elasticsearch.py similarity index 99% rename from panko/event/storage/impl_elasticsearch.py rename to panko/storage/impl_elasticsearch.py index 3922d3e6..13db3068 100644 --- a/panko/event/storage/impl_elasticsearch.py +++ b/panko/storage/impl_elasticsearch.py @@ -21,8 +21,8 @@ from oslo_utils import netutils from oslo_utils import timeutils import six -from panko.event.storage import base -from panko.event.storage import models +from panko.storage import base +from panko.storage import models from panko.i18n import _LE, _LI, _LW from panko import storage from panko import utils diff --git a/panko/event/storage/impl_hbase.py b/panko/storage/impl_hbase.py similarity index 99% rename from panko/event/storage/impl_hbase.py rename to panko/storage/impl_hbase.py index 0ef6cbff..9801da35 100644 --- a/panko/event/storage/impl_hbase.py +++ b/panko/storage/impl_hbase.py @@ -15,8 +15,8 @@ import operator from oslo_log import log -from panko.event.storage import base -from panko.event.storage import models +from panko.storage import base +from panko.storage import models from panko.i18n import _LE, _LW from panko.storage.hbase import base as hbase_base from panko.storage.hbase import utils as hbase_utils diff --git a/panko/event/storage/impl_log.py b/panko/storage/impl_log.py similarity index 96% rename from panko/event/storage/impl_log.py rename to panko/storage/impl_log.py index a4f74cf8..a8863ee0 100644 --- a/panko/event/storage/impl_log.py +++ b/panko/storage/impl_log.py @@ -13,8 +13,8 @@ from oslo_log import log -from panko.event.storage import base from panko.i18n import _LI +from panko.storage import base LOG = log.getLogger(__name__) diff --git a/panko/event/storage/impl_mongodb.py b/panko/storage/impl_mongodb.py similarity index 98% rename from panko/event/storage/impl_mongodb.py rename to panko/storage/impl_mongodb.py index a0c39dd8..d8c2ba4f 100644 --- a/panko/event/storage/impl_mongodb.py +++ b/panko/storage/impl_mongodb.py @@ -15,9 +15,9 @@ from oslo_log import log import pymongo -from panko.event.storage import pymongo_base from panko import storage from panko.storage.mongo import utils as pymongo_utils +from panko.storage import pymongo_base LOG = log.getLogger(__name__) diff --git a/panko/event/storage/impl_sqlalchemy.py b/panko/storage/impl_sqlalchemy.py similarity index 99% rename from panko/event/storage/impl_sqlalchemy.py rename to panko/storage/impl_sqlalchemy.py index 7309f546..dc7038c8 100644 --- a/panko/event/storage/impl_sqlalchemy.py +++ b/panko/storage/impl_sqlalchemy.py @@ -24,9 +24,8 @@ from oslo_log import log from oslo_utils import timeutils import sqlalchemy as sa -from panko.event import storage as event_storage -from panko.event.storage import base -from panko.event.storage import models as api_models +from panko.storage import base +from panko.storage import models as api_models from panko.i18n import _LE, _LI from panko import storage from panko.storage.sqlalchemy import models @@ -213,7 +212,7 @@ class Connection(base.Connection): marker = None if pagination.get('marker'): - marker_filter = event_storage.EventFilter( + marker_filter = storage.EventFilter( message_id=pagination.get('marker')) markers = list(self.get_events(marker_filter)) if markers: diff --git a/panko/event/storage/models.py b/panko/storage/models.py similarity index 100% rename from panko/event/storage/models.py rename to panko/storage/models.py diff --git a/panko/event/storage/pymongo_base.py b/panko/storage/pymongo_base.py similarity index 98% rename from panko/event/storage/pymongo_base.py rename to panko/storage/pymongo_base.py index 656fbd52..3dec498b 100644 --- a/panko/event/storage/pymongo_base.py +++ b/panko/storage/pymongo_base.py @@ -15,8 +15,8 @@ from oslo_log import log import pymongo -from panko.event.storage import base -from panko.event.storage import models +from panko.storage import base +from panko.storage import models from panko.i18n import _LE, _LI, _LW from panko.storage.mongo import utils as pymongo_utils from panko import utils diff --git a/panko/tests/functional/api/v2/test_acl_scenarios.py b/panko/tests/functional/api/v2/test_acl_scenarios.py index e2c8193f..7c755250 100644 --- a/panko/tests/functional/api/v2/test_acl_scenarios.py +++ b/panko/tests/functional/api/v2/test_acl_scenarios.py @@ -24,7 +24,7 @@ import six import webtest from panko.api import app -from panko.event.storage import models as ev_model +from panko.storage import models from panko.tests.functional.api import v2 VALID_TOKEN = uuidutils.generate_uuid(dashed=False) @@ -87,11 +87,11 @@ class TestBaseApiEventRBAC(v2.FunctionalTest): def setUp(self): super(TestBaseApiEventRBAC, self).setUp() - traits = [ev_model.Trait('project_id', 1, 'project-good'), - ev_model.Trait('user_id', 1, 'user-good')] + traits = [models.Trait('project_id', 1, 'project-good'), + models.Trait('user_id', 1, 'user-good')] self.message_id = uuidutils.generate_uuid() - ev = ev_model.Event(self.message_id, 'event_type', - datetime.datetime.now(), traits, {}) + ev = models.Event(self.message_id, 'event_type', + datetime.datetime.now(), traits, {}) self.event_conn.record_events([ev]) def test_get_events_without_project(self): diff --git a/panko/tests/functional/api/v2/test_event_scenarios.py b/panko/tests/functional/api/v2/test_event_scenarios.py index fc256cb2..7e9d68b7 100644 --- a/panko/tests/functional/api/v2/test_event_scenarios.py +++ b/panko/tests/functional/api/v2/test_event_scenarios.py @@ -20,7 +20,7 @@ from oslo_serialization import jsonutils from oslo_utils import uuidutils import webtest.app -from panko.event.storage import models +from panko.storage import models from panko.tests import db as tests_db from panko.tests.functional.api import v2 diff --git a/panko/tests/functional/gabbi/fixtures.py b/panko/tests/functional/gabbi/fixtures.py index 7a6c1204..082e57c4 100644 --- a/panko/tests/functional/gabbi/fixtures.py +++ b/panko/tests/functional/gabbi/fixtures.py @@ -29,9 +29,9 @@ from six.moves.urllib import parse as urlparse import sqlalchemy_utils from panko.api import app -from panko.event.storage import models from panko import service from panko import storage +from panko.storage import models # NOTE(chdent): Hack to restore semblance of global configuration to # pass to the WSGI app used per test suite. LOAD_APP_KWARGS are the olso diff --git a/panko/tests/functional/storage/test_impl_mongodb.py b/panko/tests/functional/storage/test_impl_mongodb.py index efbc0211..33b4606b 100644 --- a/panko/tests/functional/storage/test_impl_mongodb.py +++ b/panko/tests/functional/storage/test_impl_mongodb.py @@ -21,7 +21,7 @@ """ -from panko.event.storage import impl_mongodb +from panko.storage import impl_mongodb from panko.tests import base as test_base from panko.tests import db as tests_db diff --git a/panko/tests/functional/storage/test_impl_sqlalchemy.py b/panko/tests/functional/storage/test_impl_sqlalchemy.py index edb4f7c4..5c4947f9 100644 --- a/panko/tests/functional/storage/test_impl_sqlalchemy.py +++ b/panko/tests/functional/storage/test_impl_sqlalchemy.py @@ -22,8 +22,8 @@ import datetime from six.moves import reprlib -from panko.event.storage import impl_sqlalchemy as impl_sqla_event -from panko.event.storage import models +from panko.storage import impl_sqlalchemy as impl_sqla_event +from panko.storage import models from panko.storage.sqlalchemy import models as sql_models from panko.tests import base as test_base from panko.tests import db as tests_db diff --git a/panko/tests/functional/storage/test_storage_scenarios.py b/panko/tests/functional/storage/test_storage_scenarios.py index bfd60045..67d0a578 100644 --- a/panko/tests/functional/storage/test_storage_scenarios.py +++ b/panko/tests/functional/storage/test_storage_scenarios.py @@ -20,8 +20,8 @@ import operator import mock from oslo_utils import timeutils -from panko.event import storage as event_storage -from panko.event.storage import models as event_models +from panko import storage +from panko.storage import models from panko.tests import db as tests_db @@ -36,30 +36,30 @@ class EventTestBase(tests_db.TestBase): self.prepare_data() def prepare_data(self): - self.event_models = [] + self.models = [] base = 0 self.start = datetime.datetime(2013, 12, 31, 5, 0) now = self.start for event_type in ['Foo', 'Bar', 'Zoo', 'Foo', 'Bar', 'Zoo']: - trait_models = [event_models.Trait(name, dtype, value) + trait_models = [models.Trait(name, dtype, value) for name, dtype, value in [ - ('trait_A', event_models.Trait.TEXT_TYPE, + ('trait_A', models.Trait.TEXT_TYPE, "my_%s_text" % event_type), - ('trait_B', event_models.Trait.INT_TYPE, + ('trait_B', models.Trait.INT_TYPE, base + 1), - ('trait_C', event_models.Trait.FLOAT_TYPE, + ('trait_C', models.Trait.FLOAT_TYPE, float(base) + 0.123456), - ('trait_D', event_models.Trait.DATETIME_TYPE, + ('trait_D', models.Trait.DATETIME_TYPE, now)]] - self.event_models.append( - event_models.Event("id_%s_%d" % (event_type, base), - event_type, now, trait_models, - {'status': {'nested': 'started'}})) + self.models.append( + models.Event("id_%s_%d" % (event_type, base), + event_type, now, trait_models, + {'status': {'nested': 'started'}})) base += 100 now = now + datetime.timedelta(hours=1) self.end = now - self.event_conn.record_events(self.event_models) + self.event_conn.record_events(self.models) @tests_db.run_with('sqlite', 'mysql', 'pgsql') @@ -70,7 +70,7 @@ class EventTTLTest(EventTestBase): mock_utcnow.return_value = datetime.datetime(2013, 12, 31, 10, 0) self.event_conn.clear_expired_event_data(3600) - events = list(self.event_conn.get_events(event_storage.EventFilter())) + events = list(self.event_conn.get_events(storage.EventFilter())) self.assertEqual(2, len(events)) event_types = list(self.event_conn.get_event_types()) self.assertEqual(['Bar', 'Zoo'], event_types) @@ -85,8 +85,8 @@ class EventTTLTest(EventTestBase): class EventTest(EventTestBase): def test_duplicate_message_id(self): now = datetime.datetime.utcnow() - m = [event_models.Event("1", "Foo", now, None, {}), - event_models.Event("1", "Zoo", now, [], {})] + m = [models.Event("1", "Foo", now, None, {}), + models.Event("1", "Zoo", now, [], {})] with mock.patch('%s.LOG' % self.event_conn.record_events.__module__) as log: self.event_conn.record_events(m) @@ -94,7 +94,7 @@ class EventTest(EventTestBase): def test_bad_event(self): now = datetime.datetime.utcnow() - broken_event = event_models.Event("1", "Foo", now, None, {}) + broken_event = models.Event("1", "Foo", now, None, {}) del(broken_event.__dict__['raw']) m = [broken_event, broken_event] with mock.patch('%s.LOG' % @@ -108,21 +108,21 @@ class EventTest(EventTestBase): class GetEventTest(EventTestBase): def test_generated_is_datetime(self): - event_filter = event_storage.EventFilter(self.start, self.end) + event_filter = storage.EventFilter(self.start, self.end) events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(6, len(events)) for i, event in enumerate(events): self.assertIsInstance(event.generated, datetime.datetime) self.assertEqual(event.generated, - self.event_models[i].generated) - model_traits = self.event_models[i].traits + self.models[i].generated) + model_traits = self.models[i].traits for j, trait in enumerate(event.traits): - if trait.dtype == event_models.Trait.DATETIME_TYPE: + if trait.dtype == models.Trait.DATETIME_TYPE: self.assertIsInstance(trait.value, datetime.datetime) self.assertEqual(trait.value, model_traits[j].value) def test_simple_get(self): - event_filter = event_storage.EventFilter(self.start, self.end) + event_filter = storage.EventFilter(self.start, self.end) events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(6, len(events)) start_time = None @@ -151,7 +151,7 @@ class GetEventTest(EventTestBase): } } - event_filter = event_storage.EventFilter(self.start, self.end, "Bar") + event_filter = storage.EventFilter(self.start, self.end, "Bar") events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(2, len(events)) self.assertEqual("Bar", events[0].event_type) @@ -172,8 +172,8 @@ class GetEventTest(EventTestBase): def test_get_event_trait_filter(self): trait_filters = [{'key': 'trait_B', 'integer': 101}] - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) + event_filter = storage.EventFilter(self.start, self.end, + traits_filter=trait_filters) events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(1, len(events)) self.assertEqual("Bar", events[0].event_type) @@ -182,117 +182,117 @@ class GetEventTest(EventTestBase): def test_get_event_trait_filter_op_string(self): trait_filters = [{'key': 'trait_A', 'string': 'my_Foo_text', 'op': 'eq'}] - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) + event_filter = storage.EventFilter(self.start, self.end, + traits_filter=trait_filters) events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(2, len(events)) self.assertEqual("Foo", events[0].event_type) self.assertEqual(4, len(events[0].traits)) trait_filters[0].update({'key': 'trait_A', 'op': 'lt'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) + event_filter = storage.EventFilter(self.start, self.end, + traits_filter=trait_filters) events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(2, len(events)) self.assertEqual("Bar", events[0].event_type) trait_filters[0].update({'key': 'trait_A', 'op': 'le'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) + event_filter = storage.EventFilter(self.start, self.end, + traits_filter=trait_filters) events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(4, len(events)) self.assertEqual("Bar", events[1].event_type) trait_filters[0].update({'key': 'trait_A', 'op': 'ne'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) + event_filter = storage.EventFilter(self.start, self.end, + traits_filter=trait_filters) events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(4, len(events)) self.assertEqual("Zoo", events[3].event_type) trait_filters[0].update({'key': 'trait_A', 'op': 'gt'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) + event_filter = storage.EventFilter(self.start, self.end, + traits_filter=trait_filters) events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(2, len(events)) self.assertEqual("Zoo", events[0].event_type) trait_filters[0].update({'key': 'trait_A', 'op': 'ge'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) + event_filter = storage.EventFilter(self.start, self.end, + traits_filter=trait_filters) events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(4, len(events)) self.assertEqual("Foo", events[2].event_type) def test_get_event_trait_filter_op_integer(self): trait_filters = [{'key': 'trait_B', 'integer': 101, 'op': 'eq'}] - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) + event_filter = storage.EventFilter(self.start, self.end, + traits_filter=trait_filters) events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(1, len(events)) self.assertEqual("Bar", events[0].event_type) self.assertEqual(4, len(events[0].traits)) trait_filters[0].update({'key': 'trait_B', 'op': 'lt'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) + event_filter = storage.EventFilter(self.start, self.end, + traits_filter=trait_filters) events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(1, len(events)) self.assertEqual("Foo", events[0].event_type) trait_filters[0].update({'key': 'trait_B', 'op': 'le'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) + event_filter = storage.EventFilter(self.start, self.end, + traits_filter=trait_filters) events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(2, len(events)) self.assertEqual("Bar", events[1].event_type) trait_filters[0].update({'key': 'trait_B', 'op': 'ne'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) + event_filter = storage.EventFilter(self.start, self.end, + traits_filter=trait_filters) events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(5, len(events)) self.assertEqual("Zoo", events[4].event_type) trait_filters[0].update({'key': 'trait_B', 'op': 'gt'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) + event_filter = storage.EventFilter(self.start, self.end, + traits_filter=trait_filters) events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(4, len(events)) self.assertEqual("Zoo", events[0].event_type) trait_filters[0].update({'key': 'trait_B', 'op': 'ge'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) + event_filter = storage.EventFilter(self.start, self.end, + traits_filter=trait_filters) events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(5, len(events)) self.assertEqual("Foo", events[2].event_type) def test_get_event_trait_filter_op_float(self): trait_filters = [{'key': 'trait_C', 'float': 300.123456, 'op': 'eq'}] - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) + event_filter = storage.EventFilter(self.start, self.end, + traits_filter=trait_filters) events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(1, len(events)) self.assertEqual("Foo", events[0].event_type) self.assertEqual(4, len(events[0].traits)) trait_filters[0].update({'key': 'trait_C', 'op': 'lt'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) + event_filter = storage.EventFilter(self.start, self.end, + traits_filter=trait_filters) events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(3, len(events)) self.assertEqual("Zoo", events[2].event_type) trait_filters[0].update({'key': 'trait_C', 'op': 'le'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) + event_filter = storage.EventFilter(self.start, self.end, + traits_filter=trait_filters) events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(4, len(events)) self.assertEqual("Bar", events[1].event_type) trait_filters[0].update({'key': 'trait_C', 'op': 'ne'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) + event_filter = storage.EventFilter(self.start, self.end, + traits_filter=trait_filters) events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(5, len(events)) self.assertEqual("Zoo", events[2].event_type) trait_filters[0].update({'key': 'trait_C', 'op': 'gt'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) + event_filter = storage.EventFilter(self.start, self.end, + traits_filter=trait_filters) events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(2, len(events)) self.assertEqual("Bar", events[0].event_type) trait_filters[0].update({'key': 'trait_C', 'op': 'ge'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) + event_filter = storage.EventFilter(self.start, self.end, + traits_filter=trait_filters) events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(3, len(events)) self.assertEqual("Zoo", events[2].event_type) @@ -301,39 +301,39 @@ class GetEventTest(EventTestBase): trait_filters = [{'key': 'trait_D', 'datetime': self.start + datetime.timedelta(hours=2), 'op': 'eq'}] - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) + event_filter = storage.EventFilter(self.start, self.end, + traits_filter=trait_filters) events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(1, len(events)) self.assertEqual("Zoo", events[0].event_type) self.assertEqual(4, len(events[0].traits)) trait_filters[0].update({'key': 'trait_D', 'op': 'lt'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) + event_filter = storage.EventFilter(self.start, self.end, + traits_filter=trait_filters) events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(2, len(events)) trait_filters[0].update({'key': 'trait_D', 'op': 'le'}) self.assertEqual("Bar", events[1].event_type) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) + event_filter = storage.EventFilter(self.start, self.end, + traits_filter=trait_filters) events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(3, len(events)) self.assertEqual("Bar", events[1].event_type) trait_filters[0].update({'key': 'trait_D', 'op': 'ne'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) + event_filter = storage.EventFilter(self.start, self.end, + traits_filter=trait_filters) events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(5, len(events)) self.assertEqual("Foo", events[2].event_type) trait_filters[0].update({'key': 'trait_D', 'op': 'gt'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) + event_filter = storage.EventFilter(self.start, self.end, + traits_filter=trait_filters) events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(3, len(events)) self.assertEqual("Zoo", events[2].event_type) trait_filters[0].update({'key': 'trait_D', 'op': 'ge'}) - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) + event_filter = storage.EventFilter(self.start, self.end, + traits_filter=trait_filters) events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(4, len(events)) self.assertEqual("Bar", events[2].event_type) @@ -342,8 +342,8 @@ class GetEventTest(EventTestBase): trait_filters = [{'key': 'trait_B', 'integer': 1}, {'key': 'trait_A', 'string': 'my_Foo_text'}, {'key': 'trait_C', 'float': 0.123456}] - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) + event_filter = storage.EventFilter(self.start, self.end, + traits_filter=trait_filters) events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(1, len(events)) self.assertEqual("Foo", events[0].event_type) @@ -352,8 +352,8 @@ class GetEventTest(EventTestBase): def test_get_event_multiple_trait_filter_expect_none(self): trait_filters = [{'key': 'trait_B', 'integer': 1}, {'key': 'trait_A', 'string': 'my_Zoo_text'}] - event_filter = event_storage.EventFilter(self.start, self.end, - traits_filter=trait_filters) + event_filter = storage.EventFilter(self.start, self.end, + traits_filter=trait_filters) events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(0, len(events)) @@ -389,13 +389,13 @@ class GetEventTest(EventTestBase): trait_dict[trait.name] = trait.dtype self.assertIn("trait_A", trait_dict) - self.assertEqual(event_models.Trait.TEXT_TYPE, trait_dict["trait_A"]) + self.assertEqual(models.Trait.TEXT_TYPE, trait_dict["trait_A"]) self.assertIn("trait_B", trait_dict) - self.assertEqual(event_models.Trait.INT_TYPE, trait_dict["trait_B"]) + self.assertEqual(models.Trait.INT_TYPE, trait_dict["trait_B"]) self.assertIn("trait_C", trait_dict) - self.assertEqual(event_models.Trait.FLOAT_TYPE, trait_dict["trait_C"]) + self.assertEqual(models.Trait.FLOAT_TYPE, trait_dict["trait_C"]) self.assertIn("trait_D", trait_dict) - self.assertEqual(event_models.Trait.DATETIME_TYPE, + self.assertEqual(models.Trait.DATETIME_TYPE, trait_dict["trait_D"]) def test_get_all_traits(self): @@ -404,13 +404,13 @@ class GetEventTest(EventTestBase): self.assertEqual(8, len(traits)) trait = traits[0] self.assertEqual("trait_A", trait.name) - self.assertEqual(event_models.Trait.TEXT_TYPE, trait.dtype) + self.assertEqual(models.Trait.TEXT_TYPE, trait.dtype) def test_simple_get_event_no_traits(self): - new_events = [event_models.Event("id_notraits", "NoTraits", + new_events = [models.Event("id_notraits", "NoTraits", self.start, [], {})] self.event_conn.record_events(new_events) - event_filter = event_storage.EventFilter( + event_filter = storage.EventFilter( self.start, self.end, "NoTraits") events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(1, len(events)) @@ -419,41 +419,39 @@ class GetEventTest(EventTestBase): self.assertEqual(0, len(events[0].traits)) def test_simple_get_no_filters(self): - event_filter = event_storage.EventFilter(None, None, None) + event_filter = storage.EventFilter(None, None, None) events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(6, len(events)) def test_get_by_message_id(self): - new_events = [event_models.Event("id_testid", - "MessageIDTest", - self.start, - [], {})] + new_events = [models.Event("id_testid", "MessageIDTest", + self.start, [], {})] self.event_conn.record_events(new_events) - event_filter = event_storage.EventFilter(message_id="id_testid") + event_filter = storage.EventFilter(message_id="id_testid") events = [event for event in self.event_conn.get_events(event_filter)] self.assertEqual(1, len(events)) event = events[0] self.assertEqual("id_testid", event.message_id) def test_simple_get_raw(self): - event_filter = event_storage.EventFilter() + event_filter = storage.EventFilter() events = [event for event in self.event_conn.get_events(event_filter)] self.assertTrue(events) self.assertEqual({'status': {'nested': 'started'}}, events[0].raw) def test_trait_type_enforced_on_none(self): - new_events = [event_models.Event( + new_events = [models.Event( "id_testid", "MessageIDTest", self.start, - [event_models.Trait('text', event_models.Trait.TEXT_TYPE, ''), - event_models.Trait('int', event_models.Trait.INT_TYPE, 0), - event_models.Trait('float', event_models.Trait.FLOAT_TYPE, 0.0)], + [models.Trait('text', models.Trait.TEXT_TYPE, ''), + models.Trait('int', models.Trait.INT_TYPE, 0), + models.Trait('float', models.Trait.FLOAT_TYPE, 0.0)], {})] self.event_conn.record_events(new_events) - event_filter = event_storage.EventFilter(message_id="id_testid") + event_filter = storage.EventFilter(message_id="id_testid") events = [event for event in self.event_conn.get_events(event_filter)] - options = [(event_models.Trait.TEXT_TYPE, ''), - (event_models.Trait.INT_TYPE, 0.0), - (event_models.Trait.FLOAT_TYPE, 0.0)] + options = [(models.Trait.TEXT_TYPE, ''), + (models.Trait.INT_TYPE, 0.0), + (models.Trait.FLOAT_TYPE, 0.0)] for trait in events[0].traits: options.remove((trait.dtype, trait.value)) diff --git a/panko/tests/unit/dispatcher/test_db.py b/panko/tests/unit/dispatcher/test_db.py index 0abcfab1..8c66bbae 100644 --- a/panko/tests/unit/dispatcher/test_db.py +++ b/panko/tests/unit/dispatcher/test_db.py @@ -19,8 +19,8 @@ from oslo_utils import uuidutils from oslotest import base from panko.dispatcher import database -from panko.event.storage import models as event_models from panko import service +from panko.storage import models class TestDispatcherDB(base.BaseTestCase): @@ -34,9 +34,9 @@ class TestDispatcherDB(base.BaseTestCase): self.dispatcher = database.DatabaseDispatcher(None) def test_event_conn(self): - event = event_models.Event(uuidutils.generate_uuid(), 'test', - datetime.datetime(2012, 7, 2, 13, 53, 40), - [], {}).serialize() + event = models.Event(uuidutils.generate_uuid(), 'test', + datetime.datetime(2012, 7, 2, 13, 53, 40), + [], {}).serialize() with mock.patch.object(self.dispatcher.event_conn, 'record_events') as record_events: self.dispatcher.record_events(event) diff --git a/panko/tests/unit/storage/test_get_connection.py b/panko/tests/unit/storage/test_get_connection.py index b4c771b0..1de79688 100644 --- a/panko/tests/unit/storage/test_get_connection.py +++ b/panko/tests/unit/storage/test_get_connection.py @@ -17,13 +17,12 @@ import mock from oslotest import base +import six -from panko.event.storage import impl_log -from panko.event.storage import impl_sqlalchemy from panko import service from panko import storage - -import six +from panko.storage import impl_log +from panko.storage import impl_sqlalchemy class EngineTest(base.BaseTestCase): diff --git a/setup.cfg b/setup.cfg index f3a46f0b..17a65feb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,14 +29,14 @@ packages = panko [entry_points] -panko.event.storage = - es = panko.event.storage.impl_elasticsearch:Connection - log = panko.event.storage.impl_log:Connection - mongodb = panko.event.storage.impl_mongodb:Connection - mysql = panko.event.storage.impl_sqlalchemy:Connection - postgresql = panko.event.storage.impl_sqlalchemy:Connection - sqlite = panko.event.storage.impl_sqlalchemy:Connection - hbase = panko.event.storage.impl_hbase:Connection +panko.storage = + es = panko.storage.impl_elasticsearch:Connection + log = panko.storage.impl_log:Connection + mongodb = panko.storage.impl_mongodb:Connection + mysql = panko.storage.impl_sqlalchemy:Connection + postgresql = panko.storage.impl_sqlalchemy:Connection + sqlite = panko.storage.impl_sqlalchemy:Connection + hbase = panko.storage.impl_hbase:Connection console_scripts = panko-dbsync = panko.cmd.storage:dbsync diff --git a/tools/make_test_event_data.py b/tools/make_test_event_data.py index 00e5bd6e..6c6cd808 100755 --- a/tools/make_test_event_data.py +++ b/tools/make_test_event_data.py @@ -28,9 +28,9 @@ import random from oslo_utils import timeutils from oslo_utils import uuidutils -from panko.event.storage import models from panko import service from panko import storage +from panko.storage import models def make_test_data(conn, start, end, interval, event_types): diff --git a/tools/test_hbase_table_utils.py b/tools/test_hbase_table_utils.py deleted file mode 100755 index d2874dc0..00000000 --- a/tools/test_hbase_table_utils.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os -import sys - -from panko import service -from panko import storage - - -def main(argv): - if os.getenv("PANKO_TEST_STORAGE_URL", "").startswith("hbase://"): - url = ("%s?table_prefix=%s" % - (os.getenv("PANKO_TEST_STORAGE_URL"), - os.getenv("PANKO_TEST_HBASE_TABLE_PREFIX", "test"))) - event_conn = storage.get_connection(url, service.prepare_service()) - for arg in argv: - if arg == "--upgrade": - event_conn.upgrade() - if arg == "--clear": - event_conn.clear() - - -if __name__ == '__main__': - main(sys.argv[1:])