From a828ec96cd38a7da404a9205d5b6b9e8668fbae5 Mon Sep 17 00:00:00 2001 From: Flavio Percoco Date: Mon, 21 Oct 2013 16:47:03 +0200 Subject: [PATCH] Isolate tests a bit more This patch isolates mongodb tests and adds new `shortcut-methods` to base clases in order to make it easier for subclasses to access conf instances, override them or even hook them. Change-Id: I8fe05bf383198244616499adb60480859f1ec7c6 --- marconi/tests/base.py | 8 +- marconi/tests/queues/storage/base.py | 17 ++++- .../{test_catalogue.py => _test_catalogue.py} | 0 ...test_partitions.py => _test_partitions.py} | 0 .../proxy/{test_queues.py => _test_queues.py} | 0 tests/unit/proxy/base.py | 2 +- ...st_impl_memory.py => _test_impl_memory.py} | 0 ..._impl_mongodb.py => _test_impl_mongodb.py} | 0 .../unit/queues/storage/test_impl_mongodb.py | 75 +++++++++---------- .../unit/queues/storage/test_shard_catalog.py | 11 +-- tests/unit/queues/transport/wsgi/base.py | 11 ++- tests/unit/queues/transport/wsgi/test_auth.py | 2 +- .../unit/queues/transport/wsgi/test_claims.py | 6 +- .../unit/queues/transport/wsgi/test_health.py | 2 +- tests/unit/queues/transport/wsgi/test_home.py | 2 +- .../queues/transport/wsgi/test_media_type.py | 2 +- .../queues/transport/wsgi/test_messages.py | 8 +- .../transport/wsgi/test_queue_lifecycle.py | 8 +- .../unit/queues/transport/wsgi/test_shards.py | 2 +- .../queues/transport/wsgi/test_validation.py | 2 +- .../wsgi/wsgi/test_default_limits.py | 2 +- tests/unit/test_bootstrap.py | 14 ++-- 22 files changed, 93 insertions(+), 81 deletions(-) rename tests/unit/proxy/{test_catalogue.py => _test_catalogue.py} (100%) rename tests/unit/proxy/{test_partitions.py => _test_partitions.py} (100%) rename tests/unit/proxy/{test_queues.py => _test_queues.py} (100%) rename tests/unit/proxy/storage/{test_impl_memory.py => _test_impl_memory.py} (100%) rename tests/unit/proxy/storage/{test_impl_mongodb.py => _test_impl_mongodb.py} (100%) diff --git a/marconi/tests/base.py b/marconi/tests/base.py index cb43e8360..26889590b 100644 --- a/marconi/tests/base.py +++ b/marconi/tests/base.py @@ -28,6 +28,8 @@ class TestBase(testtools.TestCase): test method. """ + config_file = None + def setUp(self): super(TestBase, self).setUp() @@ -38,8 +40,10 @@ class TestBase(testtools.TestCase): stderr = self.useFixture(fixtures.StringStream('stderr')).stream self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr)) - def tearDown(self): - super(TestBase, self).tearDown() + if self.config_file: + self.conf = self.load_conf(self.config_file) + else: + self.conf = cfg.ConfigOpts() @classmethod def conf_path(cls, filename): diff --git a/marconi/tests/queues/storage/base.py b/marconi/tests/queues/storage/base.py index edeab68aa..19fc9d0d3 100644 --- a/marconi/tests/queues/storage/base.py +++ b/marconi/tests/queues/storage/base.py @@ -18,7 +18,6 @@ import time import uuid import ddt -from oslo.config import cfg import six from testtools import matchers @@ -47,9 +46,23 @@ class ControllerBaseTest(testing.TestBase): self.controller_class, self.controller_base_class)) - self.driver = self.driver_class(cfg.ConfigOpts()) + self.driver = self.driver_class(self.conf) + self._prepare_conf() + + self.addCleanup(self._purge_databases) + self.controller = self.controller_class(self.driver) + def _prepare_conf(self): + """Prepare the conf before running tests + + Classes overriding this method, must use + the `self.conf` instance and alter its state. + """ + + def _purge_databases(self): + """Override to clean databases.""" + def tearDown(self): timeutils.clear_time_override() super(ControllerBaseTest, self).tearDown() diff --git a/tests/unit/proxy/test_catalogue.py b/tests/unit/proxy/_test_catalogue.py similarity index 100% rename from tests/unit/proxy/test_catalogue.py rename to tests/unit/proxy/_test_catalogue.py diff --git a/tests/unit/proxy/test_partitions.py b/tests/unit/proxy/_test_partitions.py similarity index 100% rename from tests/unit/proxy/test_partitions.py rename to tests/unit/proxy/_test_partitions.py diff --git a/tests/unit/proxy/test_queues.py b/tests/unit/proxy/_test_queues.py similarity index 100% rename from tests/unit/proxy/test_queues.py rename to tests/unit/proxy/_test_queues.py diff --git a/tests/unit/proxy/base.py b/tests/unit/proxy/base.py index 5e5610ce3..86e9dc4d1 100644 --- a/tests/unit/proxy/base.py +++ b/tests/unit/proxy/base.py @@ -25,7 +25,7 @@ from tests.unit.queues.transport.wsgi import base class TestBase(base.TestBase): - config_filename = "wsgi_proxy_memory.conf" + config_file = "wsgi_proxy_memory.conf" @classmethod def setUpClass(cls): diff --git a/tests/unit/proxy/storage/test_impl_memory.py b/tests/unit/proxy/storage/_test_impl_memory.py similarity index 100% rename from tests/unit/proxy/storage/test_impl_memory.py rename to tests/unit/proxy/storage/_test_impl_memory.py diff --git a/tests/unit/proxy/storage/test_impl_mongodb.py b/tests/unit/proxy/storage/_test_impl_mongodb.py similarity index 100% rename from tests/unit/proxy/storage/test_impl_mongodb.py rename to tests/unit/proxy/storage/_test_impl_mongodb.py diff --git a/tests/unit/queues/storage/test_impl_mongodb.py b/tests/unit/queues/storage/test_impl_mongodb.py index 7702caad3..caf96806a 100644 --- a/tests/unit/queues/storage/test_impl_mongodb.py +++ b/tests/unit/queues/storage/test_impl_mongodb.py @@ -14,6 +14,7 @@ # limitations under the License. import time +import uuid import mock from pymongo import cursor @@ -25,11 +26,23 @@ from marconi.queues import storage from marconi.queues.storage import errors from marconi.queues.storage import mongodb from marconi.queues.storage.mongodb import controllers +from marconi.queues.storage.mongodb import options from marconi.queues.storage.mongodb import utils from marconi import tests as testing from marconi.tests.queues.storage import base +class MongodbTestMixin(object): + + def _purge_databases(self): + """Override to clean databases.""" + databases = (self.driver.message_databases + + [self.driver.queues_database]) + + for db in databases: + self.driver.connection.drop_database(db) + + class MongodbUtilsTest(testing.TestBase): def test_scope_queue_name(self): @@ -74,38 +87,31 @@ class MongodbUtilsTest(testing.TestBase): @testing.requires_mongodb -class MongodbDriverTest(testing.TestBase): +class MongodbDriverTest(testing.TestBase, MongodbTestMixin): - def setUp(self): - super(MongodbDriverTest, self).setUp() - self._conf = self.load_conf('wsgi_mongodb.conf') + config_file = 'wsgi_mongodb.conf' def test_db_instance(self): - driver = mongodb.DataDriver(self._conf) + driver = mongodb.DataDriver(self.conf) + + databases = (driver.message_databases + + [driver.queues_database]) - databases = driver.message_databases + [driver.queues_database] for db in databases: self.assertThat(db.name, matchers.StartsWith( driver.mongodb_conf.database)) @testing.requires_mongodb -class MongodbQueueTests(base.QueueControllerTest): +class MongodbQueueTests(base.QueueControllerTest, MongodbTestMixin): driver_class = mongodb.DataDriver + config_file = 'wsgi_mongodb.conf' controller_class = controllers.QueueController - def setUp(self): - super(MongodbQueueTests, self).setUp() - self.load_conf('wsgi_mongodb.conf') - - def tearDown(self): - self.controller._collection.drop() - - for collection in self.message_controller._collections: - collection.drop() - - super(MongodbQueueTests, self).tearDown() + def _prepare_conf(self): + self.config(options.MONGODB_GROUP, + database=uuid.uuid4().hex) def test_indexes(self): collection = self.controller._collection @@ -134,24 +140,18 @@ class MongodbQueueTests(base.QueueControllerTest): @testing.requires_mongodb -class MongodbMessageTests(base.MessageControllerTest): +class MongodbMessageTests(base.MessageControllerTest, MongodbTestMixin): driver_class = mongodb.DataDriver + config_file = 'wsgi_mongodb.conf' controller_class = controllers.MessageController # NOTE(kgriffs): MongoDB's TTL scavenger only runs once a minute gc_interval = 60 - def setUp(self): - super(MongodbMessageTests, self).setUp() - self.load_conf('wsgi_mongodb.conf') - - def tearDown(self): - self.queue_controller._collection.drop() - for collection in self.controller._collections: - collection.drop() - - super(MongodbMessageTests, self).tearDown() + def _prepare_conf(self): + self.config(options.MONGODB_GROUP, + database=uuid.uuid4().hex) def test_indexes(self): for collection in self.controller._collections: @@ -294,20 +294,15 @@ class MongodbMessageTests(base.MessageControllerTest): @testing.requires_mongodb -class MongodbClaimTests(base.ClaimControllerTest): +class MongodbClaimTests(base.ClaimControllerTest, MongodbTestMixin): + driver_class = mongodb.DataDriver + config_file = 'wsgi_mongodb.conf' controller_class = controllers.ClaimController - def setUp(self): - super(MongodbClaimTests, self).setUp() - self.load_conf('wsgi_mongodb.conf') - - def tearDown(self): - for collection in self.message_controller._collections: - collection.drop() - - self.queue_controller._collection.drop() - super(MongodbClaimTests, self).tearDown() + def _prepare_conf(self): + self.config(options.MONGODB_GROUP, + database=uuid.uuid4().hex) def test_claim_doesnt_exist(self): """Verifies that operations fail on expired/missing claims. diff --git a/tests/unit/queues/storage/test_shard_catalog.py b/tests/unit/queues/storage/test_shard_catalog.py index 236b3ba0c..eed16a177 100644 --- a/tests/unit/queues/storage/test_shard_catalog.py +++ b/tests/unit/queues/storage/test_shard_catalog.py @@ -29,14 +29,15 @@ from marconi import tests as testing # have shards/catalogue implementations. class TestShardCatalog(testing.TestBase): + config_file = 'wsgi_mongodb_sharded.conf' + @testing.requires_mongodb def setUp(self): super(TestShardCatalog, self).setUp() - conf = self.load_conf('wsgi_mongodb_sharded.conf') - conf.register_opts([cfg.StrOpt('storage')], - group='queues:drivers') - control = utils.load_storage_driver(conf, control_mode=True) + self.conf.register_opts([cfg.StrOpt('storage')], + group='queues:drivers') + control = utils.load_storage_driver(self.conf, control_mode=True) self.catalogue_ctrl = control.catalogue_controller self.shards_ctrl = control.shards_controller @@ -46,7 +47,7 @@ class TestShardCatalog(testing.TestBase): self.project = str(uuid.uuid1()) self.shards_ctrl.create(self.shard, 100, 'sqlite://memory') self.catalogue_ctrl.insert(self.project, self.queue, self.shard) - self.catalog = sharding.Catalog(conf, control) + self.catalog = sharding.Catalog(self.conf, control) def tearDown(self): self.catalogue_ctrl.drop_all() diff --git a/tests/unit/queues/transport/wsgi/base.py b/tests/unit/queues/transport/wsgi/base.py index 328c07080..4badbedea 100644 --- a/tests/unit/queues/transport/wsgi/base.py +++ b/tests/unit/queues/transport/wsgi/base.py @@ -23,22 +23,21 @@ from marconi import tests as testing class TestBase(testing.TestBase): - config_filename = None + config_file = None def setUp(self): - if self.config_filename is None: - self.skipTest('No config specified') - super(TestBase, self).setUp() - self.conf = self.load_conf(self.conf_path(self.config_filename)) + if not self.config_file: + self.skipTest("No config specified") + self.conf.register_opts(driver._WSGI_OPTIONS, group=driver._WSGI_GROUP) - self.wsgi_cfg = self.conf[driver._WSGI_GROUP] self.conf.admin_mode = True self.boot = bootstrap.Bootstrap(self.conf) + self.app = self.boot.transport.app self.srmock = ftest.StartResponseMock() diff --git a/tests/unit/queues/transport/wsgi/test_auth.py b/tests/unit/queues/transport/wsgi/test_auth.py index 40988592e..feae55958 100644 --- a/tests/unit/queues/transport/wsgi/test_auth.py +++ b/tests/unit/queues/transport/wsgi/test_auth.py @@ -25,7 +25,7 @@ import base # noqa class TestWSGIAuth(base.TestBase): - config_filename = 'keystone_auth.conf' + config_file = 'keystone_auth.conf' def setUp(self): super(TestWSGIAuth, self).setUp() diff --git a/tests/unit/queues/transport/wsgi/test_claims.py b/tests/unit/queues/transport/wsgi/test_claims.py index a9137c14f..80ec33e89 100644 --- a/tests/unit/queues/transport/wsgi/test_claims.py +++ b/tests/unit/queues/transport/wsgi/test_claims.py @@ -235,7 +235,7 @@ class ClaimsBaseTest(base.TestBase): @testing.requires_mongodb class ClaimsMongoDBTests(ClaimsBaseTest): - config_filename = 'wsgi_mongodb.conf' + config_file = 'wsgi_mongodb.conf' def setUp(self): super(ClaimsMongoDBTests, self).setUp() @@ -254,12 +254,12 @@ class ClaimsMongoDBTests(ClaimsBaseTest): class ClaimsSQLiteTests(ClaimsBaseTest): - config_filename = 'wsgi_sqlite.conf' + config_file = 'wsgi_sqlite.conf' class ClaimsFaultyDriverTests(base.TestBaseFaulty): - config_filename = 'wsgi_faulty.conf' + config_file = 'wsgi_faulty.conf' def test_simple(self): project_id = '480924' diff --git a/tests/unit/queues/transport/wsgi/test_health.py b/tests/unit/queues/transport/wsgi/test_health.py index 70de6b7b4..b8a6bc021 100644 --- a/tests/unit/queues/transport/wsgi/test_health.py +++ b/tests/unit/queues/transport/wsgi/test_health.py @@ -21,7 +21,7 @@ import base # noqa class TestHealth(base.TestBase): - config_filename = 'wsgi_sqlite.conf' + config_file = 'wsgi_sqlite.conf' def test_get(self): response = self.simulate_get('/v1/health') diff --git a/tests/unit/queues/transport/wsgi/test_home.py b/tests/unit/queues/transport/wsgi/test_home.py index 3143bdd31..794da0f5b 100644 --- a/tests/unit/queues/transport/wsgi/test_home.py +++ b/tests/unit/queues/transport/wsgi/test_home.py @@ -23,7 +23,7 @@ import base # noqa class TestHomeDocument(base.TestBase): - config_filename = 'wsgi_sqlite.conf' + config_file = 'wsgi_sqlite.conf' def test_json_response(self): body = self.simulate_get('/v1') diff --git a/tests/unit/queues/transport/wsgi/test_media_type.py b/tests/unit/queues/transport/wsgi/test_media_type.py index 3811819a7..2a8c64491 100644 --- a/tests/unit/queues/transport/wsgi/test_media_type.py +++ b/tests/unit/queues/transport/wsgi/test_media_type.py @@ -25,7 +25,7 @@ import base # noqa @ddt.ddt class TestWSGIMediaType(base.TestBase): - config_filename = 'wsgi_sqlite.conf' + config_file = 'wsgi_sqlite.conf' @ddt.data( ('GET', '/v1/queues'), diff --git a/tests/unit/queues/transport/wsgi/test_messages.py b/tests/unit/queues/transport/wsgi/test_messages.py index f98835853..d499aa2ec 100644 --- a/tests/unit/queues/transport/wsgi/test_messages.py +++ b/tests/unit/queues/transport/wsgi/test_messages.py @@ -434,7 +434,7 @@ class MessagesBaseTest(base.TestBase): class MessagesSQLiteTests(MessagesBaseTest): - config_filename = 'wsgi_sqlite.conf' + config_file = 'wsgi_sqlite.conf' # TODO(cpp-cabrera): restore sqlite sharded test suite once shards and @@ -444,7 +444,7 @@ class MessagesSQLiteTests(MessagesBaseTest): @testing.requires_mongodb class MessagesMongoDBTests(MessagesBaseTest): - config_filename = 'wsgi_mongodb.conf' + config_file = 'wsgi_mongodb.conf' def setUp(self): super(MessagesMongoDBTests, self).setUp() @@ -456,7 +456,7 @@ class MessagesMongoDBTests(MessagesBaseTest): @testing.requires_mongodb class MessagesMongoDBShardedTests(MessagesBaseTest): - config_filename = 'wsgi_mongodb_sharded.conf' + config_file = 'wsgi_mongodb_sharded.conf' def setUp(self): super(MessagesMongoDBShardedTests, self).setUp() @@ -472,7 +472,7 @@ class MessagesMongoDBShardedTests(MessagesBaseTest): class MessagesFaultyDriverTests(base.TestBaseFaulty): - config_filename = 'wsgi_faulty.conf' + config_file = 'wsgi_faulty.conf' def test_simple(self): project_id = 'xyz' diff --git a/tests/unit/queues/transport/wsgi/test_queue_lifecycle.py b/tests/unit/queues/transport/wsgi/test_queue_lifecycle.py index 4a7ff5def..faba19865 100644 --- a/tests/unit/queues/transport/wsgi/test_queue_lifecycle.py +++ b/tests/unit/queues/transport/wsgi/test_queue_lifecycle.py @@ -27,7 +27,7 @@ from marconi import tests as testing @ddt.ddt class QueueLifecycleBaseTest(base.TestBase): - config_filename = None + config_file = None def setUp(self): super(QueueLifecycleBaseTest, self).setUp() @@ -307,7 +307,7 @@ class QueueLifecycleBaseTest(base.TestBase): class QueueLifecycleMongoDBTests(QueueLifecycleBaseTest): - config_filename = 'wsgi_mongodb.conf' + config_file = 'wsgi_mongodb.conf' @testing.requires_mongodb def setUp(self): @@ -327,12 +327,12 @@ class QueueLifecycleMongoDBTests(QueueLifecycleBaseTest): class QueueLifecycleSQLiteTests(QueueLifecycleBaseTest): - config_filename = 'wsgi_sqlite.conf' + config_file = 'wsgi_sqlite.conf' class QueueFaultyDriverTests(base.TestBaseFaulty): - config_filename = 'wsgi_faulty.conf' + config_file = 'wsgi_faulty.conf' def test_simple(self): path = '/v1/queues/gumshoe' diff --git a/tests/unit/queues/transport/wsgi/test_shards.py b/tests/unit/queues/transport/wsgi/test_shards.py index 7cbc2aabe..e118bb872 100644 --- a/tests/unit/queues/transport/wsgi/test_shards.py +++ b/tests/unit/queues/transport/wsgi/test_shards.py @@ -285,7 +285,7 @@ class ShardsBaseTest(base.TestBase): @testing.requires_mongodb class ShardsMongoDBTests(ShardsBaseTest): - config_filename = 'wsgi_mongodb.conf' + config_file = 'wsgi_mongodb.conf' def setUp(self): super(ShardsMongoDBTests, self).setUp() diff --git a/tests/unit/queues/transport/wsgi/test_validation.py b/tests/unit/queues/transport/wsgi/test_validation.py index 6b63aab7e..3a842d233 100644 --- a/tests/unit/queues/transport/wsgi/test_validation.py +++ b/tests/unit/queues/transport/wsgi/test_validation.py @@ -23,7 +23,7 @@ import base # noqa class ValidationTest(base.TestBase): - config_filename = 'wsgi_sqlite_validation.conf' + config_file = 'wsgi_sqlite_validation.conf' def setUp(self): super(ValidationTest, self).setUp() diff --git a/tests/unit/queues/transport/wsgi/wsgi/test_default_limits.py b/tests/unit/queues/transport/wsgi/wsgi/test_default_limits.py index 5aaa666ec..4296753c2 100644 --- a/tests/unit/queues/transport/wsgi/wsgi/test_default_limits.py +++ b/tests/unit/queues/transport/wsgi/wsgi/test_default_limits.py @@ -23,7 +23,7 @@ import base # noqa class DefaultLimitsTest(base.TestBase): - config_filename = 'wsgi_sqlite_default_limits.conf' + config_file = 'wsgi_sqlite_default_limits.conf' def setUp(self): super(DefaultLimitsTest, self).setUp() diff --git a/tests/unit/test_bootstrap.py b/tests/unit/test_bootstrap.py index a731131a1..5f703c690 100644 --- a/tests/unit/test_bootstrap.py +++ b/tests/unit/test_bootstrap.py @@ -29,25 +29,25 @@ class TestBootstrap(base.TestBase): return bootstrap.Bootstrap(self.conf) def test_storage_invalid(self): - boot = self._bootstrap('etc/drivers_storage_invalid.conf') + bootstrap = self._bootstrap('drivers_storage_invalid.conf') self.assertRaises(errors.InvalidDriver, - lambda: boot.storage) + lambda: bootstrap.storage) def test_storage_sqlite(self): - bootstrap = self._bootstrap('etc/wsgi_sqlite.conf') + bootstrap = self._bootstrap('wsgi_sqlite.conf') self.assertIsInstance(bootstrap.storage, pipeline.DataDriver) self.assertIsInstance(bootstrap.storage._storage, sqlite.DataDriver) def test_storage_sqlite_sharded(self): """Makes sure we can load the shard driver.""" - bootstrap = self._bootstrap('etc/wsgi_sqlite_sharded.conf') + bootstrap = self._bootstrap('wsgi_sqlite_sharded.conf') self.assertIsInstance(bootstrap.storage._storage, sharding.DataDriver) def test_transport_invalid(self): - boot = self._bootstrap('etc/drivers_transport_invalid.conf') + bootstrap = self._bootstrap('drivers_transport_invalid.conf') self.assertRaises(errors.InvalidDriver, - lambda: boot.transport) + lambda: bootstrap.transport) def test_transport_wsgi(self): - bootstrap = self._bootstrap('etc/wsgi_sqlite.conf') + bootstrap = self._bootstrap('wsgi_sqlite.conf') self.assertIsInstance(bootstrap.transport, wsgi.Driver)