From c46e29a5f93a4bb0cddfa60b6ebb5693c70cbecb Mon Sep 17 00:00:00 2001 From: Mike Fedosin Date: Tue, 30 Aug 2016 13:44:00 +0300 Subject: [PATCH] Add tests for db migrations Change-Id: I4b445c6b0b06d3c762426e16c52513c2510dda73 --- glare/common/exception.py | 4 + glare/tests/unit/__init__.py | 0 glare/tests/unit/db/__init__.py | 0 glare/tests/unit/db/migrations/__init__.py | 0 .../unit/db/migrations/test_migrations.py | 224 ++++++++++++++++++ glare/tests/unit/glare_fixtures.py | 40 ++++ glare/tests/unit/test_fixtures.py | 37 +++ 7 files changed, 305 insertions(+) create mode 100644 glare/tests/unit/__init__.py create mode 100644 glare/tests/unit/db/__init__.py create mode 100644 glare/tests/unit/db/migrations/__init__.py create mode 100644 glare/tests/unit/db/migrations/test_migrations.py create mode 100644 glare/tests/unit/glare_fixtures.py create mode 100644 glare/tests/unit/test_fixtures.py diff --git a/glare/common/exception.py b/glare/common/exception.py index 633e486..014327d 100644 --- a/glare/common/exception.py +++ b/glare/common/exception.py @@ -153,3 +153,7 @@ class SIGHUPInterrupt(GlareException): class WorkerCreationFailure(GlareException): message = _("Server worker creation failed: %(reason)s.") + + +class DBNotAllowed(GlareException): + msg_fmt = _('This operation is not allowed with current DB') diff --git a/glare/tests/unit/__init__.py b/glare/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/glare/tests/unit/db/__init__.py b/glare/tests/unit/db/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/glare/tests/unit/db/migrations/__init__.py b/glare/tests/unit/db/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/glare/tests/unit/db/migrations/test_migrations.py b/glare/tests/unit/db/migrations/test_migrations.py new file mode 100644 index 0000000..48e7ab0 --- /dev/null +++ b/glare/tests/unit/db/migrations/test_migrations.py @@ -0,0 +1,224 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Tests for database migrations. There are "opportunistic" tests for both mysql +and postgresql in here, which allows testing against these databases in a +properly configured unit test environment. +For the opportunistic testing you need to set up a db named 'openstack_citest' +with user 'openstack_citest' and password 'openstack_citest' on localhost. +The test will then use that db and u/p combo to run the tests. +For postgres on Ubuntu this can be done with the following commands: +:: + sudo -u postgres psql + postgres=# create user openstack_citest with createdb login password + 'openstack_citest'; + postgres=# create database openstack_citest with owner openstack_citest; +""" + +import contextlib + +from alembic import script +import mock +from oslo_db.sqlalchemy import test_base +from oslo_db.sqlalchemy import utils as db_utils +from oslo_log import log as logging +import sqlalchemy +import sqlalchemy.exc + +from glare.db.migration import migration +import glare.db.sqlalchemy.api +from glare.i18n import _LE +from glare.tests.unit import glare_fixtures + +LOG = logging.getLogger(__name__) + + +@contextlib.contextmanager +def patch_with_engine(engine): + with mock.patch.object(glare.db.sqlalchemy.api, + 'get_engine') as patch_engine: + patch_engine.return_value = engine + yield + + +class WalkVersionsMixin(object): + def _walk_versions(self, engine=None, alembic_cfg=None): + # Determine latest version script from the repo, then + # upgrade from 1 through to the latest, with no data + # in the databases. This just checks that the schema itself + # upgrades successfully. + + # Place the database under version control + with patch_with_engine(engine): + + script_directory = script.ScriptDirectory.from_config(alembic_cfg) + + self.assertIsNone(self.migration_api.version(engine)) + + versions = [ver for ver in script_directory.walk_revisions()] + + for version in reversed(versions): + with glare_fixtures.BannedDBSchemaOperations(): + self._migrate_up(engine, alembic_cfg, + version.revision, with_data=True) + + def _migrate_up(self, engine, config, version, with_data=False): + """migrate up to a new version of the db. + We allow for data insertion and post checks at every + migration version with special _pre_upgrade_### and + _check_### functions in the main test. + """ + try: + if with_data: + data = None + pre_upgrade = getattr( + self, "_pre_upgrade_%s" % version, None) + if pre_upgrade: + data = pre_upgrade(engine) + + self.migration_api.upgrade(version, config=config) + self.assertEqual(version, self.migration_api.version(engine)) + if with_data: + check = getattr(self, "_check_%s" % version, None) + if check: + check(engine, data) + except Exception: + LOG.error(_LE("Failed to migrate to version %(version)s on engine " + "%(engine)s"), + {'version': version, 'engine': engine}) + raise + + +class GlareMigrationsCheckers(object): + + def setUp(self): + super(GlareMigrationsCheckers, self).setUp() + self.config = migration.get_alembic_config() + self.migration_api = migration + + def assert_table(self, engine, table_name, indices, columns): + table = db_utils.get_table(engine, table_name) + index_data = [(index.name, index.columns.keys()) for index in + table.indexes] + column_data = [column.name for column in table.columns] + self.assertItemsEqual(columns, column_data) + self.assertItemsEqual(indices, index_data) + + def test_walk_versions(self): + self._walk_versions(self.engine, self.config) + + def _pre_upgrade_001(self, engine): + self.assertRaises(sqlalchemy.exc.NoSuchTableError, + db_utils.get_table, engine, + 'glare_artifacts') + self.assertRaises(sqlalchemy.exc.NoSuchTableError, + db_utils.get_table, engine, + 'glare_artifact_tags') + self.assertRaises(sqlalchemy.exc.NoSuchTableError, + db_utils.get_table, engine, + 'glare_artifact_properties') + self.assertRaises(sqlalchemy.exc.NoSuchTableError, + db_utils.get_table, engine, + 'glare_artifact_blobs') + + def _check_001(self, engine, data): + artifacts_indices = [('ix_glare_artifact_name_and_version', + ['name', 'version_prefix', 'version_suffix']), + ('ix_glare_artifact_type', + ['type_name']), + ('ix_glare_artifact_status', ['status']), + ('ix_glare_artifact_visibility', ['visibility']), + ('ix_glare_artifact_owner', ['owner'])] + artifacts_columns = ['id', + 'name', + 'type_name', + 'version_prefix', + 'version_suffix', + 'version_meta', + 'description', + 'visibility', + 'status', + 'owner', + 'created_at', + 'updated_at', + 'activated_at'] + self.assert_table(engine, 'glare_artifacts', artifacts_indices, + artifacts_columns) + + tags_indices = [('ix_glare_artifact_tags_artifact_id', + ['artifact_id']), + ('ix_glare_artifact_tags_artifact_id_tag_value', + ['artifact_id', + 'value'])] + tags_columns = ['id', + 'artifact_id', + 'value'] + self.assert_table(engine, 'glare_artifact_tags', tags_indices, + tags_columns) + + prop_indices = [ + ('ix_glare_artifact_properties_artifact_id', + ['artifact_id']), + ('ix_glare_artifact_properties_name', ['name'])] + prop_columns = ['id', + 'artifact_id', + 'name', + 'string_value', + 'int_value', + 'numeric_value', + 'bool_value', + 'key_name', + 'position'] + self.assert_table(engine, 'glare_artifact_properties', prop_indices, + prop_columns) + + blobs_indices = [ + ('ix_glare_artifact_blobs_artifact_id', ['artifact_id']), + ('ix_glare_artifact_blobs_name', ['name'])] + blobs_columns = ['id', + 'artifact_id', + 'size', + 'checksum', + 'name', + 'key_name', + 'external', + 'status', + 'content_type', + 'url'] + self.assert_table(engine, 'glare_artifact_blobs', blobs_indices, + blobs_columns) + + locks_indices = [] + locks_columns = ['id'] + self.assert_table(engine, 'glare_artifact_locks', locks_indices, + locks_columns) + + +class TestMigrationsMySQL(GlareMigrationsCheckers, + WalkVersionsMixin, + test_base.MySQLOpportunisticTestCase): + pass + + +class TestMigrationsPostgreSQL(GlareMigrationsCheckers, + WalkVersionsMixin, + test_base.PostgreSQLOpportunisticTestCase): + pass + + +class TestMigrationsSqlite(GlareMigrationsCheckers, + WalkVersionsMixin, + test_base.DbTestCase,): + pass diff --git a/glare/tests/unit/glare_fixtures.py b/glare/tests/unit/glare_fixtures.py new file mode 100644 index 0000000..926a520 --- /dev/null +++ b/glare/tests/unit/glare_fixtures.py @@ -0,0 +1,40 @@ +# Copyright (c) 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import fixtures + +from glare.common import exception + + +class BannedDBSchemaOperations(fixtures.Fixture): + """Ban some operations for migrations""" + def __init__(self, banned_resources=None): + super(BannedDBSchemaOperations, self).__init__() + self._banned_resources = banned_resources or [] + + @staticmethod + def _explode(resource, op): + raise exception.DBNotAllowed( + 'Operation %s.%s() is not allowed in a database migration' % ( + resource, op)) + + def setUp(self): + super(BannedDBSchemaOperations, self).setUp() + for thing in self._banned_resources: + self.useFixture(fixtures.MonkeyPatch( + 'sqlalchemy.%s.drop' % thing, + lambda *a, **k: self._explode(thing, 'drop'))) + self.useFixture(fixtures.MonkeyPatch( + 'sqlalchemy.%s.alter' % thing, + lambda *a, **k: self._explode(thing, 'alter'))) diff --git a/glare/tests/unit/test_fixtures.py b/glare/tests/unit/test_fixtures.py new file mode 100644 index 0000000..eddc127 --- /dev/null +++ b/glare/tests/unit/test_fixtures.py @@ -0,0 +1,37 @@ +# Copyright (c) 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy +import testtools + +from glare.common import exception +from glare.tests.unit import glare_fixtures + + +class TestBannedDBSchemaOperations(testtools.TestCase): + def test_column(self): + column = sqlalchemy.Column() + with glare_fixtures.BannedDBSchemaOperations(['Column']): + self.assertRaises(exception.DBNotAllowed, + column.drop) + self.assertRaises(exception.DBNotAllowed, + column.alter) + + def test_table(self): + table = sqlalchemy.Table() + with glare_fixtures.BannedDBSchemaOperations(['Table']): + self.assertRaises(exception.DBNotAllowed, + table.drop) + self.assertRaises(exception.DBNotAllowed, + table.alter)