diff --git a/oslo_db/sqlalchemy/migration.py b/oslo_db/sqlalchemy/migration.py new file mode 100644 index 00000000..b29b4daf --- /dev/null +++ b/oslo_db/sqlalchemy/migration.py @@ -0,0 +1,183 @@ +# coding=utf-8 + +# Copyright (c) 2013 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Base on code in migrate/changeset/databases/sqlite.py which is under +# the following license: +# +# The MIT License +# +# Copyright (c) 2009 Evan Rosson, Jan Dittberner, Domen Kožar +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import os + +from debtcollector import removals +from migrate import exceptions as versioning_exceptions +from migrate.versioning import api as versioning_api +from migrate.versioning.repository import Repository +import sqlalchemy + +from oslo_db._i18n import _ +from oslo_db import exception + + +_removed_msg = ( + 'sqlalchemy-migrate support in oslo_db is deprecated; consider ' + 'migrating to alembic' +) + + +@removals.remove(message=_removed_msg, version='8.3.0') +def db_sync(engine, abs_path, version=None, init_version=0, sanity_check=True): + """Upgrade or downgrade a database. + + Function runs the upgrade() or downgrade() functions in change scripts. + + :param engine: SQLAlchemy engine instance for a given database + :param abs_path: Absolute path to migrate repository. + :param version: Database will upgrade/downgrade until this version. + If None - database will update to the latest + available version. + :param init_version: Initial database version + :param sanity_check: Require schema sanity checking for all tables + """ + + if version is not None: + try: + version = int(version) + except ValueError: + raise exception.DBMigrationError(_("version should be an integer")) + + current_version = db_version(engine, abs_path, init_version) + repository = _find_migrate_repo(abs_path) + if sanity_check: + _db_schema_sanity_check(engine) + if version is None or version > current_version: + try: + migration = versioning_api.upgrade(engine, repository, version) + except Exception as ex: + raise exception.DBMigrationError(ex) + else: + migration = versioning_api.downgrade(engine, repository, + version) + if sanity_check: + _db_schema_sanity_check(engine) + + return migration + + +def _db_schema_sanity_check(engine): + """Ensure all database tables were created with required parameters. + + :param engine: SQLAlchemy engine instance for a given database + + """ + + if engine.name == 'mysql': + onlyutf8_sql = ('SELECT TABLE_NAME,TABLE_COLLATION ' + 'from information_schema.TABLES ' + 'where TABLE_SCHEMA=%s and ' + 'TABLE_COLLATION NOT LIKE \'%%utf8%%\'') + + # NOTE(morganfainberg): exclude the sqlalchemy-migrate and alembic + # versioning tables from the tables we need to verify utf8 status on. + # Non-standard table names are not supported. + EXCLUDED_TABLES = ['migrate_version', 'alembic_version'] + + table_names = [res[0] for res in + engine.execute(onlyutf8_sql, engine.url.database) if + res[0].lower() not in EXCLUDED_TABLES] + + if len(table_names) > 0: + raise ValueError(_('Tables "%s" have non utf8 collation, ' + 'please make sure all tables are CHARSET=utf8' + ) % ','.join(table_names)) + + +@removals.remove(message=_removed_msg, version='8.3.0') +def db_version(engine, abs_path, init_version): + """Show the current version of the repository. + + :param engine: SQLAlchemy engine instance for a given database + :param abs_path: Absolute path to migrate repository + :param init_version: Initial database version + """ + repository = _find_migrate_repo(abs_path) + try: + return versioning_api.db_version(engine, repository) + except versioning_exceptions.DatabaseNotControlledError: + meta = sqlalchemy.MetaData() + meta.reflect(bind=engine) + tables = meta.tables + if (len(tables) == 0 or 'alembic_version' in tables or + 'migrate_version' in tables): + db_version_control(engine, abs_path, version=init_version) + return versioning_api.db_version(engine, repository) + else: + raise exception.DBMigrationError( + _("The database is not under version control, but has " + "tables. Please stamp the current version of the schema " + "manually.")) + + +@removals.remove(message=_removed_msg, version='8.3.0') +def db_version_control(engine, abs_path, version=None): + """Mark a database as under this repository's version control. + + Once a database is under version control, schema changes should + only be done via change scripts in this repository. + + :param engine: SQLAlchemy engine instance for a given database + :param abs_path: Absolute path to migrate repository + :param version: Initial database version + """ + repository = _find_migrate_repo(abs_path) + + try: + versioning_api.version_control(engine, repository, version) + except versioning_exceptions.InvalidVersionError as ex: + raise exception.DBMigrationError("Invalid version : %s" % ex) + except versioning_exceptions.DatabaseAlreadyControlledError: + raise exception.DBMigrationError("Database is already controlled.") + + return version + + +def _find_migrate_repo(abs_path): + """Get the project's change script repository + + :param abs_path: Absolute path to migrate repository + """ + if not os.path.exists(abs_path): + raise exception.DBMigrationError("Path %s not found" % abs_path) + return Repository(abs_path) diff --git a/oslo_db/sqlalchemy/test_migrations.py b/oslo_db/sqlalchemy/test_migrations.py index e1cb295a..f50c2a36 100644 --- a/oslo_db/sqlalchemy/test_migrations.py +++ b/oslo_db/sqlalchemy/test_migrations.py @@ -28,12 +28,229 @@ import sqlalchemy.exc import sqlalchemy.sql.expression as expr import sqlalchemy.types as types +from oslo_db import exception as exc from oslo_db.sqlalchemy import provision from oslo_db.sqlalchemy import utils LOG = logging.getLogger(__name__) +class WalkVersionsMixin(object, metaclass=abc.ABCMeta): + """Test mixin to check upgrade and downgrade ability of migration. + + This is only suitable for testing of migrate_ migration scripts. An + abstract class mixin. `INIT_VERSION`, `REPOSITORY` and `migration_api` + attributes must be implemented in subclasses. + + .. _auxiliary-dynamic-methods: + + Auxiliary Methods: + + `migrate_up` and `migrate_down` instance methods of the class can be + used with auxiliary methods named `_pre_upgrade_`, + `_check_`, `_post_downgrade_`. The methods + intended to check applied changes for correctness of data operations. + This methods should be implemented for every particular revision + which you want to check with data. Implementation recommendations for + `_pre_upgrade_`, `_check_`, + `_post_downgrade_` implementation: + + * `_pre_upgrade_`: provide a data appropriate to + a next revision. Should be used an id of revision which + going to be applied. + + * `_check_`: Insert, select, delete operations + with newly applied changes. The data provided by + `_pre_upgrade_` will be used. + + * `_post_downgrade_`: check for absence + (inability to use) changes provided by reverted revision. + + Execution order of auxiliary methods when revision is upgrading: + + `_pre_upgrade_###` => `upgrade` => `_check_###` + + Execution order of auxiliary methods when revision is downgrading: + + `downgrade` => `_post_downgrade_###` + + .. _migrate: https://sqlalchemy-migrate.readthedocs.org/en/latest/ + + """ + + @property + @abc.abstractmethod + def INIT_VERSION(self): + """Initial version of a migration repository. + + Can be different from 0, if a migrations were squashed. + + :rtype: int + """ + pass + + @property + @abc.abstractmethod + def REPOSITORY(self): + """Allows basic manipulation with migration repository. + + :returns: `migrate.versioning.repository.Repository` subclass. + """ + pass + + @property + @abc.abstractmethod + def migration_api(self): + """Provides API for upgrading, downgrading and version manipulations. + + :returns: `migrate.api` or overloaded analog. + """ + pass + + @property + @abc.abstractmethod + def migrate_engine(self): + """Provides engine instance. + + Should be the same instance as used when migrations are applied. In + most cases, the `engine` attribute provided by the test class in a + `setUp` method will work. + + Example of implementation: + + def migrate_engine(self): + return self.engine + + :returns: sqlalchemy engine instance + """ + pass + + def walk_versions(self, snake_walk=False, downgrade=True): + """Check if migration upgrades and downgrades successfully. + + Determine the latest version script from the repo, then + upgrade from 1 through to the latest, with no data + in the databases. This just checks that the schema itself + upgrades successfully. + + `walk_versions` calls `migrate_up` and `migrate_down` with + `with_data` argument to check changes with data, but these methods + can be called without any extra check outside of `walk_versions` + method. + + :param snake_walk: enables checking that each individual migration can + be upgraded/downgraded by itself. + + If we have ordered migrations 123abc, 456def, 789ghi and we run + upgrading with the `snake_walk` argument set to `True`, the + migrations will be applied in the following order:: + + `123abc => 456def => 123abc => + 456def => 789ghi => 456def => 789ghi` + + :type snake_walk: bool + :param downgrade: Check downgrade behavior if True. + :type downgrade: bool + """ + + # Place the database under version control + self.migration_api.version_control(self.migrate_engine, + self.REPOSITORY, + self.INIT_VERSION) + self.assertEqual(self.INIT_VERSION, + self.migration_api.db_version(self.migrate_engine, + self.REPOSITORY)) + + LOG.debug('latest version is %s', self.REPOSITORY.latest) + versions = range(int(self.INIT_VERSION) + 1, + int(self.REPOSITORY.latest) + 1) + + for version in versions: + # upgrade -> downgrade -> upgrade + self.migrate_up(version, with_data=True) + if snake_walk: + downgraded = self.migrate_down(version - 1, with_data=True) + if downgraded: + self.migrate_up(version) + + if downgrade: + # Now walk it back down to 0 from the latest, testing + # the downgrade paths. + for version in reversed(versions): + # downgrade -> upgrade -> downgrade + downgraded = self.migrate_down(version - 1) + + if snake_walk and downgraded: + self.migrate_up(version) + self.migrate_down(version - 1) + + def migrate_down(self, version, with_data=False): + """Migrate down to a previous version of the db. + + :param version: id of revision to downgrade. + :type version: str + :keyword with_data: Whether to verify the absence of changes from + migration(s) being downgraded, see + :ref:`Auxiliary Methods `. + :type with_data: Bool + """ + + try: + self.migration_api.downgrade(self.migrate_engine, + self.REPOSITORY, version) + except NotImplementedError: + # NOTE(sirp): some migrations, namely release-level + # migrations, don't support a downgrade. + return False + + self.assertEqual(version, self.migration_api.db_version( + self.migrate_engine, self.REPOSITORY)) + + # NOTE(sirp): `version` is what we're downgrading to (i.e. the 'target' + # version). So if we have any downgrade checks, they need to be run for + # the previous (higher numbered) migration. + if with_data: + post_downgrade = getattr( + self, "_post_downgrade_%03d" % (version + 1), None) + if post_downgrade: + post_downgrade(self.migrate_engine) + + return True + + def migrate_up(self, version, with_data=False): + """Migrate up to a new version of the db. + + :param version: id of revision to upgrade. + :type version: str + :keyword with_data: Whether to verify the applied changes with data, + see :ref:`Auxiliary Methods `. + :type with_data: Bool + """ + # NOTE(sdague): try block is here because it's impossible to debug + # where a failed data migration happens otherwise + try: + if with_data: + data = None + pre_upgrade = getattr( + self, "_pre_upgrade_%03d" % version, None) + if pre_upgrade: + data = pre_upgrade(self.migrate_engine) + + self.migration_api.upgrade(self.migrate_engine, + self.REPOSITORY, version) + self.assertEqual(version, + self.migration_api.db_version(self.migrate_engine, + self.REPOSITORY)) + if with_data: + check = getattr(self, "_check_%03d" % version, None) + if check: + check(self.migrate_engine, data) + except exc.DBMigrationError: + msg = "Failed to migrate to version %(ver)s on engine %(eng)s" + LOG.error(msg, {"ver": version, "eng": self.migrate_engine}) + raise + + class ModelsMigrationsSync(object, metaclass=abc.ABCMeta): """A helper class for comparison of DB migration scripts and models. diff --git a/releasenotes/notes/readd-sqlalchemy-migrate-ea7af75ef6d2f48b.yaml b/releasenotes/notes/readd-sqlalchemy-migrate-ea7af75ef6d2f48b.yaml new file mode 100644 index 00000000..4325256b --- /dev/null +++ b/releasenotes/notes/readd-sqlalchemy-migrate-ea7af75ef6d2f48b.yaml @@ -0,0 +1,9 @@ +--- +upgrade: + - | + The ``oslo_db.sqlalchemy.migration`` module and ``WalkVersionsMixin`` + test mixin in the ``oslo_db.sqlalchemy.test_migrations``, which were + removed in 13.0.0, have been re-added temporarily to allow a longer + transition time for projects. These are still deprecated as + SQLAlchemy-Migrate is not compatible with SQLAlchemy 2.x. They will + be removed again in a future release. diff --git a/requirements.txt b/requirements.txt index 7764c23c..8842a41d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,7 @@ oslo.i18n>=3.15.3 # Apache-2.0 oslo.config>=5.2.0 # Apache-2.0 oslo.utils>=3.33.0 # Apache-2.0 SQLAlchemy>=1.4.0 # MIT +sqlalchemy-migrate>=0.11.0 # Apache-2.0 stevedore>=1.20.0 # Apache-2.0 # these are used by downstream libraries that require # oslo.db as one of their test requirements - do not remove!