[stable-only] Re-add migration helpers, tests

There are a number of projects that have yet to migrate from
SQLAlchemy-Migrate to Alembic and are relying on some old
SQLAlchemy-Migrate helpers. To prevent them blocking the release,
re-add the helpers (though not the tests for same since those won't pass
with SQLAlchemy 2.x). We only do this on the stable branches since we
are going to need to address SQLAlchemy 2.x support in C.

This is effectively a partial revert of change
Ic3d6bd318038d723b0d50d39e45f8e26289e9a57 but it only re-adds the test
helper rather than the entirety of oslo.db's former SQLAlchemy-Migrate
support.

Change-Id: Icdb6a83be28e731b3a99df88b20d27829bcf50be
Signed-off-by: Stephen Finucane <stephenfin@redhat.com>

oslo_db/sqlalchemy/migration.py

@@ -0,0 +1,183 @@
+# coding=utf-8
+
+# Copyright (c) 2013 OpenStack Foundation
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Base on code in migrate/changeset/databases/sqlite.py which is under
+# the following license:
+#
+# The MIT License
+#
+# Copyright (c) 2009 Evan Rosson, Jan Dittberner, Domen Kožar
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import os
+
+from debtcollector import removals
+from migrate import exceptions as versioning_exceptions
+from migrate.versioning import api as versioning_api
+from migrate.versioning.repository import Repository
+import sqlalchemy
+
+from oslo_db._i18n import _
+from oslo_db import exception
+
+
+_removed_msg = (
+    'sqlalchemy-migrate support in oslo_db is deprecated; consider '
+    'migrating to alembic'
+)
+
+
+@removals.remove(message=_removed_msg, version='8.3.0')
+def db_sync(engine, abs_path, version=None, init_version=0, sanity_check=True):
+    """Upgrade or downgrade a database.
+
+    Function runs the upgrade() or downgrade() functions in change scripts.
+
+    :param engine:       SQLAlchemy engine instance for a given database
+    :param abs_path:     Absolute path to migrate repository.
+    :param version:      Database will upgrade/downgrade until this version.
+                         If None - database will update to the latest
+                         available version.
+    :param init_version: Initial database version
+    :param sanity_check: Require schema sanity checking for all tables
+    """
+
+    if version is not None:
+        try:
+            version = int(version)
+        except ValueError:
+            raise exception.DBMigrationError(_("version should be an integer"))
+
+    current_version = db_version(engine, abs_path, init_version)
+    repository = _find_migrate_repo(abs_path)
+    if sanity_check:
+        _db_schema_sanity_check(engine)
+    if version is None or version > current_version:
+        try:
+            migration = versioning_api.upgrade(engine, repository, version)
+        except Exception as ex:
+            raise exception.DBMigrationError(ex)
+    else:
+        migration = versioning_api.downgrade(engine, repository,
+                                             version)
+    if sanity_check:
+        _db_schema_sanity_check(engine)
+
+    return migration
+
+
+def _db_schema_sanity_check(engine):
+    """Ensure all database tables were created with required parameters.
+
+    :param engine:  SQLAlchemy engine instance for a given database
+
+    """
+
+    if engine.name == 'mysql':
+        onlyutf8_sql = ('SELECT TABLE_NAME,TABLE_COLLATION '
+                        'from information_schema.TABLES '
+                        'where TABLE_SCHEMA=%s and '
+                        'TABLE_COLLATION NOT LIKE \'%%utf8%%\'')
+
+        # NOTE(morganfainberg): exclude the sqlalchemy-migrate and alembic
+        # versioning tables from the tables we need to verify utf8 status on.
+        # Non-standard table names are not supported.
+        EXCLUDED_TABLES = ['migrate_version', 'alembic_version']
+
+        table_names = [res[0] for res in
+                       engine.execute(onlyutf8_sql, engine.url.database) if
+                       res[0].lower() not in EXCLUDED_TABLES]
+
+        if len(table_names) > 0:
+            raise ValueError(_('Tables "%s" have non utf8 collation, '
+                               'please make sure all tables are CHARSET=utf8'
+                               ) % ','.join(table_names))
+
+
+@removals.remove(message=_removed_msg, version='8.3.0')
+def db_version(engine, abs_path, init_version):
+    """Show the current version of the repository.
+
+    :param engine:  SQLAlchemy engine instance for a given database
+    :param abs_path: Absolute path to migrate repository
+    :param init_version:  Initial database version
+    """
+    repository = _find_migrate_repo(abs_path)
+    try:
+        return versioning_api.db_version(engine, repository)
+    except versioning_exceptions.DatabaseNotControlledError:
+        meta = sqlalchemy.MetaData()
+        meta.reflect(bind=engine)
+        tables = meta.tables
+        if (len(tables) == 0 or 'alembic_version' in tables or
+                'migrate_version' in tables):
+            db_version_control(engine, abs_path, version=init_version)
+            return versioning_api.db_version(engine, repository)
+        else:
+            raise exception.DBMigrationError(
+                _("The database is not under version control, but has "
+                  "tables. Please stamp the current version of the schema "
+                  "manually."))
+
+
+@removals.remove(message=_removed_msg, version='8.3.0')
+def db_version_control(engine, abs_path, version=None):
+    """Mark a database as under this repository's version control.
+
+    Once a database is under version control, schema changes should
+    only be done via change scripts in this repository.
+
+    :param engine:  SQLAlchemy engine instance for a given database
+    :param abs_path: Absolute path to migrate repository
+    :param version:  Initial database version
+    """
+    repository = _find_migrate_repo(abs_path)
+
+    try:
+        versioning_api.version_control(engine, repository, version)
+    except versioning_exceptions.InvalidVersionError as ex:
+        raise exception.DBMigrationError("Invalid version : %s" % ex)
+    except versioning_exceptions.DatabaseAlreadyControlledError:
+        raise exception.DBMigrationError("Database is already controlled.")
+
+    return version
+
+
+def _find_migrate_repo(abs_path):
+    """Get the project's change script repository
+
+    :param abs_path: Absolute path to migrate repository
+    """
+    if not os.path.exists(abs_path):
+        raise exception.DBMigrationError("Path %s not found" % abs_path)
+    return Repository(abs_path)

oslo_db/sqlalchemy/test_migrations.py

@@ -28,12 +28,229 @@ import sqlalchemy.exc
 import sqlalchemy.sql.expression as expr
 import sqlalchemy.types as types
 
+from oslo_db import exception as exc
 from oslo_db.sqlalchemy import provision
 from oslo_db.sqlalchemy import utils
 
 LOG = logging.getLogger(__name__)
 
 
+class WalkVersionsMixin(object, metaclass=abc.ABCMeta):
+    """Test mixin to check upgrade and downgrade ability of migration.
+
+    This is only suitable for testing of migrate_ migration scripts. An
+    abstract class mixin. `INIT_VERSION`, `REPOSITORY` and `migration_api`
+    attributes must be implemented in subclasses.
+
+    .. _auxiliary-dynamic-methods:
+
+    Auxiliary Methods:
+
+    `migrate_up` and `migrate_down` instance methods of the class can be
+    used with auxiliary methods named `_pre_upgrade_<revision_id>`,
+    `_check_<revision_id>`, `_post_downgrade_<revision_id>`. The methods
+    intended to check applied changes for correctness of data operations.
+    This methods should be implemented for every particular revision
+    which you want to check with data. Implementation recommendations for
+    `_pre_upgrade_<revision_id>`, `_check_<revision_id>`,
+    `_post_downgrade_<revision_id>` implementation:
+
+        * `_pre_upgrade_<revision_id>`: provide a data appropriate to
+           a next revision. Should be used an id of revision which
+           going to be applied.
+
+        * `_check_<revision_id>`: Insert, select, delete operations
+           with newly applied changes. The data provided by
+           `_pre_upgrade_<revision_id>` will be used.
+
+        * `_post_downgrade_<revision_id>`: check for absence
+          (inability to use) changes provided by reverted revision.
+
+    Execution order of auxiliary methods when revision is upgrading:
+
+        `_pre_upgrade_###` => `upgrade` => `_check_###`
+
+    Execution order of auxiliary methods when revision is downgrading:
+
+        `downgrade` => `_post_downgrade_###`
+
+    .. _migrate: https://sqlalchemy-migrate.readthedocs.org/en/latest/
+
+    """
+
+    @property
+    @abc.abstractmethod
+    def INIT_VERSION(self):
+        """Initial version of a migration repository.
+
+        Can be different from 0, if a migrations were squashed.
+
+        :rtype: int
+        """
+        pass
+
+    @property
+    @abc.abstractmethod
+    def REPOSITORY(self):
+        """Allows basic manipulation with migration repository.
+
+        :returns: `migrate.versioning.repository.Repository` subclass.
+        """
+        pass
+
+    @property
+    @abc.abstractmethod
+    def migration_api(self):
+        """Provides API for upgrading, downgrading and version manipulations.
+
+        :returns: `migrate.api` or overloaded analog.
+        """
+        pass
+
+    @property
+    @abc.abstractmethod
+    def migrate_engine(self):
+        """Provides engine instance.
+
+        Should be the same instance as used when migrations are applied. In
+        most cases, the `engine` attribute provided by the test class in a
+        `setUp` method will work.
+
+        Example of implementation:
+
+            def migrate_engine(self):
+                return self.engine
+
+        :returns: sqlalchemy engine instance
+        """
+        pass
+
+    def walk_versions(self, snake_walk=False, downgrade=True):
+        """Check if migration upgrades and downgrades successfully.
+
+        Determine the latest version script from the repo, then
+        upgrade from 1 through to the latest, with no data
+        in the databases. This just checks that the schema itself
+        upgrades successfully.
+
+        `walk_versions` calls `migrate_up` and `migrate_down` with
+        `with_data` argument to check changes with data, but these methods
+        can be called without any extra check outside of `walk_versions`
+        method.
+
+        :param snake_walk: enables checking that each individual migration can
+            be upgraded/downgraded by itself.
+
+            If we have ordered migrations 123abc, 456def, 789ghi and we run
+            upgrading with the `snake_walk` argument set to `True`, the
+            migrations will be applied in the following order::
+
+                `123abc => 456def => 123abc =>
+                 456def => 789ghi => 456def => 789ghi`
+
+        :type snake_walk: bool
+        :param downgrade: Check downgrade behavior if True.
+        :type downgrade: bool
+        """
+
+        # Place the database under version control
+        self.migration_api.version_control(self.migrate_engine,
+                                           self.REPOSITORY,
+                                           self.INIT_VERSION)
+        self.assertEqual(self.INIT_VERSION,
+                         self.migration_api.db_version(self.migrate_engine,
+                                                       self.REPOSITORY))
+
+        LOG.debug('latest version is %s', self.REPOSITORY.latest)
+        versions = range(int(self.INIT_VERSION) + 1,
+                         int(self.REPOSITORY.latest) + 1)
+
+        for version in versions:
+            # upgrade -> downgrade -> upgrade
+            self.migrate_up(version, with_data=True)
+            if snake_walk:
+                downgraded = self.migrate_down(version - 1, with_data=True)
+                if downgraded:
+                    self.migrate_up(version)
+
+        if downgrade:
+            # Now walk it back down to 0 from the latest, testing
+            # the downgrade paths.
+            for version in reversed(versions):
+                # downgrade -> upgrade -> downgrade
+                downgraded = self.migrate_down(version - 1)
+
+                if snake_walk and downgraded:
+                    self.migrate_up(version)
+                    self.migrate_down(version - 1)
+
+    def migrate_down(self, version, with_data=False):
+        """Migrate down to a previous version of the db.
+
+        :param version: id of revision to downgrade.
+        :type version: str
+        :keyword with_data: Whether to verify the absence of changes from
+            migration(s) being downgraded, see
+            :ref:`Auxiliary Methods <auxiliary-dynamic-methods>`.
+        :type with_data: Bool
+        """
+
+        try:
+            self.migration_api.downgrade(self.migrate_engine,
+                                         self.REPOSITORY, version)
+        except NotImplementedError:
+            # NOTE(sirp): some migrations, namely release-level
+            # migrations, don't support a downgrade.
+            return False
+
+        self.assertEqual(version, self.migration_api.db_version(
+            self.migrate_engine, self.REPOSITORY))
+
+        # NOTE(sirp): `version` is what we're downgrading to (i.e. the 'target'
+        # version). So if we have any downgrade checks, they need to be run for
+        # the previous (higher numbered) migration.
+        if with_data:
+            post_downgrade = getattr(
+                self, "_post_downgrade_%03d" % (version + 1), None)
+            if post_downgrade:
+                post_downgrade(self.migrate_engine)
+
+        return True
+
+    def migrate_up(self, version, with_data=False):
+        """Migrate up to a new version of the db.
+
+        :param version: id of revision to upgrade.
+        :type version: str
+        :keyword with_data: Whether to verify the applied changes with data,
+            see :ref:`Auxiliary Methods <auxiliary-dynamic-methods>`.
+        :type with_data: Bool
+        """
+        # NOTE(sdague): try block is here because it's impossible to debug
+        # where a failed data migration happens otherwise
+        try:
+            if with_data:
+                data = None
+                pre_upgrade = getattr(
+                    self, "_pre_upgrade_%03d" % version, None)
+                if pre_upgrade:
+                    data = pre_upgrade(self.migrate_engine)
+
+            self.migration_api.upgrade(self.migrate_engine,
+                                       self.REPOSITORY, version)
+            self.assertEqual(version,
+                             self.migration_api.db_version(self.migrate_engine,
+                                                           self.REPOSITORY))
+            if with_data:
+                check = getattr(self, "_check_%03d" % version, None)
+                if check:
+                    check(self.migrate_engine, data)
+        except exc.DBMigrationError:
+            msg = "Failed to migrate to version %(ver)s on engine %(eng)s"
+            LOG.error(msg, {"ver": version, "eng": self.migrate_engine})
+            raise
+
+
 class ModelsMigrationsSync(object, metaclass=abc.ABCMeta):
     """A helper class for comparison of DB migration scripts and models.
 

releasenotes/notes/readd-sqlalchemy-migrate-ea7af75ef6d2f48b.yaml

@@ -0,0 +1,9 @@
+---
+upgrade:
+  - |
+    The ``oslo_db.sqlalchemy.migration`` module and ``WalkVersionsMixin``
+    test mixin in the ``oslo_db.sqlalchemy.test_migrations``, which were
+    removed in 13.0.0, have been re-added temporarily to allow a longer
+    transition time for projects. These are still deprecated as
+    SQLAlchemy-Migrate is not compatible with SQLAlchemy 2.x. They will
+    be removed again in a future release.

requirements.txt

@@ -9,6 +9,7 @@ oslo.i18n>=3.15.3 # Apache-2.0
 oslo.config>=5.2.0 # Apache-2.0
 oslo.utils>=3.33.0 # Apache-2.0
 SQLAlchemy>=1.4.0 # MIT
+sqlalchemy-migrate>=0.11.0 # Apache-2.0
 stevedore>=1.20.0 # Apache-2.0
 # these are used by downstream libraries that require
 # oslo.db as one of their test requirements - do not remove!