diff --git a/doc/source/contributor/database-migrations.rst b/doc/source/contributor/database-migrations.rst index f8e8a7a5e3..3827ea8e61 100644 --- a/doc/source/contributor/database-migrations.rst +++ b/doc/source/contributor/database-migrations.rst @@ -17,10 +17,17 @@ Database Migrations =================== +.. note:: + + The framework being used is currently being migrated from + SQLAlchemy-Migrate to Alembic, meaning this information will change in the + near-term. + Starting with Newton, keystone supports upgrading both with and without downtime. In order to support this, there are three separate migration -repositories (all under ``keystone/common/sql/``) that match the three phases -of an upgrade (schema expansion, data migration, and schema contraction): +repositories (all under ``keystone/common/sql/legacy_migrations``) that match +the three phases of an upgrade (schema expansion, data migration, and schema +contraction): ``expand_repo`` For additive schema modifications and triggers to ensure data is kept in diff --git a/doc/source/contributor/programming-exercises.rst b/doc/source/contributor/programming-exercises.rst index 5af344467d..b51725d088 100644 --- a/doc/source/contributor/programming-exercises.rst +++ b/doc/source/contributor/programming-exercises.rst @@ -53,9 +53,9 @@ Refer to the :doc:`API Change tutorial `. In short, you wil steps: #. Create a SQL migration to add the parameter to the database table - (:py:mod:`keystone.common.sql.expand_repo.versions`, - :py:mod:`keystone.common.sql.data_migration_repo.versions`, - :py:mod:`keystone.common.sql.contract_repo.versions`) + (:py:mod:`keystone.common.sql.legacy_migration.expand_repo.versions`, + :py:mod:`keystone.common.sql.legacy_migration.data_migration_repo.versions`, + :py:mod:`keystone.common.sql.legacy_migration.contract_repo.versions`) #. Add a SQL migration unit test (`keystone/tests/unit/test_sql_upgrade.py`) diff --git a/doc/source/contributor/testing-keystone.rst b/doc/source/contributor/testing-keystone.rst index 893d67e2a4..72575fbcbd 100644 --- a/doc/source/contributor/testing-keystone.rst +++ b/doc/source/contributor/testing-keystone.rst @@ -138,6 +138,12 @@ Identity module. Testing Schema Migrations ------------------------- +.. note:: + + The framework being used is currently being migrated from + SQLAlchemy-Migrate to Alembic, meaning this information will change in the + near-term. + The application of schema migrations can be tested using SQLAlchemy Migrate's built-in test runner, one migration at a time. @@ -151,9 +157,9 @@ version control: .. code-block:: bash - $ python keystone/common/sql/expand_repo/manage.py test \ + $ python keystone/common/sql/legacy_migrations/expand_repo/manage.py test \ --url=sqlite:///test.db \ - --repository=keystone/common/sql/expand_repo/ + --repository=keystone/common/sql/legacy_migrations/expand_repo/ This command references to a SQLite database (test.db) to be used. Depending on the migration, this command alone does not make assertions as to the integrity diff --git a/keystone/common/sql/contract_repo/README b/keystone/common/sql/contract_repo/README deleted file mode 100644 index 1311171041..0000000000 --- a/keystone/common/sql/contract_repo/README +++ /dev/null @@ -1,4 +0,0 @@ -This is a database migration repository. - -More information at -https://opendev.org/openstack/sqlalchemy-migrate diff --git a/keystone/common/sql/data_migration_repo/README b/keystone/common/sql/data_migration_repo/README deleted file mode 100644 index 1311171041..0000000000 --- a/keystone/common/sql/data_migration_repo/README +++ /dev/null @@ -1,4 +0,0 @@ -This is a database migration repository. - -More information at -https://opendev.org/openstack/sqlalchemy-migrate diff --git a/keystone/common/sql/expand_repo/README b/keystone/common/sql/expand_repo/README deleted file mode 100644 index 1311171041..0000000000 --- a/keystone/common/sql/expand_repo/README +++ /dev/null @@ -1,4 +0,0 @@ -This is a database migration repository. - -More information at -https://opendev.org/openstack/sqlalchemy-migrate diff --git a/keystone/common/sql/contract_repo/__init__.py b/keystone/common/sql/legacy_migrations/__init__.py similarity index 100% rename from keystone/common/sql/contract_repo/__init__.py rename to keystone/common/sql/legacy_migrations/__init__.py diff --git a/keystone/common/sql/legacy_migrations/contract_repo/README.rst b/keystone/common/sql/legacy_migrations/contract_repo/README.rst new file mode 100644 index 0000000000..6ecb178cca --- /dev/null +++ b/keystone/common/sql/legacy_migrations/contract_repo/README.rst @@ -0,0 +1,13 @@ +Contract repo migrations +======================== + +.. warning:: + + This repo is deprecated and will be removed in a future release. All new + migrations should be alembic-based and placed in + ``keystone/common/sql/migrations``. + +Contract-style or destructive migrations for the database. + +This is a database migration repository. More information at +https://opendev.org/x/sqlalchemy-migrate diff --git a/keystone/common/sql/contract_repo/versions/__init__.py b/keystone/common/sql/legacy_migrations/contract_repo/__init__.py similarity index 100% rename from keystone/common/sql/contract_repo/versions/__init__.py rename to keystone/common/sql/legacy_migrations/contract_repo/__init__.py diff --git a/keystone/common/sql/contract_repo/manage.py b/keystone/common/sql/legacy_migrations/contract_repo/manage.py similarity index 100% rename from keystone/common/sql/contract_repo/manage.py rename to keystone/common/sql/legacy_migrations/contract_repo/manage.py diff --git a/keystone/common/sql/contract_repo/migrate.cfg b/keystone/common/sql/legacy_migrations/contract_repo/migrate.cfg similarity index 100% rename from keystone/common/sql/contract_repo/migrate.cfg rename to keystone/common/sql/legacy_migrations/contract_repo/migrate.cfg diff --git a/keystone/common/sql/contract_repo/versions/073_contract_initial_migration.py b/keystone/common/sql/legacy_migrations/contract_repo/versions/073_contract_initial_migration.py similarity index 100% rename from keystone/common/sql/contract_repo/versions/073_contract_initial_migration.py rename to keystone/common/sql/legacy_migrations/contract_repo/versions/073_contract_initial_migration.py diff --git a/keystone/common/sql/contract_repo/versions/074_placeholder.py b/keystone/common/sql/legacy_migrations/contract_repo/versions/074_placeholder.py similarity index 100% rename from keystone/common/sql/contract_repo/versions/074_placeholder.py rename to keystone/common/sql/legacy_migrations/contract_repo/versions/074_placeholder.py diff --git a/keystone/common/sql/contract_repo/versions/075_placeholder.py b/keystone/common/sql/legacy_migrations/contract_repo/versions/075_placeholder.py similarity index 100% rename from keystone/common/sql/contract_repo/versions/075_placeholder.py rename to keystone/common/sql/legacy_migrations/contract_repo/versions/075_placeholder.py diff --git a/keystone/common/sql/contract_repo/versions/076_placeholder.py b/keystone/common/sql/legacy_migrations/contract_repo/versions/076_placeholder.py similarity index 100% rename from keystone/common/sql/contract_repo/versions/076_placeholder.py rename to keystone/common/sql/legacy_migrations/contract_repo/versions/076_placeholder.py diff --git a/keystone/common/sql/contract_repo/versions/077_placeholder.py b/keystone/common/sql/legacy_migrations/contract_repo/versions/077_placeholder.py similarity index 100% rename from keystone/common/sql/contract_repo/versions/077_placeholder.py rename to keystone/common/sql/legacy_migrations/contract_repo/versions/077_placeholder.py diff --git a/keystone/common/sql/contract_repo/versions/078_placeholder.py b/keystone/common/sql/legacy_migrations/contract_repo/versions/078_placeholder.py similarity index 100% rename from keystone/common/sql/contract_repo/versions/078_placeholder.py rename to keystone/common/sql/legacy_migrations/contract_repo/versions/078_placeholder.py diff --git a/keystone/common/sql/contract_repo/versions/079_contract_update_local_id_limit.py b/keystone/common/sql/legacy_migrations/contract_repo/versions/079_contract_update_local_id_limit.py similarity index 100% rename from keystone/common/sql/contract_repo/versions/079_contract_update_local_id_limit.py rename to keystone/common/sql/legacy_migrations/contract_repo/versions/079_contract_update_local_id_limit.py diff --git a/keystone/common/sql/data_migration_repo/__init__.py b/keystone/common/sql/legacy_migrations/contract_repo/versions/__init__.py similarity index 100% rename from keystone/common/sql/data_migration_repo/__init__.py rename to keystone/common/sql/legacy_migrations/contract_repo/versions/__init__.py diff --git a/keystone/common/sql/legacy_migrations/data_migration_repo/README.rst b/keystone/common/sql/legacy_migrations/data_migration_repo/README.rst new file mode 100644 index 0000000000..0b4202f9f0 --- /dev/null +++ b/keystone/common/sql/legacy_migrations/data_migration_repo/README.rst @@ -0,0 +1,13 @@ +Data migration repo migrations +============================== + +.. warning:: + + This repo is deprecated and will be removed in a future release. All new + migrations should be alembic-based and placed in + ``keystone/common/sql/migrations``. + +Data migrations for the database. + +This is a database migration repository. More information at +https://opendev.org/x/sqlalchemy-migrate diff --git a/keystone/common/sql/data_migration_repo/versions/__init__.py b/keystone/common/sql/legacy_migrations/data_migration_repo/__init__.py similarity index 100% rename from keystone/common/sql/data_migration_repo/versions/__init__.py rename to keystone/common/sql/legacy_migrations/data_migration_repo/__init__.py diff --git a/keystone/common/sql/data_migration_repo/manage.py b/keystone/common/sql/legacy_migrations/data_migration_repo/manage.py similarity index 100% rename from keystone/common/sql/data_migration_repo/manage.py rename to keystone/common/sql/legacy_migrations/data_migration_repo/manage.py diff --git a/keystone/common/sql/data_migration_repo/migrate.cfg b/keystone/common/sql/legacy_migrations/data_migration_repo/migrate.cfg similarity index 100% rename from keystone/common/sql/data_migration_repo/migrate.cfg rename to keystone/common/sql/legacy_migrations/data_migration_repo/migrate.cfg diff --git a/keystone/common/sql/data_migration_repo/versions/073_migrate_initial_migration.py b/keystone/common/sql/legacy_migrations/data_migration_repo/versions/073_migrate_initial_migration.py similarity index 100% rename from keystone/common/sql/data_migration_repo/versions/073_migrate_initial_migration.py rename to keystone/common/sql/legacy_migrations/data_migration_repo/versions/073_migrate_initial_migration.py diff --git a/keystone/common/sql/data_migration_repo/versions/074_placeholder.py b/keystone/common/sql/legacy_migrations/data_migration_repo/versions/074_placeholder.py similarity index 100% rename from keystone/common/sql/data_migration_repo/versions/074_placeholder.py rename to keystone/common/sql/legacy_migrations/data_migration_repo/versions/074_placeholder.py diff --git a/keystone/common/sql/data_migration_repo/versions/075_placeholder.py b/keystone/common/sql/legacy_migrations/data_migration_repo/versions/075_placeholder.py similarity index 100% rename from keystone/common/sql/data_migration_repo/versions/075_placeholder.py rename to keystone/common/sql/legacy_migrations/data_migration_repo/versions/075_placeholder.py diff --git a/keystone/common/sql/data_migration_repo/versions/076_placeholder.py b/keystone/common/sql/legacy_migrations/data_migration_repo/versions/076_placeholder.py similarity index 100% rename from keystone/common/sql/data_migration_repo/versions/076_placeholder.py rename to keystone/common/sql/legacy_migrations/data_migration_repo/versions/076_placeholder.py diff --git a/keystone/common/sql/data_migration_repo/versions/077_placeholder.py b/keystone/common/sql/legacy_migrations/data_migration_repo/versions/077_placeholder.py similarity index 100% rename from keystone/common/sql/data_migration_repo/versions/077_placeholder.py rename to keystone/common/sql/legacy_migrations/data_migration_repo/versions/077_placeholder.py diff --git a/keystone/common/sql/data_migration_repo/versions/078_placeholder.py b/keystone/common/sql/legacy_migrations/data_migration_repo/versions/078_placeholder.py similarity index 100% rename from keystone/common/sql/data_migration_repo/versions/078_placeholder.py rename to keystone/common/sql/legacy_migrations/data_migration_repo/versions/078_placeholder.py diff --git a/keystone/common/sql/data_migration_repo/versions/079_migrate_update_local_id_limit.py b/keystone/common/sql/legacy_migrations/data_migration_repo/versions/079_migrate_update_local_id_limit.py similarity index 100% rename from keystone/common/sql/data_migration_repo/versions/079_migrate_update_local_id_limit.py rename to keystone/common/sql/legacy_migrations/data_migration_repo/versions/079_migrate_update_local_id_limit.py diff --git a/keystone/common/sql/legacy_migrations/data_migration_repo/versions/__init__.py b/keystone/common/sql/legacy_migrations/data_migration_repo/versions/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/keystone/common/sql/legacy_migrations/expand_repo/README.rst b/keystone/common/sql/legacy_migrations/expand_repo/README.rst new file mode 100644 index 0000000000..6019e2d6c7 --- /dev/null +++ b/keystone/common/sql/legacy_migrations/expand_repo/README.rst @@ -0,0 +1,13 @@ +Expand repo migrations +====================== + +.. warning:: + + This repo is deprecated and will be removed in a future release. All new + migrations should be alembic-based and placed in + ``keystone/common/sql/migrations``. + +Expand-style or additive migrations for the database. + +This is a database migration repository. More information at +https://opendev.org/x/sqlalchemy-migrate diff --git a/keystone/common/sql/expand_repo/__init__.py b/keystone/common/sql/legacy_migrations/expand_repo/__init__.py similarity index 100% rename from keystone/common/sql/expand_repo/__init__.py rename to keystone/common/sql/legacy_migrations/expand_repo/__init__.py diff --git a/keystone/common/sql/expand_repo/manage.py b/keystone/common/sql/legacy_migrations/expand_repo/manage.py similarity index 100% rename from keystone/common/sql/expand_repo/manage.py rename to keystone/common/sql/legacy_migrations/expand_repo/manage.py diff --git a/keystone/common/sql/expand_repo/migrate.cfg b/keystone/common/sql/legacy_migrations/expand_repo/migrate.cfg similarity index 100% rename from keystone/common/sql/expand_repo/migrate.cfg rename to keystone/common/sql/legacy_migrations/expand_repo/migrate.cfg diff --git a/keystone/common/sql/expand_repo/versions/073_expand_initial_migration.py b/keystone/common/sql/legacy_migrations/expand_repo/versions/073_expand_initial_migration.py similarity index 100% rename from keystone/common/sql/expand_repo/versions/073_expand_initial_migration.py rename to keystone/common/sql/legacy_migrations/expand_repo/versions/073_expand_initial_migration.py diff --git a/keystone/common/sql/expand_repo/versions/074_placeholder.py b/keystone/common/sql/legacy_migrations/expand_repo/versions/074_placeholder.py similarity index 100% rename from keystone/common/sql/expand_repo/versions/074_placeholder.py rename to keystone/common/sql/legacy_migrations/expand_repo/versions/074_placeholder.py diff --git a/keystone/common/sql/expand_repo/versions/075_placeholder.py b/keystone/common/sql/legacy_migrations/expand_repo/versions/075_placeholder.py similarity index 100% rename from keystone/common/sql/expand_repo/versions/075_placeholder.py rename to keystone/common/sql/legacy_migrations/expand_repo/versions/075_placeholder.py diff --git a/keystone/common/sql/expand_repo/versions/076_placeholder.py b/keystone/common/sql/legacy_migrations/expand_repo/versions/076_placeholder.py similarity index 100% rename from keystone/common/sql/expand_repo/versions/076_placeholder.py rename to keystone/common/sql/legacy_migrations/expand_repo/versions/076_placeholder.py diff --git a/keystone/common/sql/expand_repo/versions/077_placeholder.py b/keystone/common/sql/legacy_migrations/expand_repo/versions/077_placeholder.py similarity index 100% rename from keystone/common/sql/expand_repo/versions/077_placeholder.py rename to keystone/common/sql/legacy_migrations/expand_repo/versions/077_placeholder.py diff --git a/keystone/common/sql/expand_repo/versions/078_placeholder.py b/keystone/common/sql/legacy_migrations/expand_repo/versions/078_placeholder.py similarity index 100% rename from keystone/common/sql/expand_repo/versions/078_placeholder.py rename to keystone/common/sql/legacy_migrations/expand_repo/versions/078_placeholder.py diff --git a/keystone/common/sql/expand_repo/versions/079_expand_update_local_id_limit.py b/keystone/common/sql/legacy_migrations/expand_repo/versions/079_expand_update_local_id_limit.py similarity index 100% rename from keystone/common/sql/expand_repo/versions/079_expand_update_local_id_limit.py rename to keystone/common/sql/legacy_migrations/expand_repo/versions/079_expand_update_local_id_limit.py diff --git a/keystone/common/sql/expand_repo/versions/__init__.py b/keystone/common/sql/legacy_migrations/expand_repo/versions/__init__.py similarity index 100% rename from keystone/common/sql/expand_repo/versions/__init__.py rename to keystone/common/sql/legacy_migrations/expand_repo/versions/__init__.py diff --git a/keystone/common/sql/upgrades.py b/keystone/common/sql/upgrades.py index 742748e7ea..4e959df3fa 100644 --- a/keystone/common/sql/upgrades.py +++ b/keystone/common/sql/upgrades.py @@ -68,8 +68,13 @@ class Repository(object): def find_repo(repo_name): """Return the absolute path to the named repository.""" - path = os.path.abspath(os.path.join( - os.path.dirname(sql.__file__), repo_name)) + path = os.path.abspath( + os.path.join( + os.path.dirname(sql.__file__), + 'legacy_migrations', + repo_name, + ) + ) if not os.path.isdir(path): raise exception.MigrationNotProvided(sql.__name__, path) diff --git a/keystone/tests/unit/test_sql_banned_operations.py b/keystone/tests/unit/test_sql_banned_operations.py index 74c6034120..2a9be1029d 100644 --- a/keystone/tests/unit/test_sql_banned_operations.py +++ b/keystone/tests/unit/test_sql_banned_operations.py @@ -12,7 +12,6 @@ # License for the specific language governing permissions and limitations # under the License. - import os import fixtures @@ -25,9 +24,9 @@ from oslotest import base as test_base import sqlalchemy import testtools -from keystone.common.sql import contract_repo -from keystone.common.sql import data_migration_repo -from keystone.common.sql import expand_repo +from keystone.common.sql.legacy_migrations import contract_repo +from keystone.common.sql.legacy_migrations import data_migration_repo +from keystone.common.sql.legacy_migrations import expand_repo from keystone.common.sql import upgrades diff --git a/tools/generate-schemas b/tools/generate-schemas index 9d9821c565..59fbb226c6 100755 --- a/tools/generate-schemas +++ b/tools/generate-schemas @@ -28,7 +28,7 @@ set -e source .tox/py38/bin/activate -INIT_VERSION=$(ls -1 keystone/common/sql/expand_repo/versions/ | head -1 | awk -F_ '{print $1}' | sed 's/^0*//') +INIT_VERSION=$(ls -1 keystone/common/sql/legacy_migrations/expand_repo/versions/ | head -1 | awk -F_ '{print $1}' | sed 's/^0*//') INIT_VERSION=$(($INIT_VERSION-1)) echo "Detected init version of $INIT_VERSION" @@ -43,28 +43,28 @@ rm -f "/tmp/keystone-schemas/$INIT_VERSION-*.sql" function sync () { DB_URL=$1 - python keystone/common/sql/expand_repo/manage.py version_control \ + python keystone/common/sql/legacy_migrations/expand_repo/manage.py version_control \ --database "$DB_URL" \ --version "$INIT_VERSION" \ - --repository keystone/common/sql/expand_repo/ - python keystone/common/sql/data_migration_repo/manage.py version_control \ + --repository keystone/common/sql/legacy_migrations/expand_repo/ + python keystone/common/sql/legacy_migrations/data_migration_repo/manage.py version_control \ --database "$DB_URL" \ --version "$INIT_VERSION" \ - --repository keystone/common/sql/data_migration_repo/ - python keystone/common/sql/contract_repo/manage.py version_control \ + --repository keystone/common/sql/legacy_migrations/data_migration_repo/ + python keystone/common/sql/legacy_migrations/contract_repo/manage.py version_control \ --database "$DB_URL" \ --version "$INIT_VERSION" \ - --repository keystone/common/sql/contract_repo/ + --repository keystone/common/sql/legacy_migrations/contract_repo/ - python keystone/common/sql/expand_repo/manage.py upgrade \ + python keystone/common/sql/legacy_migrations/expand_repo/manage.py upgrade \ --database "$DB_URL" \ - --repository keystone/common/sql/expand_repo/ - python keystone/common/sql/data_migration_repo/manage.py upgrade \ + --repository keystone/common/sql/legacy_migrations/expand_repo/ + python keystone/common/sql/legacy_migrations/data_migration_repo/manage.py upgrade \ --database "$DB_URL" \ - --repository keystone/common/sql/data_migration_repo/ - python keystone/common/sql/contract_repo/manage.py upgrade \ + --repository keystone/common/sql/legacy_migrations/data_migration_repo/ + python keystone/common/sql/legacy_migrations/contract_repo/manage.py upgrade \ --database "$DB_URL" \ - --repository keystone/common/sql/contract_repo/ + --repository keystone/common/sql/legacy_migrations/contract_repo/ } #