db: Remove legacy migrations

sqlalchemy-migrate does not (and will not) support sqlalchemy 2.0. We
need to drop these migrations to ensure we can upgrade our sqlalchemy
version.

Change-Id: I31ba9e4f129a7cc28744e814b5fd28eb284ae3de
Signed-off-by: Stephen Finucane <sfinucan@redhat.com>
This commit is contained in:
Stephen Finucane 2023-02-28 13:17:31 +00:00
parent 0bbaf63a5a
commit 8c9462f6fa
44 changed files with 8 additions and 2071 deletions

View File

@ -286,14 +286,6 @@ class DbSync(BaseApp):
) )
return 2 return 2
if isinstance(expand_version, int):
# we're still using sqlalchemy-migrate
LOG.info(
'Your database is currently using legacy version control. '
'Your first step is to run `keystone-manage db_sync --expand`.'
)
return 2
try: try:
contract_version = upgrades.get_db_version(branch='contract') contract_version = upgrades.get_db_version(branch='contract')
except db_exception.DBMigrationError: except db_exception.DBMigrationError:

View File

@ -1,13 +0,0 @@
Contract repo migrations
========================
.. warning::
This repo is deprecated and will be removed in a future release. All new
migrations should be alembic-based and placed in
``keystone/common/sql/migrations``.
Contract-style or destructive migrations for the database.
This is a database migration repository. More information at
https://opendev.org/x/sqlalchemy-migrate

View File

@ -1,18 +0,0 @@
#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from migrate.versioning.shell import main
if __name__ == '__main__':
main(debug='False')

View File

@ -1,25 +0,0 @@
[db_settings]
# Used to identify which repository this database is versioned under.
# You can use the name of your project.
repository_id=keystone_contract
# The name of the database table used to track the schema version.
# This name shouldn't already be used by your project.
# If this is changed once a database is under version control, you'll need to
# change the table name in each database too.
version_table=migrate_version
# When committing a change script, Migrate will attempt to generate the
# sql for all supported databases; normally, if one of them fails - probably
# because you don't have that database installed - it is ignored and the
# commit continues, perhaps ending successfully.
# Databases in this list MUST compile successfully during a commit, or the
# entire commit will fail. List the databases your application will actually
# be using to ensure your updates to that database work properly.
# This must be a list; example: ['postgres','sqlite']
required_dbs=[]
# When creating new change scripts, Migrate will stamp the new script with
# a version number. By default this is latest_version + 1. You can set this
# to 'true' to tell Migrate to use the UTC timestamp instead.
use_timestamp_numbering=False

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# A null initial migration to open this repo. Do not re-use replace this with
# a real migration, add additional ones in subsequent version scripts.
def upgrade(migrate_engine):
pass

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ussuri backports. Do not use this number for new
# Victoria work. New Victoria work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ussuri backports. Do not use this number for new
# Victoria work. New Victoria work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ussuri backports. Do not use this number for new
# Victoria work. New Victoria work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ussuri backports. Do not use this number for new
# Victoria work. New Victoria work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ussuri backports. Do not use this number for new
# Victoria work. New Victoria work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ussuri backports. Do not use this number for new
# Victoria work. New Victoria work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,13 +0,0 @@
Data migration repo migrations
==============================
.. warning::
This repo is deprecated and will be removed in a future release. All new
migrations should be alembic-based and placed in
``keystone/common/sql/migrations``.
Data migrations for the database.
This is a database migration repository. More information at
https://opendev.org/x/sqlalchemy-migrate

View File

@ -1,18 +0,0 @@
#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from migrate.versioning.shell import main
if __name__ == '__main__':
main(debug='False')

View File

@ -1,25 +0,0 @@
[db_settings]
# Used to identify which repository this database is versioned under.
# You can use the name of your project.
repository_id=keystone_data_migrate
# The name of the database table used to track the schema version.
# This name shouldn't already be used by your project.
# If this is changed once a database is under version control, you'll need to
# change the table name in each database too.
version_table=migrate_version
# When committing a change script, Migrate will attempt to generate the
# sql for all supported databases; normally, if one of them fails - probably
# because you don't have that database installed - it is ignored and the
# commit continues, perhaps ending successfully.
# Databases in this list MUST compile successfully during a commit, or the
# entire commit will fail. List the databases your application will actually
# be using to ensure your updates to that database work properly.
# This must be a list; example: ['postgres','sqlite']
required_dbs=[]
# When creating new change scripts, Migrate will stamp the new script with
# a version number. By default this is latest_version + 1. You can set this
# to 'true' to tell Migrate to use the UTC timestamp instead.
use_timestamp_numbering=False

View File

@ -1,59 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# A null initial migration to open this repo. Do not re-use replace this with
# a real migration, add additional ones in subsequent version scripts.
import sqlalchemy as sql
import sqlalchemy.orm
NULL_DOMAIN_ID = '<<keystone.domain.root>>'
def upgrade(migrate_engine):
def _generate_root_domain_project():
# Generate a project that will act as a root for all domains, in order
# for use to be able to use a FK constraint on domain_id. Projects
# acting as a domain will not reference this as their parent_id, just
# as domain_id.
#
# This special project is filtered out by the driver, so is never
# visible to the manager or API.
project_ref = {
'id': NULL_DOMAIN_ID,
'name': NULL_DOMAIN_ID,
'enabled': False,
'description': '',
'domain_id': NULL_DOMAIN_ID,
'is_domain': True,
'parent_id': None,
'extra': '{}',
}
return project_ref
meta = sql.MetaData()
# NOTE(stephenfin): This is not compatible with SQLAlchemy 2.0 but neither
# is sqlalchemy-migrate which requires this. We'll remove these migrations
# when dropping SQLAlchemy < 2.x support
meta.bind = migrate_engine
session = sql.orm.sessionmaker(bind=migrate_engine)()
project = sql.Table('project', meta, autoload_with=migrate_engine)
root_domain_project = _generate_root_domain_project()
new_entry = project.insert().values(**root_domain_project)
session.execute(new_entry)
session.commit()
session.close()

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ussuri backports. Do not use this number for new
# Victoria work. New Victoria work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ussuri backports. Do not use this number for new
# Victoria work. New Victoria work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ussuri backports. Do not use this number for new
# Victoria work. New Victoria work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ussuri backports. Do not use this number for new
# Victoria work. New Victoria work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ussuri backports. Do not use this number for new
# Victoria work. New Victoria work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ussuri backports. Do not use this number for new
# Victoria work. New Victoria work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,13 +0,0 @@
Expand repo migrations
======================
.. warning::
This repo is deprecated and will be removed in a future release. All new
migrations should be alembic-based and placed in
``keystone/common/sql/migrations``.
Expand-style or additive migrations for the database.
This is a database migration repository. More information at
https://opendev.org/x/sqlalchemy-migrate

View File

@ -1,15 +0,0 @@
# Copyright 2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from keystone.common.sql.core import * # noqa

View File

@ -1,18 +0,0 @@
#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from migrate.versioning.shell import main
if __name__ == '__main__':
main(debug='False')

View File

@ -1,25 +0,0 @@
[db_settings]
# Used to identify which repository this database is versioned under.
# You can use the name of your project.
repository_id=keystone_expand
# The name of the database table used to track the schema version.
# This name shouldn't already be used by your project.
# If this is changed once a database is under version control, you'll need to
# change the table name in each database too.
version_table=migrate_version
# When committing a change script, Migrate will attempt to generate the
# sql for all supported databases; normally, if one of them fails - probably
# because you don't have that database installed - it is ignored and the
# commit continues, perhaps ending successfully.
# Databases in this list MUST compile successfully during a commit, or the
# entire commit will fail. List the databases your application will actually
# be using to ensure your updates to that database work properly.
# This must be a list; example: ['postgres','sqlite']
required_dbs=[]
# When creating new change scripts, Migrate will stamp the new script with
# a version number. By default this is latest_version + 1. You can set this
# to 'true' to tell Migrate to use the UTC timestamp instead.
use_timestamp_numbering=False

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ussuri backports. Do not use this number for new
# Victoria work. New Victoria work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ussuri backports. Do not use this number for new
# Victoria work. New Victoria work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ussuri backports. Do not use this number for new
# Victoria work. New Victoria work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ussuri backports. Do not use this number for new
# Victoria work. New Victoria work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for Ussuri backports. Do not use this number for new
# Victoria work. New Victoria work starts after all the placeholders.
def upgrade(migrate_engine):
pass

View File

@ -1,28 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sqlalchemy as sql
def upgrade(migrate_engine):
meta = sql.MetaData()
# NOTE(stephenfin): This is not compatible with SQLAlchemy 2.0 but neither
# is sqlalchemy-migrate which requires this. We'll remove these migrations
# when dropping SQLAlchemy < 2.x support
meta.bind = migrate_engine
id_mapping_table = sql.Table(
'id_mapping', meta, autoload_with=migrate_engine,
)
id_mapping_table.c.local_id.alter(type=sql.String(255))

View File

@ -1,15 +0,0 @@
# Copyright 2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from keystone.common.sql.core import * # noqa

View File

@ -20,9 +20,6 @@ from alembic import command as alembic_api
from alembic import config as alembic_config from alembic import config as alembic_config
from alembic import migration as alembic_migration from alembic import migration as alembic_migration
from alembic import script as alembic_script from alembic import script as alembic_script
from migrate import exceptions as migrate_exceptions
from migrate.versioning import api as migrate_api
from migrate.versioning import repository as migrate_repository
from oslo_db import exception as db_exception from oslo_db import exception as db_exception
from oslo_log import log as logging from oslo_log import log as logging
@ -33,7 +30,6 @@ CONF = keystone.conf.CONF
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
ALEMBIC_INIT_VERSION = '27e647c0fad4' ALEMBIC_INIT_VERSION = '27e647c0fad4'
MIGRATE_INIT_VERSION = 72
EXPAND_BRANCH = 'expand' EXPAND_BRANCH = 'expand'
DATA_MIGRATION_BRANCH = 'data_migration' DATA_MIGRATION_BRANCH = 'data_migration'
@ -50,25 +46,6 @@ VERSIONS_PATH = os.path.join(
) )
def _find_migrate_repo(branch):
"""Get the project's change script repository.
:param branch: Name of the repository "branch" to be used; this will be
transformed to repository path.
:returns: An instance of ``migrate.versioning.repository.Repository``
"""
abs_path = os.path.abspath(
os.path.join(
os.path.dirname(sql.__file__),
'legacy_migrations',
f'{branch}_repo',
)
)
if not os.path.exists(abs_path):
raise db_exception.DBMigrationError("Path %s not found" % abs_path)
return migrate_repository.Repository(abs_path)
def _find_alembic_conf(): def _find_alembic_conf():
"""Get the project's alembic configuration. """Get the project's alembic configuration.
@ -140,45 +117,12 @@ def get_current_heads():
return heads return heads
def _is_database_under_migrate_control(engine):
# if any of the repos is present, they're all present (in theory, at least)
repository = _find_migrate_repo('expand')
try:
migrate_api.db_version(engine, repository)
return True
except migrate_exceptions.DatabaseNotControlledError:
return False
def _is_database_under_alembic_control(engine): def _is_database_under_alembic_control(engine):
with engine.connect() as conn: with engine.connect() as conn:
context = alembic_migration.MigrationContext.configure(conn) context = alembic_migration.MigrationContext.configure(conn)
return bool(context.get_current_heads()) return bool(context.get_current_heads())
def _init_alembic_on_legacy_database(engine, config):
"""Init alembic in an existing environment with sqlalchemy-migrate."""
LOG.info(
'The database is still under sqlalchemy-migrate control; '
'applying any remaining sqlalchemy-migrate-based migrations '
'and fake applying the initial alembic migration'
)
# bring all repos up to date; note that we're relying on the fact that
# there aren't any "real" contract migrations left (since the great squash
# of migrations in yoga) so we're really only applying the expand side of
# '079_expand_update_local_id_limit' and the rest are for completeness'
# sake
for branch in (EXPAND_BRANCH, DATA_MIGRATION_BRANCH, CONTRACT_BRANCH):
repository = _find_migrate_repo(branch or 'expand')
migrate_api.upgrade(engine, repository)
# re-use the connection rather than creating a new one
with engine.begin() as connection:
config.attributes['connection'] = connection
alembic_api.stamp(config, ALEMBIC_INIT_VERSION)
def _upgrade_alembic(engine, config, branch): def _upgrade_alembic(engine, config, branch):
revision = 'heads' revision = 'heads'
if branch: if branch:
@ -209,17 +153,10 @@ def get_db_version(branch=EXPAND_BRANCH, *, engine=None):
engine_url = str(engine.url).replace('%', '%%') engine_url = str(engine.url).replace('%', '%%')
config.set_main_option('sqlalchemy.url', str(engine_url)) config.set_main_option('sqlalchemy.url', str(engine_url))
migrate_version = None # we use '.get' since the particular branch might not have been created
if _is_database_under_migrate_control(engine): alembic_version = _get_current_heads(engine, config).get(branch)
repository = _find_migrate_repo(branch)
migrate_version = migrate_api.db_version(engine, repository)
alembic_version = None return alembic_version
if _is_database_under_alembic_control(engine):
# we use '.get' since the particular branch might not have been created
alembic_version = _get_current_heads(engine, config).get(branch)
return alembic_version or migrate_version
def _db_sync(branch=None, *, engine=None): def _db_sync(branch=None, *, engine=None):
@ -241,16 +178,6 @@ def _db_sync(branch=None, *, engine=None):
engine_url = str(engine.url).replace('%', '%%') engine_url = str(engine.url).replace('%', '%%')
config.set_main_option('sqlalchemy.url', str(engine_url)) config.set_main_option('sqlalchemy.url', str(engine_url))
# if we're in a deployment where sqlalchemy-migrate is already present,
# then apply all the updates for that and fake apply the initial
# alembic migration; if we're not then 'upgrade' will take care of
# everything this should be a one-time operation
if (
not _is_database_under_alembic_control(engine) and
_is_database_under_migrate_control(engine)
):
_init_alembic_on_legacy_database(engine, config)
_upgrade_alembic(engine, config, branch) _upgrade_alembic(engine, config, branch)

View File

@ -21,7 +21,6 @@ test will then use that DB and username/password combo to run the tests.
""" """
import fixtures import fixtures
from migrate.versioning import api as migrate_api
from oslo_db import options as db_options from oslo_db import options as db_options
from oslo_db.sqlalchemy import enginefacade from oslo_db.sqlalchemy import enginefacade
from oslo_db.sqlalchemy import test_fixtures from oslo_db.sqlalchemy import test_fixtures
@ -292,49 +291,3 @@ class TestModelsSyncPostgreSQL(
base.BaseTestCase, base.BaseTestCase,
): ):
FIXTURE = test_fixtures.PostgresqlOpportunisticFixture FIXTURE = test_fixtures.PostgresqlOpportunisticFixture
class KeystoneModelsMigrationsLegacySync(KeystoneModelsMigrationsSync):
"""Test that the models match the database after old migrations are run."""
def db_sync(self, engine):
# the 'upgrades._db_sync' method will not use the legacy
# sqlalchemy-migrate-based migration flow unless the database is
# already controlled with sqlalchemy-migrate, so we need to manually
# enable version controlling with this tool to test this code path
for branch in (
upgrades.EXPAND_BRANCH,
upgrades.DATA_MIGRATION_BRANCH,
upgrades.CONTRACT_BRANCH,
):
repository = upgrades._find_migrate_repo(branch)
migrate_api.version_control(
engine, repository, upgrades.MIGRATE_INIT_VERSION)
# now we can apply migrations as expected and the legacy path will be
# followed
super().db_sync(engine)
class TestModelsLegacySyncSQLite(
KeystoneModelsMigrationsLegacySync,
test_fixtures.OpportunisticDBTestMixin,
base.BaseTestCase,
):
pass
class TestModelsLegacySyncMySQL(
KeystoneModelsMigrationsLegacySync,
test_fixtures.OpportunisticDBTestMixin,
base.BaseTestCase,
):
FIXTURE = test_fixtures.MySQLOpportunisticFixture
class TestModelsLegacySyncPostgreSQL(
KeystoneModelsMigrationsLegacySync,
test_fixtures.OpportunisticDBTestMixin,
base.BaseTestCase,
):
FIXTURE = test_fixtures.PostgresqlOpportunisticFixture

View File

@ -40,7 +40,6 @@ For further information, see `oslo.db documentation
""" """
import fixtures import fixtures
from migrate.versioning import script
from oslo_db import options as db_options from oslo_db import options as db_options
from oslo_db.sqlalchemy import enginefacade from oslo_db.sqlalchemy import enginefacade
from oslo_db.sqlalchemy import test_fixtures as db_fixtures from oslo_db.sqlalchemy import test_fixtures as db_fixtures
@ -239,11 +238,6 @@ class MigrateBase(
self.engine = enginefacade.writer.get_engine() self.engine = enginefacade.writer.get_engine()
self.sessionmaker = enginefacade.writer.get_sessionmaker() self.sessionmaker = enginefacade.writer.get_sessionmaker()
# NOTE(dstanek): Clear out sqlalchemy-migrate's script cache to allow
# us to have multiple repos (expand, migrate, contract) where the
# modules have the same name (001_awesome.py).
self.addCleanup(script.PythonScript.clear)
db_options.set_defaults(CONF, connection=self.engine.url) db_options.set_defaults(CONF, connection=self.engine.url)
# Override keystone's context manager to be oslo.db's global context # Override keystone's context manager to be oslo.db's global context

View File

@ -0,0 +1,5 @@
---
upgrade:
- |
The legacy ``sqlalchemy-migrate`` migrations, which have been deprecated
since Zed, have been removed. There should be no end-user impact.

View File

@ -12,7 +12,6 @@ Flask!=0.11,>=1.0.2 # BSD
Flask-RESTful>=0.3.5 # BSD Flask-RESTful>=0.3.5 # BSD
cryptography>=2.7 # BSD/Apache-2.0 cryptography>=2.7 # BSD/Apache-2.0
SQLAlchemy>=1.4.0 # MIT SQLAlchemy>=1.4.0 # MIT
sqlalchemy-migrate>=0.13.0 # Apache-2.0
stevedore>=1.20.0 # Apache-2.0 stevedore>=1.20.0 # Apache-2.0
passlib>=1.7.0 # BSD passlib>=1.7.0 # BSD
python-keystoneclient>=3.8.0 # Apache-2.0 python-keystoneclient>=3.8.0 # Apache-2.0

View File

@ -1,134 +0,0 @@
#!/usr/bin/env bash
#
# Script to generate schemas for the various versions.
#
# Some setup is required, similar to the opportunistic tests.
#
# MySQL ->
#
# $ mysql -uroot
# MariaDB [(none)]> CREATE DATABASE keystone
# MariaDB [(none)]> GRANT ALL PRIVILEGES ON keystone.* TO 'keystone'@'localhost' IDENTIFIED BY 'password';
# MariaDB [(none)]> quit;
#
# Postgres ->
#
# $ sudo -u postgres psql
# postgres=# create user keystone with createdb login password 'password';
# postgres=# create database keystone with owner keystone;
# postgres=# quit;
#
# Note that you may also have to configure 'pg_hba.conf' to use password-based
# auth instead of "ident", if you haven't done so already. You can locate this
# with 'locate pg_hba.conf'. More details at
# https://ubuntu.com/server/docs/databases-postgresql
set -o xtrace
set -e
source .tox/py38/bin/activate
INIT_VERSION=$(ls -1 keystone/common/sql/legacy_migrations/expand_repo/versions/ | head -1 | awk -F_ '{print $1}' | sed 's/^0*//')
INIT_VERSION=$(($INIT_VERSION-1))
echo "Detected init version of $INIT_VERSION"
mkdir -p /tmp/keystone-schemas
rm -f "/tmp/keystone-schemas/$INIT_VERSION-*.sql"
#
# functions
#
function sync () {
DB_URL=$1
python keystone/common/sql/legacy_migrations/expand_repo/manage.py version_control \
--database "$DB_URL" \
--version "$INIT_VERSION" \
--repository keystone/common/sql/legacy_migrations/expand_repo/
python keystone/common/sql/legacy_migrations/data_migration_repo/manage.py version_control \
--database "$DB_URL" \
--version "$INIT_VERSION" \
--repository keystone/common/sql/legacy_migrations/data_migration_repo/
python keystone/common/sql/legacy_migrations/contract_repo/manage.py version_control \
--database "$DB_URL" \
--version "$INIT_VERSION" \
--repository keystone/common/sql/legacy_migrations/contract_repo/
python keystone/common/sql/legacy_migrations/expand_repo/manage.py upgrade \
--database "$DB_URL" \
--repository keystone/common/sql/legacy_migrations/expand_repo/
python keystone/common/sql/legacy_migrations/data_migration_repo/manage.py upgrade \
--database "$DB_URL" \
--repository keystone/common/sql/legacy_migrations/data_migration_repo/
python keystone/common/sql/legacy_migrations/contract_repo/manage.py upgrade \
--database "$DB_URL" \
--repository keystone/common/sql/legacy_migrations/contract_repo/
}
#
# sqlite
#
# cleanup from previous runs
rm -f /tmp/keystone.db
# sync schema
sync 'sqlite:////tmp/keystone.db'
# dump the schema
sqlite3 /tmp/keystone.db << EOF
.output "/tmp/keystone-schemas/${INIT_VERSION}-sqlite.sql"
.schema
.quit
EOF
rm -f /tmp/keystone.db
#
# mysql
#
# cleanup from previous runs
mysql -u keystone -ppassword << EOF
DROP DATABASE IF EXISTS keystone;
CREATE DATABASE keystone;
EOF
# sync schema
sync 'mysql+pymysql://keystone:password@localhost/keystone'
# dump the schema
mysqldump --no-data --skip-comments -u keystone -ppassword \
keystone > "/tmp/keystone-schemas/${INIT_VERSION}-mysql.sql"
mysql -u keystone -ppassword << EOF
DROP DATABASE IF EXISTS keystone;
EOF
#
# postgres
#
# cleanup from previous runs
sudo -u postgres dropdb --if-exists keystone
sudo -u postgres createdb --owner=keystone keystone
# sync to initial version
sync 'postgresql://keystone:password@localhost/keystone'
# dump the schema
pg_dump postgresql://keystone:password@localhost/keystone \
--schema-only > "/tmp/keystone-schemas/${INIT_VERSION}-postgres.sql"
sudo -u postgres dropdb --if-exists keystone