openstack
/
keystone
Code Issues Proposed changes

Merge "sql: Remove legacy 'migrate_repo' migration repo"

This commit is contained in:
Zuul 2022-02-04 22:41:29 +00:00 committed by Gerrit Code Review
parent 2961208028 503421d3d4
commit ac3a779e10
16 changed files with 867 additions and 1063 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
doc/source/conf.py
View File

@ -32,18 +32,20 @@
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = ['sphinx.ext.coverage',
'sphinx.ext.viewcode',
'oslo_config.sphinxconfiggen',
'oslo_config.sphinxext',
'oslo_policy.sphinxpolicygen',
'openstackdocstheme',
'oslo_policy.sphinxext',
'sphinxcontrib.apidoc',
'sphinxcontrib.seqdiag',
'sphinx_feature_classification.support_matrix',
'sphinxcontrib.blockdiag'
]
extensions = [
'sphinx.ext.coverage',
'sphinx.ext.viewcode',
'sphinx.ext.todo',
'oslo_config.sphinxconfiggen',
'oslo_config.sphinxext',
'oslo_policy.sphinxpolicygen',
'openstackdocstheme',
'oslo_policy.sphinxext',
'sphinxcontrib.apidoc',
'sphinxcontrib.seqdiag',
'sphinx_feature_classification.support_matrix',
'sphinxcontrib.blockdiag'
]
blockdiag_html_image_format = 'SVG'

13
doc/source/contributor/api_change_tutorial.rst
View File

@ -87,6 +87,19 @@ files, respectively (currently only the SQL driver is supported).
Changing the SQL Model and Driver
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. note::
The below guidance is out-of-date and refers to the legacy ``migrate_repo``
migration repository, which was removed in 21.0.0 (Yoga). Nowadays, for a
change like this, you would create an additive or "expand" migration in the
``expand_repo`` repository along with null migrations in the
``contract_repo`` and ``data_migration_repo`` repositories. For more
information, refer to :doc:`/contributor/database-migrations`.
.. todo::
Update this section to reflect the new migration model.
First, you need to change the role model to include the description attribute.
Go to `keystone/assignment/role_backends/sql.py` and update it like::

10
doc/source/contributor/database-migrations.rst
View File

@ -43,14 +43,6 @@ do in a specific phase, then include a no-op migration to simply ``pass`` (in
fact the ``001`` migration in each of these repositories is a no-op migration,
so that can be used as a template).
.. NOTE::
Since rolling upgrade support was added part way through the Newton cycle,
some migrations had already been added to the legacy repository
(``keystone/common/sql/migrate_repo``). This repository is now closed and
no new migrations should be added (except for backporting of previous
placeholders).
In order to support rolling upgrades, where two releases of keystone briefly
operate side-by-side using the same database without downtime, each phase of
the migration must adhere to following constraints:
@ -79,7 +71,7 @@ Data Migration phase:
No schema changes are allowed.
Contract phase:
Only contractive schema changes are allowed, such as dropping or altering
Only destructive schema changes are allowed, such as dropping or altering
columns, tables, indices, and triggers.
Data insertion, modification, and removal is not allowed.

6
doc/source/contributor/testing-keystone.rst
View File

@ -151,9 +151,9 @@ version control:
.. code-block:: bash
$ python keystone/common/sql/migrate_repo/manage.py test \
--url=sqlite:///test.db \
--repository=keystone/common/sql/migrate_repo/
$ python keystone/common/sql/expand_repo/manage.py test \
--url=sqlite:///test.db \
--repository=keystone/common/sql/expand_repo/
This command references to a SQLite database (test.db) to be used. Depending on
the migration, this command alone does not make assertions as to the integrity

40
keystone/common/sql/data_migration_repo/versions/001_data_initial_null_migration.py
View File

@ -13,6 +13,44 @@
# A null initial migration to open this repo. Do not re-use replace this with
# a real migration, add additional ones in subsequent version scripts.
import sqlalchemy as sql
import sqlalchemy.orm
NULL_DOMAIN_ID = '<<keystone.domain.root>>'
def upgrade(migrate_engine):
pass
def _generate_root_domain_project():
# Generate a project that will act as a root for all domains, in order
# for use to be able to use a FK constraint on domain_id. Projects
# acting as a domain will not reference this as their parent_id, just
# as domain_id.
#
# This special project is filtered out by the driver, so is never
# visible to the manager or API.
project_ref = {
'id': NULL_DOMAIN_ID,
'name': NULL_DOMAIN_ID,
'enabled': False,
'description': '',
'domain_id': NULL_DOMAIN_ID,
'is_domain': True,
'parent_id': None,
'extra': '{}',
}
return project_ref
meta = sql.MetaData()
meta.bind = migrate_engine
session = sql.orm.sessionmaker(bind=migrate_engine)()
project = sql.Table('project', meta, autoload=True)
root_domain_project = _generate_root_domain_project()
new_entry = project.insert().values(**root_domain_project)
session.execute(new_entry)
session.commit()
session.close()

728
keystone/common/sql/expand_repo/versions/001_expand_initial_null_migration.py
View File

@ -10,9 +10,731 @@
# License for the specific language governing permissions and limitations
# under the License.
# A null initial migration to open this repo. Do not re-use replace this with
# a real migration, add additional ones in subsequent version scripts.
import migrate
from oslo_log import log
import sqlalchemy as sql
from keystone.assignment.backends import sql as assignment_sql
from keystone.common import sql as ks_sql
import keystone.conf
from keystone.identity.mapping_backends import mapping as mapping_backend
CONF = keystone.conf.CONF
LOG = log.getLogger(__name__)
def upgrade(migrate_engine):
pass
meta = sql.MetaData()
meta.bind = migrate_engine
if migrate_engine.name == 'mysql':
# In Folsom we explicitly converted migrate_version to UTF8.
migrate_engine.execute(
'ALTER TABLE migrate_version CONVERT TO CHARACTER SET utf8'
)
# Set default DB charset to UTF8.
migrate_engine.execute(
'ALTER DATABASE %s DEFAULT CHARACTER SET utf8'
% migrate_engine.url.database
)
access_token = sql.Table(
'access_token',
meta,
sql.Column('id', sql.String(64), primary_key=True, nullable=False),
sql.Column('access_secret', sql.String(64), nullable=False),
sql.Column(
'authorizing_user_id', sql.String(64), nullable=False, index=True
),
sql.Column('project_id', sql.String(64), nullable=False),
sql.Column('role_ids', sql.Text(), nullable=False),
sql.Column(
'consumer_id',
sql.String(64),
sql.ForeignKey('consumer.id'),
nullable=False,
index=True,
),
sql.Column('expires_at', sql.String(64), nullable=True),
)
consumer = sql.Table(
'consumer',
meta,
sql.Column('id', sql.String(64), primary_key=True, nullable=False),
sql.Column('description', sql.String(64), nullable=True),
sql.Column('secret', sql.String(64), nullable=False),
sql.Column('extra', sql.Text(), nullable=False),
)
credential = sql.Table(
'credential',
meta,
sql.Column('id', sql.String(length=64), primary_key=True),
sql.Column('user_id', sql.String(length=64), nullable=False),
sql.Column('project_id', sql.String(length=64)),
sql.Column('blob', ks_sql.JsonBlob, nullable=False),
sql.Column('type', sql.String(length=255), nullable=False),
sql.Column('extra', ks_sql.JsonBlob.impl),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
endpoint = sql.Table(
'endpoint',
meta,
sql.Column('id', sql.String(length=64), primary_key=True),
sql.Column('legacy_endpoint_id', sql.String(length=64)),
sql.Column('interface', sql.String(length=8), nullable=False),
sql.Column('service_id', sql.String(length=64), nullable=False),
sql.Column('url', sql.Text, nullable=False),
sql.Column('extra', ks_sql.JsonBlob.impl),
sql.Column(
'enabled',
sql.Boolean,
nullable=False,
default=True,
server_default='1',
),
sql.Column('region_id', sql.String(length=255), nullable=True),
# NOTE(stevemar): The index was named 'service_id' in
# 050_fk_consistent_indexes.py and needs to be preserved
sql.Index('service_id', 'service_id'),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
endpoint_group = sql.Table(
'endpoint_group',
meta,
sql.Column('id', sql.String(64), primary_key=True),
sql.Column('name', sql.String(255), nullable=False),
sql.Column('description', sql.Text, nullable=True),
sql.Column('filters', sql.Text(), nullable=False),
)
federated_user = sql.Table(
'federated_user',
meta,
sql.Column('id', sql.Integer, primary_key=True, nullable=False),
sql.Column(
'user_id',
sql.String(64),
sql.ForeignKey('user.id', ondelete='CASCADE'),
nullable=False,
),
sql.Column(
'idp_id',
sql.String(64),
sql.ForeignKey('identity_provider.id', ondelete='CASCADE'),
nullable=False,
),
sql.Column('protocol_id', sql.String(64), nullable=False),
sql.Column('unique_id', sql.String(255), nullable=False),
sql.Column('display_name', sql.String(255), nullable=True),
sql.UniqueConstraint('idp_id', 'protocol_id', 'unique_id'),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
federation_protocol = sql.Table(
'federation_protocol',
meta,
sql.Column('id', sql.String(64), primary_key=True),
sql.Column(
'idp_id',
sql.String(64),
sql.ForeignKey('identity_provider.id', ondelete='CASCADE'),
primary_key=True,
),
sql.Column('mapping_id', sql.String(64), nullable=False),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
group = sql.Table(
'group',
meta,
sql.Column('id', sql.String(length=64), primary_key=True),
sql.Column('domain_id', sql.String(length=64), nullable=False),
sql.Column('name', sql.String(length=64), nullable=False),
sql.Column('description', sql.Text),
sql.Column('extra', ks_sql.JsonBlob.impl),
migrate.UniqueConstraint(
'domain_id',
'name',
name='ixu_group_name_domain_id',
),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
identity_provider = sql.Table(
'identity_provider',
meta,
sql.Column('id', sql.String(64), primary_key=True),
sql.Column('enabled', sql.Boolean, nullable=False),
sql.Column('description', sql.Text(), nullable=True),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
idp_remote_ids = sql.Table(
'idp_remote_ids',
meta,
sql.Column(
'idp_id',
sql.String(64),
sql.ForeignKey('identity_provider.id', ondelete='CASCADE'),
),
sql.Column('remote_id', sql.String(255), primary_key=True),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
implied_role = sql.Table(
'implied_role',
meta,
sql.Column('prior_role_id', sql.String(length=64), primary_key=True),
sql.Column('implied_role_id', sql.String(length=64), primary_key=True),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
local_user = sql.Table(
'local_user',
meta,
sql.Column('id', sql.Integer, primary_key=True, nullable=False),
sql.Column(
'user_id',
sql.String(64),
sql.ForeignKey('user.id', ondelete='CASCADE'),
nullable=False,
unique=True,
),
sql.Column('domain_id', sql.String(64), nullable=False),
sql.Column('name', sql.String(255), nullable=False),
sql.Column('failed_auth_count', sql.Integer, nullable=True),
sql.Column('failed_auth_at', sql.DateTime(), nullable=True),
sql.UniqueConstraint('domain_id', 'name'),
)
mapping = sql.Table(
'mapping',
meta,
sql.Column('id', sql.String(64), primary_key=True),
sql.Column('rules', sql.Text(), nullable=False),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
password = sql.Table(
'password',
meta,
sql.Column('id', sql.Integer, primary_key=True, nullable=False),
sql.Column(
'local_user_id',
sql.Integer,
sql.ForeignKey(local_user.c.id, ondelete='CASCADE'),
nullable=False,
),
sql.Column('password', sql.String(128), nullable=True),
sql.Column('created_at', sql.DateTime(), nullable=True),
sql.Column('expires_at', sql.DateTime(), nullable=True),
sql.Column(
'self_service',
sql.Boolean,
nullable=False,
server_default='0',
default=False,
),
)
policy = sql.Table(
'policy',
meta,
sql.Column('id', sql.String(length=64), primary_key=True),
sql.Column('type', sql.String(length=255), nullable=False),
sql.Column('blob', ks_sql.JsonBlob, nullable=False),
sql.Column('extra', ks_sql.JsonBlob.impl),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
policy_association = sql.Table(
'policy_association',
meta,
sql.Column('id', sql.String(64), primary_key=True),
sql.Column('policy_id', sql.String(64), nullable=False),
sql.Column('endpoint_id', sql.String(64), nullable=True),
sql.Column('service_id', sql.String(64), nullable=True),
sql.Column('region_id', sql.String(64), nullable=True),
sql.UniqueConstraint('endpoint_id', 'service_id', 'region_id'),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
project = sql.Table(
'project',
meta,
sql.Column('id', sql.String(length=64), primary_key=True),
sql.Column('name', sql.String(length=64), nullable=False),
sql.Column('extra', ks_sql.JsonBlob.impl),
sql.Column('description', sql.Text),
sql.Column('enabled', sql.Boolean),
sql.Column('domain_id', sql.String(length=64), nullable=False),
sql.Column('parent_id', sql.String(64), nullable=True),
sql.Column(
'is_domain',
sql.Boolean,
nullable=False,
server_default='0',
default=False,
),
migrate.UniqueConstraint(
'domain_id',
'name',
name='ixu_project_name_domain_id',
),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
project_endpoint = sql.Table(
'project_endpoint',
meta,
sql.Column(
'endpoint_id', sql.String(64), primary_key=True, nullable=False
),
sql.Column(
'project_id', sql.String(64), primary_key=True, nullable=False
),
)
project_endpoint_group = sql.Table(
'project_endpoint_group',
meta,
sql.Column(
'endpoint_group_id',
sql.String(64),
sql.ForeignKey('endpoint_group.id'),
nullable=False,
),
sql.Column('project_id', sql.String(64), nullable=False),
sql.PrimaryKeyConstraint('endpoint_group_id', 'project_id'),
)
config_register = sql.Table(
'config_register',
meta,
sql.Column('type', sql.String(64), primary_key=True),
sql.Column('domain_id', sql.String(64), nullable=False),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
request_token = sql.Table(
'request_token',
meta,
sql.Column('id', sql.String(64), primary_key=True, nullable=False),
sql.Column('request_secret', sql.String(64), nullable=False),
sql.Column('verifier', sql.String(64), nullable=True),
sql.Column('authorizing_user_id', sql.String(64), nullable=True),
sql.Column('requested_project_id', sql.String(64), nullable=False),
sql.Column('role_ids', sql.Text(), nullable=True),
sql.Column(
'consumer_id',
sql.String(64),
sql.ForeignKey('consumer.id'),
nullable=False,
index=True,
),
sql.Column('expires_at', sql.String(64), nullable=True),
)
revocation_event = sql.Table(
'revocation_event',
meta,
sql.Column('id', sql.Integer, primary_key=True),
sql.Column('domain_id', sql.String(64)),
sql.Column('project_id', sql.String(64)),
sql.Column('user_id', sql.String(64)),
sql.Column('role_id', sql.String(64)),
sql.Column('trust_id', sql.String(64)),
sql.Column('consumer_id', sql.String(64)),
sql.Column('access_token_id', sql.String(64)),
sql.Column('issued_before', sql.DateTime(), nullable=False),
sql.Column('expires_at', sql.DateTime()),
sql.Column('revoked_at', sql.DateTime(), nullable=False),
sql.Column('audit_id', sql.String(32), nullable=True),
sql.Column('audit_chain_id', sql.String(32), nullable=True),
# NOTE(stephenfin): The '_new' suffix here is due to migration 095,
# which changed the 'id' column from String(64) to Integer. It did this
# by creating a 'revocation_event_new' table and populating it with
# data from the 'revocation_event' table before deleting the
# 'revocation_event' table and renaming the 'revocation_event_new'
# table to 'revocation_event'. Because the 'revoked_at' column had
# 'index=True', sqlalchemy automatically generated the index name as
# 'ix_{table}_{column}'. However, when intitially created, '{table}'
# was 'revocation_event_new' so the index got that name. We may wish to
# rename this eventually.
sql.Index('ix_revocation_event_new_revoked_at', 'revoked_at'),
)
role = sql.Table(
'role',
meta,
sql.Column('id', sql.String(length=64), primary_key=True),
sql.Column('name', sql.String(length=255), nullable=False),
sql.Column('extra', ks_sql.JsonBlob.impl),
sql.Column(
'domain_id',
sql.String(64),
nullable=False,
server_default='<<null>>',
),
migrate.UniqueConstraint(
'name',
'domain_id',
name='ixu_role_name_domain_id',
),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
service = sql.Table(
'service',
meta,
sql.Column('id', sql.String(length=64), primary_key=True),
sql.Column('type', sql.String(length=255)),
sql.Column(
'enabled',
sql.Boolean,
nullable=False,
default=True,
server_default='1',
),
sql.Column('extra', ks_sql.JsonBlob.impl),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
service_provider = sql.Table(
'service_provider',
meta,
sql.Column('auth_url', sql.String(256), nullable=False),
sql.Column('id', sql.String(64), primary_key=True),
sql.Column('enabled', sql.Boolean, nullable=False),
sql.Column('description', sql.Text(), nullable=True),
sql.Column('sp_url', sql.String(256), nullable=False),
sql.Column(
'relay_state_prefix',
sql.String(256),
nullable=False,
server_default=CONF.saml.relay_state_prefix,
),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
token = sql.Table(
'token',
meta,
sql.Column('id', sql.String(length=64), primary_key=True),
sql.Column('expires', sql.DateTime, default=None),
sql.Column('extra', ks_sql.JsonBlob.impl),
sql.Column('valid', sql.Boolean, default=True, nullable=False),
sql.Column('trust_id', sql.String(length=64)),
sql.Column('user_id', sql.String(length=64)),
sql.Index('ix_token_expires', 'expires'),
sql.Index(
'ix_token_expires_valid', 'expires', 'valid'
),
sql.Index('ix_token_user_id', 'user_id'),
sql.Index('ix_token_trust_id', 'trust_id'),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
trust = sql.Table(
'trust',
meta,
sql.Column('id', sql.String(length=64), primary_key=True),
sql.Column('trustor_user_id', sql.String(length=64), nullable=False),
sql.Column('trustee_user_id', sql.String(length=64), nullable=False),
sql.Column('project_id', sql.String(length=64)),
sql.Column('impersonation', sql.Boolean, nullable=False),
sql.Column('deleted_at', sql.DateTime),
sql.Column('expires_at', sql.DateTime),
sql.Column('remaining_uses', sql.Integer, nullable=True),
sql.Column('extra', ks_sql.JsonBlob.impl),
sql.UniqueConstraint(
'trustor_user_id',
'trustee_user_id',
'project_id',
'impersonation',
'expires_at',
name='duplicate_trust_constraint',
),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
trust_role = sql.Table(
'trust_role',
meta,
sql.Column(
'trust_id', sql.String(length=64), primary_key=True, nullable=False
),
sql.Column(
'role_id', sql.String(length=64), primary_key=True, nullable=False
),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
user = sql.Table(
'user',
meta,
sql.Column('id', sql.String(length=64), primary_key=True),
sql.Column('extra', ks_sql.JsonBlob.impl),
sql.Column('enabled', sql.Boolean),
sql.Column('default_project_id', sql.String(length=64)),
sql.Column('created_at', sql.DateTime(), nullable=True),
sql.Column('last_active_at', sql.Date(), nullable=True),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
nonlocal_user = sql.Table(
'nonlocal_user',
meta,
sql.Column('domain_id', sql.String(64), primary_key=True),
sql.Column('name', sql.String(255), primary_key=True),
sql.Column(
'user_id',
sql.String(64),
sql.ForeignKey(user.c.id, ondelete='CASCADE'),
nullable=False,
),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
user_group_membership = sql.Table(
'user_group_membership',
meta,
sql.Column('user_id', sql.String(length=64), primary_key=True),
sql.Column('group_id', sql.String(length=64), primary_key=True),
# NOTE(stevemar): The index was named 'group_id' in
# 050_fk_consistent_indexes.py and needs to be preserved
sql.Index('group_id', 'group_id'),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
region = sql.Table(
'region',
meta,
sql.Column('id', sql.String(255), primary_key=True),
sql.Column('description', sql.String(255), nullable=False),
sql.Column('parent_region_id', sql.String(255), nullable=True),
sql.Column('extra', sql.Text()),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
assignment = sql.Table(
'assignment',
meta,
sql.Column(
'type',
sql.Enum(
assignment_sql.AssignmentType.USER_PROJECT,
assignment_sql.AssignmentType.GROUP_PROJECT,
assignment_sql.AssignmentType.USER_DOMAIN,
assignment_sql.AssignmentType.GROUP_DOMAIN,
name='type',
),
nullable=False,
),
sql.Column('actor_id', sql.String(64), nullable=False),
sql.Column('target_id', sql.String(64), nullable=False),
sql.Column('role_id', sql.String(64), nullable=False),
sql.Column('inherited', sql.Boolean, default=False, nullable=False),
sql.PrimaryKeyConstraint(
'type',
'actor_id',
'target_id',
'role_id',
'inherited',
),
sql.Index('ix_actor_id', 'actor_id'),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
id_mapping = sql.Table(
'id_mapping',
meta,
sql.Column('public_id', sql.String(64), primary_key=True),
sql.Column('domain_id', sql.String(64), nullable=False),
sql.Column('local_id', sql.String(64), nullable=False),
sql.Column(
'entity_type',
sql.Enum(
mapping_backend.EntityType.USER,
mapping_backend.EntityType.GROUP,
name='entity_type',
),
nullable=False,
),
migrate.UniqueConstraint(
'domain_id',
'local_id',
'entity_type',
name='domain_id',
),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
whitelisted_config = sql.Table(
'whitelisted_config',
meta,
sql.Column('domain_id', sql.String(64), primary_key=True),
sql.Column('group', sql.String(255), primary_key=True),
sql.Column('option', sql.String(255), primary_key=True),
sql.Column('value', ks_sql.JsonBlob.impl, nullable=False),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
sensitive_config = sql.Table(
'sensitive_config',
meta,
sql.Column('domain_id', sql.String(64), primary_key=True),
sql.Column('group', sql.String(255), primary_key=True),
sql.Column('option', sql.String(255), primary_key=True),
sql.Column('value', ks_sql.JsonBlob.impl, nullable=False),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
# create all tables
tables = [
credential,
endpoint,
group,
policy,
project,
role,
service,
token,
trust,
trust_role,
user,
user_group_membership,
region,
assignment,
id_mapping,
whitelisted_config,
sensitive_config,
config_register,
policy_association,
identity_provider,
federation_protocol,
mapping,
service_provider,
idp_remote_ids,
consumer,
request_token,
access_token,
revocation_event,
project_endpoint,
endpoint_group,
project_endpoint_group,
implied_role,
local_user,
password,
federated_user,
nonlocal_user,
]
for table in tables:
try:
table.create()
except Exception:
LOG.exception('Exception while creating table: %r', table)
raise
fkeys = [
{
'columns': [endpoint.c.service_id],
'references': [service.c.id],
},
{
'columns': [user_group_membership.c.group_id],
'references': [group.c.id],
'name': 'fk_user_group_membership_group_id',
},
{
'columns': [user_group_membership.c.user_id],
'references': [user.c.id],
'name': 'fk_user_group_membership_user_id',
},
{
'columns': [project.c.domain_id],
'references': [project.c.id],
},
{
'columns': [endpoint.c.region_id],
'references': [region.c.id],
'name': 'fk_endpoint_region_id',
},
{
'columns': [project.c.parent_id],
'references': [project.c.id],
'name': 'project_parent_id_fkey',
},
{
'columns': [implied_role.c.prior_role_id],
'references': [role.c.id],
'ondelete': 'CASCADE',
},
{
'columns': [implied_role.c.implied_role_id],
'references': [role.c.id],
'ondelete': 'CASCADE',
},
{
'columns': [
federated_user.c.protocol_id,
federated_user.c.idp_id,
],
'references': [
federation_protocol.c.id,
federation_protocol.c.idp_id,
],
},
]
if migrate_engine.name == 'sqlite':
# NOTE(stevemar): We need to keep this FK constraint due to 073, but
# only for sqlite, once we collapse 073 we can remove this constraint
fkeys.append(
{
'columns': [assignment.c.role_id],
'references': [role.c.id],
'name': 'fk_assignment_role_id',
},
)
for fkey in fkeys:
migrate.ForeignKeyConstraint(
columns=fkey['columns'],
refcolumns=fkey['references'],
name=fkey.get('name'),
ondelete=fkey.get('ondelete'),
).create()

4
keystone/common/sql/migrate_repo/README
View File

@ -1,4 +0,0 @@
This is a database migration repository.
More information at
https://opendev.org/openstack/sqlalchemy-migrate

0
keystone/common/sql/migrate_repo/__init__.py
View File

18
keystone/common/sql/migrate_repo/manage.py
View File

@ -1,18 +0,0 @@
#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from migrate.versioning.shell import main
if __name__ == '__main__':
main(debug='False')

25
keystone/common/sql/migrate_repo/migrate.cfg
View File

@ -1,25 +0,0 @@
[db_settings]
# Used to identify which repository this database is versioned under.
# You can use the name of your project.
repository_id=keystone
# The name of the database table used to track the schema version.
# This name shouldn't already be used by your project.
# If this is changed once a database is under version control, you'll need to
# change the table name in each database too.
version_table=migrate_version
# When committing a change script, Migrate will attempt to generate the
# sql for all supported databases; normally, if one of them fails - probably
# because you don't have that database installed - it is ignored and the
# commit continues, perhaps ending successfully.
# Databases in this list MUST compile successfully during a commit, or the
# entire commit will fail. List the databases your application will actually
# be using to ensure your updates to that database work properly.
# This must be a list; example: ['postgres','sqlite']
required_dbs=[]
# When creating new change scripts, Migrate will stamp the new script with
# a version number. By default this is latest_version + 1. You can set this
# to 'true' to tell Migrate to use the UTC timestamp instead.
use_timestamp_numbering=False

776
keystone/common/sql/migrate_repo/versions/109_newton.py
View File

@ -1,776 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import migrate
from oslo_log import log
import sqlalchemy as sql
from keystone.assignment.backends import sql as assignment_sql
from keystone.common import sql as ks_sql
import keystone.conf
from keystone.identity.mapping_backends import mapping as mapping_backend
CONF = keystone.conf.CONF
LOG = log.getLogger(__name__)
NULL_DOMAIN_ID = '<<keystone.domain.root>>'
def upgrade(migrate_engine):
meta = sql.MetaData()
meta.bind = migrate_engine
if migrate_engine.name == 'mysql':
# In Folsom we explicitly converted migrate_version to UTF8.
migrate_engine.execute(
'ALTER TABLE migrate_version CONVERT TO CHARACTER SET utf8'
)
# Set default DB charset to UTF8.
migrate_engine.execute(
'ALTER DATABASE %s DEFAULT CHARACTER SET utf8'
% migrate_engine.url.database
)
access_token = sql.Table(
'access_token',
meta,
sql.Column('id', sql.String(64), primary_key=True, nullable=False),
sql.Column('access_secret', sql.String(64), nullable=False),
sql.Column(
'authorizing_user_id', sql.String(64), nullable=False, index=True
),
sql.Column('project_id', sql.String(64), nullable=False),
sql.Column('role_ids', sql.Text(), nullable=False),
sql.Column(
'consumer_id',
sql.String(64),
sql.ForeignKey('consumer.id'),
nullable=False,
index=True,
),
sql.Column('expires_at', sql.String(64), nullable=True),
)
consumer = sql.Table(
'consumer',
meta,
sql.Column('id', sql.String(64), primary_key=True, nullable=False),
sql.Column('description', sql.String(64), nullable=True),
sql.Column('secret', sql.String(64), nullable=False),
sql.Column('extra', sql.Text(), nullable=False),
)
credential = sql.Table(
'credential',
meta,
sql.Column('id', sql.String(length=64), primary_key=True),
sql.Column('user_id', sql.String(length=64), nullable=False),
sql.Column('project_id', sql.String(length=64)),
sql.Column('blob', ks_sql.JsonBlob, nullable=False),
sql.Column('type', sql.String(length=255), nullable=False),
sql.Column('extra', ks_sql.JsonBlob.impl),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
endpoint = sql.Table(
'endpoint',
meta,
sql.Column('id', sql.String(length=64), primary_key=True),
sql.Column('legacy_endpoint_id', sql.String(length=64)),
sql.Column('interface', sql.String(length=8), nullable=False),
sql.Column('service_id', sql.String(length=64), nullable=False),
sql.Column('url', sql.Text, nullable=False),
sql.Column('extra', ks_sql.JsonBlob.impl),
sql.Column(
'enabled',
sql.Boolean,
nullable=False,
default=True,
server_default='1',
),
sql.Column('region_id', sql.String(length=255), nullable=True),
# NOTE(stevemar): The index was named 'service_id' in
# 050_fk_consistent_indexes.py and needs to be preserved
sql.Index('service_id', 'service_id'),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
endpoint_group = sql.Table(
'endpoint_group',
meta,
sql.Column('id', sql.String(64), primary_key=True),
sql.Column('name', sql.String(255), nullable=False),
sql.Column('description', sql.Text, nullable=True),
sql.Column('filters', sql.Text(), nullable=False),
)
federated_user = sql.Table(
'federated_user',
meta,
sql.Column('id', sql.Integer, primary_key=True, nullable=False),
sql.Column(
'user_id',
sql.String(64),
sql.ForeignKey('user.id', ondelete='CASCADE'),
nullable=False,
),
sql.Column(
'idp_id',
sql.String(64),
sql.ForeignKey('identity_provider.id', ondelete='CASCADE'),
nullable=False,
),
sql.Column('protocol_id', sql.String(64), nullable=False),
sql.Column('unique_id', sql.String(255), nullable=False),
sql.Column('display_name', sql.String(255), nullable=True),
sql.UniqueConstraint('idp_id', 'protocol_id', 'unique_id'),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
federation_protocol = sql.Table(
'federation_protocol',
meta,
sql.Column('id', sql.String(64), primary_key=True),
sql.Column(
'idp_id',
sql.String(64),
sql.ForeignKey('identity_provider.id', ondelete='CASCADE'),
primary_key=True,
),
sql.Column('mapping_id', sql.String(64), nullable=False),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
group = sql.Table(
'group',
meta,
sql.Column('id', sql.String(length=64), primary_key=True),
sql.Column('domain_id', sql.String(length=64), nullable=False),
sql.Column('name', sql.String(length=64), nullable=False),
sql.Column('description', sql.Text),
sql.Column('extra', ks_sql.JsonBlob.impl),
migrate.UniqueConstraint(
'domain_id',
'name',
name='ixu_group_name_domain_id',
),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
identity_provider = sql.Table(
'identity_provider',
meta,
sql.Column('id', sql.String(64), primary_key=True),
sql.Column('enabled', sql.Boolean, nullable=False),
sql.Column('description', sql.Text(), nullable=True),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
idp_remote_ids = sql.Table(
'idp_remote_ids',
meta,
sql.Column(
'idp_id',
sql.String(64),
sql.ForeignKey('identity_provider.id', ondelete='CASCADE'),
),
sql.Column('remote_id', sql.String(255), primary_key=True),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
implied_role = sql.Table(
'implied_role',
meta,
sql.Column('prior_role_id', sql.String(length=64), primary_key=True),
sql.Column('implied_role_id', sql.String(length=64), primary_key=True),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
local_user = sql.Table(
'local_user',
meta,
sql.Column('id', sql.Integer, primary_key=True, nullable=False),
sql.Column(
'user_id',
sql.String(64),
sql.ForeignKey('user.id', ondelete='CASCADE'),
nullable=False,
unique=True,
),
sql.Column('domain_id', sql.String(64), nullable=False),
sql.Column('name', sql.String(255), nullable=False),
sql.Column('failed_auth_count', sql.Integer, nullable=True),
sql.Column('failed_auth_at', sql.DateTime(), nullable=True),
sql.UniqueConstraint('domain_id', 'name'),
)
mapping = sql.Table(
'mapping',
meta,
sql.Column('id', sql.String(64), primary_key=True),
sql.Column('rules', sql.Text(), nullable=False),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
password = sql.Table(
'password',
meta,
sql.Column('id', sql.Integer, primary_key=True, nullable=False),
sql.Column(
'local_user_id',
sql.Integer,
sql.ForeignKey(local_user.c.id, ondelete='CASCADE'),
nullable=False,
),
sql.Column('password', sql.String(128), nullable=True),
sql.Column('created_at', sql.DateTime(), nullable=True),
sql.Column('expires_at', sql.DateTime(), nullable=True),
sql.Column(
'self_service',
sql.Boolean,
nullable=False,
server_default='0',
default=False,
),
)
policy = sql.Table(
'policy',
meta,
sql.Column('id', sql.String(length=64), primary_key=True),
sql.Column('type', sql.String(length=255), nullable=False),
sql.Column('blob', ks_sql.JsonBlob, nullable=False),
sql.Column('extra', ks_sql.JsonBlob.impl),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
policy_association = sql.Table(
'policy_association',
meta,
sql.Column('id', sql.String(64), primary_key=True),
sql.Column('policy_id', sql.String(64), nullable=False),
sql.Column('endpoint_id', sql.String(64), nullable=True),
sql.Column('service_id', sql.String(64), nullable=True),
sql.Column('region_id', sql.String(64), nullable=True),
sql.UniqueConstraint('endpoint_id', 'service_id', 'region_id'),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
project = sql.Table(
'project',
meta,
sql.Column('id', sql.String(length=64), primary_key=True),
sql.Column('name', sql.String(length=64), nullable=False),
sql.Column('extra', ks_sql.JsonBlob.impl),
sql.Column('description', sql.Text),
sql.Column('enabled', sql.Boolean),
sql.Column('domain_id', sql.String(length=64), nullable=False),
sql.Column('parent_id', sql.String(64), nullable=True),
sql.Column(
'is_domain',
sql.Boolean,
nullable=False,
server_default='0',
default=False,
),
migrate.UniqueConstraint(
'domain_id',
'name',
name='ixu_project_name_domain_id',
),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
project_endpoint = sql.Table(
'project_endpoint',
meta,
sql.Column(
'endpoint_id', sql.String(64), primary_key=True, nullable=False
),
sql.Column(
'project_id', sql.String(64), primary_key=True, nullable=False
),
)
project_endpoint_group = sql.Table(
'project_endpoint_group',
meta,
sql.Column(
'endpoint_group_id',
sql.String(64),
sql.ForeignKey('endpoint_group.id'),
nullable=False,
),
sql.Column('project_id', sql.String(64), nullable=False),
sql.PrimaryKeyConstraint('endpoint_group_id', 'project_id'),
)
config_register = sql.Table(
'config_register',
meta,
sql.Column('type', sql.String(64), primary_key=True),
sql.Column('domain_id', sql.String(64), nullable=False),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
request_token = sql.Table(
'request_token',
meta,
sql.Column('id', sql.String(64), primary_key=True, nullable=False),
sql.Column('request_secret', sql.String(64), nullable=False),
sql.Column('verifier', sql.String(64), nullable=True),
sql.Column('authorizing_user_id', sql.String(64), nullable=True),
sql.Column('requested_project_id', sql.String(64), nullable=False),
sql.Column('role_ids', sql.Text(), nullable=True),
sql.Column(
'consumer_id',
sql.String(64),
sql.ForeignKey('consumer.id'),
nullable=False,
index=True,
),
sql.Column('expires_at', sql.String(64), nullable=True),
)
revocation_event = sql.Table(
'revocation_event',
meta,
sql.Column('id', sql.Integer, primary_key=True),
sql.Column('domain_id', sql.String(64)),
sql.Column('project_id', sql.String(64)),
sql.Column('user_id', sql.String(64)),
sql.Column('role_id', sql.String(64)),
sql.Column('trust_id', sql.String(64)),
sql.Column('consumer_id', sql.String(64)),
sql.Column('access_token_id', sql.String(64)),
sql.Column('issued_before', sql.DateTime(), nullable=False),
sql.Column('expires_at', sql.DateTime()),
sql.Column('revoked_at', sql.DateTime(), nullable=False),
sql.Column('audit_id', sql.String(32), nullable=True),
sql.Column('audit_chain_id', sql.String(32), nullable=True),
# NOTE(stephenfin): The '_new' suffix here is due to migration 095,
# which changed the 'id' column from String(64) to Integer. It did this
# by creating a 'revocation_event_new' table and populating it with
# data from the 'revocation_event' table before deleting the
# 'revocation_event' table and renaming the 'revocation_event_new'
# table to 'revocation_event'. Because the 'revoked_at' column had
# 'index=True', sqlalchemy automatically generated the index name as
# 'ix_{table}_{column}'. However, when intitially created, '{table}'
# was 'revocation_event_new' so the index got that name. We may wish to
# rename this eventually.
sql.Index('ix_revocation_event_new_revoked_at', 'revoked_at'),
)
role = sql.Table(
'role',
meta,
sql.Column('id', sql.String(length=64), primary_key=True),
sql.Column('name', sql.String(length=255), nullable=False),
sql.Column('extra', ks_sql.JsonBlob.impl),
sql.Column(
'domain_id',
sql.String(64),
nullable=False,
server_default='<<null>>',
),
migrate.UniqueConstraint(
'name',
'domain_id',
name='ixu_role_name_domain_id',
),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
service = sql.Table(
'service',
meta,
sql.Column('id', sql.String(length=64), primary_key=True),
sql.Column('type', sql.String(length=255)),
sql.Column(
'enabled',
sql.Boolean,
nullable=False,
default=True,
server_default='1',
),
sql.Column('extra', ks_sql.JsonBlob.impl),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
service_provider = sql.Table(
'service_provider',
meta,
sql.Column('auth_url', sql.String(256), nullable=False),
sql.Column('id', sql.String(64), primary_key=True),
sql.Column('enabled', sql.Boolean, nullable=False),
sql.Column('description', sql.Text(), nullable=True),
sql.Column('sp_url', sql.String(256), nullable=False),
sql.Column(
'relay_state_prefix',
sql.String(256),
nullable=False,
server_default=CONF.saml.relay_state_prefix,
),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
token = sql.Table(
'token',
meta,
sql.Column('id', sql.String(length=64), primary_key=True),
sql.Column('expires', sql.DateTime, default=None),
sql.Column('extra', ks_sql.JsonBlob.impl),
sql.Column('valid', sql.Boolean, default=True, nullable=False),
sql.Column('trust_id', sql.String(length=64)),
sql.Column('user_id', sql.String(length=64)),
sql.Index('ix_token_expires', 'expires'),
sql.Index(
'ix_token_expires_valid', 'expires', 'valid'
),
sql.Index('ix_token_user_id', 'user_id'),
sql.Index('ix_token_trust_id', 'trust_id'),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
trust = sql.Table(
'trust',
meta,
sql.Column('id', sql.String(length=64), primary_key=True),
sql.Column('trustor_user_id', sql.String(length=64), nullable=False),
sql.Column('trustee_user_id', sql.String(length=64), nullable=False),
sql.Column('project_id', sql.String(length=64)),
sql.Column('impersonation', sql.Boolean, nullable=False),
sql.Column('deleted_at', sql.DateTime),
sql.Column('expires_at', sql.DateTime),
sql.Column('remaining_uses', sql.Integer, nullable=True),
sql.Column('extra', ks_sql.JsonBlob.impl),
sql.UniqueConstraint(
'trustor_user_id',
'trustee_user_id',
'project_id',
'impersonation',
'expires_at',
name='duplicate_trust_constraint',
),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
trust_role = sql.Table(
'trust_role',
meta,
sql.Column(
'trust_id', sql.String(length=64), primary_key=True, nullable=False
),
sql.Column(
'role_id', sql.String(length=64), primary_key=True, nullable=False
),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
user = sql.Table(
'user',
meta,
sql.Column('id', sql.String(length=64), primary_key=True),
sql.Column('extra', ks_sql.JsonBlob.impl),
sql.Column('enabled', sql.Boolean),
sql.Column('default_project_id', sql.String(length=64)),
sql.Column('created_at', sql.DateTime(), nullable=True),
sql.Column('last_active_at', sql.Date(), nullable=True),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
nonlocal_user = sql.Table(
'nonlocal_user',
meta,
sql.Column('domain_id', sql.String(64), primary_key=True),
sql.Column('name', sql.String(255), primary_key=True),
sql.Column(
'user_id',
sql.String(64),
sql.ForeignKey(user.c.id, ondelete='CASCADE'),
nullable=False,
),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
user_group_membership = sql.Table(
'user_group_membership',
meta,
sql.Column('user_id', sql.String(length=64), primary_key=True),
sql.Column('group_id', sql.String(length=64), primary_key=True),
# NOTE(stevemar): The index was named 'group_id' in
# 050_fk_consistent_indexes.py and needs to be preserved
sql.Index('group_id', 'group_id'),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
region = sql.Table(
'region',
meta,
sql.Column('id', sql.String(255), primary_key=True),
sql.Column('description', sql.String(255), nullable=False),
sql.Column('parent_region_id', sql.String(255), nullable=True),
sql.Column('extra', sql.Text()),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
assignment = sql.Table(
'assignment',
meta,
sql.Column(
'type',
sql.Enum(
assignment_sql.AssignmentType.USER_PROJECT,
assignment_sql.AssignmentType.GROUP_PROJECT,
assignment_sql.AssignmentType.USER_DOMAIN,
assignment_sql.AssignmentType.GROUP_DOMAIN,
name='type',
),
nullable=False,
),
sql.Column('actor_id', sql.String(64), nullable=False),
sql.Column('target_id', sql.String(64), nullable=False),
sql.Column('role_id', sql.String(64), nullable=False),
sql.Column('inherited', sql.Boolean, default=False, nullable=False),
sql.PrimaryKeyConstraint(
'type',
'actor_id',
'target_id',
'role_id',
'inherited',
),
sql.Index('ix_actor_id', 'actor_id'),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
id_mapping = sql.Table(
'id_mapping',
meta,
sql.Column('public_id', sql.String(64), primary_key=True),
sql.Column('domain_id', sql.String(64), nullable=False),
sql.Column('local_id', sql.String(64), nullable=False),
sql.Column(
'entity_type',
sql.Enum(
mapping_backend.EntityType.USER,
mapping_backend.EntityType.GROUP,
name='entity_type',
),
nullable=False,
),
migrate.UniqueConstraint(
'domain_id',
'local_id',
'entity_type',
name='domain_id',
),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
whitelisted_config = sql.Table(
'whitelisted_config',
meta,
sql.Column('domain_id', sql.String(64), primary_key=True),
sql.Column('group', sql.String(255), primary_key=True),
sql.Column('option', sql.String(255), primary_key=True),
sql.Column('value', ks_sql.JsonBlob.impl, nullable=False),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
sensitive_config = sql.Table(
'sensitive_config',
meta,
sql.Column('domain_id', sql.String(64), primary_key=True),
sql.Column('group', sql.String(255), primary_key=True),
sql.Column('option', sql.String(255), primary_key=True),
sql.Column('value', ks_sql.JsonBlob.impl, nullable=False),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
# create all tables
tables = [
credential,
endpoint,
group,
policy,
project,
role,
service,
token,
trust,
trust_role,
user,
user_group_membership,
region,
assignment,
id_mapping,
whitelisted_config,
sensitive_config,
config_register,
policy_association,
identity_provider,
federation_protocol,
mapping,
service_provider,
idp_remote_ids,
consumer,
request_token,
access_token,
revocation_event,
project_endpoint,
endpoint_group,
project_endpoint_group,
implied_role,
local_user,
password,
federated_user,
nonlocal_user,
]
for table in tables:
try:
table.create()
except Exception:
LOG.exception('Exception while creating table: %r', table)
raise
fkeys = [
{
'columns': [endpoint.c.service_id],
'references': [service.c.id],
},
{
'columns': [user_group_membership.c.group_id],
'references': [group.c.id],
'name': 'fk_user_group_membership_group_id',
},
{
'columns': [user_group_membership.c.user_id],
'references': [user.c.id],
'name': 'fk_user_group_membership_user_id',
},
{
'columns': [project.c.domain_id],
'references': [project.c.id],
},
{
'columns': [endpoint.c.region_id],
'references': [region.c.id],
'name': 'fk_endpoint_region_id',
},
{
'columns': [project.c.parent_id],
'references': [project.c.id],
'name': 'project_parent_id_fkey',
},
{
'columns': [implied_role.c.prior_role_id],
'references': [role.c.id],
'ondelete': 'CASCADE',
},
{
'columns': [implied_role.c.implied_role_id],
'references': [role.c.id],
'ondelete': 'CASCADE',
},
{
'columns': [
federated_user.c.protocol_id,
federated_user.c.idp_id,
],
'references': [
federation_protocol.c.id,
federation_protocol.c.idp_id,
],
},
]
if migrate_engine.name == 'sqlite':
# NOTE(stevemar): We need to keep this FK constraint due to 073, but
# only for sqlite, once we collapse 073 we can remove this constraint
fkeys.append(
{
'columns': [assignment.c.role_id],
'references': [role.c.id],
'name': 'fk_assignment_role_id',
},
)
for fkey in fkeys:
migrate.ForeignKeyConstraint(
columns=fkey['columns'],
refcolumns=fkey['references'],
name=fkey.get('name'),
ondelete=fkey.get('ondelete'),
).create()
# data generation
def _generate_root_domain_project():
# Generate a project that will act as a root for all domains, in order
# for use to be able to use a FK constraint on domain_id. Projects
# acting as a domain will not reference this as their parent_id, just
# as domain_id.
#
# This special project is filtered out by the driver, so is never
# visible to the manager or API.
project_ref = {
'id': NULL_DOMAIN_ID,
'name': NULL_DOMAIN_ID,
'enabled': False,
'description': '',
'domain_id': NULL_DOMAIN_ID,
'is_domain': True,
'parent_id': None,
'extra': '{}',
}
return project_ref
meta = sql.MetaData()
meta.bind = migrate_engine
session = sql.orm.sessionmaker(bind=migrate_engine)()
root_domain_project = _generate_root_domain_project()
new_entry = project.insert().values(**root_domain_project)
session.execute(new_entry)
session.commit()
session.close()

0
keystone/common/sql/migrate_repo/versions/__init__.py
View File

42
keystone/common/sql/upgrades.py
View File

@ -30,7 +30,6 @@ from keystone.i18n import _
USE_TRIGGERS = True
LEGACY_REPO = 'migrate_repo'
EXPAND_REPO = 'expand_repo'
DATA_MIGRATION_REPO = 'data_migration_repo'
CONTRACT_REPO = 'contract_repo'
@ -136,16 +135,6 @@ def find_repo(repo_name):
return path
def _sync_common_repo(version):
abs_path = find_repo(LEGACY_REPO)
init_version = get_init_version()
with sql.session_for_write() as session:
engine = session.get_bind()
_assert_not_schema_downgrade(version=version)
migration.db_sync(engine, abs_path, version=version,
init_version=init_version, sanity_check=False)
def _sync_repo(repo_name):
abs_path = find_repo(repo_name)
with sql.session_for_write() as session:
@ -170,7 +159,7 @@ def get_init_version(abs_path=None):
:return: initial version number or None, if DB is empty.
"""
if abs_path is None:
abs_path = find_repo(LEGACY_REPO)
abs_path = find_repo(EXPAND_REPO)
repo = migrate.versioning.repository.Repository(abs_path)
@ -221,14 +210,14 @@ def offline_sync_database_to_version(version=None):
USE_TRIGGERS = False
if version:
_sync_common_repo(version)
else:
expand_schema()
migrate_data()
contract_schema()
raise Exception('Specifying a version is no longer supported')
expand_schema()
migrate_data()
contract_schema()
def get_db_version(repo=LEGACY_REPO):
def get_db_version(repo=EXPAND_REPO):
with sql.session_for_read() as session:
repo = find_repo(repo)
return migration.db_version(
@ -254,19 +243,7 @@ def validate_upgrade_order(repo_name, target_repo_version=None):
db_sync_order = {DATA_MIGRATION_REPO: EXPAND_REPO,
CONTRACT_REPO: DATA_MIGRATION_REPO}
if repo_name == LEGACY_REPO:
return
# If expand is being run, we validate that Legacy repo is at the maximum
# version before running the additional schema expansions.
elif repo_name == EXPAND_REPO:
abs_path = find_repo(LEGACY_REPO)
repo = migrate.versioning.repository.Repository(abs_path)
if int(repo.latest) != get_db_version():
raise db_exception.DBMigrationError(
'Your Legacy repo version is not up to date. Please refer to '
'https://docs.openstack.org/keystone/latest/admin/'
'identity-upgrading.html '
'to see the proper steps for rolling upgrades.')
if repo_name == EXPAND_REPO:
return
# find the latest version that the current command will upgrade to if there
@ -295,9 +272,6 @@ def expand_schema():
keystone node is migrated to the latest release.
"""
# Make sure all the legacy migrations are run before we run any new
# expand migrations.
_sync_common_repo(version=None)
validate_upgrade_order(EXPAND_REPO)
_sync_repo(repo_name=EXPAND_REPO)

86
keystone/tests/unit/test_sql_banned_operations.py
View File

@ -28,7 +28,6 @@ import testtools
from keystone.common.sql import contract_repo
from keystone.common.sql import data_migration_repo
from keystone.common.sql import expand_repo
from keystone.common.sql import migrate_repo
from keystone.common.sql import upgrades
@ -39,9 +38,8 @@ class DBOperationNotAllowed(Exception):
class BannedDBSchemaOperations(fixtures.Fixture):
"""Ban some operations for migrations."""
def __init__(self, banned_ops=None,
migration_repo=migrate_repo.__file__):
super(BannedDBSchemaOperations, self).__init__()
def __init__(self, banned_ops, migration_repo):
super().__init__()
self._banned_ops = banned_ops or {}
self._migration_repo = migration_repo
@ -54,7 +52,7 @@ class BannedDBSchemaOperations(fixtures.Fixture):
resource_op, repo_name))
def setUp(self):
super(BannedDBSchemaOperations, self).setUp()
super().setUp()
explode_lambda = {
'Table.create': lambda *a, **k: self._explode(
'Table.create', self._migration_repo),
@ -91,7 +89,9 @@ class TestBannedDBSchemaOperations(testtools.TestCase):
"""Test column operations raise DBOperationNotAllowed."""
column = sqlalchemy.Column()
with BannedDBSchemaOperations(
banned_ops={'Column': ['create', 'alter', 'drop']}):
banned_ops={'Column': ['create', 'alter', 'drop']},
migration_repo=expand_repo.__file__,
):
self.assertRaises(DBOperationNotAllowed, column.drop)
self.assertRaises(DBOperationNotAllowed, column.alter)
self.assertRaises(DBOperationNotAllowed, column.create)
@ -100,8 +100,10 @@ class TestBannedDBSchemaOperations(testtools.TestCase):
"""Test table operations raise DBOperationNotAllowed."""
table = sqlalchemy.Table()
with BannedDBSchemaOperations(
banned_ops={'Table': ['create', 'alter', 'drop',
'insert', 'update', 'delete']}):
banned_ops={'Table': ['create', 'alter', 'drop',
'insert', 'update', 'delete']},
migration_repo=expand_repo.__file__,
):
self.assertRaises(DBOperationNotAllowed, table.drop)
self.assertRaises(DBOperationNotAllowed, table.alter)
self.assertRaises(DBOperationNotAllowed, table.create)
@ -113,29 +115,14 @@ class TestBannedDBSchemaOperations(testtools.TestCase):
class KeystoneMigrationsCheckers(test_migrations.WalkVersionsMixin):
"""Walk over and test all sqlalchemy-migrate migrations."""
# NOTE(xek): We start requiring things be additive in Newton, so
# ignore all migrations before the first version in Newton.
migrate_file = migrate_repo.__file__
first_version = 101
# NOTE(henry-nash): We don't ban data modification in the legacy repo,
# since there are already migrations that do this for Newton (and these
# do not cause us issues, or are already worked around).
banned_ops = {'Table': ['alter', 'drop'],
'Column': ['alter', 'drop']}
migrate_file = None
first_version = 1
# A mapping of entity (Table, Column, ...) to operation
banned_ops = {}
exceptions = [
# NOTE(xek): Reviewers: DO NOT ALLOW THINGS TO BE ADDED HERE UNLESS
# JUSTIFICATION CAN BE PROVIDED AS TO WHY THIS WILL NOT CAUSE
# PROBLEMS FOR ROLLING UPGRADES.
# Migration 102 drops the domain table in the Newton release. All
# code that referenced the domain table was removed in the Mitaka
# release, hence this migration will not cause problems when
# running a mixture of Mitaka and Newton versions of keystone.
102,
# Migration 106 simply allows the password column to be nullable.
# This change would not impact a rolling upgrade.
106
]
@property
@ -190,8 +177,7 @@ class KeystoneMigrationsCheckers(test_migrations.WalkVersionsMixin):
else:
banned_ops = None
with BannedDBSchemaOperations(banned_ops, self.migrate_file):
super(KeystoneMigrationsCheckers,
self).migrate_up(version, with_data)
super().migrate_up(version, with_data)
snake_walk = False
downgrade = False
@ -200,43 +186,7 @@ class KeystoneMigrationsCheckers(test_migrations.WalkVersionsMixin):
self.walk_versions(self.snake_walk, self.downgrade)
class TestKeystoneMigrationsMySQL(
KeystoneMigrationsCheckers,
db_fixtures.OpportunisticDBTestMixin,
test_base.BaseTestCase):
FIXTURE = db_fixtures.MySQLOpportunisticFixture
def setUp(self):
super(TestKeystoneMigrationsMySQL, self).setUp()
self.engine = enginefacade.writer.get_engine()
self.sessionmaker = enginefacade.writer.get_sessionmaker()
class TestKeystoneMigrationsPostgreSQL(
KeystoneMigrationsCheckers,
db_fixtures.OpportunisticDBTestMixin,
test_base.BaseTestCase):
FIXTURE = db_fixtures.PostgresqlOpportunisticFixture
def setUp(self):
super(TestKeystoneMigrationsPostgreSQL, self).setUp()
self.engine = enginefacade.writer.get_engine()
self.sessionmaker = enginefacade.writer.get_sessionmaker()
class TestKeystoneMigrationsSQLite(
KeystoneMigrationsCheckers,
db_fixtures.OpportunisticDBTestMixin,
test_base.BaseTestCase):
def setUp(self):
super(TestKeystoneMigrationsSQLite, self).setUp()
self.engine = enginefacade.writer.get_engine()
self.sessionmaker = enginefacade.writer.get_sessionmaker()
class TestKeystoneExpandSchemaMigrations(
KeystoneMigrationsCheckers):
class TestKeystoneExpandSchemaMigrations(KeystoneMigrationsCheckers):
migrate_file = expand_repo.__file__
first_version = 1
@ -285,7 +235,6 @@ class TestKeystoneExpandSchemaMigrationsMySQL(
super(TestKeystoneExpandSchemaMigrationsMySQL, self).setUp()
self.engine = enginefacade.writer.get_engine()
self.sessionmaker = enginefacade.writer.get_sessionmaker()
self.migrate_fully(migrate_repo.__file__)
class TestKeystoneExpandSchemaMigrationsPostgreSQL(
@ -298,7 +247,6 @@ class TestKeystoneExpandSchemaMigrationsPostgreSQL(
super(TestKeystoneExpandSchemaMigrationsPostgreSQL, self).setUp()
self.engine = enginefacade.writer.get_engine()
self.sessionmaker = enginefacade.writer.get_sessionmaker()
self.migrate_fully(migrate_repo.__file__)
class TestKeystoneDataMigrations(
@ -326,7 +274,6 @@ class TestKeystoneDataMigrations(
def setUp(self):
super(TestKeystoneDataMigrations, self).setUp()
self.migrate_fully(migrate_repo.__file__)
self.migrate_fully(expand_repo.__file__)
@ -387,7 +334,6 @@ class TestKeystoneContractSchemaMigrations(
def setUp(self):
super(TestKeystoneContractSchemaMigrations, self).setUp()
self.migrate_fully(migrate_repo.__file__)
self.migrate_fully(expand_repo.__file__)
self.migrate_fully(data_migration_repo.__file__)

149
keystone/tests/unit/test_sql_upgrade.py
View File

@ -194,7 +194,6 @@ INITIAL_TABLE_STRUCTURE = {
],
}
LEGACY_REPO = 'migrate_repo'
EXPAND_REPO = 'expand_repo'
DATA_MIGRATION_REPO = 'data_migration_repo'
CONTRACT_REPO = 'contract_repo'
@ -222,7 +221,7 @@ class SqlUpgradeGetInitVersionTests(unit.TestCase):
# first invocation of repo. Cannot match the full path because it is
# based on where the test is run.
param = repo.call_args_list[0][0][0]
self.assertTrue(param.endswith('/sql/' + LEGACY_REPO))
self.assertTrue(param.endswith('/sql/' + EXPAND_REPO))
@mock.patch.object(repository, 'Repository')
def test_get_init_version_with_path_initial_version_0(self, repo):
@ -235,7 +234,7 @@ class SqlUpgradeGetInitVersionTests(unit.TestCase):
# os.path.isdir() is called by `find_repo()`. Mock it to avoid
# an exception.
with mock.patch('os.path.isdir', return_value=True):
path = '/keystone/' + LEGACY_REPO + '/'
path = '/keystone/' + EXPAND_REPO + '/'
# since 0 is the smallest version expect None
version = upgrades.get_init_version(abs_path=path)
@ -253,16 +252,18 @@ class SqlUpgradeGetInitVersionTests(unit.TestCase):
# os.path.isdir() is called by `find_repo()`. Mock it to avoid
# an exception.
with mock.patch('os.path.isdir', return_value=True):
path = '/keystone/' + LEGACY_REPO + '/'
path = '/keystone/' + EXPAND_REPO + '/'
version = upgrades.get_init_version(abs_path=path)
self.assertEqual(initial_version, version)
class SqlMigrateBase(db_fixtures.OpportunisticDBTestMixin,
test_base.BaseTestCase):
class MigrateBase(
db_fixtures.OpportunisticDBTestMixin,
test_base.BaseTestCase,
):
def setUp(self):
super(SqlMigrateBase, self).setUp()
super().setUp()
self.engine = enginefacade.writer.get_engine()
self.sessionmaker = enginefacade.writer.get_sessionmaker()
@ -284,15 +285,12 @@ class SqlMigrateBase(db_fixtures.OpportunisticDBTestMixin,
self.addCleanup(sql.cleanup)
self.repos = {
LEGACY_REPO: upgrades.Repository(self.engine, LEGACY_REPO),
EXPAND_REPO: upgrades.Repository(self.engine, EXPAND_REPO),
DATA_MIGRATION_REPO: upgrades.Repository(
self.engine, DATA_MIGRATION_REPO),
CONTRACT_REPO: upgrades.Repository(self.engine, CONTRACT_REPO)}
def upgrade(self, *args, **kwargs):
"""Upgrade the legacy migration repository."""
self.repos[LEGACY_REPO].upgrade(*args, **kwargs)
self.engine, DATA_MIGRATION_REPO,
),
CONTRACT_REPO: upgrades.Repository(self.engine, CONTRACT_REPO),
}
def expand(self, *args, **kwargs):
"""Expand database schema."""
@ -403,21 +401,18 @@ class SqlMigrateBase(db_fixtures.OpportunisticDBTestMixin,
return False
class SqlLegacyRepoUpgradeTests(SqlMigrateBase):
class ExpandSchemaUpgradeTests(MigrateBase):
def test_start_version_db_init_version(self):
self.assertEqual(
self.repos[EXPAND_REPO].min_version,
self.repos[EXPAND_REPO].version)
def test_blank_db_to_start(self):
self.assertTableDoesNotExist('user')
def test_start_version_db_init_version(self):
self.assertEqual(
self.repos[LEGACY_REPO].min_version,
self.repos[LEGACY_REPO].version,
'DB is not at version %s' % (
self.repos[LEGACY_REPO].min_version)
)
def test_upgrade_add_initial_tables(self):
self.upgrade(self.repos[LEGACY_REPO].min_version + 1)
self.expand(1)
self.check_initial_table_structure()
def check_initial_table_structure(self):
@ -425,45 +420,24 @@ class SqlLegacyRepoUpgradeTests(SqlMigrateBase):
self.assertTableColumns(table, INITIAL_TABLE_STRUCTURE[table])
class MySQLOpportunisticUpgradeTestCase(SqlLegacyRepoUpgradeTests):
FIXTURE = db_fixtures.MySQLOpportunisticFixture
class PostgreSQLOpportunisticUpgradeTestCase(SqlLegacyRepoUpgradeTests):
FIXTURE = db_fixtures.PostgresqlOpportunisticFixture
class SqlExpandSchemaUpgradeTests(SqlMigrateBase):
def setUp(self):
# Make sure the main repo is fully upgraded for this release since the
# expand phase is only run after such an upgrade
super(SqlExpandSchemaUpgradeTests, self).setUp()
self.upgrade()
def test_start_version_db_init_version(self):
self.assertEqual(
self.repos[EXPAND_REPO].min_version,
self.repos[EXPAND_REPO].version)
class MySQLOpportunisticExpandSchemaUpgradeTestCase(
SqlExpandSchemaUpgradeTests):
ExpandSchemaUpgradeTests,
):
FIXTURE = db_fixtures.MySQLOpportunisticFixture
class PostgreSQLOpportunisticExpandSchemaUpgradeTestCase(
SqlExpandSchemaUpgradeTests):
ExpandSchemaUpgradeTests,
):
FIXTURE = db_fixtures.PostgresqlOpportunisticFixture
class SqlDataMigrationUpgradeTests(SqlMigrateBase):
class DataMigrationUpgradeTests(MigrateBase):
def setUp(self):
# Make sure the legacy and expand repos are fully upgraded, since the
# data migration phase is only run after these are upgraded
super(SqlDataMigrationUpgradeTests, self).setUp()
self.upgrade()
# Make sure the expand repo is fully upgraded, since the data migration
# phase is only run after this is upgraded
super().setUp()
self.expand()
def test_start_version_db_init_version(self):
@ -473,22 +447,24 @@ class SqlDataMigrationUpgradeTests(SqlMigrateBase):
class MySQLOpportunisticDataMigrationUpgradeTestCase(
SqlDataMigrationUpgradeTests):
DataMigrationUpgradeTests,
):
FIXTURE = db_fixtures.MySQLOpportunisticFixture
class PostgreSQLOpportunisticDataMigrationUpgradeTestCase(
SqlDataMigrationUpgradeTests):
DataMigrationUpgradeTests,
):
FIXTURE = db_fixtures.PostgresqlOpportunisticFixture
class SqlContractSchemaUpgradeTests(SqlMigrateBase, unit.TestCase):
class ContractSchemaUpgradeTests(MigrateBase, unit.TestCase):
def setUp(self):
# Make sure the legacy, expand and data migration repos are fully
# Make sure the expand and data migration repos are fully
# upgraded, since the contract phase is only run after these are
# upgraded.
super(SqlContractSchemaUpgradeTests, self).setUp()
super().setUp()
self.useFixture(
ksfixtures.KeyRepository(
self.config_fixture,
@ -496,7 +472,6 @@ class SqlContractSchemaUpgradeTests(SqlMigrateBase, unit.TestCase):
credential_fernet.MAX_ACTIVE_KEYS
)
)
self.upgrade()
self.expand()
self.migrate()
@ -507,53 +482,18 @@ class SqlContractSchemaUpgradeTests(SqlMigrateBase, unit.TestCase):
class MySQLOpportunisticContractSchemaUpgradeTestCase(
SqlContractSchemaUpgradeTests):
ContractSchemaUpgradeTests,
):
FIXTURE = db_fixtures.MySQLOpportunisticFixture
class PostgreSQLOpportunisticContractSchemaUpgradeTestCase(
SqlContractSchemaUpgradeTests):
ContractSchemaUpgradeTests,
):
FIXTURE = db_fixtures.PostgresqlOpportunisticFixture
class VersionTests(SqlMigrateBase):
def test_core_initial(self):
"""Get the version before migrated, it's the initial DB version."""
self.assertEqual(
self.repos[LEGACY_REPO].min_version,
self.repos[LEGACY_REPO].version)
def test_core_max(self):
"""When get the version after upgrading, it's the new version."""
self.upgrade()
self.assertEqual(
self.repos[LEGACY_REPO].max_version,
self.repos[LEGACY_REPO].version)
def test_assert_not_schema_downgrade(self):
self.upgrade()
self.assertRaises(
db_exception.DBMigrationError,
upgrades._sync_common_repo,
self.repos[LEGACY_REPO].max_version - 1)
def test_these_are_not_the_migrations_you_are_looking_for(self):
"""Keystone has shifted to rolling upgrades.
New database migrations should no longer land in the legacy migration
repository. Instead, new database migrations should be divided into
three discrete steps: schema expansion, data migration, and schema
contraction. These migrations live in a new set of database migration
repositories, called ``expand_repo``, ``data_migration_repo``, and
``contract_repo``.
For more information, see "Database Migrations" here:
https://docs.openstack.org/keystone/latest/contributor/database-migrations.html
"""
# Note to reviewers: this version number should never change.
self.assertEqual(109, self.repos[LEGACY_REPO].max_version)
class VersionTests(MigrateBase):
def test_migrate_repos_stay_in_lockstep(self):
"""Rolling upgrade repositories should always stay in lockstep.
@ -618,7 +558,7 @@ class VersionTests(SqlMigrateBase):
self.assertRegex(file_name, pattern, msg)
class MigrationValidation(SqlMigrateBase, unit.TestCase):
class MigrationValidation(MigrateBase, unit.TestCase):
"""Test validation of database between database phases."""
def _set_db_sync_command_versions(self):
@ -630,7 +570,6 @@ class MigrationValidation(SqlMigrateBase, unit.TestCase):
self.assertEqual(upgrades.get_db_version('contract_repo'), 1)
def test_running_db_sync_migrate_ahead_of_expand_fails(self):
self.upgrade()
self._set_db_sync_command_versions()
self.assertRaises(
db_exception.DBMigrationError,
@ -639,7 +578,6 @@ class MigrationValidation(SqlMigrateBase, unit.TestCase):
"You are attempting to upgrade migrate ahead of expand")
def test_running_db_sync_contract_ahead_of_migrate_fails(self):
self.upgrade()
self._set_db_sync_command_versions()
self.assertRaises(
db_exception.DBMigrationError,
@ -648,14 +586,9 @@ class MigrationValidation(SqlMigrateBase, unit.TestCase):
"You are attempting to upgrade contract ahead of migrate")
class FullMigration(SqlMigrateBase, unit.TestCase):
class FullMigration(MigrateBase, unit.TestCase):
"""Test complete orchestration between all database phases."""
def setUp(self):
super(FullMigration, self).setUp()
# Upgrade the legacy repository
self.upgrade()
def test_db_sync_check(self):
checker = cli.DbSync()
latest_version = self.repos[EXPAND_REPO].max_version

7
releasenotes/notes/remove-legacy-migrations-647f60019c8dd9e8.yaml Normal file
View File

@ -0,0 +1,7 @@
---
upgrade:
- |
The legacy migrations that existed before the split into separate expand
schema, contract schema, and data migration migration have now been
removed. These have been deprecated since 10.0.0 (Newton). This should
have no user-facing impact.