diff --git a/keystone/common/sql/migrate_repo/versions/001_add_initial_tables.py b/keystone/common/sql/migrate_repo/versions/001_add_initial_tables.py deleted file mode 100644 index db8779ff5f..0000000000 --- a/keystone/common/sql/migrate_repo/versions/001_add_initial_tables.py +++ /dev/null @@ -1,155 +0,0 @@ -# Copyright 2012 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql - - -def upgrade(migrate_engine): - # Upgrade operations go here. Don't create your own engine; bind - # migrate_engine to your metadata - meta = sql.MetaData() - meta.bind = migrate_engine - - # catalog - - service_table = sql.Table( - 'service', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('type', sql.String(255)), - sql.Column('extra', sql.Text())) - service_table.create(migrate_engine, checkfirst=True) - - endpoint_table = sql.Table( - 'endpoint', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('region', sql.String(255)), - sql.Column('service_id', - sql.String(64), - sql.ForeignKey('service.id'), - nullable=False), - sql.Column('extra', sql.Text())) - endpoint_table.create(migrate_engine, checkfirst=True) - - # identity - - role_table = sql.Table( - 'role', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('name', sql.String(255), unique=True, nullable=False)) - role_table.create(migrate_engine, checkfirst=True) - - if migrate_engine.name == 'ibm_db_sa': - # NOTE(blk-u): SQLAlchemy for PostgreSQL picks the name tenant_name_key - # for the unique constraint, but for DB2 doesn't give the UC a name - # unless we tell it to and there is no DDL to alter a column to drop - # an unnamed unique constraint, so this code creates a named unique - # constraint on the name column rather than an unnamed one. - # (This is used in migration 16.) - tenant_table = sql.Table( - 'tenant', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('name', sql.String(64), nullable=False), - sql.Column('extra', sql.Text()), - sql.UniqueConstraint('name', name='tenant_name_key')) - else: - tenant_table = sql.Table( - 'tenant', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('name', sql.String(64), unique=True, nullable=False), - sql.Column('extra', sql.Text())) - - tenant_table.create(migrate_engine, checkfirst=True) - - metadata_table = sql.Table( - 'metadata', - meta, - sql.Column('user_id', sql.String(64), primary_key=True), - sql.Column('tenant_id', sql.String(64), primary_key=True), - sql.Column('data', sql.Text())) - metadata_table.create(migrate_engine, checkfirst=True) - - ec2_credential_table = sql.Table( - 'ec2_credential', - meta, - sql.Column('access', sql.String(64), primary_key=True), - sql.Column('secret', sql.String(64)), - sql.Column('user_id', sql.String(64)), - sql.Column('tenant_id', sql.String(64))) - ec2_credential_table.create(migrate_engine, checkfirst=True) - - if migrate_engine.name == 'ibm_db_sa': - # NOTE(blk-u): SQLAlchemy for PostgreSQL picks the name user_name_key - # for the unique constraint, but for DB2 doesn't give the UC a name - # unless we tell it to and there is no DDL to alter a column to drop - # an unnamed unique constraint, so this code creates a named unique - # constraint on the name column rather than an unnamed one. - # (This is used in migration 16.) - user_table = sql.Table( - 'user', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('name', sql.String(64), nullable=False), - sql.Column('extra', sql.Text()), - sql.UniqueConstraint('name', name='user_name_key')) - else: - user_table = sql.Table( - 'user', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('name', sql.String(64), unique=True, nullable=False), - sql.Column('extra', sql.Text())) - - user_table.create(migrate_engine, checkfirst=True) - - user_tenant_membership_table = sql.Table( - 'user_tenant_membership', - meta, - sql.Column( - 'user_id', - sql.String(64), - sql.ForeignKey('user.id'), - primary_key=True), - sql.Column( - 'tenant_id', - sql.String(64), - sql.ForeignKey('tenant.id'), - primary_key=True)) - user_tenant_membership_table.create(migrate_engine, checkfirst=True) - - # token - - token_table = sql.Table( - 'token', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('expires', sql.DateTime()), - sql.Column('extra', sql.Text())) - token_table.create(migrate_engine, checkfirst=True) - - -def downgrade(migrate_engine): - # Operations to reverse the above upgrade go here. - meta = sql.MetaData() - meta.bind = migrate_engine - - tables = ['user_tenant_membership', 'token', 'user', 'tenant', 'role', - 'metadata', 'ec2_credential', 'endpoint', 'service'] - for t in tables: - table = sql.Table(t, meta, autoload=True) - table.drop(migrate_engine, checkfirst=True) diff --git a/keystone/common/sql/migrate_repo/versions/002_sqlite_downgrade.sql b/keystone/common/sql/migrate_repo/versions/002_sqlite_downgrade.sql deleted file mode 100644 index d3ebe92a88..0000000000 --- a/keystone/common/sql/migrate_repo/versions/002_sqlite_downgrade.sql +++ /dev/null @@ -1,8 +0,0 @@ -drop table token; - -CREATE TABLE token ( - id VARCHAR(64) NOT NULL, - expires DATETIME, - extra TEXT, - PRIMARY KEY (id) -); diff --git a/keystone/common/sql/migrate_repo/versions/002_sqlite_upgrade.sql b/keystone/common/sql/migrate_repo/versions/002_sqlite_upgrade.sql deleted file mode 100644 index 41225dae89..0000000000 --- a/keystone/common/sql/migrate_repo/versions/002_sqlite_upgrade.sql +++ /dev/null @@ -1,27 +0,0 @@ -CREATE TABLE token_backup ( - id_hash VARCHAR(64) NOT NULL, - id VARCHAR(1024), - expires DATETIME, - extra TEXT, - PRIMARY KEY (id_hash) -); - -insert into token_backup - select id as old_id, - '', - expires as old_expires, - extra as old_extra from token; - -drop table token; - -CREATE TABLE token ( - id_hash VARCHAR(64) NOT NULL, - id VARCHAR(1024), - expires DATETIME, - extra TEXT, - PRIMARY KEY (id_hash) -); - -insert into token select * from token_backup; - -drop table token_backup; diff --git a/keystone/common/sql/migrate_repo/versions/002_token_id_hash.py b/keystone/common/sql/migrate_repo/versions/002_token_id_hash.py deleted file mode 100644 index d2b6d47300..0000000000 --- a/keystone/common/sql/migrate_repo/versions/002_token_id_hash.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from sqlalchemy import Column, MetaData, String, Table - - -def upgrade(migrate_engine): - meta = MetaData() - meta.bind = migrate_engine - token = Table('token', meta, autoload=True) - old_id_col = token.c.id - old_id_col.alter(name='id_hash') - # Note: We obtain a new metadata reference to avoid - # sqlalchemy.exc.ArgumentError: - # Trying to redefine primary-key column 'id' as a non-primary-key... - meta = MetaData() - meta.bind = migrate_engine - token = Table('token', meta, autoload=True) - new_id = Column("id", String(2048)) - token.create_column(new_id) - - -def downgrade(migrate_engine): - meta = MetaData() - meta.bind = migrate_engine - token = Table('token', meta, autoload=True) - token.drop_column('id') - token = Table('token', meta, autoload=True) - id_col = token.c.id_hash - id_col.alter(name='id') diff --git a/keystone/common/sql/migrate_repo/versions/003_sqlite_upgrade.sql b/keystone/common/sql/migrate_repo/versions/003_sqlite_upgrade.sql deleted file mode 100644 index 963bfa0ab9..0000000000 --- a/keystone/common/sql/migrate_repo/versions/003_sqlite_upgrade.sql +++ /dev/null @@ -1,3 +0,0 @@ -alter TABLE token ADD valid integer; -update token set valid = 1; - diff --git a/keystone/common/sql/migrate_repo/versions/003_token_valid.py b/keystone/common/sql/migrate_repo/versions/003_token_valid.py deleted file mode 100644 index 5fefc05150..0000000000 --- a/keystone/common/sql/migrate_repo/versions/003_token_valid.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2012 OpenStack LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql - - -def upgrade(migrate_engine): - # Upgrade operations go here. Don't create your own engine; bind - - meta = sql.MetaData() - meta.bind = migrate_engine - token = sql.Table('token', meta, autoload=True) - # creating the column immediately with nullable=False fails with - # PostgreSQL (LP 1068181), so do it in two steps instead - valid = sql.Column( - 'valid', sql.Boolean(), sql.ColumnDefault(True), nullable=True) - valid.create(token, populate_default=True) - valid.alter(type=sql.Boolean(), default=True, nullable=False) - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - token = sql.Table('token', meta, autoload=True) - token.drop_column('valid') diff --git a/keystone/common/sql/migrate_repo/versions/004_undo_token_id_hash.py b/keystone/common/sql/migrate_repo/versions/004_undo_token_id_hash.py deleted file mode 100644 index cfe1fce814..0000000000 --- a/keystone/common/sql/migrate_repo/versions/004_undo_token_id_hash.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from sqlalchemy import Column, MetaData, String, Table - - -def downgrade(migrate_engine): - meta = MetaData() - meta.bind = migrate_engine - token = Table('token', meta, autoload=True) - old_id_col = token.c.id - old_id_col.alter(name='id_hash') - # Note: We obtain a new metadata reference to avoid - # sqlalchemy.exc.ArgumentError: - # Trying to redefine primary-key column 'id' as a non-primary-key... - meta = MetaData() - meta.bind = migrate_engine - token = Table('token', meta, autoload=True) - new_id = Column("id", String(2048)) - token.create_column(new_id) - - -def upgrade(migrate_engine): - meta = MetaData() - meta.bind = migrate_engine - token = Table('token', meta, autoload=True) - token.drop_column('id') - token = Table('token', meta, autoload=True) - id_col = token.c.id_hash - id_col.alter(name='id') diff --git a/keystone/common/sql/migrate_repo/versions/005_set_utf8_character_set.py b/keystone/common/sql/migrate_repo/versions/005_set_utf8_character_set.py deleted file mode 100644 index 72cac71f36..0000000000 --- a/keystone/common/sql/migrate_repo/versions/005_set_utf8_character_set.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2012 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from sqlalchemy import MetaData - - -def upgrade(migrate_engine): - # Upgrade operations go here. Don't create your own engine; bind - # migrate_engine to your metadata - meta = MetaData() - meta.bind = migrate_engine - - if migrate_engine.name == "mysql": - tables = ['tenant', 'user', 'role', 'token', 'service', 'metadata', - 'ec2_credential', 'endpoint', 'user_tenant_membership'] - sql = "SET foreign_key_checks = 0;" - - for table in tables: - sql += "ALTER TABLE %s CONVERT TO CHARACTER SET utf8;" % table - sql += "SET foreign_key_checks = 1;" - sql += "ALTER DATABASE %s DEFAULT CHARACTER SET utf8;" \ - % migrate_engine.url.database - migrate_engine.execute(sql) - - -def downgrade(migrate_engine): - # Operations to reverse the above upgrade go here. - if migrate_engine.name == "mysql": - tables = ['tenant', 'user', 'role', 'token', 'service', 'metadata', - 'ec2_credential', 'endpoint', 'user_tenant_membership'] - sql = "SET foreign_key_checks = 0;" - - for table in tables: - sql += "ALTER TABLE %s CONVERT TO CHARACTER SET latin1;" % table - sql += "SET foreign_key_checks = 1;" - sql += "ALTER DATABASE %s DEFAULT CHARACTER SET latin1;" \ - % migrate_engine.url.database diff --git a/keystone/common/sql/migrate_repo/versions/006_add_policy_table.py b/keystone/common/sql/migrate_repo/versions/006_add_policy_table.py deleted file mode 100644 index 62f87d427b..0000000000 --- a/keystone/common/sql/migrate_repo/versions/006_add_policy_table.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2012 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - policy_table = sql.Table( - 'policy', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('type', sql.String(255), nullable=False), - sql.Column('blob', sql.Text(), nullable=False), - sql.Column('extra', sql.Text())) - policy_table.create(migrate_engine, checkfirst=True) - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - policy_table = sql.Table('policy', meta, autoload=True) - policy_table.drop(migrate_engine, checkfirst=True) diff --git a/keystone/common/sql/migrate_repo/versions/007_add_domain_tables.py b/keystone/common/sql/migrate_repo/versions/007_add_domain_tables.py deleted file mode 100644 index adf350b4c6..0000000000 --- a/keystone/common/sql/migrate_repo/versions/007_add_domain_tables.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright 2012 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql - - -def upgrade(migrate_engine): - # Upgrade operations go here. Don't create your own engine; bind - # migrate_engine to your metadata - meta = sql.MetaData() - meta.bind = migrate_engine - - domain_table = sql.Table( - 'domain', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('name', sql.String(64), unique=True, nullable=False), - sql.Column('enabled', sql.Boolean, nullable=False, default=True), - sql.Column('extra', sql.Text())) - domain_table.create(migrate_engine, checkfirst=True) - - sql.Table('user', meta, autoload=True) - user_domain_metadata_table = sql.Table( - 'user_domain_metadata', - meta, - sql.Column( - 'user_id', - sql.String(64), - sql.ForeignKey('user.id'), - primary_key=True), - sql.Column( - 'domain_id', - sql.String(64), - sql.ForeignKey('domain.id'), - primary_key=True), - sql.Column('data', sql.Text())) - user_domain_metadata_table.create(migrate_engine, checkfirst=True) - - sql.Table('tenant', meta, autoload=True) - credential_table = sql.Table( - 'credential', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('user_id', - sql.String(64), - sql.ForeignKey('user.id'), - nullable=False), - sql.Column('project_id', - sql.String(64), - sql.ForeignKey('tenant.id')), - sql.Column('blob', sql.Text(), nullable=False), - sql.Column('type', sql.String(255), nullable=False), - sql.Column('extra', sql.Text())) - credential_table.create(migrate_engine, checkfirst=True) - - role = sql.Table('role', meta, autoload=True) - extra = sql.Column('extra', sql.Text()) - role.create_column(extra) - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - role = sql.Table('role', meta, autoload=True) - role.drop_column('extra') - - tables = ['user_domain_metadata', 'credential', 'domain'] - for t in tables: - table = sql.Table(t, meta, autoload=True) - table.drop(migrate_engine, checkfirst=True) diff --git a/keystone/common/sql/migrate_repo/versions/008_create_default_domain.py b/keystone/common/sql/migrate_repo/versions/008_create_default_domain.py deleted file mode 100644 index 09e16cbf5e..0000000000 --- a/keystone/common/sql/migrate_repo/versions/008_create_default_domain.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2012 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import json - -import sqlalchemy as sql -from sqlalchemy import orm - -from keystone import config - - -CONF = config.CONF - - -def upgrade(migrate_engine): - """Creates the default domain.""" - meta = sql.MetaData() - meta.bind = migrate_engine - - domain_table = sql.Table('domain', meta, autoload=True) - - domain = { - 'id': CONF.identity.default_domain_id, - 'name': 'Default', - 'enabled': True, - 'extra': json.dumps({ - 'description': 'Owns users and tenants (i.e. projects) available ' - 'on Identity API v2.'})} - - session = orm.sessionmaker(bind=migrate_engine)() - insert = domain_table.insert() - insert.execute(domain) - session.commit() - - -def downgrade(migrate_engine): - """Delete the default domain.""" - meta = sql.MetaData() - meta.bind = migrate_engine - - sql.Table('domain', meta, autoload=True) - session = orm.sessionmaker(bind=migrate_engine)() - session.execute( - 'DELETE FROM domain WHERE id=:id', - {'id': CONF.identity.default_domain_id}) - session.commit() diff --git a/keystone/common/sql/migrate_repo/versions/009_normalize_identity.py b/keystone/common/sql/migrate_repo/versions/009_normalize_identity.py deleted file mode 100644 index c3fa4cf47c..0000000000 --- a/keystone/common/sql/migrate_repo/versions/009_normalize_identity.py +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright 2012 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -from sqlalchemy import Column, MetaData, String, Table, Text, types -from sqlalchemy.orm import sessionmaker - - -# sqlite doesn't support dropping columns. Copy to a new table instead -def downgrade_user_table_with_copy(meta, migrate_engine): - maker = sessionmaker(bind=migrate_engine) - session = maker() - session.execute("ALTER TABLE user RENAME TO orig_user;") - - user_table = Table( - 'user', - meta, - Column('id', String(64), primary_key=True), - Column('name', String(64), unique=True, nullable=False), - Column('extra', Text())) - user_table.create(migrate_engine, checkfirst=True) - - orig_user_table = Table('orig_user', meta, autoload=True) - for user in session.query(orig_user_table): - session.execute("insert into user (id, name, extra) " - "values ( :id, :name, :extra);", - {'id': user.id, - 'name': user.name, - 'extra': user.extra}) - session.execute("drop table orig_user;") - session.close() - - -def downgrade_tenant_table_with_copy(meta, migrate_engine): - maker = sessionmaker(bind=migrate_engine) - session = maker() - session.execute("ALTER TABLE tenant RENAME TO orig_tenant;") - - tenant_table = Table( - 'tenant', - meta, - Column('id', String(64), primary_key=True), - Column('name', String(64), unique=True, nullable=False), - Column('extra', Text())) - tenant_table.create(migrate_engine, checkfirst=True) - - orig_tenant_table = Table('orig_tenant', meta, autoload=True) - for tenant in session.query(orig_tenant_table): - session.execute("insert into tenant (id, name, extra) " - "values ( :id, :name, :extra);", - {'id': tenant.id, - 'name': tenant.name, - 'extra': tenant.extra}) - session.execute("drop table orig_tenant;") - session.close() - - -def downgrade_user_table_with_column_drop(meta, migrate_engine): - user_table = Table('user', meta, autoload=True) - user_table.drop_column(Column('password', String(128))) - user_table.drop_column(Column('enabled', types.Boolean, - default=True)) - - -def downgrade_tenant_table_with_column_drop(meta, migrate_engine): - tenant_table = Table('tenant', meta, autoload=True) - tenant_table.drop_column(Column('description', Text())) - tenant_table.drop_column(Column('enabled', types.Boolean)) - - -def upgrade_user_table(meta, migrate_engine): - user_table = Table('user', meta, autoload=True) - user_table.create_column(Column('password', String(128))) - user_table.create_column(Column('enabled', types.Boolean, - default=True)) - - -def upgrade_tenant_table(meta, migrate_engine): - tenant_table = Table('tenant', meta, autoload=True) - tenant_table.create_column(Column('description', Text())) - tenant_table.create_column(Column('enabled', types.Boolean)) - - -def upgrade(migrate_engine): - meta = MetaData() - meta.bind = migrate_engine - upgrade_user_table(meta, migrate_engine) - upgrade_tenant_table(meta, migrate_engine) - - -def downgrade(migrate_engine): - meta = MetaData() - meta.bind = migrate_engine - if migrate_engine.name == 'sqlite': - downgrade_user_table_with_copy(meta, migrate_engine) - downgrade_tenant_table_with_copy(meta, migrate_engine) - else: - downgrade_user_table_with_column_drop(meta, migrate_engine) - downgrade_tenant_table_with_column_drop(meta, migrate_engine) diff --git a/keystone/common/sql/migrate_repo/versions/010_normalize_identity_migration.py b/keystone/common/sql/migrate_repo/versions/010_normalize_identity_migration.py deleted file mode 100644 index 41b609ad72..0000000000 --- a/keystone/common/sql/migrate_repo/versions/010_normalize_identity_migration.py +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright 2012 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import json - -import six - -from sqlalchemy import MetaData, Table -from sqlalchemy.orm import sessionmaker - - -DISABLED_VALUES = ['false', 'disabled', 'no', '0'] - - -def is_enabled(enabled): - # no explicit value means enabled - if enabled is True or enabled is None: - return True - if (isinstance(enabled, six.string_types) - and enabled.lower() in DISABLED_VALUES): - return False - return bool(enabled) - - -def downgrade_user_table(meta, migrate_engine, session): - user_table = Table('user', meta, autoload=True) - for user in session.query(user_table).all(): - extra = json.loads(user.extra) - extra['password'] = user.password - extra['enabled'] = '%r' % is_enabled(user.enabled) - values = {'extra': json.dumps(extra)} - update = user_table.update().\ - where(user_table.c.id == user.id).\ - values(values) - migrate_engine.execute(update) - - -def downgrade_tenant_table(meta, migrate_engine, session): - tenant_table = Table('tenant', meta, autoload=True) - for tenant in session.query(tenant_table).all(): - extra = json.loads(tenant.extra) - extra['description'] = tenant.description - extra['enabled'] = '%r' % is_enabled(tenant.enabled) - values = {'extra': json.dumps(extra)} - update = tenant_table.update().\ - where(tenant_table.c.id == tenant.id).\ - values(values) - migrate_engine.execute(update) - - -def upgrade_user_table(meta, migrate_engine, session): - user_table = Table('user', meta, autoload=True) - for user in session.query(user_table).all(): - extra = json.loads(user.extra) - values = {'password': extra.pop('password', None), - 'enabled': is_enabled(extra.pop('enabled', True)), - 'extra': json.dumps(extra)} - update = user_table.update().\ - where(user_table.c.id == user.id).\ - values(values) - migrate_engine.execute(update) - - -def upgrade_tenant_table(meta, migrate_engine, session): - tenant_table = Table('tenant', meta, autoload=True) - for tenant in session.query(tenant_table): - extra = json.loads(tenant.extra) - values = {'description': extra.pop('description', None), - 'enabled': is_enabled(extra.pop('enabled', True)), - 'extra': json.dumps(extra)} - update = tenant_table.update().\ - where(tenant_table.c.id == tenant.id).\ - values(values) - migrate_engine.execute(update) - - -def upgrade(migrate_engine): - meta = MetaData() - meta.bind = migrate_engine - session = sessionmaker(bind=migrate_engine)() - upgrade_user_table(meta, migrate_engine, session) - upgrade_tenant_table(meta, migrate_engine, session) - session.commit() - - -def downgrade(migrate_engine): - meta = MetaData() - meta.bind = migrate_engine - session = sessionmaker(bind=migrate_engine)() - downgrade_user_table(meta, migrate_engine, session) - downgrade_tenant_table(meta, migrate_engine, session) - session.commit() diff --git a/keystone/common/sql/migrate_repo/versions/011_endpoints_v3.py b/keystone/common/sql/migrate_repo/versions/011_endpoints_v3.py deleted file mode 100644 index d9dc00acb1..0000000000 --- a/keystone/common/sql/migrate_repo/versions/011_endpoints_v3.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright 2012 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql - -from keystone.common.sql import migration_helpers - - -def upgrade(migrate_engine): - """Create API-version specific endpoint tables.""" - meta = sql.MetaData() - meta.bind = migrate_engine - - legacy_table = sql.Table('endpoint', meta, autoload=True) - - renames = {'endpoint_v2': legacy_table} - service_table = sql.Table('service', meta, autoload=True) - constraints = [{'table': legacy_table, - 'fk_column': 'service_id', - 'ref_column': service_table.c.id}] - migration_helpers.rename_tables_with_constraints(renames, constraints, - migrate_engine) - - sql.Table('service', meta, autoload=True) - new_table = sql.Table( - 'endpoint_v3', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('legacy_endpoint_id', sql.String(64)), - sql.Column('interface', sql.String(8), nullable=False), - sql.Column('region', sql.String(255)), - sql.Column('service_id', - sql.String(64), - sql.ForeignKey('service.id'), - nullable=False), - sql.Column('url', sql.Text(), nullable=False), - sql.Column('extra', sql.Text())) - new_table.create(migrate_engine, checkfirst=True) - - -def downgrade(migrate_engine): - """Replace API-version specific endpoint tables with one based on v2.""" - meta = sql.MetaData() - meta.bind = migrate_engine - - new_table = sql.Table('endpoint_v3', meta, autoload=True) - new_table.drop() - - legacy_table = sql.Table('endpoint_v2', meta, autoload=True) - - renames = {'endpoint': legacy_table} - service_table = sql.Table('service', meta, autoload=True) - constraints = [{'table': legacy_table, - 'fk_column': 'service_id', - 'ref_column': service_table.c.id}] - migration_helpers.rename_tables_with_constraints(renames, constraints, - migrate_engine) diff --git a/keystone/common/sql/migrate_repo/versions/012_populate_endpoint_type.py b/keystone/common/sql/migrate_repo/versions/012_populate_endpoint_type.py deleted file mode 100644 index 1759e16c14..0000000000 --- a/keystone/common/sql/migrate_repo/versions/012_populate_endpoint_type.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright 2012 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import json -import uuid - -import sqlalchemy as sql -from sqlalchemy import orm - - -ENDPOINT_TYPES = ['public', 'internal', 'admin'] - - -def upgrade(migrate_engine): - """Split each legacy endpoint into separate records for each interface.""" - meta = sql.MetaData() - meta.bind = migrate_engine - - legacy_table = sql.Table('endpoint_v2', meta, autoload=True) - new_table = sql.Table('endpoint_v3', meta, autoload=True) - - session = orm.sessionmaker(bind=migrate_engine)() - for ref in session.query(legacy_table).all(): - # pull urls out of extra - extra = json.loads(ref.extra) - urls = dict((i, extra.pop('%surl' % i)) for i in ENDPOINT_TYPES) - - for interface in ENDPOINT_TYPES: - endpoint = { - 'id': uuid.uuid4().hex, - 'legacy_endpoint_id': ref.id, - 'interface': interface, - 'region': ref.region, - 'service_id': ref.service_id, - 'url': urls[interface], - 'extra': json.dumps(extra), - } - insert = new_table.insert().values(endpoint) - migrate_engine.execute(insert) - session.commit() - session.close() - - -def downgrade(migrate_engine): - """Re-create the v2 endpoints table based on v3 endpoints.""" - meta = sql.MetaData() - meta.bind = migrate_engine - - legacy_table = sql.Table('endpoint_v2', meta, autoload=True) - new_table = sql.Table('endpoint_v3', meta, autoload=True) - - session = orm.sessionmaker(bind=migrate_engine)() - for ref in session.query(new_table).all(): - extra = json.loads(ref.extra) - legacy_id = ref.legacy_endpoint_id or extra.get('legacy_endpoint_id') - if not legacy_id: - continue - - q = session.query(legacy_table) - q = q.filter_by(id=legacy_id) - legacy_ref = q.first() - if legacy_ref: - # We already have one, so just update the extra - # attribute with the urls. - extra = json.loads(legacy_ref.extra) - extra['%surl' % ref.interface] = ref.url - values = {'extra': json.dumps(extra)} - update = legacy_table.update().\ - where(legacy_table.c.id == legacy_ref.id).\ - values(values) - migrate_engine.execute(update) - else: - # This is the first one of this legacy ID, so - # we can insert instead. - extra = json.loads(ref.extra) - extra['%surl' % ref.interface] = ref.url - endpoint = { - 'id': legacy_id, - 'region': ref.region, - 'service_id': ref.service_id, - 'extra': json.dumps(extra), - } - insert = legacy_table.insert().values(endpoint) - migrate_engine.execute(insert) - session.commit() - session.close() diff --git a/keystone/common/sql/migrate_repo/versions/013_drop_legacy_endpoints.py b/keystone/common/sql/migrate_repo/versions/013_drop_legacy_endpoints.py deleted file mode 100644 index ca83faa29c..0000000000 --- a/keystone/common/sql/migrate_repo/versions/013_drop_legacy_endpoints.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2012 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql - -from keystone.common.sql import migration_helpers - - -def upgrade(migrate_engine): - """Replace API-version specific endpoint tables with one based on v3.""" - meta = sql.MetaData() - meta.bind = migrate_engine - - legacy_table = sql.Table('endpoint_v2', meta, autoload=True) - legacy_table.drop() - - new_table = sql.Table('endpoint_v3', meta, autoload=True) - - renames = {'endpoint': new_table} - service_table = sql.Table('service', meta, autoload=True) - constraints = [{'table': new_table, - 'fk_column': 'service_id', - 'ref_column': service_table.c.id}] - migration_helpers.rename_tables_with_constraints(renames, constraints, - migrate_engine) - - -def downgrade(migrate_engine): - """Create API-version specific endpoint tables.""" - meta = sql.MetaData() - meta.bind = migrate_engine - - new_table = sql.Table('endpoint', meta, autoload=True) - - renames = {'endpoint_v3': new_table} - service_table = sql.Table('service', meta, autoload=True) - constraints = [{'table': new_table, - 'fk_column': 'service_id', - 'ref_column': service_table.c.id}] - migration_helpers.rename_tables_with_constraints(renames, constraints, - migrate_engine) - - sql.Table('service', meta, autoload=True) - legacy_table = sql.Table( - 'endpoint_v2', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('region', sql.String(255)), - sql.Column('service_id', - sql.String(64), - sql.ForeignKey('service.id'), - nullable=False), - sql.Column('extra', sql.Text())) - legacy_table.create(migrate_engine, checkfirst=True) diff --git a/keystone/common/sql/migrate_repo/versions/014_add_group_tables.py b/keystone/common/sql/migrate_repo/versions/014_add_group_tables.py deleted file mode 100644 index 5fdf3d4441..0000000000 --- a/keystone/common/sql/migrate_repo/versions/014_add_group_tables.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright 2012 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - sql.Table('domain', meta, autoload=True) - group_table = sql.Table( - 'group', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('domain_id', sql.String(64), sql.ForeignKey('domain.id'), - nullable=False), - sql.Column('name', sql.String(64), nullable=False), - sql.Column('description', sql.Text()), - sql.Column('extra', sql.Text()), - sql.UniqueConstraint('domain_id', 'name')) - group_table.create(migrate_engine, checkfirst=True) - - sql.Table('user', meta, autoload=True) - user_group_membership_table = sql.Table( - 'user_group_membership', - meta, - sql.Column( - 'user_id', - sql.String(64), - sql.ForeignKey('user.id'), - primary_key=True), - sql.Column( - 'group_id', - sql.String(64), - sql.ForeignKey('group.id'), - primary_key=True)) - user_group_membership_table.create(migrate_engine, checkfirst=True) - - sql.Table('tenant', meta, autoload=True) - group_project_metadata_table = sql.Table( - 'group_project_metadata', - meta, - sql.Column( - 'group_id', - sql.String(64), - sql.ForeignKey('group.id'), - primary_key=True), - sql.Column( - 'project_id', - sql.String(64), - sql.ForeignKey('tenant.id'), - primary_key=True), - sql.Column('data', sql.Text())) - group_project_metadata_table.create(migrate_engine, checkfirst=True) - - group_domain_metadata_table = sql.Table( - 'group_domain_metadata', - meta, - sql.Column( - 'group_id', - sql.String(64), - sql.ForeignKey('group.id'), - primary_key=True), - sql.Column( - 'domain_id', - sql.String(64), - sql.ForeignKey('domain.id'), - primary_key=True), - sql.Column('data', sql.Text())) - group_domain_metadata_table.create(migrate_engine, checkfirst=True) - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - tables = ['user_group_membership', 'group_project_metadata', - 'group_domain_metadata', 'group'] - for t in tables: - table = sql.Table(t, meta, autoload=True) - table.drop(migrate_engine, checkfirst=True) diff --git a/keystone/common/sql/migrate_repo/versions/015_tenant_to_project.py b/keystone/common/sql/migrate_repo/versions/015_tenant_to_project.py deleted file mode 100644 index 7338aaee7f..0000000000 --- a/keystone/common/sql/migrate_repo/versions/015_tenant_to_project.py +++ /dev/null @@ -1,194 +0,0 @@ -# Copyright 2013 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql -from sqlalchemy.orm import sessionmaker - -from keystone.common.sql import migration_helpers - - -def rename_with_constraints(meta, legacy_project_table_name, - new_project_table_name, - legacy_user_project_membership_table_name, - new_user_project_membership_table_name): - # Not all RDBMSs support renaming a table that has foreign key constraints - # on it, so drop FK constraints before renaming and then replace FKs - # afterwards. - - credential_table = sql.Table('credential', meta, autoload=True) - group_project_meta_table = sql.Table('group_project_metadata', meta, - autoload=True) - project_table = sql.Table(legacy_project_table_name, meta, autoload=True) - user_project_membership_table = sql.Table( - legacy_user_project_membership_table_name, meta, autoload=True) - user_table = sql.Table('user', meta, autoload=True) - - constraints = [{'table': credential_table, - 'fk_column': 'project_id', - 'ref_column': project_table.c.id}, - {'table': group_project_meta_table, - 'fk_column': 'project_id', - 'ref_column': project_table.c.id}, - {'table': user_project_membership_table, - 'fk_column': 'tenant_id', - 'ref_column': project_table.c.id}, - {'table': user_project_membership_table, - 'fk_column': 'user_id', - 'ref_column': user_table.c.id}] - - renames = { - new_project_table_name: project_table, - new_user_project_membership_table_name: user_project_membership_table} - - migration_helpers.rename_tables_with_constraints(renames, constraints, - meta.bind) - - -def upgrade_with_rename(meta, migrate_engine): - legacy_project_table_name = 'tenant' - new_project_table_name = 'project' - legacy_user_project_membership_table_name = 'user_tenant_membership' - new_user_project_membership_table_name = 'user_project_membership' - rename_with_constraints(meta, legacy_project_table_name, - new_project_table_name, - legacy_user_project_membership_table_name, - new_user_project_membership_table_name) - - -def downgrade_with_rename(meta, migrate_engine): - legacy_project_table_name = 'project' - new_project_table_name = 'tenant' - legacy_user_project_membership_table_name = 'user_project_membership' - new_user_project_membership_table_name = 'user_tenant_membership' - rename_with_constraints(meta, legacy_project_table_name, - new_project_table_name, - legacy_user_project_membership_table_name, - new_user_project_membership_table_name) - - -def upgrade_with_copy(meta, migrate_engine): - sql.Table('user', meta, autoload=True) - project_table = sql.Table( - 'project', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('name', sql.String(64), unique=True, nullable=False), - sql.Column('extra', sql.Text()), - sql.Column('description', sql.Text(), nullable=True), - sql.Column('enabled', sql.types.Boolean, default=True)) - project_table.create(migrate_engine, checkfirst=True) - - user_project_membership_table = sql.Table( - 'user_project_membership', - meta, - sql.Column( - 'user_id', - sql.String(64), - sql.ForeignKey('user.id'), - primary_key=True), - sql.Column( - 'tenant_id', - sql.String(64), - sql.ForeignKey('project.id'), - primary_key=True)) - user_project_membership_table.create(migrate_engine, checkfirst=True) - - session = sessionmaker(bind=migrate_engine)() - - tenant_table = sql.Table('tenant', meta, autoload=True) - insert = project_table.insert() - for tenant in session.query(tenant_table): - insert.execute({'id': tenant.id, - 'name': tenant.name, - 'extra': tenant.extra, - 'description': tenant.description, - 'enabled': tenant.enabled}) - - user_tenant_membership_table = sql.Table('user_tenant_membership', - meta, - autoload=True) - insert = user_project_membership_table.insert() - for user_id, tenant_id in session.query(user_tenant_membership_table): - insert.execute({'user_id': user_id, 'tenant_id': tenant_id}) - - session.commit() - session.close() - - user_tenant_membership_table.drop() - tenant_table.drop() - - -def downgrade_with_copy(meta, migrate_engine): - sql.Table('user', meta, autoload=True) - tenant_table = sql.Table( - 'tenant', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('name', sql.String(64), unique=True, nullable=False), - sql.Column('extra', sql.Text()), - sql.Column('description', sql.Text(), nullable=True), - sql.Column('enabled', sql.types.Boolean)) - tenant_table.create(migrate_engine, checkfirst=True) - - user_tenant_membership_table = sql.Table( - 'user_tenant_membership', - meta, - sql.Column( - 'user_id', - sql.String(64), - sql.ForeignKey('user.id'), - primary_key=True), - sql.Column( - 'tenant_id', - sql.String(64), - sql.ForeignKey('tenant.id'), - primary_key=True)) - user_tenant_membership_table.create(migrate_engine, checkfirst=True) - - session = sessionmaker(bind=migrate_engine)() - - project_table = sql.Table('project', meta, autoload=True) - insert = tenant_table.insert() - for project in session.query(project_table): - insert.values(project).execute() - project_table.drop() - - user_project_membership_table = sql.Table('user_project_membership', - meta, - autoload=True) - insert = user_tenant_membership_table.insert() - for membership in session.query(user_project_membership_table): - insert.execute(membership) - user_project_membership_table.drop() - - session.commit() - session.close() - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - if migrate_engine.name == "sqlite": - upgrade_with_copy(meta, migrate_engine) - else: - upgrade_with_rename(meta, migrate_engine) - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - if migrate_engine.name == "sqlite": - downgrade_with_copy(meta, migrate_engine) - else: - downgrade_with_rename(meta, migrate_engine) diff --git a/keystone/common/sql/migrate_repo/versions/016_normalize_domain_ids.py b/keystone/common/sql/migrate_repo/versions/016_normalize_domain_ids.py deleted file mode 100644 index 22bdc7078a..0000000000 --- a/keystone/common/sql/migrate_repo/versions/016_normalize_domain_ids.py +++ /dev/null @@ -1,435 +0,0 @@ -# Copyright 2012 OpenStack Foundation -# Copyright 2013 IBM Corp. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Normalize for domain_id, i.e. ensure User and Project entities have the -domain_id as a first class attribute. - -Both User and Project (as well as Group) entities are owned by a -domain, which is implemented as each having a domain_id foreign key -in their sql representation that points back to the respective -domain in the domain table. This domain_id attribute should also -be required (i.e. not nullable) - -Adding a non_nullable foreign key attribute to a table with existing -data causes a few problems since not all DB engines support the -ability to either control the triggering of integrity constraints -or the ability to modify columns after they are created. - -To get round the above inconsistencies, two versions of the -upgrade/downgrade functions are supplied, one for those engines -that support dropping columns, and one for those that don't. For -the latter we are forced to do table copy AND control the triggering -of integrity constraints. -""" - -import sqlalchemy as sql -from sqlalchemy.orm import sessionmaker - -from keystone import config - - -CONF = config.CONF - - -def _disable_foreign_constraints(session, migrate_engine): - if migrate_engine.name == 'mysql': - session.execute('SET foreign_key_checks = 0;') - - -def _enable_foreign_constraints(session, migrate_engine): - if migrate_engine.name == 'mysql': - session.execute('SET foreign_key_checks = 1;') - - -def upgrade_user_table_with_copy(meta, migrate_engine, session): - # We want to add the domain_id attribute to the user table. Since - # it is non nullable and the table may have data, easiest way is - # a table copy. Further, in order to keep foreign key constraints - # pointing at the right table, we need to be able and do a table - # DROP then CREATE, rather than ALTERing the name of the table. - - # First make a copy of the user table - temp_user_table = sql.Table( - 'temp_user', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('name', sql.String(64), unique=True, nullable=False), - sql.Column('extra', sql.Text()), - sql.Column('password', sql.String(128)), - sql.Column('enabled', sql.Boolean, default=True)) - temp_user_table.create(migrate_engine, checkfirst=True) - - user_table = sql.Table('user', meta, autoload=True) - for user in session.query(user_table): - session.execute('insert into temp_user (id, name, extra, ' - 'password, enabled) ' - 'values ( :id, :name, :extra, ' - ':password, :enabled);', - {'id': user.id, - 'name': user.name, - 'extra': user.extra, - 'password': user.password, - 'enabled': user.enabled}) - - # Now switch off constraints while we drop and then re-create the - # user table, with the additional domain_id column - _disable_foreign_constraints(session, migrate_engine) - session.execute('drop table user;') - # Need to create a new metadata stream since we are going to load a - # different version of the user table - meta2 = sql.MetaData() - meta2.bind = migrate_engine - sql.Table('domain', meta2, autoload=True) - user_table = sql.Table( - 'user', - meta2, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('name', sql.String(64), nullable=False), - sql.Column('extra', sql.Text()), - sql.Column("password", sql.String(128)), - sql.Column("enabled", sql.Boolean, default=True), - sql.Column('domain_id', sql.String(64), sql.ForeignKey('domain.id'), - nullable=False), - sql.UniqueConstraint('domain_id', 'name')) - user_table.create(migrate_engine, checkfirst=True) - - # Finally copy in the data from our temp table and then clean - # up by deleting our temp table - for user in session.query(temp_user_table): - session.execute('insert into user (id, name, extra, ' - 'password, enabled, domain_id) ' - 'values ( :id, :name, :extra, ' - ':password, :enabled, :domain_id);', - {'id': user.id, - 'name': user.name, - 'extra': user.extra, - 'password': user.password, - 'enabled': user.enabled, - 'domain_id': CONF.identity.default_domain_id}) - _enable_foreign_constraints(session, migrate_engine) - session.execute('drop table temp_user;') - - -def upgrade_project_table_with_copy(meta, migrate_engine, session): - # We want to add the domain_id attribute to the project table. Since - # it is non nullable and the table may have data, easiest way is - # a table copy. Further, in order to keep foreign key constraints - # pointing at the right table, we need to be able and do a table - # DROP then CREATE, rather than ALTERing the name of the table. - - # Fist make a copy of the project table - temp_project_table = sql.Table( - 'temp_project', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('name', sql.String(64), unique=True, nullable=False), - sql.Column('extra', sql.Text()), - sql.Column('description', sql.Text()), - sql.Column('enabled', sql.Boolean, default=True)) - temp_project_table.create(migrate_engine, checkfirst=True) - - project_table = sql.Table('project', meta, autoload=True) - for project in session.query(project_table): - session.execute('insert into temp_project (id, name, extra, ' - 'description, enabled) ' - 'values ( :id, :name, :extra, ' - ':description, :enabled);', - {'id': project.id, - 'name': project.name, - 'extra': project.extra, - 'description': project.description, - 'enabled': project.enabled}) - - # Now switch off constraints while we drop and then re-create the - # project table, with the additional domain_id column - _disable_foreign_constraints(session, migrate_engine) - session.execute('drop table project;') - # Need to create a new metadata stream since we are going to load a - # different version of the project table - meta2 = sql.MetaData() - meta2.bind = migrate_engine - sql.Table('domain', meta2, autoload=True) - project_table = sql.Table( - 'project', - meta2, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('name', sql.String(64), nullable=False), - sql.Column('extra', sql.Text()), - sql.Column('description', sql.Text()), - sql.Column('enabled', sql.Boolean, default=True), - sql.Column('domain_id', sql.String(64), sql.ForeignKey('domain.id'), - nullable=False), - sql.UniqueConstraint('domain_id', 'name')) - project_table.create(migrate_engine, checkfirst=True) - - # Finally copy in the data from our temp table and then clean - # up by deleting our temp table - for project in session.query(temp_project_table): - session.execute('insert into project (id, name, extra, ' - 'description, enabled, domain_id) ' - 'values ( :id, :name, :extra, ' - ':description, :enabled, :domain_id);', - {'id': project.id, - 'name': project.name, - 'extra': project.extra, - 'description': project.description, - 'enabled': project.enabled, - 'domain_id': CONF.identity.default_domain_id}) - _enable_foreign_constraints(session, migrate_engine) - session.execute('drop table temp_project;') - - -def downgrade_user_table_with_copy(meta, migrate_engine, session): - # For engines that don't support dropping columns, we need to do this - # as a table copy. Further, in order to keep foreign key constraints - # pointing at the right table, we need to be able and do a table - # DROP then CREATE, rather than ALTERing the name of the table. - - # Fist make a copy of the user table - temp_user_table = sql.Table( - 'temp_user', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('name', sql.String(64), unique=True, nullable=False), - sql.Column('password', sql.String(128)), - sql.Column('enabled', sql.Boolean, default=True), - sql.Column('extra', sql.Text())) - temp_user_table.create(migrate_engine, checkfirst=True) - - user_table = sql.Table('user', meta, autoload=True) - for user in session.query(user_table): - session.execute('insert into temp_user (id, name, ' - 'password, enabled, extra) ' - 'values ( :id, :name, ' - ':password, :enabled, :extra);', - {'id': user.id, - 'name': user.name, - 'password': user.password, - 'enabled': user.enabled, - 'extra': user.extra}) - - # Now switch off constraints while we drop and then re-create the - # user table, less the columns we wanted to drop - _disable_foreign_constraints(session, migrate_engine) - session.execute('drop table user;') - # Need to create a new metadata stream since we are going to load a - # different version of the user table - meta2 = sql.MetaData() - meta2.bind = migrate_engine - user_table = sql.Table( - 'user', - meta2, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('name', sql.String(64), unique=True, nullable=False), - sql.Column('extra', sql.Text()), - sql.Column('password', sql.String(128)), - sql.Column('enabled', sql.Boolean, default=True)) - user_table.create(migrate_engine, checkfirst=True) - _enable_foreign_constraints(session, migrate_engine) - - # Finally copy in the data from our temp table and then clean - # up by deleting our temp table - for user in session.query(temp_user_table): - session.execute('insert into user (id, name, extra, ' - 'password, enabled) ' - 'values ( :id, :name, :extra, ' - ':password, :enabled);', - {'id': user.id, - 'name': user.name, - 'extra': user.extra, - 'password': user.password, - 'enabled': user.enabled}) - session.execute('drop table temp_user;') - - -def downgrade_project_table_with_copy(meta, migrate_engine, session): - # For engines that don't support dropping columns, we need to do this - # as a table copy. Further, in order to keep foreign key constraints - # pointing at the right table, we need to be able and do a table - # DROP then CREATE, rather than ALTERing the name of the table. - - # Fist make a copy of the project table - temp_project_table = sql.Table( - 'temp_project', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('name', sql.String(64), unique=True, nullable=False), - sql.Column('description', sql.Text()), - sql.Column('enabled', sql.Boolean, default=True), - sql.Column('extra', sql.Text())) - temp_project_table.create(migrate_engine, checkfirst=True) - - project_table = sql.Table('project', meta, autoload=True) - for project in session.query(project_table): - session.execute('insert into temp_project (id, name, ' - 'description, enabled, extra) ' - 'values ( :id, :name, ' - ':description, :enabled, :extra);', - {'id': project.id, - 'name': project.name, - 'description': project.description, - 'enabled': project.enabled, - 'extra': project.extra}) - - # Now switch off constraints while we drop and then re-create the - # project table, less the columns we wanted to drop - _disable_foreign_constraints(session, migrate_engine) - session.execute('drop table project;') - # Need to create a new metadata stream since we are going to load a - # different version of the project table - meta2 = sql.MetaData() - meta2.bind = migrate_engine - project_table = sql.Table( - 'project', - meta2, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('name', sql.String(64), unique=True, nullable=False), - sql.Column('extra', sql.Text()), - sql.Column('description', sql.Text()), - sql.Column('enabled', sql.Boolean, default=True)) - project_table.create(migrate_engine, checkfirst=True) - _enable_foreign_constraints(session, migrate_engine) - - # Finally copy in the data from our temp table and then clean - # up by deleting our temp table - for project in session.query(temp_project_table): - session.execute('insert into project (id, name, extra, ' - 'description, enabled) ' - 'values ( :id, :name, :extra, ' - ':description, :enabled);', - {'id': project.id, - 'name': project.name, - 'extra': project.extra, - 'description': project.description, - 'enabled': project.enabled}) - session.execute("drop table temp_project;") - - -def upgrade_user_table_with_col_create(meta, migrate_engine, session): - # Create the domain_id column. We want this to be not nullable - # but also a foreign key. We can't create this right off the - # bat since any existing rows would cause an Integrity Error. - # We therefore create it nullable, fill the column with the - # default data and then set it to non nullable. - sql.Table('domain', meta, autoload=True) - user_table = sql.Table('user', meta, autoload=True) - user_table.create_column( - sql.Column('domain_id', sql.String(64), - sql.ForeignKey('domain.id'), nullable=True)) - for user in session.query(user_table).all(): - values = {'domain_id': CONF.identity.default_domain_id} - update = user_table.update().\ - where(user_table.c.id == user.id).\ - values(values) - migrate_engine.execute(update) - # Need to commit this or setting nullable to False will fail - session.commit() - user_table.columns.domain_id.alter(nullable=False) - - # Finally, change the uniqueness settings for the name attribute - session.execute('ALTER TABLE "user" DROP CONSTRAINT user_name_key;') - session.execute('ALTER TABLE "user" ADD CONSTRAINT user_dom_name_unique ' - 'UNIQUE (domain_id, name);') - - session.commit() - - -def upgrade_project_table_with_col_create(meta, migrate_engine, session): - # Create the domain_id column. We want this to be not nullable - # but also a foreign key. We can't create this right off the - # bat since any existing rows would cause an Integrity Error. - # We therefore create it nullable, fill the column with the - # default data and then set it to non nullable. - sql.Table('domain', meta, autoload=True) - project_table = sql.Table('project', meta, autoload=True) - project_table.create_column( - sql.Column('domain_id', sql.String(64), - sql.ForeignKey('domain.id'), nullable=True)) - for project in session.query(project_table).all(): - values = {'domain_id': CONF.identity.default_domain_id} - update = project_table.update().\ - where(project_table.c.id == project.id).\ - values(values) - migrate_engine.execute(update) - # Need to commit this or setting nullable to False will fail - session.commit() - project_table.columns.domain_id.alter(nullable=False) - - # Finally, change the uniqueness settings for the name attribute - session.execute('ALTER TABLE project DROP CONSTRAINT tenant_name_key;') - session.execute('ALTER TABLE project ADD CONSTRAINT proj_dom_name_unique ' - 'UNIQUE (domain_id, name);') - - -def downgrade_user_table_with_col_drop(meta, migrate_engine, session): - # Revert uniqueness settings for the name attribute - session.execute('ALTER TABLE "user" DROP CONSTRAINT ' - 'user_dom_name_unique;') - # specify the constraint name so it can be referenced later - session.execute('ALTER TABLE "user" ADD CONSTRAINT user_name_key ' - 'UNIQUE (name);') - session.commit() - # And now go ahead an drop the domain_id column - sql.Table('domain', meta, autoload=True) - user_table = sql.Table('user', meta, autoload=True) - column = sql.Column('domain_id', sql.String(64), - sql.ForeignKey('domain.id'), nullable=False) - column.drop(user_table) - - -def downgrade_project_table_with_col_drop(meta, migrate_engine, session): - # Revert uniqueness settings for the name attribute - session.execute('ALTER TABLE project DROP CONSTRAINT ' - 'proj_dom_name_unique;') - session.execute('ALTER TABLE project ADD CONSTRAINT tenant_name_key ' - 'UNIQUE (name);') - session.commit() - # And now go ahead an drop the domain_id column - sql.Table('domain', meta, autoload=True) - project_table = sql.Table('project', meta, autoload=True) - column = sql.Column('domain_id', sql.String(64), - sql.ForeignKey('domain.id'), nullable=False) - column.drop(project_table) - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - session = sessionmaker(bind=migrate_engine)() - if migrate_engine.name in ['sqlite', 'mysql']: - upgrade_user_table_with_copy(meta, migrate_engine, session) - upgrade_project_table_with_copy(meta, migrate_engine, session) - else: - upgrade_user_table_with_col_create(meta, migrate_engine, session) - upgrade_project_table_with_col_create(meta, migrate_engine, session) - session.commit() - session.close() - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - session = sessionmaker(bind=migrate_engine)() - if migrate_engine.name in ['sqlite', 'mysql']: - downgrade_user_table_with_copy(meta, migrate_engine, session) - downgrade_project_table_with_copy(meta, migrate_engine, session) - else: - # MySQL should in theory be able to use this path, but seems to - # have problems dropping columns which are foreign keys - downgrade_user_table_with_col_drop(meta, migrate_engine, session) - downgrade_project_table_with_col_drop(meta, migrate_engine, session) - session.commit() - session.close() diff --git a/keystone/common/sql/migrate_repo/versions/017_membership_role.py b/keystone/common/sql/migrate_repo/versions/017_membership_role.py deleted file mode 100644 index 64c9c4740a..0000000000 --- a/keystone/common/sql/migrate_repo/versions/017_membership_role.py +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright 2013 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import json - -import sqlalchemy as sql - -from keystone import config - - -CONF = config.CONF - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - sql.Table('user', meta, autoload=True) - sql.Table('project', meta, autoload=True) - role_table = sql.Table('role', meta, autoload=True) - - user_project_role_table = sql.Table( - 'user_project_metadata', - meta, - sql.Column('user_id', - sql.String(64), - sql.ForeignKey('user.id'), - primary_key=True), - sql.Column('project_id', - sql.String(64), - sql.ForeignKey('project.id'), - primary_key=True), - sql.Column('data', sql.Text())) - user_project_role_table.create(migrate_engine, checkfirst=True) - - conn = migrate_engine.connect() - conn.execute(role_table.insert(), - id=CONF.member_role_id, - name=CONF.member_role_name, - extra=json.dumps({'description': - 'Default role for project membership', - 'enabled': 'True'})) - - user_project_membership_table = sql.Table('user_project_membership', - meta, autoload=True) - session = sql.orm.sessionmaker(bind=migrate_engine)() - for membership in session.query(user_project_membership_table): - data = {'roles': [config.CONF.member_role_id]} - ins = user_project_role_table.insert().values( - user_id=membership.user_id, - project_id=membership.tenant_id, - data=json.dumps(data)) - conn.execute(ins) - session.close() - user_project_membership_table.drop() - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - sql.Table('user', meta, autoload=True) - sql.Table('project', meta, autoload=True) - - user_project_membership_table = sql.Table( - 'user_project_membership', - meta, - sql.Column( - 'user_id', - sql.String(64), - sql.ForeignKey('user.id'), - primary_key=True), - sql.Column( - 'tenant_id', - sql.String(64), - sql.ForeignKey('project.id'), - primary_key=True)) - user_project_membership_table.create(migrate_engine, checkfirst=True) - - user_project_metadata_table = sql.Table( - 'user_project_metadata', - meta, - autoload=True) - - session = sql.orm.sessionmaker(bind=migrate_engine)() - for membership in session.query(user_project_metadata_table): - if 'roles' in membership: - roles = membership['roles'] - if config.CONF.member_role_id in roles: - user_project_membership_table.insert().values( - user_id=membership.user_id, - tenant_id=membership.project_id) - session.close() - role_table = sql.Table('role', meta, autoload=True) - conn = migrate_engine.connect() - user_project_membership_table = sql.Table( - 'user_project_membership', meta, autoload=True) - - role_table = sql.Table('role', meta, autoload=True) - conn.execute(role_table.delete().where(role_table.c.id == - config.CONF.member_role_id)) - user_project_metadata_table.drop() diff --git a/keystone/common/sql/migrate_repo/versions/018_add_trust_tables.py b/keystone/common/sql/migrate_repo/versions/018_add_trust_tables.py deleted file mode 100644 index cec39a74c3..0000000000 --- a/keystone/common/sql/migrate_repo/versions/018_add_trust_tables.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2012 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql - - -def upgrade(migrate_engine): - # Upgrade operations go here. Don't create your own engine; bind - # migrate_engine to your metadata - meta = sql.MetaData() - meta.bind = migrate_engine - - sql.Table('user', meta, autoload=True) - sql.Table('role', meta, autoload=True) - sql.Table('project', meta, autoload=True) - - trust_table = sql.Table( - 'trust', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('trustor_user_id', - sql.String(64), - unique=False, - nullable=False,), - sql.Column('trustee_user_id', - sql.String(64), - unique=False, - nullable=False), - sql.Column('project_id', sql.String(64), - unique=False, - nullable=True), - sql.Column("impersonation", sql.types.Boolean, nullable=False), - sql.Column("deleted_at", sql.types.DateTime, nullable=True), - sql.Column("expires_at", sql.types.DateTime, nullable=True), - sql.Column('extra', sql.Text())) - trust_table.create(migrate_engine, checkfirst=True) - - trust_role_table = sql.Table( - 'trust_role', - meta, - sql.Column('trust_id', sql.String(64), primary_key=True, - nullable=False), - sql.Column('role_id', sql.String(64), primary_key=True, - nullable=False)) - trust_role_table.create(migrate_engine, checkfirst=True) - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - # Operations to reverse the above upgrade go here. - for table_name in ['trust_role', 'trust']: - table = sql.Table(table_name, meta, autoload=True) - table.drop() diff --git a/keystone/common/sql/migrate_repo/versions/019_fixup_role.py b/keystone/common/sql/migrate_repo/versions/019_fixup_role.py deleted file mode 100644 index bbd0cae5db..0000000000 --- a/keystone/common/sql/migrate_repo/versions/019_fixup_role.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2013 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql - -from keystone import config - - -CONF = config.CONF - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - role_table = sql.Table('role', meta, autoload=True) - # name should be 255 characters to match fresh database - role_table.c.name.alter(type=sql.String(length=255)) - - # blank 'extra' field should be "{}" - none = None - update = role_table.update().where(role_table.c.extra == none).values( - {role_table.c.extra: "{}"}) - migrate_engine.execute(update) - - -def downgrade(migrate_engine): - # this fixes bugs in migration 001 and 007 that result in discrepancies - # between fresh databases and databases updated from 004 (folsom). - # the changes fixing 007 will be rolled back in 007's rollback if - # the user desires to return to a state before the existence of the extra - # column. - # the name length change reflects the current default and should not be - # rolled back. - pass diff --git a/keystone/common/sql/migrate_repo/versions/020_migrate_metadata_table_roles.py b/keystone/common/sql/migrate_repo/versions/020_migrate_metadata_table_roles.py deleted file mode 100644 index 05237c0d9b..0000000000 --- a/keystone/common/sql/migrate_repo/versions/020_migrate_metadata_table_roles.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright 2013 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import json - -import sqlalchemy as sql - -from keystone import config - - -CONF = config.CONF - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - sql.Table('user', meta, autoload=True) - sql.Table('role', meta, autoload=True) - sql.Table('project', meta, autoload=True) - new_metadata_table = sql.Table('user_project_metadata', - meta, - autoload=True) - - old_metadata_table = sql.Table('metadata', meta, autoload=True) - session = sql.orm.sessionmaker(bind=migrate_engine)() - - for metadata in session.query(old_metadata_table): - data = json.loads(metadata.data) - if config.CONF.member_role_id not in metadata.data: - data['roles'].append(config.CONF.member_role_id) - - r = session.query(new_metadata_table).filter_by( - user_id=metadata.user_id, - project_id=metadata.tenant_id).first() - - if r is not None: - # roles should be the union of the two role lists - old_roles = data['roles'] - new_roles = json.loads(r.data)['roles'] - data['roles'] = list(set(old_roles) | set(new_roles)) - q = new_metadata_table.update().where( - new_metadata_table.c.user_id == metadata.user_id).where( - new_metadata_table.c.project_id == - metadata.tenant_id).values(data=json.dumps(data)) - else: - q = new_metadata_table.insert().values( - user_id=metadata.user_id, - project_id=metadata.tenant_id, - data=json.dumps(data)) - - session.execute(q) - - session.commit() - old_metadata_table.drop() - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - sql.Table('user', meta, autoload=True) - sql.Table('project', meta, autoload=True) - - metadata_table = sql.Table( - 'metadata', - meta, - sql.Column( - u'user_id', - sql.String(64), - primary_key=True), - sql.Column( - u'tenant_id', - sql.String(64), - primary_key=True), - sql.Column('data', - sql.Text())) - metadata_table.create(migrate_engine, checkfirst=True) - - user_project_metadata_table = sql.Table( - 'user_project_metadata', - meta, - autoload=True) - - metadata_table = sql.Table( - 'metadata', - meta, - autoload=True) - - session = sql.orm.sessionmaker(bind=migrate_engine)() - - for metadata in session.query(user_project_metadata_table): - if 'roles' in metadata: - metadata_table.insert().values( - user_id=metadata.user_id, - tenant_id=metadata.project_id) - - session.close() diff --git a/keystone/common/sql/migrate_repo/versions/021_add_trust_to_token.py b/keystone/common/sql/migrate_repo/versions/021_add_trust_to_token.py deleted file mode 100644 index 1e189e9336..0000000000 --- a/keystone/common/sql/migrate_repo/versions/021_add_trust_to_token.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright 2013 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy -from sqlalchemy import exc - - -def downgrade_token_table_with_column_drop(meta, migrate_engine): - token_table = sqlalchemy.Table('token', meta, autoload=True) - # delete old tokens, as the format has changed. - # We don't guarantee that existing tokens will be - # usable after a migration - token_table.delete() - token_table.drop_column( - sqlalchemy.Column('trust_id', - sqlalchemy.String(64), - nullable=True)) - token_table.drop_column( - sqlalchemy.Column('user_id', - sqlalchemy.String(64))) - - -def create_column_forgiving(migrate_engine, table, column): - try: - table.create_column(column) - except exc.OperationalError as e: - if (e.args[0].endswith('duplicate column name: %s' % column.name) - and migrate_engine.name == "sqlite"): - # sqlite does not drop columns, so if we have already - # done a downgrade and are now upgrading, we will hit - # this: the SQLite driver previously reported success - # dropping the columns but it hasn't. - pass - else: - raise - - -def upgrade_token_table(meta, migrate_engine): - # delete old tokens, as the format has changed. - # The existing tokens will not - # support some of the list functions - - token_table = sqlalchemy.Table('token', meta, autoload=True) - token_table.delete() - - create_column_forgiving( - migrate_engine, token_table, - sqlalchemy.Column('trust_id', - sqlalchemy.String(64), - nullable=True)) - create_column_forgiving( - migrate_engine, token_table, - sqlalchemy.Column('user_id', sqlalchemy.String(64))) - - -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - upgrade_token_table(meta, migrate_engine) - - -def downgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - downgrade_token_table_with_column_drop(meta, migrate_engine) diff --git a/keystone/common/sql/migrate_repo/versions/022_move_legacy_endpoint_id.py b/keystone/common/sql/migrate_repo/versions/022_move_legacy_endpoint_id.py deleted file mode 100644 index 58737f5795..0000000000 --- a/keystone/common/sql/migrate_repo/versions/022_move_legacy_endpoint_id.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright 2013 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import json - -import sqlalchemy as sql -from sqlalchemy import orm - -from keystone import config - - -CONF = config.CONF - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - endpoint_table = sql.Table('endpoint', meta, autoload=True) - - session = orm.sessionmaker(bind=migrate_engine)() - for endpoint in session.query(endpoint_table).all(): - try: - extra = json.loads(endpoint.extra) - legacy_endpoint_id = extra.pop('legacy_endpoint_id') - except KeyError: - # if there is no legacy_endpoint_id, there's nothing to do - pass - else: - q = endpoint_table.update() - q = q.where(endpoint_table.c.id == endpoint.id) - q = q.values({ - endpoint_table.c.extra: json.dumps(extra), - endpoint_table.c.legacy_endpoint_id: legacy_endpoint_id}) - migrate_engine.execute(q) - session.close() - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - endpoint_table = sql.Table('endpoint', meta, autoload=True) - - session = orm.sessionmaker(bind=migrate_engine)() - for endpoint in session.query(endpoint_table).all(): - if endpoint.legacy_endpoint_id is not None: - extra = json.loads(endpoint.extra) - extra['legacy_endpoint_id'] = endpoint.legacy_endpoint_id - - q = endpoint_table.update() - q = q.where(endpoint_table.c.id == endpoint.id) - q = q.values({ - endpoint_table.c.extra: json.dumps(extra), - endpoint_table.c.legacy_endpoint_id: None}) - migrate_engine.execute(q) - session.close() diff --git a/keystone/common/sql/migrate_repo/versions/023_drop_credential_constraints.py b/keystone/common/sql/migrate_repo/versions/023_drop_credential_constraints.py deleted file mode 100644 index 6893842692..0000000000 --- a/keystone/common/sql/migrate_repo/versions/023_drop_credential_constraints.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2013 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy - -from keystone.common.sql import migration_helpers - - -def list_constraints(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - user_table = sqlalchemy.Table('user', meta, autoload=True) - proj_table = sqlalchemy.Table('project', meta, autoload=True) - cred_table = sqlalchemy.Table('credential', meta, autoload=True) - - constraints = [{'table': cred_table, - 'fk_column': 'user_id', - 'ref_column': user_table.c.id}, - {'table': cred_table, - 'fk_column': 'project_id', - 'ref_column': proj_table.c.id}] - return constraints - - -def upgrade(migrate_engine): - # SQLite does not support constraints, and querying the constraints - # raises an exception - if migrate_engine.name == 'sqlite': - return - migration_helpers.remove_constraints(list_constraints(migrate_engine)) - - -def downgrade(migrate_engine): - if migrate_engine.name == 'sqlite': - return - migration_helpers.add_constraints(list_constraints(migrate_engine)) diff --git a/keystone/common/sql/migrate_repo/versions/024_add_index_to_expires.py b/keystone/common/sql/migrate_repo/versions/024_add_index_to_expires.py deleted file mode 100644 index 24f98043d0..0000000000 --- a/keystone/common/sql/migrate_repo/versions/024_add_index_to_expires.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2013 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - token = sql.Table('token', meta, autoload=True) - idx = sql.Index('ix_token_expires', token.c.expires) - idx.create(migrate_engine) - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - token = sql.Table('token', meta, autoload=True) - idx = sql.Index('ix_token_expires', token.c.expires) - idx.drop(migrate_engine) diff --git a/keystone/common/sql/migrate_repo/versions/025_add_index_to_valid.py b/keystone/common/sql/migrate_repo/versions/025_add_index_to_valid.py deleted file mode 100644 index 62bee18aad..0000000000 --- a/keystone/common/sql/migrate_repo/versions/025_add_index_to_valid.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2013 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - token = sql.Table('token', meta, autoload=True) - idx = sql.Index('ix_token_valid', token.c.valid) - idx.create(migrate_engine) - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - token = sql.Table('token', meta, autoload=True) - idx = sql.Index('ix_token_valid', token.c.valid) - idx.drop(migrate_engine) diff --git a/keystone/common/sql/migrate_repo/versions/026_drop_user_group_constraints.py b/keystone/common/sql/migrate_repo/versions/026_drop_user_group_constraints.py deleted file mode 100644 index f6f236216e..0000000000 --- a/keystone/common/sql/migrate_repo/versions/026_drop_user_group_constraints.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2013 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy - -from keystone.common.sql import migration_helpers - - -def list_constraints(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - user_table = sqlalchemy.Table('user', meta, autoload=True) - sqlalchemy.Table('project', meta, autoload=True) - group_table = sqlalchemy.Table('group', meta, autoload=True) - user_domain_metadata_table = sqlalchemy.Table('user_domain_metadata', - meta, autoload=True) - group_domain_metadata_table = sqlalchemy.Table('group_domain_metadata', - meta, autoload=True) - user_project_metadata_table = sqlalchemy.Table('user_project_metadata', - meta, autoload=True) - group_project_metadata_table = sqlalchemy.Table('group_project_metadata', - meta, autoload=True) - - constraints = [{'table': user_domain_metadata_table, - 'fk_column': 'user_id', - 'ref_column': user_table.c.id}, - {'table': group_domain_metadata_table, - 'fk_column': 'group_id', - 'ref_column': group_table.c.id}, - {'table': user_project_metadata_table, - 'fk_column': 'user_id', - 'ref_column': user_table.c.id}, - {'table': group_project_metadata_table, - 'fk_column': 'group_id', - 'ref_column': group_table.c.id}, - ] - return constraints - - -def upgrade(migrate_engine): - if migrate_engine.name == 'sqlite': - return - migration_helpers.remove_constraints(list_constraints(migrate_engine)) - - -def downgrade(migrate_engine): - if migrate_engine.name == 'sqlite': - return - migration_helpers.add_constraints(list_constraints(migrate_engine)) diff --git a/keystone/common/sql/migrate_repo/versions/027_set_engine_mysql_innodb.py b/keystone/common/sql/migrate_repo/versions/027_set_engine_mysql_innodb.py deleted file mode 100644 index 48d77f5262..0000000000 --- a/keystone/common/sql/migrate_repo/versions/027_set_engine_mysql_innodb.py +++ /dev/null @@ -1,156 +0,0 @@ -# Copyright 2013 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql -from sqlalchemy import MetaData - -from keystone.common.sql import migration_helpers - - -def upgrade(migrate_engine): - # Upgrade operations go here. Don't create your own engine; - # bind migrate_engine to your metadata - - if migrate_engine.name != 'mysql': - # InnoDB / MyISAM only applies to MySQL. - return - - # This is a list of all the tables that might have been created with MyISAM - # rather than InnoDB. - tables = [ - 'credential', - 'domain', - 'ec2_credential', - 'endpoint', - 'group', - 'group_domain_metadata', - 'group_project_metadata', - 'policy', - 'project', - 'role', - 'service', - 'token', - 'trust', - 'trust_role', - 'user', - 'user_domain_metadata', - 'user_group_membership', - 'user_project_metadata', - ] - - meta = MetaData() - meta.bind = migrate_engine - - domain_table = sql.Table('domain', meta, autoload=True) - endpoint_table = sql.Table('endpoint', meta, autoload=True) - group_table = sql.Table('group', meta, autoload=True) - group_domain_metadata_table = sql.Table('group_domain_metadata', meta, - autoload=True) - group_project_metadata_table = sql.Table('group_project_metadata', meta, - autoload=True) - project_table = sql.Table('project', meta, autoload=True) - service_table = sql.Table('service', meta, autoload=True) - user_table = sql.Table('user', meta, autoload=True) - user_domain_metadata_table = sql.Table('user_domain_metadata', meta, - autoload=True) - user_group_membership_table = sql.Table('user_group_membership', meta, - autoload=True) - - # Mapping of table name to the constraints on that table, - # so we can create them. - table_constraints = { - 'endpoint': [{'table': endpoint_table, - 'fk_column': 'service_id', - 'ref_column': service_table.c.id}, - ], - 'group': [{'table': group_table, - 'fk_column': 'domain_id', - 'ref_column': domain_table.c.id}, - ], - 'group_domain_metadata': [{'table': group_domain_metadata_table, - 'fk_column': 'domain_id', - 'ref_column': domain_table.c.id}, - ], - 'group_project_metadata': [{'table': group_project_metadata_table, - 'fk_column': 'project_id', - 'ref_column': project_table.c.id}, - ], - 'project': [{'table': project_table, - 'fk_column': 'domain_id', - 'ref_column': domain_table.c.id}, - ], - 'user': [{'table': user_table, - 'fk_column': 'domain_id', - 'ref_column': domain_table.c.id}, - ], - 'user_domain_metadata': [{'table': user_domain_metadata_table, - 'fk_column': 'domain_id', - 'ref_column': domain_table.c.id}, - ], - 'user_group_membership': [{'table': user_group_membership_table, - 'fk_column': 'user_id', - 'ref_column': user_table.c.id}, - {'table': user_group_membership_table, - 'fk_column': 'group_id', - 'ref_column': group_table.c.id}, - ], - 'user_project_metadata': [{'table': group_project_metadata_table, - 'fk_column': 'project_id', - 'ref_column': project_table.c.id}, - ], - } - - # Maps a table name to the tables that reference it as a FK constraint - # (See the map above). - ref_tables_map = { - 'service': ['endpoint', ], - 'domain': ['group', 'group_domain_metadata', 'project', 'user', - 'user_domain_metadata', ], - 'project': ['group_project_metadata', 'user_project_metadata', ], - 'user': ['user_group_membership', ], - 'group': ['user_group_membership', ], - } - - # The names of tables that need to have their FKs added. - fk_table_names = set() - - d = migrate_engine.execute("SHOW TABLE STATUS WHERE Engine!='InnoDB';") - for row in d.fetchall(): - table_name = row[0] - - if table_name not in tables: - # Skip this table since it's not a Keystone table. - continue - - migrate_engine.execute("ALTER TABLE `%s` Engine=InnoDB" % table_name) - - # Will add the FKs to the table if any of - # a) the table itself was converted - # b) the tables that the table referenced were converted - - if table_name in table_constraints: - fk_table_names.add(table_name) - - ref_tables = ref_tables_map.get(table_name, []) - for other_table_name in ref_tables: - fk_table_names.add(other_table_name) - - # Now add all the FK constraints to those tables - for table_name in fk_table_names: - constraints = table_constraints.get(table_name) - migration_helpers.add_constraints(constraints) - - -def downgrade(migrate_engine): - pass diff --git a/keystone/common/sql/migrate_repo/versions/028_fixup_group_metadata.py b/keystone/common/sql/migrate_repo/versions/028_fixup_group_metadata.py deleted file mode 100644 index 61fce39eb3..0000000000 --- a/keystone/common/sql/migrate_repo/versions/028_fixup_group_metadata.py +++ /dev/null @@ -1,188 +0,0 @@ -# Copyright 2013 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql - - -def upgrade(migrate_engine): - # The group_project_metadata table was not updated in terms of its - # FK to the tenant table when the tenant->project change was made at - # the 015 migration for sqlite. This upgrade fixes that. - # We need to create a fake tenant table so that we can first load - # the group_project_metadata at all, then do a dance of copying tables - # to get us to the correct schema. - meta = sql.MetaData() - meta.bind = migrate_engine - - if migrate_engine.name != 'sqlite': - return - - temp_tenant_table = sql.Table( - 'tenant', - meta, - sql.Column('id', sql.String(64), primary_key=True)) - temp_tenant_table.create(migrate_engine, checkfirst=True) - - sql.Table('user', meta, autoload=True) - old_group_metadata_table = sql.Table('group_project_metadata', - meta, autoload=True) - - # OK, we now have the table loaded, create a first - # temporary table of a different name with the correct FK - sql.Table('project', meta, autoload=True) - temp_group_project_metadata_table = sql.Table( - 'temp_group_project_metadata', - meta, - sql.Column( - 'group_id', - sql.String(64), - primary_key=True), - sql.Column( - 'project_id', - sql.String(64), - sql.ForeignKey('project.id'), - primary_key=True), - sql.Column('data', sql.Text())) - temp_group_project_metadata_table.create(migrate_engine, checkfirst=True) - - # Populate the new temporary table, and then drop the old one - session = sql.orm.sessionmaker(bind=migrate_engine)() - - for metadata in session.query(old_group_metadata_table): - q = temp_group_project_metadata_table.insert().values( - group_id=metadata.group_id, - project_id=metadata.project_id, - data=metadata.data) - session.execute(q) - session.commit() - old_group_metadata_table.drop() - temp_tenant_table.drop() - - # Now do a final table copy to get the table of the right name. - # Re-init the metadata so that sqlalchemy does not get confused with - # multiple versions of the same named table. - meta2 = sql.MetaData() - meta2.bind = migrate_engine - - sql.Table('project', meta2, autoload=True) - new_group_project_metadata_table = sql.Table( - 'group_project_metadata', - meta2, - sql.Column( - 'group_id', - sql.String(64), - primary_key=True), - sql.Column( - 'project_id', - sql.String(64), - sql.ForeignKey('project.id'), - primary_key=True), - sql.Column('data', sql.Text())) - new_group_project_metadata_table.create(migrate_engine, checkfirst=True) - - for metadata in session.query(temp_group_project_metadata_table): - q = new_group_project_metadata_table.insert().values( - group_id=metadata.group_id, - project_id=metadata.project_id, - data=metadata.data) - session.execute(q) - session.commit() - - temp_group_project_metadata_table.drop() - - -def downgrade(migrate_engine): - # Put the group_project_metadata table back the way it was in its rather - # broken state. We don't try and re-write history, since otherwise people - # get out of step. - meta = sql.MetaData() - meta.bind = migrate_engine - - if migrate_engine.name != 'sqlite': - return - - sql.Table('user', meta, autoload=True) - sql.Table('project', meta, autoload=True) - group_metadata_table = sql.Table('group_project_metadata', - meta, autoload=True) - - # We want to create a temp group meta table with the FK - # set to the wrong place. - temp_tenant_table = sql.Table( - 'tenant', - meta, - sql.Column('id', sql.String(64), primary_key=True)) - temp_tenant_table.create(migrate_engine, checkfirst=True) - - temp_group_project_metadata_table = sql.Table( - 'temp_group_project_metadata', - meta, - sql.Column( - 'group_id', - sql.String(64), - primary_key=True), - sql.Column( - 'project_id', - sql.String(64), - sql.ForeignKey('tenant.id'), - primary_key=True), - sql.Column('data', sql.Text())) - temp_group_project_metadata_table.create(migrate_engine, checkfirst=True) - - # Now populate the temp table and drop the real one - session = sql.orm.sessionmaker(bind=migrate_engine)() - - for metadata in session.query(group_metadata_table): - q = temp_group_project_metadata_table.insert().values( - group_id=metadata.group_id, - project_id=metadata.project_id, - data=metadata.data) - session.execute(q) - - session.commit() - group_metadata_table.drop() - - # Now copy again into the correctly named table. Re-init the metadata - # so that sqlalchemy does not get confused with multiple versions of the - # same named table. - meta2 = sql.MetaData() - meta2.bind = migrate_engine - - sql.Table('tenant', meta2, autoload=True) - new_group_project_metadata_table = sql.Table( - 'group_project_metadata', - meta2, - sql.Column( - 'group_id', - sql.String(64), - primary_key=True), - sql.Column( - 'project_id', - sql.String(64), - sql.ForeignKey('tenant.id'), - primary_key=True), - sql.Column('data', sql.Text())) - new_group_project_metadata_table.create(migrate_engine, checkfirst=True) - - for metadata in session.query(temp_group_project_metadata_table): - q = new_group_project_metadata_table.insert().values( - group_id=metadata.group_id, - project_id=metadata.project_id, - data=metadata.data) - session.execute(q) - - session.commit() - - temp_group_project_metadata_table.drop() - temp_tenant_table.drop() diff --git a/keystone/common/sql/migrate_repo/versions/029_update_assignment_metadata.py b/keystone/common/sql/migrate_repo/versions/029_update_assignment_metadata.py deleted file mode 100644 index a02f9d8a1a..0000000000 --- a/keystone/common/sql/migrate_repo/versions/029_update_assignment_metadata.py +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright 2013 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import json - -import sqlalchemy as sql - - -def build_update(table_name, upgrade_table, row, values): - if table_name == 'user_project_metadata': - update = upgrade_table.update().where( - upgrade_table.c.user_id == row.user_id).where( - upgrade_table.c.project_id == row.project_id).values(values) - elif table_name == 'group_project_metadata': - update = upgrade_table.update().where( - upgrade_table.c.group_id == row.group_id).where( - upgrade_table.c.project_id == row.project_id).values(values) - elif table_name == 'user_domain_metadata': - update = upgrade_table.update().where( - upgrade_table.c.user_id == row.user_id).where( - upgrade_table.c.domain_id == row.domain_id).values(values) - else: - update = upgrade_table.update().where( - upgrade_table.c.group_id == row.group_id).where( - upgrade_table.c.domain_id == row.domain_id).values(values) - return update - - -def upgrade_grant_table(meta, migrate_engine, session, table_name): - - # Convert the roles component of the metadata from a list - # of ids to a list of dicts - - def list_to_dict_list(metadata): - json_metadata = json.loads(metadata) - if 'roles' in json_metadata: - json_metadata['roles'] = ( - [{'id': x} for x in json_metadata['roles']]) - return json.dumps(json_metadata) - - upgrade_table = sql.Table(table_name, meta, autoload=True) - for assignment in session.query(upgrade_table): - values = {'data': list_to_dict_list(assignment.data)} - update = build_update(table_name, upgrade_table, assignment, values) - migrate_engine.execute(update) - - -def downgrade_grant_table(meta, migrate_engine, session, table_name): - - # Convert the roles component of the metadata from a list - # of dicts to a simple list of ids. Any inherited roles are deleted - # since they would have no meaning - - def dict_list_to_list(metadata): - json_metadata = json.loads(metadata) - if 'roles' in json_metadata: - json_metadata['roles'] = ([x['id'] for x in json_metadata['roles'] - if 'inherited_to' not in x]) - return json.dumps(json_metadata) - - downgrade_table = sql.Table(table_name, meta, autoload=True) - for assignment in session.query(downgrade_table): - values = {'data': dict_list_to_list(assignment.data)} - update = build_update(table_name, downgrade_table, assignment, values) - migrate_engine.execute(update) - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - session = sql.orm.sessionmaker(bind=migrate_engine)() - - for grant_table in ['user_project_metadata', 'user_domain_metadata', - 'group_project_metadata', 'group_domain_metadata']: - upgrade_grant_table(meta, migrate_engine, session, grant_table) - session.commit() - session.close() - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - session = sql.orm.sessionmaker(bind=migrate_engine)() - - for grant_table in ['user_project_metadata', 'user_domain_metadata', - 'group_project_metadata', 'group_domain_metadata']: - downgrade_grant_table(meta, migrate_engine, session, grant_table) - session.commit() - session.close() diff --git a/keystone/common/sql/migrate_repo/versions/030_drop_credential_constraint_sqlite.py b/keystone/common/sql/migrate_repo/versions/030_drop_credential_constraint_sqlite.py deleted file mode 100644 index aff1713fa3..0000000000 --- a/keystone/common/sql/migrate_repo/versions/030_drop_credential_constraint_sqlite.py +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright 2013 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy -from sqlalchemy.orm import sessionmaker - - -def upgrade(migrate_engine): - if migrate_engine.name == 'sqlite': - drop_credential_table_foreign_key_constraints_for_sqlite( - migrate_engine) - - -def downgrade(migrate_engine): - if migrate_engine.name == 'sqlite': - add_credential_table_foreign_key_constraints_for_sqlite(migrate_engine) - - -def drop_credential_table_foreign_key_constraints_for_sqlite(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - - # NOTE(nachiappan): SQLite does not support ALTER TABLE DROP constraint. - # So we need to move the data to new credenital table - # created without constraints, drop the old table and - # rename the new table to credential. - sqlalchemy.Table('user', meta, autoload=True) - tenant_table = sqlalchemy.Table( - 'tenant', - meta, - sqlalchemy.Column('id', sqlalchemy.String(64), primary_key=True), - sqlalchemy.Column( - 'name', sqlalchemy.String(64), unique=True, nullable=False), - sqlalchemy.Column('extra', sqlalchemy.Text())) - tenant_table.create(migrate_engine, checkfirst=True) - cred_table = sqlalchemy.Table('credential', meta, autoload=True) - - session = sessionmaker(bind=migrate_engine)() - new_credential_table = sqlalchemy.Table( - 'new_credential', - meta, - sqlalchemy.Column('id', sqlalchemy.String(64), primary_key=True), - sqlalchemy.Column('user_id', - sqlalchemy.String(64), - nullable=False), - sqlalchemy.Column('project_id', - sqlalchemy.String(64)), - sqlalchemy.Column('blob', sqlalchemy.Text(), nullable=False), - sqlalchemy.Column('type', sqlalchemy.String(255), nullable=False), - sqlalchemy.Column('extra', sqlalchemy.Text())) - new_credential_table.create(migrate_engine, checkfirst=True) - - insert = new_credential_table.insert() - for credential in session.query(cred_table): - insert.execute({'id': credential.id, - 'user_id': credential.user_id, - 'project_id': credential.project_id, - 'blob': credential.blob, - 'type': credential.type, - 'extra': credential.extra}) - cred_table.drop() - tenant_table.drop() - new_credential_table.rename('credential') - session.commit() - session.close() - - -def add_credential_table_foreign_key_constraints_for_sqlite(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - - cred_table = sqlalchemy.Table('credential', meta, autoload=True) - sqlalchemy.Table('user', meta, autoload=True) - - session = sessionmaker(bind=migrate_engine)() - old_credential_table = sqlalchemy.Table( - 'old_credential', - meta, - sqlalchemy.Column('id', sqlalchemy.String(64), primary_key=True), - sqlalchemy.Column('user_id', - sqlalchemy.String(64), - sqlalchemy.ForeignKey('user.id'), - nullable=False), - # NOTE(nachiappan): Not creating the foreign key constraint with - # project table as version 15 conflicts with - # version 7. - sqlalchemy.Column('project_id', - sqlalchemy.String(64)), - sqlalchemy.Column('blob', sqlalchemy.Text(), nullable=False), - sqlalchemy.Column('type', sqlalchemy.String(255), nullable=False), - sqlalchemy.Column('extra', sqlalchemy.Text())) - old_credential_table.create(migrate_engine, checkfirst=True) - - insert = old_credential_table.insert() - for credential in session.query(cred_table): - insert.execute({'id': credential.id, - 'user_id': credential.user_id, - 'project_id': credential.project_id, - 'blob': credential.blob, - 'type': credential.type, - 'extra': credential.extra}) - cred_table.drop() - old_credential_table.rename('credential') - session.commit() - session.close() diff --git a/keystone/common/sql/migrate_repo/versions/031_drop_credential_indexes.py b/keystone/common/sql/migrate_repo/versions/031_drop_credential_indexes.py deleted file mode 100644 index bb9a2574f4..0000000000 --- a/keystone/common/sql/migrate_repo/versions/031_drop_credential_indexes.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2013 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy - - -def upgrade(migrate_engine): - # This migration is relevant only for mysql because for all other - # migrate engines these indexes were successfully dropped. - if migrate_engine.name != 'mysql': - return - meta = sqlalchemy.MetaData(bind=migrate_engine) - table = sqlalchemy.Table('credential', meta, autoload=True) - for index in table.indexes: - index.drop() - - -def downgrade(migrate_engine): - if migrate_engine.name != 'mysql': - return - meta = sqlalchemy.MetaData(bind=migrate_engine) - table = sqlalchemy.Table('credential', meta, autoload=True) - index = sqlalchemy.Index('user_id', table.c['user_id']) - index.create() - index = sqlalchemy.Index('credential_project_id_fkey', - table.c['project_id']) - index.create() diff --git a/keystone/common/sql/migrate_repo/versions/032_username_length.py b/keystone/common/sql/migrate_repo/versions/032_username_length.py deleted file mode 100644 index 26530eb6ba..0000000000 --- a/keystone/common/sql/migrate_repo/versions/032_username_length.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2013 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql -from sqlalchemy.orm import sessionmaker - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - user_table = sql.Table('user', meta, autoload=True) - user_table.c.name.alter(type=sql.String(255)) - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - user_table = sql.Table('user', meta, autoload=True) - if migrate_engine.name != 'mysql': - # NOTE(aloga): sqlite does not enforce length on the - # VARCHAR types: http://www.sqlite.org/faq.html#q9 - # postgresql and DB2 do not truncate. - maker = sessionmaker(bind=migrate_engine) - session = maker() - for user in session.query(user_table).all(): - values = {'name': user.name[:64]} - update = (user_table.update(). - where(user_table.c.id == user.id). - values(values)) - migrate_engine.execute(update) - - session.commit() - session.close() - user_table.c.name.alter(type=sql.String(64)) diff --git a/keystone/common/sql/migrate_repo/versions/033_migrate_ec2credentials_table_credentials.py b/keystone/common/sql/migrate_repo/versions/033_migrate_ec2credentials_table_credentials.py deleted file mode 100644 index a16904533b..0000000000 --- a/keystone/common/sql/migrate_repo/versions/033_migrate_ec2credentials_table_credentials.py +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright 2013 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql - -from keystone.common import utils -from keystone import exception -from keystone.openstack.common.gettextutils import _ - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - credential_table = sql.Table('credential', - meta, - autoload=True) - - ec2_cred_table = sql.Table('ec2_credential', - meta, - autoload=True) - - session = sql.orm.sessionmaker(bind=migrate_engine)() - insert = credential_table.insert() - for ec2credential in session.query(ec2_cred_table): - cred_exist = check_credential_exists(ec2credential, - credential_table, session) - - if not cred_exist: - credential = utils.convert_ec2_to_v3_credential(ec2credential) - insert.execute(credential) - - session.commit() - session.close() - - ec2_cred_table.drop() - - -def check_credential_exists(ec2credential, credential_table, session): - credential = session.query(credential_table).filter_by( - id=utils.hash_access_key(ec2credential.access)).first() - if credential is None: - return False - blob = utils.get_blob_from_credential(credential) - # check if credential with same access key but different - # secret key already exists in credential table. - # If exists raise an exception - if blob['secret'] != ec2credential.secret: - msg = _('Credential %(access)s already exists with different secret' - ' in %(table)s table') - message = msg % {'access': ec2credential.access, - 'table': credential_table.name} - raise exception.Conflict(type='credential', details=message) - # check if credential with same access and secret key but - # associated with a different project exists. If exists raise - # an exception - elif credential.project_id is not None and ( - credential.project_id != ec2credential.tenant_id): - msg = _('Credential %(access)s already exists with different project' - ' in %(table)s table') - message = msg % {'access': ec2credential.access, - 'table': credential_table.name} - raise exception.Conflict(type='credential', details=message) - # if credential with same access and secret key and not associated - # with any projects already exists in the credential table, then - # return true. - else: - return True - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - session = sql.orm.sessionmaker(bind=migrate_engine)() - - ec2_credential_table = sql.Table( - 'ec2_credential', - meta, - sql.Column('access', sql.String(64), primary_key=True), - sql.Column('secret', sql.String(64)), - sql.Column('user_id', sql.String(64)), - sql.Column('tenant_id', sql.String(64)), - mysql_engine='InnoDB', - mysql_charset='utf8') - - ec2_credential_table.create(migrate_engine, checkfirst=True) - credential_table = sql.Table('credential', - meta, - autoload=True) - insert = ec2_credential_table.insert() - for credential in session.query(credential_table).filter( - sql.and_(credential_table.c.type == 'ec2', - credential_table.c.project_id is not None)).all(): - ec2_credential = utils.convert_v3_to_ec2_credential(credential) - insert.execute(ec2_credential) - - session.commit() - session.close() diff --git a/keystone/common/sql/migrate_repo/versions/034_add_default_project_id_column_to_user.py b/keystone/common/sql/migrate_repo/versions/034_add_default_project_id_column_to_user.py deleted file mode 100644 index 17ad21713a..0000000000 --- a/keystone/common/sql/migrate_repo/versions/034_add_default_project_id_column_to_user.py +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright 2013 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import json - -import sqlalchemy as sql -from sqlalchemy.orm import sessionmaker - - -def migrate_default_project_from_extra_json(meta, migrate_engine): - user_table = sql.Table('user', meta, autoload=True) - - user_list = list(user_table.select().execute()) - session = sessionmaker(bind=migrate_engine)() - for user in user_list: - try: - data = json.loads(user.extra) - default_project_id = data.pop('default_project_id', None) - v2_tenant_id = data.pop('tenantId', None) - alt_v2_tenant_id = data.pop('tenant_id', None) - except (ValueError, TypeError): - # NOTE(morganfainberg): Somehow we have non-json data here. This - # is a broken user, but it was broken beforehand. Cleaning it up - # is not in the scope of this migration. - continue - - values = {} - if default_project_id is not None: - values['default_project_id'] = default_project_id - elif v2_tenant_id is not None: - values['default_project_id'] = v2_tenant_id - elif alt_v2_tenant_id is not None: - values['default_project_id'] = alt_v2_tenant_id - - if 'default_project_id' in values: - values['extra'] = json.dumps(data) - update = user_table.update().where( - user_table.c.id == user['id']).values(values) - migrate_engine.execute(update) - - session.commit() - session.close() - - -def migrate_default_project_to_extra_json(meta, migrate_engine): - user_table = sql.Table('user', meta, autoload=True) - - user_list = list(user_table.select().execute()) - session = sessionmaker(bind=migrate_engine)() - for user in user_list: - try: - data = json.loads(user.extra) - except (ValueError, TypeError): - # NOTE(morganfainberg): Somehow we have non-json data here. This - # is a broken user, but it was broken beforehand. Cleaning it up - # is not in the scope of this migration. - continue - - # NOTE(morganfainberg): We don't really know what the original 'extra' - # property was here. Populate all of the possible variants we may have - # originally used. - if user.default_project_id is not None: - data['default_project_id'] = user.default_project_id - data['tenantId'] = user.default_project_id - data['tenant_id'] = user.default_project_id - - values = {'extra': json.dumps(data)} - update = user_table.update().where( - user_table.c.id == user.id).values(values) - migrate_engine.execute(update) - session.commit() - session.close() - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - user_table = sql.Table('user', meta, autoload=True) - default_project_id = sql.Column('default_project_id', sql.String(64)) - user_table.create_column(default_project_id) - migrate_default_project_from_extra_json(meta, migrate_engine) - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - migrate_default_project_to_extra_json(meta, migrate_engine) - user_table = sql.Table('user', meta, autoload=True) - user_table.drop_column('default_project_id') diff --git a/keystone/common/sql/migrate_repo/versions/035_add_compound_revoked_token_index.py b/keystone/common/sql/migrate_repo/versions/035_add_compound_revoked_token_index.py deleted file mode 100644 index 1716819195..0000000000 --- a/keystone/common/sql/migrate_repo/versions/035_add_compound_revoked_token_index.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2013 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - token = sql.Table('token', meta, autoload=True) - idx = sql.Index('ix_token_expires_valid', token.c.expires, token.c.valid) - idx.create(migrate_engine) - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - token = sql.Table('token', meta, autoload=True) - idx = sql.Index('ix_token_expires_valid', token.c.expires, token.c.valid) - idx.drop(migrate_engine) diff --git a/keystone/common/sql/migrate_repo/versions/036_havana.py b/keystone/common/sql/migrate_repo/versions/036_havana.py new file mode 100644 index 0000000000..1604d61d02 --- /dev/null +++ b/keystone/common/sql/migrate_repo/versions/036_havana.py @@ -0,0 +1,288 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import migrate +import sqlalchemy as sql +from sqlalchemy import orm + +from keystone.common import sql as ks_sql +from keystone.common.sql import migration_helpers +from keystone import config +from keystone.openstack.common import log + + +LOG = log.getLogger(__name__) +CONF = config.CONF + + +def upgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + if migrate_engine.name == 'mysql': + # In Folsom we explicitly converted migrate_version to UTF8. + sql_stmt = 'ALTER TABLE migrate_version CONVERT TO CHARACTER SET utf8;' + # Set default DB charset to UTF8. + sql_stmt += ('ALTER DATABASE %s DEFAULT CHARACTER SET utf8;' % + migrate_engine.url.database) + migrate_engine.execute(sql_stmt) + + credential = sql.Table( + 'credential', meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('user_id', sql.String(length=64), nullable=False), + sql.Column('project_id', sql.String(length=64)), + sql.Column('blob', ks_sql.JsonBlob, nullable=False), + sql.Column('type', sql.String(length=255), nullable=False), + sql.Column('extra', ks_sql.JsonBlob.impl), + mysql_engine='InnoDB', + mysql_charset='utf8') + + domain = sql.Table( + 'domain', meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('name', sql.String(length=64), nullable=False), + sql.Column('enabled', sql.Boolean, default=True, nullable=False), + sql.Column('extra', ks_sql.JsonBlob.impl), + mysql_engine='InnoDB', + mysql_charset='utf8') + + endpoint = sql.Table( + 'endpoint', meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('legacy_endpoint_id', sql.String(length=64)), + sql.Column('interface', sql.String(length=8), nullable=False), + sql.Column('region', sql.String(length=255)), + sql.Column('service_id', sql.String(length=64), nullable=False), + sql.Column('url', sql.Text, nullable=False), + sql.Column('extra', ks_sql.JsonBlob.impl), + mysql_engine='InnoDB', + mysql_charset='utf8') + + group = sql.Table( + 'group', meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('domain_id', sql.String(length=64), nullable=False), + sql.Column('name', sql.String(length=64), nullable=False), + sql.Column('description', sql.Text), + sql.Column('extra', ks_sql.JsonBlob.impl), + mysql_engine='InnoDB', + mysql_charset='utf8') + + group_domain_metadata = sql.Table( + 'group_domain_metadata', meta, + sql.Column('group_id', sql.String(length=64), primary_key=True), + sql.Column('domain_id', sql.String(length=64), primary_key=True), + sql.Column('data', ks_sql.JsonBlob.impl), + mysql_engine='InnoDB', + mysql_charset='utf8') + + group_project_metadata = sql.Table( + 'group_project_metadata', meta, + sql.Column('group_id', sql.String(length=64), primary_key=True), + sql.Column('project_id', sql.String(length=64), primary_key=True), + sql.Column('data', ks_sql.JsonBlob.impl), + mysql_engine='InnoDB', + mysql_charset='utf8') + + policy = sql.Table( + 'policy', meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('type', sql.String(length=255), nullable=False), + sql.Column('blob', ks_sql.JsonBlob, nullable=False), + sql.Column('extra', ks_sql.JsonBlob.impl), + mysql_engine='InnoDB', + mysql_charset='utf8') + + project = sql.Table( + 'project', meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('name', sql.String(length=64), nullable=False), + sql.Column('extra', ks_sql.JsonBlob.impl), + sql.Column('description', sql.Text), + sql.Column('enabled', sql.Boolean), + sql.Column('domain_id', sql.String(length=64), nullable=False), + mysql_engine='InnoDB', + mysql_charset='utf8') + + role = sql.Table( + 'role', meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('name', sql.String(length=255), nullable=False), + sql.Column('extra', ks_sql.JsonBlob.impl), + mysql_engine='InnoDB', + mysql_charset='utf8') + + service = sql.Table( + 'service', meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('type', sql.String(length=255)), + sql.Column('extra', ks_sql.JsonBlob.impl), + mysql_engine='InnoDB', + mysql_charset='utf8') + + token = sql.Table( + 'token', meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('expires', sql.DateTime, default=None), + sql.Column('extra', ks_sql.JsonBlob.impl), + sql.Column('valid', sql.Boolean, default=True, nullable=False), + sql.Column('trust_id', sql.String(length=64)), + sql.Column('user_id', sql.String(length=64)), + mysql_engine='InnoDB', + mysql_charset='utf8') + + trust = sql.Table( + 'trust', meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('trustor_user_id', sql.String(length=64), nullable=False), + sql.Column('trustee_user_id', sql.String(length=64), nullable=False), + sql.Column('project_id', sql.String(length=64)), + sql.Column('impersonation', sql.Boolean, nullable=False), + sql.Column('deleted_at', sql.DateTime), + sql.Column('expires_at', sql.DateTime), + sql.Column('extra', ks_sql.JsonBlob.impl), + mysql_engine='InnoDB', + mysql_charset='utf8') + + trust_role = sql.Table( + 'trust_role', meta, + sql.Column('trust_id', sql.String(length=64), primary_key=True, + nullable=False), + sql.Column('role_id', sql.String(length=64), primary_key=True, + nullable=False), + mysql_engine='InnoDB', + mysql_charset='utf8') + + user = sql.Table( + 'user', meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('name', sql.String(length=255), nullable=False), + sql.Column('extra', ks_sql.JsonBlob.impl), + sql.Column('password', sql.String(length=128)), + sql.Column('enabled', sql.Boolean), + sql.Column('domain_id', sql.String(length=64), nullable=False), + sql.Column('default_project_id', sql.String(length=64)), + mysql_engine='InnoDB', + mysql_charset='utf8') + + user_domain_metadata = sql.Table( + 'user_domain_metadata', meta, + sql.Column('user_id', sql.String(length=64), primary_key=True), + sql.Column('domain_id', sql.String(length=64), primary_key=True), + sql.Column('data', ks_sql.JsonBlob.impl), + mysql_engine='InnoDB', + mysql_charset='utf8') + + user_group_membership = sql.Table( + 'user_group_membership', meta, + sql.Column('user_id', sql.String(length=64), primary_key=True), + sql.Column('group_id', sql.String(length=64), primary_key=True), + mysql_engine='InnoDB', + mysql_charset='utf8') + + user_project_metadata = sql.Table( + 'user_project_metadata', meta, + sql.Column('user_id', sql.String(length=64), primary_key=True), + sql.Column('project_id', sql.String(length=64), primary_key=True), + sql.Column('data', ks_sql.JsonBlob.impl), + mysql_engine='InnoDB', + mysql_charset='utf8') + + # create all tables + tables = [credential, domain, endpoint, group, group_domain_metadata, + group_project_metadata, policy, project, role, service, + token, trust, trust_role, user, user_domain_metadata, + user_group_membership, user_project_metadata] + + for table in tables: + try: + table.create() + except Exception: + LOG.exception('Exception while creating table: %r', table) + raise + + # Unique Constraints + migrate.UniqueConstraint(user.c.domain_id, + user.c.name, + name='ixu_user_name_domain_id').create() + migrate.UniqueConstraint(group.c.domain_id, + group.c.name, + name='ixu_group_name_domain_id').create() + migrate.UniqueConstraint(role.c.name, + name='ixu_role_name').create() + migrate.UniqueConstraint(project.c.domain_id, + project.c.name, + name='ixu_project_name_domain_id').create() + migrate.UniqueConstraint(domain.c.name, + name='ixu_domain_name').create() + + # Indexes + sql.Index('ix_token_expires', token.c.expires).create() + sql.Index('ix_token_expires_valid', token.c.expires, + token.c.valid).create() + + fkeys = [ + {'columns': [user_project_metadata.c.project_id], + 'references': [project.c.id], + 'name': 'fk_user_project_metadata_project_id'}, + + {'columns': [user_domain_metadata.c.domain_id], + 'references': [domain.c.id], + 'name': 'fk_user_domain_metadata_domain_id'}, + + {'columns': [group_project_metadata.c.project_id], + 'references': [project.c.id], + 'name': 'fk_group_project_metadata_project_id'}, + + {'columns': [group_domain_metadata.c.domain_id], + 'references': [domain.c.id], + 'name': 'fk_group_domain_metadata_domain_id'}, + + {'columns': [endpoint.c.service_id], + 'references': [service.c.id]}, + + {'columns': [user_group_membership.c.group_id], + 'references': [group.c.id], + 'name': 'fk_user_group_membership_group_id'}, + + {'columns': [user_group_membership.c.user_id], + 'references':[user.c.id], + 'name': 'fk_user_group_membership_user_id'}, + + {'columns': [user.c.domain_id], + 'references': [domain.c.id], + 'name': 'fk_user_domain_id'}, + + {'columns': [group.c.domain_id], + 'references': [domain.c.id], + 'name': 'fk_group_domain_id'}, + + {'columns': [project.c.domain_id], + 'references': [domain.c.id], + 'name': 'fk_project_domain_id'} + ] + + for fkey in fkeys: + migrate.ForeignKeyConstraint(columns=fkey['columns'], + refcolumns=fkey['references'], + name=fkey.get('name')).create() + + # Create the default domain. + session = orm.sessionmaker(bind=migrate_engine)() + domain.insert(migration_helpers.get_default_domain()).execute() + session.commit() + + +def downgrade(migrate_engine): + raise NotImplementedError('Downgrade to pre-Havana release db schema is ' + 'unsupported.') diff --git a/keystone/common/sql/migrate_repo/versions/036_token_drop_valid_index.py b/keystone/common/sql/migrate_repo/versions/036_token_drop_valid_index.py deleted file mode 100644 index 89e03b7503..0000000000 --- a/keystone/common/sql/migrate_repo/versions/036_token_drop_valid_index.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2013 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - token = sql.Table('token', meta, autoload=True) - idx = sql.Index('ix_token_valid', token.c.valid) - idx.drop(migrate_engine) - - -def downgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - token = sql.Table('token', meta, autoload=True) - idx = sql.Index('ix_token_valid', token.c.valid) - idx.create(migrate_engine) diff --git a/keystone/common/sql/migration_helpers.py b/keystone/common/sql/migration_helpers.py index 0072651ac7..63135fdbe6 100644 --- a/keystone/common/sql/migration_helpers.py +++ b/keystone/common/sql/migration_helpers.py @@ -22,11 +22,29 @@ from migrate import exceptions import sqlalchemy from keystone.common import sql +from keystone import config from keystone import contrib from keystone import exception from keystone.openstack.common.db.sqlalchemy import migration from keystone.openstack.common.gettextutils import _ from keystone.openstack.common import importutils +from keystone.openstack.common import jsonutils + + +DB_INIT_VERSION = 35 +CONF = config.CONF + + +def get_default_domain(): + # Return the reference used for the default domain structure during + # sql migrations. + return { + 'id': CONF.identity.default_domain_id, + 'name': 'Default', + 'enabled': True, + 'extra': jsonutils.dumps({'description': 'Owns users and tenants ' + '(i.e. projects) available ' + 'on Identity API v2.'})} # Different RDBMSs use different schemes for naming the Foreign Key @@ -117,7 +135,9 @@ def find_migrate_repo(package=None, repo_name='migrate_repo'): def sync_database_to_version(extension=None, version=None): if not extension: abs_path = find_migrate_repo() + init_version = DB_INIT_VERSION else: + init_version = 0 try: package_name = '.'.join((contrib.__name__, extension)) package = importutils.import_module(package_name) @@ -136,7 +156,8 @@ def sync_database_to_version(extension=None, version=None): except exception.MigrationNotProvided as e: print(e) sys.exit(1) - migration.db_sync(sql.get_engine(), abs_path, version=version) + migration.db_sync(sql.get_engine(), abs_path, version=version, + init_version=init_version) def get_db_version(extension=None): diff --git a/keystone/tests/test_sql_upgrade.py b/keystone/tests/test_sql_upgrade.py index 60770f72df..d85893730b 100644 --- a/keystone/tests/test_sql_upgrade.py +++ b/keystone/tests/test_sql_upgrade.py @@ -40,7 +40,6 @@ import sqlalchemy.exc from keystone.assignment.backends import sql as assignment_sql from keystone.common import sql from keystone.common.sql import migration_helpers -from keystone.common import utils from keystone import config from keystone.contrib import federation from keystone import exception @@ -54,6 +53,66 @@ from keystone.tests import default_fixtures CONF = config.CONF DEFAULT_DOMAIN_ID = CONF.identity.default_domain_id +# NOTE(morganfainberg): This should be updated when each DB migration collapse +# is done to mirror the expected structure of the DB in the format of +# { : [, , ...], ... } +INITIAL_TABLE_STRUCTURE = { + 'credential': [ + 'id', 'user_id', 'project_id', 'blob', 'type', 'extra', + ], + 'domain': [ + 'id', 'name', 'enabled', 'extra', + ], + 'endpoint': [ + 'id', 'legacy_endpoint_id', 'interface', 'region', 'service_id', 'url', + 'extra', + ], + 'group': [ + 'id', 'domain_id', 'name', 'description', 'extra', + ], + 'group_domain_metadata': [ + 'group_id', 'domain_id', 'data', + ], + 'group_project_metadata': [ + 'group_id', 'project_id', 'data', + ], + 'policy': [ + 'id', 'type', 'blob', 'extra', + ], + 'project': [ + 'id', 'name', 'extra', 'description', 'enabled', 'domain_id', + ], + 'role': [ + 'id', 'name', 'extra', + ], + 'service': [ + 'id', 'type', 'extra', + ], + 'token': [ + 'id', 'expires', 'extra', 'valid', 'trust_id', 'user_id', + ], + 'trust': [ + 'id', 'trustor_user_id', 'trustee_user_id', 'project_id', + 'impersonation', 'deleted_at', 'expires_at', 'extra', + ], + 'trust_role': [ + 'trust_id', 'role_id', + ], + 'user': [ + 'id', 'name', 'extra', 'password', 'enabled', 'domain_id', + 'default_project_id', + ], + 'user_domain_metadata': [ + 'user_id', 'domain_id', 'data', + ], + 'user_group_membership': [ + 'user_id', 'group_id', + ], + 'user_project_metadata': [ + 'user_id', 'project_id', 'data', + ], +} + class SqlMigrateBase(tests.SQLDriverOverrides, tests.TestCase): def initialize_sql(self): @@ -93,17 +152,19 @@ class SqlMigrateBase(tests.SQLDriverOverrides, tests.TestCase): self.repo_package()) self.schema = versioning_api.ControlledSchema.create( self.engine, - self.repo_path, 0) + self.repo_path, self.initial_db_version) # auto-detect the highest available schema version in the migrate_repo self.max_version = self.schema.repository.version().version def tearDown(self): sqlalchemy.orm.session.Session.close_all() - table = sqlalchemy.Table("migrate_version", self.metadata, - autoload=True) - self.downgrade(0) - table.drop(self.engine, checkfirst=True) + meta = sqlalchemy.MetaData() + meta.bind = self.engine + meta.reflect(self.engine) + for table in list(meta.tables.keys()): + table = sqlalchemy.Table(table, meta, autoload=True) + table.drop(self.engine, checkfirst=True) sql.cleanup() super(SqlMigrateBase, self).tearDown() @@ -161,449 +222,111 @@ class SqlMigrateBase(tests.SQLDriverOverrides, tests.TestCase): actual_cols = [col.name for col in table.columns] self.assertEqual(expected_cols, actual_cols, '%s table' % table_name) + @property + def initial_db_version(self): + return getattr(self, '_initial_db_version', 0) + class SqlUpgradeTests(SqlMigrateBase): + _initial_db_version = migration_helpers.DB_INIT_VERSION + def test_blank_db_to_start(self): self.assertTableDoesNotExist('user') - def test_start_version_0(self): - version = migration.db_version(sql.get_engine(), self.repo_path, 0) - self.assertEqual(version, 0, "DB is not at version 0") + def test_start_version_db_init_version(self): + version = migration.db_version(sql.get_engine(), self.repo_path, + migration_helpers.DB_INIT_VERSION) + self.assertEqual( + version, + migration_helpers.DB_INIT_VERSION, + 'DB is not at version %s' % migration_helpers.DB_INIT_VERSION) def test_two_steps_forward_one_step_back(self): """You should be able to cleanly undo and re-apply all upgrades. Upgrades are run in the following order:: - 0 -> 1 -> 0 -> 1 -> 2 -> 1 -> 2 -> 3 -> 2 -> 3 ... - ^---------^ ^---------^ ^---------^ + Starting with the initial version defined at + keystone.common.migration_helpers.DB_INIT_VERSION + + INIT +1 -> INIT +2 -> INIT +1 -> INIT +2 -> INIT +3 -> INIT +2 ... + ^---------------------^ ^---------------------^ + + Downgrade to the DB_INIT_VERSION does not occur based on the + requirement that the base version be DB_INIT_VERSION + 1 before + migration can occur. Downgrade below DB_INIT_VERSION + 1 is no longer + supported. + + DB_INIT_VERSION is the number preceding the release schema version from + two releases prior. Example, Juno releases with the DB_INIT_VERSION + being 35 where Havana (Havana was two releases before Juno) release + schema version is 36. + + The migrate utility requires the db must be initialized under version + control with the revision directly before the first version to be + applied. """ - for x in range(1, self.max_version + 1): + for x in range(migration_helpers.DB_INIT_VERSION + 1, + self.max_version + 1): self.upgrade(x) - self.downgrade(x - 1) + downgrade_ver = x - 1 + # Don't actually downgrade to the init version. This will raise + # a not-implemented error. + if downgrade_ver != migration_helpers.DB_INIT_VERSION: + self.downgrade(x - 1) self.upgrade(x) def test_upgrade_add_initial_tables(self): - self.upgrade(1) - self.assertTableColumns("user", ["id", "name", "extra"]) - self.assertTableColumns("tenant", ["id", "name", "extra"]) - self.assertTableColumns("role", ["id", "name"]) - self.assertTableColumns("user_tenant_membership", - ["user_id", "tenant_id"]) - self.assertTableColumns("metadata", ["user_id", "tenant_id", "data"]) - self.populate_user_table() + self.upgrade(migration_helpers.DB_INIT_VERSION + 1) + self.check_initial_table_structure() - def test_upgrade_add_policy(self): - self.upgrade(5) - self.assertTableDoesNotExist('policy') + def check_initial_table_structure(self): + for table in INITIAL_TABLE_STRUCTURE: + self.assertTableColumns(table, INITIAL_TABLE_STRUCTURE[table]) - self.upgrade(6) - self.assertTableExists('policy') - self.assertTableColumns('policy', ['id', 'type', 'blob', 'extra']) + # Ensure the default domain was properly created. + default_domain = migration_helpers.get_default_domain() - def test_upgrade_normalize_identity(self): - self.upgrade(8) - self.populate_user_table() - self.populate_tenant_table() - self.upgrade(10) - self.assertTableColumns("user", - ["id", "name", "extra", - "password", "enabled"]) - self.assertTableColumns("tenant", - ["id", "name", "extra", "description", - "enabled"]) - self.assertTableColumns("role", ["id", "name", "extra"]) - self.assertTableColumns("user_tenant_membership", - ["user_id", "tenant_id"]) - self.assertTableColumns("metadata", ["user_id", "tenant_id", "data"]) - session = self.Session() - user_table = sqlalchemy.Table("user", - self.metadata, - autoload=True) - a_user = session.query(user_table).filter("id='foo'").one() - self.assertTrue(a_user.enabled) - a_user = session.query(user_table).filter("id='badguy'").one() - self.assertFalse(a_user.enabled) - tenant_table = sqlalchemy.Table("tenant", - self.metadata, - autoload=True) - a_tenant = session.query(tenant_table).filter("id='baz'").one() - self.assertEqual(a_tenant.description, 'description') - session.commit() - session.close() + meta = sqlalchemy.MetaData() + meta.bind = self.engine - def test_upgrade_user_tenant_membership_to_metadata(self): - self.upgrade(16) - self.assertTableColumns( - 'user_project_membership', - ['user_id', 'tenant_id']) - - user = { - 'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'domain_id': 'default', - 'extra': json.dumps({}), - } - project = { - 'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'domain_id': 'default', - 'extra': json.dumps({}), - } - metadata = { - 'user_id': user['id'], - 'tenant_id': project['id'], - } - session = self.Session() - self.insert_dict(session, 'user', user) - self.insert_dict(session, 'project', project) - self.insert_dict(session, 'user_project_membership', metadata) - - self.upgrade(17) - user_project_metadata_table = sqlalchemy.Table( - 'user_project_metadata', self.metadata, autoload=True) - - result = session.query(user_project_metadata_table).one() - self.assertEqual(result.user_id, user['id']) - self.assertEqual(result.project_id, project['id']) - self.assertEqual( - json.loads(result.data), - {'roles': [CONF.member_role_id]}) - - def test_normalized_enabled_states(self): - self.upgrade(8) - - users = { - 'bool_enabled_user': { - 'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'password': uuid.uuid4().hex, - 'extra': json.dumps({'enabled': True})}, - 'bool_disabled_user': { - 'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'password': uuid.uuid4().hex, - 'extra': json.dumps({'enabled': False})}, - 'str_enabled_user': { - 'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'password': uuid.uuid4().hex, - 'extra': json.dumps({'enabled': 'True'})}, - 'str_disabled_user': { - 'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'password': uuid.uuid4().hex, - 'extra': json.dumps({'enabled': 'False'})}, - 'int_enabled_user': { - 'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'password': uuid.uuid4().hex, - 'extra': json.dumps({'enabled': 1})}, - 'int_disabled_user': { - 'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'password': uuid.uuid4().hex, - 'extra': json.dumps({'enabled': 0})}, - 'null_enabled_user': { - 'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'password': uuid.uuid4().hex, - 'extra': json.dumps({'enabled': None})}, - 'unset_enabled_user': { - 'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'password': uuid.uuid4().hex, - 'extra': json.dumps({})}} + domain_table = sqlalchemy.Table('domain', meta, autoload=True) session = self.Session() - for user in users.values(): - self.insert_dict(session, 'user', user) - session.commit() + q = session.query(domain_table) + refs = q.all() - self.upgrade(10) + self.assertEqual(1, len(refs)) + for k in default_domain.keys(): + self.assertEqual(default_domain[k], getattr(refs[0], k)) - user_table = sqlalchemy.Table('user', self.metadata, autoload=True) - q = session.query(user_table, 'enabled') + def test_downgrade_to_db_init_version(self): + self.upgrade(self.max_version) - user = q.filter_by(id=users['bool_enabled_user']['id']).one() - self.assertTrue(user.enabled) + if self.engine.name == 'mysql': + self._mysql_check_all_tables_innodb() - user = q.filter_by(id=users['bool_disabled_user']['id']).one() - self.assertFalse(user.enabled) + self.downgrade(migration_helpers.DB_INIT_VERSION + 1) + self.check_initial_table_structure() - user = q.filter_by(id=users['str_enabled_user']['id']).one() - self.assertTrue(user.enabled) + meta = sqlalchemy.MetaData() + meta.bind = self.engine + meta.reflect(self.engine) - user = q.filter_by(id=users['str_disabled_user']['id']).one() - self.assertFalse(user.enabled) + initial_table_set = set(INITIAL_TABLE_STRUCTURE.keys()) + table_set = set(meta.tables.keys()) + # explicitly remove the migrate_version table, this is not controlled + # by the migration scripts and should be exempt from this check. + table_set.remove('migrate_version') - user = q.filter_by(id=users['int_enabled_user']['id']).one() - self.assertTrue(user.enabled) - - user = q.filter_by(id=users['int_disabled_user']['id']).one() - self.assertFalse(user.enabled) - - user = q.filter_by(id=users['null_enabled_user']['id']).one() - self.assertTrue(user.enabled) - - user = q.filter_by(id=users['unset_enabled_user']['id']).one() - self.assertTrue(user.enabled) - - def test_downgrade_10_to_8(self): - self.upgrade(10) - self.populate_user_table(with_pass_enab=True) - self.populate_tenant_table(with_desc_enab=True) - self.downgrade(8) - self.assertTableColumns('user', - ['id', 'name', 'extra']) - self.assertTableColumns('tenant', - ['id', 'name', 'extra']) - session = self.Session() - user_table = sqlalchemy.Table("user", - self.metadata, - autoload=True) - a_user = session.query(user_table).filter("id='badguy'").one() - self.assertEqual(a_user.name, default_fixtures.USERS[2]['name']) - tenant_table = sqlalchemy.Table("tenant", - self.metadata, - autoload=True) - a_tenant = session.query(tenant_table).filter("id='baz'").one() - self.assertEqual(a_tenant.name, default_fixtures.TENANTS[1]['name']) - session.commit() - session.close() - - def test_upgrade_endpoints(self): - self.upgrade(10) - service_extra = { - 'name': uuid.uuid4().hex, - } - service = { - 'id': uuid.uuid4().hex, - 'type': uuid.uuid4().hex, - 'extra': json.dumps(service_extra), - } - endpoint_extra = { - 'publicurl': uuid.uuid4().hex, - 'internalurl': uuid.uuid4().hex, - 'adminurl': uuid.uuid4().hex, - } - endpoint = { - 'id': uuid.uuid4().hex, - 'region': uuid.uuid4().hex, - 'service_id': service['id'], - 'extra': json.dumps(endpoint_extra), - } - - session = self.Session() - self.insert_dict(session, 'service', service) - self.insert_dict(session, 'endpoint', endpoint) - session.commit() - session.close() - - self.upgrade(13) - self.assertTableColumns( - 'service', - ['id', 'type', 'extra']) - self.assertTableColumns( - 'endpoint', - ['id', 'legacy_endpoint_id', 'interface', 'region', 'service_id', - 'url', 'extra']) - - endpoint_table = sqlalchemy.Table( - 'endpoint', self.metadata, autoload=True) - - session = self.Session() - self.assertEqual(session.query(endpoint_table).count(), 3) - for interface in ['public', 'internal', 'admin']: - q = session.query(endpoint_table) - q = q.filter_by(legacy_endpoint_id=endpoint['id']) - q = q.filter_by(interface=interface) - ref = q.one() - self.assertNotEqual(ref.id, endpoint['id']) - self.assertEqual(ref.legacy_endpoint_id, endpoint['id']) - self.assertEqual(ref.interface, interface) - self.assertEqual(ref.region, endpoint['region']) - self.assertEqual(ref.service_id, endpoint['service_id']) - self.assertEqual(ref.url, endpoint_extra['%surl' % interface]) - self.assertEqual(ref.extra, '{}') - session.commit() - session.close() - - def assertTenantTables(self): - self.assertTableExists('tenant') - self.assertTableExists('user_tenant_membership') - self.assertTableDoesNotExist('project') - self.assertTableDoesNotExist('user_project_membership') - - def assertProjectTables(self): - self.assertTableExists('project') - self.assertTableExists('user_project_membership') - self.assertTableDoesNotExist('tenant') - self.assertTableDoesNotExist('user_tenant_membership') - - def test_upgrade_tenant_to_project(self): - self.upgrade(14) - self.assertTenantTables() - self.upgrade(15) - self.assertProjectTables() - - def test_downgrade_project_to_tenant(self): - # TODO(henry-nash): Debug why we need to re-load the tenant - # or user_tenant_membership ahead of upgrading to project - # in order for the assertProjectTables to work on sqlite - # (MySQL is fine without it) - self.upgrade(14) - self.assertTenantTables() - self.upgrade(15) - self.assertProjectTables() - self.downgrade(14) - self.assertTenantTables() - - def test_upgrade_add_group_tables(self): - self.upgrade(13) - self.upgrade(14) - self.assertTableExists('group') - self.assertTableExists('group_project_metadata') - self.assertTableExists('group_domain_metadata') - self.assertTableExists('user_group_membership') - - def test_upgrade_14_to_16(self): - self.upgrade(14) - self.populate_user_table(with_pass_enab=True) - self.populate_tenant_table(with_desc_enab=True) - self.upgrade(16) - - self.assertTableColumns("user", - ["id", "name", "extra", - "password", "enabled", "domain_id"]) - session = self.Session() - user_table = sqlalchemy.Table("user", - self.metadata, - autoload=True) - a_user = session.query(user_table).filter("id='foo'").one() - self.assertTrue(a_user.enabled) - self.assertEqual(a_user.domain_id, DEFAULT_DOMAIN_ID) - a_user = session.query(user_table).filter("id='badguy'").one() - self.assertEqual(a_user.name, default_fixtures.USERS[2]['name']) - self.assertEqual(a_user.domain_id, DEFAULT_DOMAIN_ID) - project_table = sqlalchemy.Table("project", - self.metadata, - autoload=True) - a_project = session.query(project_table).filter("id='baz'").one() - self.assertEqual(a_project.description, - default_fixtures.TENANTS[1]['description']) - self.assertEqual(a_project.domain_id, DEFAULT_DOMAIN_ID) - - session.commit() - session.close() - - self.check_uniqueness_constraints() - - def test_downgrade_16_to_14(self): - self.upgrade(16) - self.populate_user_table(with_pass_enab_domain=True) - self.populate_tenant_table(with_desc_enab_domain=True) - self.downgrade(14) - self.assertTableColumns("user", - ["id", "name", "extra", - "password", "enabled"]) - session = self.Session() - user_table = sqlalchemy.Table("user", - self.metadata, - autoload=True) - a_user = session.query(user_table).filter("id='foo'").one() - self.assertTrue(a_user.enabled) - a_user = session.query(user_table).filter("id='badguy'").one() - self.assertEqual(a_user.name, default_fixtures.USERS[2]['name']) - tenant_table = sqlalchemy.Table("tenant", - self.metadata, - autoload=True) - a_tenant = session.query(tenant_table).filter("id='baz'").one() - self.assertEqual(a_tenant.description, - default_fixtures.TENANTS[1]['description']) - session.commit() - session.close() - - def test_downgrade_remove_group_tables(self): - self.upgrade(14) - self.downgrade(13) - self.assertTableDoesNotExist('group') - self.assertTableDoesNotExist('group_project_metadata') - self.assertTableDoesNotExist('group_domain_metadata') - self.assertTableDoesNotExist('user_group_membership') - - def test_downgrade_endpoints(self): - self.upgrade(13) - - service_extra = { - 'name': uuid.uuid4().hex, - } - service = { - 'id': uuid.uuid4().hex, - 'type': uuid.uuid4().hex, - 'extra': json.dumps(service_extra), - } - - common_endpoint_attrs = { - 'legacy_endpoint_id': uuid.uuid4().hex, - 'region': uuid.uuid4().hex, - 'service_id': service['id'], - 'extra': json.dumps({}), - } - endpoints = { - 'public': { - 'id': uuid.uuid4().hex, - 'interface': 'public', - 'url': uuid.uuid4().hex, - }, - 'internal': { - 'id': uuid.uuid4().hex, - 'interface': 'internal', - 'url': uuid.uuid4().hex, - }, - 'admin': { - 'id': uuid.uuid4().hex, - 'interface': 'admin', - 'url': uuid.uuid4().hex, - }, - } - - session = self.Session() - self.insert_dict(session, 'service', service) - for endpoint in endpoints.values(): - endpoint.update(common_endpoint_attrs) - self.insert_dict(session, 'endpoint', endpoint) - session.commit() - session.close() - - self.downgrade(9) - - self.assertTableColumns( - 'service', - ['id', 'type', 'extra']) - self.assertTableColumns( - 'endpoint', - ['id', 'region', 'service_id', 'extra']) - - endpoint_table = sqlalchemy.Table( - 'endpoint', self.metadata, autoload=True) - - session = self.Session() - self.assertEqual(session.query(endpoint_table).count(), 1) - q = session.query(endpoint_table) - q = q.filter_by(id=common_endpoint_attrs['legacy_endpoint_id']) - ref = q.one() - self.assertEqual(ref.id, common_endpoint_attrs['legacy_endpoint_id']) - self.assertEqual(ref.region, endpoint['region']) - self.assertEqual(ref.service_id, endpoint['service_id']) - extra = json.loads(ref.extra) - for interface in ['public', 'internal', 'admin']: - expected_url = endpoints[interface]['url'] - self.assertEqual(extra['%surl' % interface], expected_url) - session.commit() - session.close() + self.assertSetEqual(initial_table_set, table_set) + # Downgrade to before Havana's release schema version (036) is not + # supported. A NotImplementedError should be raised when attempting to + # downgrade. + self.assertRaises(NotImplementedError, self.downgrade, + migration_helpers.DB_INIT_VERSION) def insert_dict(self, session, table_name, d, table=None): """Naively inserts key-value pairs into a table, given a dictionary.""" @@ -616,1079 +339,7 @@ class SqlUpgradeTests(SqlMigrateBase): insert.execute(d) session.commit() - def test_upgrade_31_to_32(self): - self.upgrade(32) - - user_table = self.select_table("user") - self.assertEqual(user_table.c.name.type.length, 255) - - def test_downgrade_32_to_31(self): - self.upgrade(32) - session = self.Session() - # NOTE(aloga): we need a different metadata object - user_table = sqlalchemy.Table('user', - sqlalchemy.MetaData(), - autoload=True, - autoload_with=self.engine) - user_id = uuid.uuid4().hex - ins = user_table.insert().values( - {'id': user_id, - 'name': 'a' * 255, - 'password': uuid.uuid4().hex, - 'enabled': True, - 'domain_id': DEFAULT_DOMAIN_ID, - 'extra': '{}'}) - session.execute(ins) - session.commit() - - self.downgrade(31) - # Check that username has been truncated - q = session.query(user_table.c.name) - q = q.filter(user_table.c.id == user_id) - r = q.one() - user_name = r[0] - self.assertEqual(len(user_name), 64) - - user_table = self.select_table("user") - self.assertEqual(user_table.c.name.type.length, 64) - - def test_downgrade_to_0(self): - self.upgrade(self.max_version) - - if self.engine.name == 'mysql': - self._mysql_check_all_tables_innodb() - - self.downgrade(0) - for table_name in ["user", "token", "role", "user_tenant_membership", - "metadata"]: - self.assertTableDoesNotExist(table_name) - - def test_upgrade_add_domain_tables(self): - self.upgrade(6) - self.assertTableDoesNotExist('credential') - self.assertTableDoesNotExist('domain') - self.assertTableDoesNotExist('user_domain_metadata') - - self.upgrade(7) - self.assertTableExists('credential') - self.assertTableColumns('credential', ['id', 'user_id', 'project_id', - 'blob', 'type', 'extra']) - self.assertTableExists('domain') - self.assertTableColumns('domain', ['id', 'name', 'enabled', 'extra']) - self.assertTableExists('user_domain_metadata') - self.assertTableColumns('user_domain_metadata', - ['user_id', 'domain_id', 'data']) - - def test_metadata_table_migration(self): - # Scaffolding - session = self.Session() - - self.upgrade(16) - domain_table = sqlalchemy.Table('domain', self.metadata, autoload=True) - user_table = sqlalchemy.Table('user', self.metadata, autoload=True) - role_table = sqlalchemy.Table('role', self.metadata, autoload=True) - project_table = sqlalchemy.Table( - 'project', self.metadata, autoload=True) - metadata_table = sqlalchemy.Table( - 'metadata', self.metadata, autoload=True) - - # Create a Domain - domain = {'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'enabled': True} - session.execute(domain_table.insert().values(domain)) - - # Create a Project - project = {'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'domain_id': domain['id'], - 'extra': "{}"} - session.execute(project_table.insert().values(project)) - - # Create another Project - project2 = {'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'domain_id': domain['id'], - 'extra': "{}"} - session.execute(project_table.insert().values(project2)) - - # Create a User - user = {'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'domain_id': domain['id'], - 'password': uuid.uuid4().hex, - 'enabled': True, - 'extra': json.dumps({})} - session.execute(user_table.insert().values(user)) - - # Create a Role - role = {'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex} - session.execute(role_table.insert().values(role)) - - # And another role - role2 = {'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex} - session.execute(role_table.insert().values(role2)) - - # Grant Role to User - role_grant = {'user_id': user['id'], - 'tenant_id': project['id'], - 'data': json.dumps({"roles": [role['id']]})} - session.execute(metadata_table.insert().values(role_grant)) - - role_grant = {'user_id': user['id'], - 'tenant_id': project2['id'], - 'data': json.dumps({"roles": [role2['id']]})} - session.execute(metadata_table.insert().values(role_grant)) - - # Create another user to test the case where member_role_id is already - # assigned. - user2 = {'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'domain_id': domain['id'], - 'password': uuid.uuid4().hex, - 'enabled': True, - 'extra': json.dumps({})} - session.execute(user_table.insert().values(user2)) - - # Grant CONF.member_role_id to User2 - role_grant = {'user_id': user2['id'], - 'tenant_id': project['id'], - 'data': json.dumps({"roles": [CONF.member_role_id]})} - session.execute(metadata_table.insert().values(role_grant)) - - session.commit() - - self.upgrade(17) - - user_project_metadata_table = sqlalchemy.Table( - 'user_project_metadata', self.metadata, autoload=True) - - s = sqlalchemy.select([metadata_table.c.data]).where( - (metadata_table.c.user_id == user['id']) & - (metadata_table.c.tenant_id == project['id'])) - r = session.execute(s) - test_project1 = json.loads(r.fetchone()['data']) - self.assertEqual(len(test_project1['roles']), 1) - self.assertIn(role['id'], test_project1['roles']) - - # Test user in project2 has role2 - s = sqlalchemy.select([metadata_table.c.data]).where( - (metadata_table.c.user_id == user['id']) & - (metadata_table.c.tenant_id == project2['id'])) - r = session.execute(s) - test_project2 = json.loads(r.fetchone()['data']) - self.assertEqual(len(test_project2['roles']), 1) - self.assertIn(role2['id'], test_project2['roles']) - - # Test for user in project has role in user_project_metadata - # Migration 17 does not properly migrate this data, so this should - # be None. - s = sqlalchemy.select([user_project_metadata_table.c.data]).where( - (user_project_metadata_table.c.user_id == user['id']) & - (user_project_metadata_table.c.project_id == project['id'])) - r = session.execute(s) - self.assertIsNone(r.fetchone()) - - # Create a conflicting user-project in user_project_metadata with - # a different role - data = json.dumps({"roles": [role2['id']]}) - role_grant = {'user_id': user['id'], - 'project_id': project['id'], - 'data': data} - cmd = user_project_metadata_table.insert().values(role_grant) - self.engine.execute(cmd) - - # Create another conflicting user-project for User2 - data = json.dumps({"roles": [role2['id']]}) - role_grant = {'user_id': user2['id'], - 'project_id': project['id'], - 'data': data} - cmd = user_project_metadata_table.insert().values(role_grant) - self.engine.execute(cmd) - # End Scaffolding - - session.commit() - - # Migrate to 20 - self.upgrade(20) - - # The user-project pairs should have all roles from the previous - # metadata table in addition to any roles currently in - # user_project_metadata - s = sqlalchemy.select([user_project_metadata_table.c.data]).where( - (user_project_metadata_table.c.user_id == user['id']) & - (user_project_metadata_table.c.project_id == project['id'])) - r = session.execute(s) - role_ids = json.loads(r.fetchone()['data'])['roles'] - self.assertEqual(len(role_ids), 3) - self.assertIn(CONF.member_role_id, role_ids) - self.assertIn(role['id'], role_ids) - self.assertIn(role2['id'], role_ids) - - # pairs that only existed in old metadata table should be in - # user_project_metadata - s = sqlalchemy.select([user_project_metadata_table.c.data]).where( - (user_project_metadata_table.c.user_id == user['id']) & - (user_project_metadata_table.c.project_id == project2['id'])) - r = session.execute(s) - role_ids = json.loads(r.fetchone()['data'])['roles'] - self.assertEqual(len(role_ids), 2) - self.assertIn(CONF.member_role_id, role_ids) - self.assertIn(role2['id'], role_ids) - - self.assertTableDoesNotExist('metadata') - - def test_upgrade_default_roles(self): - def count_member_roles(): - session = self.Session() - role_table = sqlalchemy.Table("role", self.metadata, autoload=True) - return session.query(role_table).filter_by( - name=config.CONF.member_role_name).count() - - self.upgrade(16) - self.assertEqual(0, count_member_roles()) - self.upgrade(17) - self.assertEqual(1, count_member_roles()) - self.downgrade(16) - self.assertEqual(0, count_member_roles()) - - def check_uniqueness_constraints(self): - # Check uniqueness constraints for User & Project tables are - # correct following schema modification. The Group table's - # schema is never modified, so we don't bother to check that. - domain_table = sqlalchemy.Table('domain', - self.metadata, - autoload=True) - domain1 = {'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'enabled': True} - domain2 = {'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'enabled': True} - cmd = domain_table.insert().values(domain1) - self.engine.execute(cmd) - cmd = domain_table.insert().values(domain2) - self.engine.execute(cmd) - - # First, the User table. - this_table = sqlalchemy.Table('user', - self.metadata, - autoload=True) - user = {'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'domain_id': domain1['id'], - 'password': uuid.uuid4().hex, - 'enabled': True, - 'extra': json.dumps({})} - cmd = this_table.insert().values(user) - self.engine.execute(cmd) - # now insert a user with the same name into a different - # domain - which should work. - user['id'] = uuid.uuid4().hex - user['domain_id'] = domain2['id'] - cmd = this_table.insert().values(user) - self.engine.execute(cmd) - # TODO(henry-nash): For now, as part of clean-up we delete one of these - # users. Although not part of this test, unless we do so the - # downgrade(16->15) that is part of teardown with fail due to having - # two uses with clashing name as we try to revert to a single global - # name space. This limitation is raised as Bug #1125046 and the delete - # could be removed depending on how that bug is resolved. - cmd = this_table.delete().where(this_table.c.id == user['id']) - self.engine.execute(cmd) - - # Now, the Project table. - this_table = sqlalchemy.Table('project', - self.metadata, - autoload=True) - project = {'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'domain_id': domain1['id'], - 'description': uuid.uuid4().hex, - 'enabled': True, - 'extra': json.dumps({})} - cmd = this_table.insert().values(project) - self.engine.execute(cmd) - # now insert a project with the same name into a different - # domain - which should work. - project['id'] = uuid.uuid4().hex - project['domain_id'] = domain2['id'] - cmd = this_table.insert().values(project) - self.engine.execute(cmd) - # TODO(henry-nash): For now, we delete one of the projects for the same - # reason as we delete one of the users (Bug #1125046). This delete - # could be removed depending on that bug resolution. - cmd = this_table.delete().where(this_table.c.id == project['id']) - self.engine.execute(cmd) - - def test_upgrade_trusts(self): - self.assertEqual(self.schema.version, 0, "DB is at version 0") - self.upgrade(20) - self.assertTableColumns("token", - ["id", "expires", "extra", "valid"]) - self.upgrade(21) - self.assertTableColumns("trust", - ["id", "trustor_user_id", - "trustee_user_id", - "project_id", "impersonation", - "deleted_at", - "expires_at", "extra"]) - self.assertTableColumns("trust_role", - ["trust_id", "role_id"]) - self.assertTableColumns("token", - ["id", "expires", "extra", "valid", - "trust_id", "user_id"]) - - def test_fixup_role(self): - def count_role(): - session = self.Session() - self.initialize_sql() - role_table = sqlalchemy.Table("role", self.metadata, autoload=True) - return session.query(role_table).filter_by(extra=None).count() - - session = self.Session() - self.assertEqual(self.schema.version, 0, "DB is at version 0") - self.upgrade(1) - self.insert_dict(session, "role", {"id": "test", "name": "test"}) - self.upgrade(18) - self.insert_dict(session, "role", {"id": "test2", - "name": "test2", - "extra": None}) - self.assertEqual(count_role(), 2) - self.upgrade(19) - self.assertEqual(count_role(), 0) - - def test_legacy_endpoint_id(self): - session = self.Session() - self.upgrade(21) - - service = { - 'id': uuid.uuid4().hex, - 'name': 'keystone', - 'type': 'identity'} - self.insert_dict(session, 'service', service) - - legacy_endpoint_id = uuid.uuid4().hex - endpoint = { - 'id': uuid.uuid4().hex, - 'service_id': service['id'], - 'interface': uuid.uuid4().hex[:8], - 'url': uuid.uuid4().hex, - 'extra': json.dumps({ - 'legacy_endpoint_id': legacy_endpoint_id})} - self.insert_dict(session, 'endpoint', endpoint) - - session.commit() - self.upgrade(22) - - endpoint_table = sqlalchemy.Table( - 'endpoint', self.metadata, autoload=True) - - self.assertEqual(session.query(endpoint_table).count(), 1) - ref = session.query(endpoint_table).one() - self.assertEqual(ref.id, endpoint['id'], ref) - self.assertEqual(ref.service_id, endpoint['service_id']) - self.assertEqual(ref.interface, endpoint['interface']) - self.assertEqual(ref.url, endpoint['url']) - self.assertEqual(ref.legacy_endpoint_id, legacy_endpoint_id) - self.assertEqual(ref.extra, '{}') - - def test_group_project_FK_fixup(self): - # To create test data we must start before we broke in the - # group_project_metadata table in 015. - self.upgrade(14) - session = self.Session() - - domain_table = sqlalchemy.Table('domain', self.metadata, autoload=True) - group_table = sqlalchemy.Table('group', self.metadata, autoload=True) - tenant_table = sqlalchemy.Table('tenant', self.metadata, autoload=True) - role_table = sqlalchemy.Table('role', self.metadata, autoload=True) - group_project_metadata_table = sqlalchemy.Table( - 'group_project_metadata', self.metadata, autoload=True) - - # Create a Domain - domain = {'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'enabled': True} - session.execute(domain_table.insert().values(domain)) - - # Create two Tenants - tenant = {'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'extra': "{}"} - session.execute(tenant_table.insert().values(tenant)) - - tenant1 = {'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'extra': "{}"} - session.execute(tenant_table.insert().values(tenant1)) - - # Create a Group - group = {'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'domain_id': domain['id'], - 'extra': json.dumps({})} - session.execute(group_table.insert().values(group)) - - # Create roles - role_list = [] - for _ in range(2): - role = {'id': uuid.uuid4().hex, 'name': uuid.uuid4().hex} - session.execute(role_table.insert().values(role)) - role_list.append(role) - - # Grant Role to User on Project - role_grant = {'group_id': group['id'], - 'project_id': tenant['id'], - 'data': json.dumps({'roles': [role_list[0]['id']]})} - session.execute( - group_project_metadata_table.insert().values(role_grant)) - - role_grant = {'group_id': group['id'], - 'project_id': tenant1['id'], - 'data': json.dumps({'roles': [role_list[1]['id']]})} - session.execute( - group_project_metadata_table.insert().values(role_grant)) - - session.commit() - - # Now upgrade and fix up the FKs - self.upgrade(28) - self.assertTableExists('group_project_metadata') - self.assertTableExists('project') - self.assertTableDoesNotExist('tenant') - - s = sqlalchemy.select([group_project_metadata_table.c.data]).where( - (group_project_metadata_table.c.group_id == group['id']) & - (group_project_metadata_table.c.project_id == tenant['id'])) - r = session.execute(s) - data = json.loads(r.fetchone()['data']) - self.assertEqual(len(data['roles']), 1) - self.assertIn(role_list[0]['id'], data['roles']) - - s = sqlalchemy.select([group_project_metadata_table.c.data]).where( - (group_project_metadata_table.c.group_id == group['id']) & - (group_project_metadata_table.c.project_id == tenant1['id'])) - r = session.execute(s) - data = json.loads(r.fetchone()['data']) - self.assertEqual(len(data['roles']), 1) - self.assertIn(role_list[1]['id'], data['roles']) - - self.downgrade(27) - self.assertTableExists('group_project_metadata') - self.assertTableExists('project') - self.assertTableDoesNotExist('tenant') - - def test_assignment_metadata_migration(self): - self.upgrade(28) - # Scaffolding - session = self.Session() - - domain_table = sqlalchemy.Table('domain', self.metadata, autoload=True) - user_table = sqlalchemy.Table('user', self.metadata, autoload=True) - group_table = sqlalchemy.Table('group', self.metadata, autoload=True) - role_table = sqlalchemy.Table('role', self.metadata, autoload=True) - project_table = sqlalchemy.Table( - 'project', self.metadata, autoload=True) - user_project_metadata_table = sqlalchemy.Table( - 'user_project_metadata', self.metadata, autoload=True) - user_domain_metadata_table = sqlalchemy.Table( - 'user_domain_metadata', self.metadata, autoload=True) - group_project_metadata_table = sqlalchemy.Table( - 'group_project_metadata', self.metadata, autoload=True) - group_domain_metadata_table = sqlalchemy.Table( - 'group_domain_metadata', self.metadata, autoload=True) - - # Create a Domain - domain = {'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'enabled': True} - session.execute(domain_table.insert().values(domain)) - - # Create anther Domain - domain2 = {'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'enabled': True} - session.execute(domain_table.insert().values(domain2)) - - # Create a Project - project = {'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'domain_id': domain['id'], - 'extra': "{}"} - session.execute(project_table.insert().values(project)) - - # Create another Project - project2 = {'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'domain_id': domain['id'], - 'extra': "{}"} - session.execute(project_table.insert().values(project2)) - - # Create a User - user = {'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'domain_id': domain['id'], - 'password': uuid.uuid4().hex, - 'enabled': True, - 'extra': json.dumps({})} - session.execute(user_table.insert().values(user)) - - # Create a Group - group = {'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'domain_id': domain['id'], - 'extra': json.dumps({})} - session.execute(group_table.insert().values(group)) - - # Create roles - role_list = [] - for _ in range(7): - role = {'id': uuid.uuid4().hex, 'name': uuid.uuid4().hex} - session.execute(role_table.insert().values(role)) - role_list.append(role) - - # Grant Role to User on Project - role_grant = {'user_id': user['id'], - 'project_id': project['id'], - 'data': json.dumps({'roles': [role_list[0]['id']]})} - session.execute( - user_project_metadata_table.insert().values(role_grant)) - - role_grant = {'user_id': user['id'], - 'project_id': project2['id'], - 'data': json.dumps({'roles': [role_list[1]['id']]})} - session.execute( - user_project_metadata_table.insert().values(role_grant)) - - # Grant Role to Group on different Project - role_grant = {'group_id': group['id'], - 'project_id': project2['id'], - 'data': json.dumps({'roles': [role_list[2]['id']]})} - session.execute( - group_project_metadata_table.insert().values(role_grant)) - - # Grant Role to User on Domain - role_grant = {'user_id': user['id'], - 'domain_id': domain['id'], - 'data': json.dumps({'roles': [role_list[3]['id']]})} - session.execute(user_domain_metadata_table.insert().values(role_grant)) - - # Grant Role to Group on Domain - role_grant = {'group_id': group['id'], - 'domain_id': domain['id'], - 'data': json.dumps( - {'roles': [role_list[4]['id']], - 'other': 'somedata'})} - session.execute( - group_domain_metadata_table.insert().values(role_grant)) - - session.commit() - - self.upgrade(29) - s = sqlalchemy.select([user_project_metadata_table.c.data]).where( - (user_project_metadata_table.c.user_id == user['id']) & - (user_project_metadata_table.c.project_id == project['id'])) - r = session.execute(s) - data = json.loads(r.fetchone()['data']) - self.assertEqual(len(data['roles']), 1) - self.assertIn({'id': role_list[0]['id']}, data['roles']) - - s = sqlalchemy.select([user_project_metadata_table.c.data]).where( - (user_project_metadata_table.c.user_id == user['id']) & - (user_project_metadata_table.c.project_id == project2['id'])) - r = session.execute(s) - data = json.loads(r.fetchone()['data']) - self.assertEqual(len(data['roles']), 1) - self.assertIn({'id': role_list[1]['id']}, data['roles']) - - s = sqlalchemy.select([group_project_metadata_table.c.data]).where( - (group_project_metadata_table.c.group_id == group['id']) & - (group_project_metadata_table.c.project_id == project2['id'])) - r = session.execute(s) - data = json.loads(r.fetchone()['data']) - self.assertEqual(len(data['roles']), 1) - self.assertIn({'id': role_list[2]['id']}, data['roles']) - - s = sqlalchemy.select([user_domain_metadata_table.c.data]).where( - (user_domain_metadata_table.c.user_id == user['id']) & - (user_domain_metadata_table.c.domain_id == domain['id'])) - r = session.execute(s) - data = json.loads(r.fetchone()['data']) - self.assertEqual(len(data['roles']), 1) - self.assertIn({'id': role_list[3]['id']}, data['roles']) - - s = sqlalchemy.select([group_domain_metadata_table.c.data]).where( - (group_domain_metadata_table.c.group_id == group['id']) & - (group_domain_metadata_table.c.domain_id == domain['id'])) - r = session.execute(s) - data = json.loads(r.fetchone()['data']) - self.assertEqual(len(data['roles']), 1) - self.assertIn({'id': role_list[4]['id']}, data['roles']) - self.assertIn('other', data) - - # Now add an entry that has one regular and one inherited role - role_grant = {'user_id': user['id'], - 'domain_id': domain2['id'], - 'data': json.dumps( - {'roles': [{'id': role_list[5]['id']}, - {'id': role_list[6]['id'], - 'inherited_to': 'projects'}]})} - session.execute(user_domain_metadata_table.insert().values(role_grant)) - - session.commit() - self.downgrade(28) - - s = sqlalchemy.select([user_project_metadata_table.c.data]).where( - (user_project_metadata_table.c.user_id == user['id']) & - (user_project_metadata_table.c.project_id == project['id'])) - r = session.execute(s) - data = json.loads(r.fetchone()['data']) - self.assertEqual(len(data['roles']), 1) - self.assertIn(role_list[0]['id'], data['roles']) - - s = sqlalchemy.select([user_project_metadata_table.c.data]).where( - (user_project_metadata_table.c.user_id == user['id']) & - (user_project_metadata_table.c.project_id == project2['id'])) - r = session.execute(s) - data = json.loads(r.fetchone()['data']) - self.assertEqual(len(data['roles']), 1) - self.assertIn(role_list[1]['id'], data['roles']) - - s = sqlalchemy.select([group_project_metadata_table.c.data]).where( - (group_project_metadata_table.c.group_id == group['id']) & - (group_project_metadata_table.c.project_id == project2['id'])) - r = session.execute(s) - data = json.loads(r.fetchone()['data']) - self.assertEqual(len(data['roles']), 1) - self.assertIn(role_list[2]['id'], data['roles']) - - s = sqlalchemy.select([user_domain_metadata_table.c.data]).where( - (user_domain_metadata_table.c.user_id == user['id']) & - (user_domain_metadata_table.c.domain_id == domain['id'])) - r = session.execute(s) - data = json.loads(r.fetchone()['data']) - self.assertEqual(len(data['roles']), 1) - self.assertIn(role_list[3]['id'], data['roles']) - - s = sqlalchemy.select([group_domain_metadata_table.c.data]).where( - (group_domain_metadata_table.c.group_id == group['id']) & - (group_domain_metadata_table.c.domain_id == domain['id'])) - r = session.execute(s) - data = json.loads(r.fetchone()['data']) - self.assertEqual(len(data['roles']), 1) - self.assertIn(role_list[4]['id'], data['roles']) - self.assertIn('other', data) - - # For user-domain2, where we had one regular and one inherited role, - # only the direct role should remain, the inherited role should - # have been deleted during the downgrade - s = sqlalchemy.select([user_domain_metadata_table.c.data]).where( - (user_domain_metadata_table.c.user_id == user['id']) & - (user_domain_metadata_table.c.domain_id == domain2['id'])) - r = session.execute(s) - data = json.loads(r.fetchone()['data']) - self.assertEqual(len(data['roles']), 1) - self.assertIn(role_list[5]['id'], data['roles']) - - def test_drop_credential_constraint(self): - ec2_credential = { - 'id': '100', - 'user_id': 'foo', - 'project_id': 'bar', - 'type': 'ec2', - 'blob': json.dumps({ - "access": "12345", - "secret": "12345" - }) - } - user = { - 'id': 'foo', - 'name': 'FOO', - 'password': 'foo2', - 'enabled': True, - 'email': 'foo@bar.com', - 'extra': json.dumps({'enabled': True}) - } - tenant = { - 'id': 'bar', - 'name': 'BAR', - 'description': 'description', - 'enabled': True, - 'extra': json.dumps({'enabled': True}) - } - session = self.Session() - self.upgrade(7) - self.insert_dict(session, 'user', user) - self.insert_dict(session, 'tenant', tenant) - self.insert_dict(session, 'credential', ec2_credential) - session.commit() - self.upgrade(30) - cred_table = sqlalchemy.Table('credential', - self.metadata, - autoload=True) - cred = session.query(cred_table).filter("id='100'").one() - self.assertEqual(cred.user_id, - ec2_credential['user_id']) - - def test_drop_credential_indexes(self): - self.upgrade(31) - table = sqlalchemy.Table('credential', self.metadata, autoload=True) - self.assertEqual(len(table.indexes), 0) - - def test_downgrade_30(self): - self.upgrade(31) - self.downgrade(30) - table = sqlalchemy.Table('credential', self.metadata, autoload=True) - index_data = [(idx.name, idx.columns.keys()) - for idx in table.indexes] - if self.engine.name == 'mysql': - self.assertIn(('user_id', ['user_id']), index_data) - self.assertIn(('credential_project_id_fkey', ['project_id']), - index_data) - else: - self.assertEqual(len(index_data), 0) - - def test_revoked_token_index(self): - self.upgrade(35) - table = sqlalchemy.Table('token', self.metadata, autoload=True) - index_data = [(idx.name, idx.columns.keys()) - for idx in table.indexes] - self.assertIn(('ix_token_expires_valid', ['expires', 'valid']), - index_data) - - def test_dropped_valid_index(self): - self.upgrade(36) - table = sqlalchemy.Table('token', self.metadata, autoload=True) - index_data = [(idx.name, idx.columns.keys()) - for idx in table.indexes] - self.assertNotIn(('ix_token_valid', ['valid']), index_data) - - def test_migrate_ec2_credential(self): - user = { - 'id': 'foo', - 'name': 'FOO', - 'password': 'foo2', - 'enabled': True, - 'email': 'foo@bar.com', - 'extra': json.dumps({'enabled': True}) - } - project = { - 'id': 'bar', - 'name': 'BAR', - 'description': 'description', - 'enabled': True, - 'extra': json.dumps({'enabled': True}) - } - ec2_credential = { - 'access': uuid.uuid4().hex, - 'secret': uuid.uuid4().hex, - 'user_id': user['id'], - 'tenant_id': project['id'], - } - session = self.Session() - self.upgrade(7) - self.insert_dict(session, 'ec2_credential', ec2_credential) - self.insert_dict(session, 'user', user) - self.insert_dict(session, 'tenant', project) - self.upgrade(33) - self.assertTableDoesNotExist('ec2_credential') - cred_table = sqlalchemy.Table('credential', - self.metadata, - autoload=True) - expected_credential_id = utils.hash_access_key( - ec2_credential['access']) - cred = session.query(cred_table).filter_by( - id=expected_credential_id).one() - self.assertEqual(cred.user_id, ec2_credential['user_id']) - self.assertEqual(cred.project_id, ec2_credential['tenant_id']) - credential_list = session.query(cred_table).filter_by( - user_id=ec2_credential['user_id']).all() - self.assertNotEmpty(credential_list) - self.downgrade(32) - session.commit() - self.assertTableExists('ec2_credential') - ec2_cred_table = sqlalchemy.Table('ec2_credential', - self.metadata, - autoload=True) - ec2_cred = session.query(ec2_cred_table).filter_by( - access=ec2_credential['access']).one() - self.assertEqual(ec2_cred.user_id, ec2_credential['user_id']) - - def test_migrate_ec2_credential_with_conflict_project(self): - user = { - 'id': 'foo', - 'name': 'FOO', - 'password': 'foo2', - 'enabled': True, - 'email': 'foo@bar.com', - 'extra': json.dumps({'enabled': True}) - } - project_1 = { - 'id': 'bar', - 'name': 'BAR', - 'description': 'description', - 'enabled': True, - 'extra': json.dumps({'enabled': True}) - } - project_2 = { - 'id': 'baz', - 'name': 'BAZ', - 'description': 'description', - 'enabled': True, - 'extra': json.dumps({'enabled': True}) - } - ec2_credential = { - 'access': uuid.uuid4().hex, - 'secret': uuid.uuid4().hex, - 'user_id': user['id'], - 'tenant_id': project_1['id'], - } - blob = {'access': ec2_credential['access'], - 'secret': ec2_credential['secret']} - v3_credential = { - 'id': utils.hash_access_key(ec2_credential['access']), - 'user_id': user['id'], - # set the project id to simulate a conflict - 'project_id': project_2['id'], - 'blob': json.dumps(blob), - 'type': 'ec2', - 'extra': json.dumps({}) - } - session = self.Session() - self.upgrade(7) - self.insert_dict(session, 'ec2_credential', ec2_credential) - self.insert_dict(session, 'user', user) - self.insert_dict(session, 'tenant', project_1) - self.insert_dict(session, 'tenant', project_2) - self.upgrade(32) - self.insert_dict(session, 'credential', v3_credential) - self.assertRaises(exception.Conflict, self.upgrade, 33) - - def test_migrate_ec2_credential_with_conflict_secret(self): - user = { - 'id': 'foo', - 'name': 'FOO', - 'password': 'foo2', - 'enabled': True, - 'email': 'foo@bar.com', - 'extra': json.dumps({'enabled': True}) - } - project_1 = { - 'id': 'bar', - 'name': 'BAR', - 'description': 'description', - 'enabled': True, - 'extra': json.dumps({'enabled': True}) - } - project_2 = { - 'id': 'baz', - 'name': 'BAZ', - 'description': 'description', - 'enabled': True, - 'extra': json.dumps({'enabled': True}) - } - ec2_credential = { - 'access': uuid.uuid4().hex, - 'secret': uuid.uuid4().hex, - 'user_id': user['id'], - 'tenant_id': project_1['id'], - } - blob = {'access': ec2_credential['access'], - 'secret': 'different secret'} - v3_cred_different_secret = { - 'id': utils.hash_access_key(ec2_credential['access']), - 'user_id': user['id'], - 'project_id': project_1['id'], - 'blob': json.dumps(blob), - 'type': 'ec2', - 'extra': json.dumps({}) - } - - session = self.Session() - self.upgrade(7) - self.insert_dict(session, 'ec2_credential', ec2_credential) - self.insert_dict(session, 'user', user) - self.insert_dict(session, 'tenant', project_1) - self.insert_dict(session, 'tenant', project_2) - self.upgrade(32) - self.insert_dict(session, 'credential', v3_cred_different_secret) - self.assertRaises(exception.Conflict, self.upgrade, 33) - - def test_migrate_ec2_credential_with_invalid_blob(self): - user = { - 'id': 'foo', - 'name': 'FOO', - 'password': 'foo2', - 'enabled': True, - 'email': 'foo@bar.com', - 'extra': json.dumps({'enabled': True}) - } - project_1 = { - 'id': 'bar', - 'name': 'BAR', - 'description': 'description', - 'enabled': True, - 'extra': json.dumps({'enabled': True}) - } - project_2 = { - 'id': 'baz', - 'name': 'BAZ', - 'description': 'description', - 'enabled': True, - 'extra': json.dumps({'enabled': True}) - } - ec2_credential = { - 'access': uuid.uuid4().hex, - 'secret': uuid.uuid4().hex, - 'user_id': user['id'], - 'tenant_id': project_1['id'], - } - blob = '{"abc":"def"d}' - v3_cred_invalid_blob = { - 'id': utils.hash_access_key(ec2_credential['access']), - 'user_id': user['id'], - 'project_id': project_1['id'], - 'blob': json.dumps(blob), - 'type': 'ec2', - 'extra': json.dumps({}) - } - - session = self.Session() - self.upgrade(7) - self.insert_dict(session, 'ec2_credential', ec2_credential) - self.insert_dict(session, 'user', user) - self.insert_dict(session, 'tenant', project_1) - self.insert_dict(session, 'tenant', project_2) - self.upgrade(32) - self.insert_dict(session, 'credential', v3_cred_invalid_blob) - self.assertRaises(exception.ValidationError, self.upgrade, 33) - - def test_migrate_add_default_project_id_column_upgrade(self): - user1 = { - 'id': 'foo1', - 'name': 'FOO1', - 'password': 'foo2', - 'enabled': True, - 'email': 'foo@bar.com', - 'extra': json.dumps({'tenantId': 'bar'}), - 'domain_id': DEFAULT_DOMAIN_ID - } - user2 = { - 'id': 'foo2', - 'name': 'FOO2', - 'password': 'foo2', - 'enabled': True, - 'email': 'foo@bar.com', - 'extra': json.dumps({'tenant_id': 'bar'}), - 'domain_id': DEFAULT_DOMAIN_ID - } - user3 = { - 'id': 'foo3', - 'name': 'FOO3', - 'password': 'foo2', - 'enabled': True, - 'email': 'foo@bar.com', - 'extra': json.dumps({'default_project_id': 'bar'}), - 'domain_id': DEFAULT_DOMAIN_ID - } - user4 = { - 'id': 'foo4', - 'name': 'FOO4', - 'password': 'foo2', - 'enabled': True, - 'email': 'foo@bar.com', - 'extra': json.dumps({'tenantId': 'baz', - 'default_project_id': 'bar'}), - 'domain_id': DEFAULT_DOMAIN_ID - } - - session = self.Session() - self.upgrade(33) - self.insert_dict(session, 'user', user1) - self.insert_dict(session, 'user', user2) - self.insert_dict(session, 'user', user3) - self.insert_dict(session, 'user', user4) - self.assertTableColumns('user', - ['id', 'name', 'extra', 'password', - 'enabled', 'domain_id']) - session.commit() - session.close() - self.upgrade(34) - session = self.Session() - self.assertTableColumns('user', - ['id', 'name', 'extra', 'password', - 'enabled', 'domain_id', 'default_project_id']) - - user_table = sqlalchemy.Table('user', self.metadata, autoload=True) - updated_user1 = session.query(user_table).filter_by(id='foo1').one() - old_json_data = json.loads(user1['extra']) - new_json_data = json.loads(updated_user1.extra) - self.assertNotIn('tenantId', new_json_data) - self.assertEqual(old_json_data['tenantId'], - updated_user1.default_project_id) - updated_user2 = session.query(user_table).filter_by(id='foo2').one() - old_json_data = json.loads(user2['extra']) - new_json_data = json.loads(updated_user2.extra) - self.assertNotIn('tenant_id', new_json_data) - self.assertEqual(old_json_data['tenant_id'], - updated_user2.default_project_id) - updated_user3 = session.query(user_table).filter_by(id='foo3').one() - old_json_data = json.loads(user3['extra']) - new_json_data = json.loads(updated_user3.extra) - self.assertNotIn('default_project_id', new_json_data) - self.assertEqual(old_json_data['default_project_id'], - updated_user3.default_project_id) - updated_user4 = session.query(user_table).filter_by(id='foo4').one() - old_json_data = json.loads(user4['extra']) - new_json_data = json.loads(updated_user4.extra) - self.assertNotIn('default_project_id', new_json_data) - self.assertNotIn('tenantId', new_json_data) - self.assertEqual(old_json_data['default_project_id'], - updated_user4.default_project_id) - - def test_migrate_add_default_project_id_column_downgrade(self): - user1 = { - 'id': 'foo1', - 'name': 'FOO1', - 'password': 'foo2', - 'enabled': True, - 'email': 'foo@bar.com', - 'extra': json.dumps({}), - 'default_project_id': 'bar', - 'domain_id': DEFAULT_DOMAIN_ID - } - - self.upgrade(34) - session = self.Session() - self.insert_dict(session, 'user', user1) - self.assertTableColumns('user', - ['id', 'name', 'extra', 'password', - 'enabled', 'domain_id', 'default_project_id']) - session.commit() - session.close() - self.downgrade(33) - session = self.Session() - self.assertTableColumns('user', - ['id', 'name', 'extra', 'password', - 'enabled', 'domain_id']) - - user_table = sqlalchemy.Table('user', self.metadata, autoload=True) - updated_user1 = session.query(user_table).filter_by(id='foo1').one() - new_json_data = json.loads(updated_user1.extra) - self.assertIn('tenantId', new_json_data) - self.assertIn('default_project_id', new_json_data) - self.assertEqual(user1['default_project_id'], - new_json_data['tenantId']) - self.assertEqual(user1['default_project_id'], - new_json_data['default_project_id']) - self.assertEqual(user1['default_project_id'], - new_json_data['tenant_id']) - def test_region_migration(self): - self.upgrade(36) self.assertTableDoesNotExist('region') self.upgrade(37) self.assertTableExists('region') @@ -2672,10 +1323,13 @@ class SqlUpgradeTests(SqlMigrateBase): class VersionTests(SqlMigrateBase): + + _initial_db_version = migration_helpers.DB_INIT_VERSION + def test_core_initial(self): - """When get the version before migrated, it's 0.""" + """Get the version before migrated, it's the initial DB version.""" version = migration_helpers.get_db_version() - self.assertEqual(0, version) + self.assertEqual(migration_helpers.DB_INIT_VERSION, version) def test_core_max(self): """When get the version after upgrading, it's the new version."""