Move federation sql migrations to common

This is the most basic way to move the federation migrations into
the keystone core migrations path. Check to see if the migration was
performed already via the extension and if so skip performing it again.

Change-Id: Id437bb68d281ee51e9a671591051c30c35f41bf0
Implements: bp move-extensions
This commit is contained in:
Steve Martinelli 2015-10-13 23:59:25 -04:00 committed by Dave Chen
parent cbefe7c7b8
commit 9f3abc6983
12 changed files with 181 additions and 397 deletions

View File

@ -0,0 +1,97 @@
# Copyright 2014 IBM Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
import sqlalchemy as sql
from keystone.common.sql import migration_helpers
CONF = cfg.CONF
_RELAY_STATE_PREFIX = 'relay_state_prefix'
def upgrade(migrate_engine):
try:
extension_version = migration_helpers.get_db_version(
extension='federation',
engine=migrate_engine)
except Exception:
extension_version = 0
# This migration corresponds to federation extension migration 8. Only
# update if it has not been run.
if extension_version >= 8:
return
# Upgrade operations go here. Don't create your own engine; bind
# migrate_engine to your metadata
meta = sql.MetaData()
meta.bind = migrate_engine
idp_table = sql.Table(
'identity_provider',
meta,
sql.Column('id', sql.String(64), primary_key=True),
sql.Column('enabled', sql.Boolean, nullable=False),
sql.Column('description', sql.Text(), nullable=True),
mysql_engine='InnoDB',
mysql_charset='utf8')
idp_table.create(migrate_engine, checkfirst=True)
federation_protocol_table = sql.Table(
'federation_protocol',
meta,
sql.Column('id', sql.String(64), primary_key=True),
sql.Column('idp_id', sql.String(64),
sql.ForeignKey('identity_provider.id', ondelete='CASCADE'),
primary_key=True),
sql.Column('mapping_id', sql.String(64), nullable=False),
mysql_engine='InnoDB',
mysql_charset='utf8')
federation_protocol_table.create(migrate_engine, checkfirst=True)
mapping_table = sql.Table(
'mapping',
meta,
sql.Column('id', sql.String(64), primary_key=True),
sql.Column('rules', sql.Text(), nullable=False),
mysql_engine='InnoDB',
mysql_charset='utf8')
mapping_table.create(migrate_engine, checkfirst=True)
relay_state_prefix_default = CONF.saml.relay_state_prefix
sp_table = sql.Table(
'service_provider',
meta,
sql.Column('auth_url', sql.String(256), nullable=False),
sql.Column('id', sql.String(64), primary_key=True),
sql.Column('enabled', sql.Boolean, nullable=False),
sql.Column('description', sql.Text(), nullable=True),
sql.Column('sp_url', sql.String(256), nullable=False),
sql.Column(_RELAY_STATE_PREFIX, sql.String(256), nullable=False,
server_default=relay_state_prefix_default),
mysql_engine='InnoDB',
mysql_charset='utf8')
sp_table.create(migrate_engine, checkfirst=True)
idp_table = sql.Table('identity_provider', meta, autoload=True)
remote_id_table = sql.Table(
'idp_remote_ids',
meta,
sql.Column('idp_id', sql.String(64),
sql.ForeignKey('identity_provider.id', ondelete='CASCADE')),
sql.Column('remote_id', sql.String(255), primary_key=True),
mysql_engine='InnoDB',
mysql_charset='utf8')
remote_id_table.create(migrate_engine, checkfirst=True)

View File

@ -35,12 +35,13 @@ from keystone.i18n import _
CONF = cfg.CONF
DEFAULT_EXTENSIONS = ['endpoint_filter',
'federation',
'oauth1',
'revoke',
]
MIGRATED_EXTENSIONS = ['endpoint_policy']
MIGRATED_EXTENSIONS = ['endpoint_policy',
'federation',
]
def get_default_domain():

View File

@ -10,33 +10,8 @@
# License for the specific language governing permissions and limitations
# under the License.
import sqlalchemy as sql
from keystone import exception
def upgrade(migrate_engine):
meta = sql.MetaData()
meta.bind = migrate_engine
idp_table = sql.Table(
'identity_provider',
meta,
sql.Column('id', sql.String(64), primary_key=True),
sql.Column('enabled', sql.Boolean, nullable=False),
sql.Column('description', sql.Text(), nullable=True),
mysql_engine='InnoDB',
mysql_charset='utf8')
idp_table.create(migrate_engine, checkfirst=True)
federation_protocol_table = sql.Table(
'federation_protocol',
meta,
sql.Column('id', sql.String(64), primary_key=True),
sql.Column('idp_id', sql.String(64),
sql.ForeignKey('identity_provider.id', ondelete='CASCADE'),
primary_key=True),
sql.Column('mapping_id', sql.String(64), nullable=True),
mysql_engine='InnoDB',
mysql_charset='utf8')
federation_protocol_table.create(migrate_engine, checkfirst=True)
raise exception.MigrationMovedFailure(extension='federation')

View File

@ -10,18 +10,8 @@
# License for the specific language governing permissions and limitations
# under the License.
import sqlalchemy as sql
from keystone import exception
def upgrade(migrate_engine):
meta = sql.MetaData()
meta.bind = migrate_engine
mapping_table = sql.Table(
'mapping',
meta,
sql.Column('id', sql.String(64), primary_key=True),
sql.Column('rules', sql.Text(), nullable=False),
mysql_engine='InnoDB',
mysql_charset='utf8')
mapping_table.create(migrate_engine, checkfirst=True)
raise exception.MigrationMovedFailure(extension='federation')

View File

@ -13,17 +13,8 @@
# License for the specific language governing permissions and limitations
# under the License.
import sqlalchemy as sa
from keystone import exception
def upgrade(migrate_engine):
meta = sa.MetaData(bind=migrate_engine)
federation_protocol = sa.Table('federation_protocol', meta, autoload=True)
# NOTE(i159): The column is changed to non-nullable. To prevent
# database errors when the column will be altered, all the existing
# null-records should be filled with not null values.
stmt = (federation_protocol.update().
where(federation_protocol.c.mapping_id.is_(None)).
values(mapping_id=''))
migrate_engine.execute(stmt)
federation_protocol.c.mapping_id.alter(nullable=False)
raise exception.MigrationMovedFailure(extension='federation')

View File

@ -10,14 +10,8 @@
# License for the specific language governing permissions and limitations
# under the License.
from oslo_db.sqlalchemy import utils
import sqlalchemy as sql
from keystone import exception
def upgrade(migrate_engine):
meta = sql.MetaData()
meta.bind = migrate_engine
idp_table = utils.get_table(migrate_engine, 'identity_provider')
remote_id = sql.Column('remote_id', sql.String(256), nullable=True)
idp_table.create_column(remote_id)
raise exception.MigrationMovedFailure(extension='federation')

View File

@ -10,22 +10,8 @@
# License for the specific language governing permissions and limitations
# under the License.
import sqlalchemy as sql
from keystone import exception
def upgrade(migrate_engine):
meta = sql.MetaData()
meta.bind = migrate_engine
sp_table = sql.Table(
'service_provider',
meta,
sql.Column('auth_url', sql.String(256), nullable=True),
sql.Column('id', sql.String(64), primary_key=True),
sql.Column('enabled', sql.Boolean, nullable=False),
sql.Column('description', sql.Text(), nullable=True),
sql.Column('sp_url', sql.String(256), nullable=True),
mysql_engine='InnoDB',
mysql_charset='utf8')
sp_table.create(migrate_engine, checkfirst=True)
raise exception.MigrationMovedFailure(extension='federation')

View File

@ -10,31 +10,8 @@
# License for the specific language governing permissions and limitations
# under the License.
import sqlalchemy as sql
_SP_TABLE_NAME = 'service_provider'
def _update_null_columns(migrate_engine, sp_table):
stmt = (sp_table.update().
where(sp_table.c.auth_url.is_(None)).
values(auth_url=''))
migrate_engine.execute(stmt)
stmt = (sp_table.update().
where(sp_table.c.sp_url.is_(None)).
values(sp_url=''))
migrate_engine.execute(stmt)
from keystone import exception
def upgrade(migrate_engine):
meta = sql.MetaData()
meta.bind = migrate_engine
sp_table = sql.Table(_SP_TABLE_NAME, meta, autoload=True)
# The columns are being changed to non-nullable. To prevent
# database errors when both are altered, all the existing
# null-records should be filled with not null values.
_update_null_columns(migrate_engine, sp_table)
sp_table.c.auth_url.alter(nullable=False)
sp_table.c.sp_url.alter(nullable=False)
raise exception.MigrationMovedFailure(extension='federation')

View File

@ -10,34 +10,8 @@
# License for the specific language governing permissions and limitations
# under the License.
import sqlalchemy as orm
from keystone import exception
def upgrade(migrate_engine):
meta = orm.MetaData()
meta.bind = migrate_engine
idp_table = orm.Table('identity_provider', meta, autoload=True)
remote_id_table = orm.Table(
'idp_remote_ids',
meta,
orm.Column('idp_id',
orm.String(64),
orm.ForeignKey('identity_provider.id',
ondelete='CASCADE')),
orm.Column('remote_id',
orm.String(255),
primary_key=True),
mysql_engine='InnoDB',
mysql_charset='utf8')
remote_id_table.create(migrate_engine, checkfirst=True)
select = orm.sql.select([idp_table.c.id, idp_table.c.remote_id]).where(
idp_table.c.remote_id.isnot(None))
for identity in migrate_engine.execute(select):
remote_idp_entry = {'idp_id': identity.id,
'remote_id': identity.remote_id}
remote_id_table.insert(remote_idp_entry).execute()
idp_table.drop_column('remote_id')
raise exception.MigrationMovedFailure(extension='federation')

View File

@ -10,30 +10,8 @@
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
from oslo_db.sqlalchemy import utils
import sqlalchemy as sql
CONF = cfg.CONF
_SP_TABLE_NAME = 'service_provider'
_RELAY_STATE_PREFIX = 'relay_state_prefix'
from keystone import exception
def upgrade(migrate_engine):
meta = sql.MetaData()
meta.bind = migrate_engine
idp_table = utils.get_table(migrate_engine, _SP_TABLE_NAME)
relay_state_prefix_default = CONF.saml.relay_state_prefix
relay_state_prefix = sql.Column(_RELAY_STATE_PREFIX, sql.String(256),
nullable=False,
server_default=relay_state_prefix_default)
idp_table.create_column(relay_state_prefix)
def downgrade(migrate_engine):
meta = sql.MetaData()
meta.bind = migrate_engine
idp_table = utils.get_table(migrate_engine, _SP_TABLE_NAME)
idp_table.drop_column(_RELAY_STATE_PREFIX)
raise exception.MigrationMovedFailure(extension='federation')

View File

@ -29,12 +29,6 @@ WARNING::
all data will be lost.
"""
import sqlalchemy
import uuid
from oslo_db import exception as db_exception
from oslo_db.sqlalchemy import utils
from keystone.contrib import endpoint_filter
from keystone.contrib import endpoint_policy
from keystone.contrib import example
@ -171,212 +165,18 @@ class EndpointPolicyExtension(test_sql_upgrade.SqlMigrateBase):
class FederationExtension(test_sql_upgrade.SqlMigrateBase):
"""Test class for ensuring the Federation SQL."""
def setUp(self):
super(FederationExtension, self).setUp()
self.identity_provider = 'identity_provider'
self.federation_protocol = 'federation_protocol'
self.service_provider = 'service_provider'
self.mapping = 'mapping'
self.remote_id_table = 'idp_remote_ids'
FEDERATION_MIGRATIONS = 8
def repo_package(self):
return federation
def insert_dict(self, session, table_name, d):
"""Naively inserts key-value pairs into a table, given a dictionary."""
table = sqlalchemy.Table(table_name, self.metadata, autoload=True)
insert = table.insert().values(**d)
session.execute(insert)
session.commit()
def test_upgrade(self):
self.assertTableDoesNotExist(self.identity_provider)
self.assertTableDoesNotExist(self.federation_protocol)
self.assertTableDoesNotExist(self.mapping)
self.upgrade(1, repository=self.repo_path)
self.assertTableColumns(self.identity_provider,
['id',
'enabled',
'description'])
self.assertTableColumns(self.federation_protocol,
['id',
'idp_id',
'mapping_id'])
self.upgrade(2, repository=self.repo_path)
self.assertTableColumns(self.mapping,
['id', 'rules'])
federation_protocol = utils.get_table(
self.engine,
'federation_protocol')
with self.engine.begin() as conn:
conn.execute(federation_protocol.insert(), id=0, idp_id=1)
self.upgrade(3, repository=self.repo_path)
federation_protocol = utils.get_table(
self.engine,
'federation_protocol')
self.assertFalse(federation_protocol.c.mapping_id.nullable)
def test_service_provider_attributes_cannot_be_null(self):
self.upgrade(6, repository=self.repo_path)
self.assertTableColumns(self.service_provider,
['id', 'description', 'enabled', 'auth_url',
'sp_url'])
session = self.Session()
sp1 = {'id': uuid.uuid4().hex,
'auth_url': None,
'sp_url': uuid.uuid4().hex,
'description': uuid.uuid4().hex,
'enabled': True}
sp2 = {'id': uuid.uuid4().hex,
'auth_url': uuid.uuid4().hex,
'sp_url': None,
'description': uuid.uuid4().hex,
'enabled': True}
sp3 = {'id': uuid.uuid4().hex,
'auth_url': None,
'sp_url': None,
'description': uuid.uuid4().hex,
'enabled': True}
# Insert with 'auth_url' or 'sp_url' set to null must fail
self.assertRaises(db_exception.DBError,
self.insert_dict,
session,
self.service_provider,
sp1)
self.assertRaises(db_exception.DBError,
self.insert_dict,
session,
self.service_provider,
sp2)
self.assertRaises(db_exception.DBError,
self.insert_dict,
session,
self.service_provider,
sp3)
session.close()
def test_fixup_service_provider_attributes(self):
session = self.Session()
sp1 = {'id': uuid.uuid4().hex,
'auth_url': None,
'sp_url': uuid.uuid4().hex,
'description': uuid.uuid4().hex,
'enabled': True}
sp2 = {'id': uuid.uuid4().hex,
'auth_url': uuid.uuid4().hex,
'sp_url': None,
'description': uuid.uuid4().hex,
'enabled': True}
sp3 = {'id': uuid.uuid4().hex,
'auth_url': None,
'sp_url': None,
'description': uuid.uuid4().hex,
'enabled': True}
self.upgrade(5, repository=self.repo_path)
self.assertTableColumns(self.service_provider,
['id', 'description', 'enabled', 'auth_url',
'sp_url'])
# Before the migration, the table should accept null values
self.insert_dict(session, self.service_provider, sp1)
self.insert_dict(session, self.service_provider, sp2)
self.insert_dict(session, self.service_provider, sp3)
# Check if null values are updated to empty string when migrating
session.close()
self.upgrade(6, repository=self.repo_path)
sp_table = sqlalchemy.Table(self.service_provider,
self.metadata,
autoload=True)
session = self.Session()
self.metadata.clear()
sp = session.query(sp_table).filter(sp_table.c.id == sp1['id'])[0]
self.assertEqual('', sp.auth_url)
sp = session.query(sp_table).filter(sp_table.c.id == sp2['id'])[0]
self.assertEqual('', sp.sp_url)
sp = session.query(sp_table).filter(sp_table.c.id == sp3['id'])[0]
self.assertEqual('', sp.auth_url)
self.assertEqual('', sp.sp_url)
def test_propagate_remote_id_to_separate_column(self):
"""Make sure empty remote_id is not propagated.
Test scenario:
- Upgrade database to version 6 where identity_provider table has a
remote_id column
- Add 3 identity provider objects, where idp1 and idp2 have valid
remote_id parameter set, and idp3 has it empty (None).
- Upgrade database to version 7 and expect migration scripts to
properly move data rom identity_provider.remote_id column into
separate table idp_remote_ids.
- In the idp_remote_ids table expect to find entries for idp1 and idp2
and not find anything for idp3 (identified by idp's id)
"""
session = self.Session()
idp1 = {'id': uuid.uuid4().hex,
'remote_id': uuid.uuid4().hex,
'description': uuid.uuid4().hex,
'enabled': True}
idp2 = {'id': uuid.uuid4().hex,
'remote_id': uuid.uuid4().hex,
'description': uuid.uuid4().hex,
'enabled': True}
idp3 = {'id': uuid.uuid4().hex,
'remote_id': None,
'description': uuid.uuid4().hex,
'enabled': True}
self.upgrade(6, repository=self.repo_path)
self.assertTableColumns(self.identity_provider,
['id', 'description', 'enabled', 'remote_id'])
self.insert_dict(session, self.identity_provider, idp1)
self.insert_dict(session, self.identity_provider, idp2)
self.insert_dict(session, self.identity_provider, idp3)
session.close()
self.upgrade(7, repository=self.repo_path)
self.assertTableColumns(self.identity_provider,
['id', 'description', 'enabled'])
remote_id_table = sqlalchemy.Table(self.remote_id_table,
self.metadata,
autoload=True)
session = self.Session()
self.metadata.clear()
idp = session.query(remote_id_table).filter(
remote_id_table.c.idp_id == idp1['id'])[0]
self.assertEqual(idp1['remote_id'], idp.remote_id)
idp = session.query(remote_id_table).filter(
remote_id_table.c.idp_id == idp2['id'])[0]
self.assertEqual(idp2['remote_id'], idp.remote_id)
idp = session.query(remote_id_table).filter(
remote_id_table.c.idp_id == idp3['id'])
# NOTE(marek-denis): As idp3 had empty 'remote_id' attribute we expect
# not to find it in the 'remote_id_table' table, hence count should be
# 0.real
self.assertEqual(0, idp.count())
def test_add_relay_state_column(self):
self.upgrade(8, repository=self.repo_path)
self.assertTableColumns(self.service_provider,
['id', 'description', 'enabled', 'auth_url',
'relay_state_prefix', 'sp_url'])
for version in range(self.FEDERATION_MIGRATIONS):
v = version + 1
self.assertRaises(exception.MigrationMovedFailure,
self.upgrade, version=v,
repository=self.repo_path)
class RevokeExtension(test_sql_upgrade.SqlMigrateBase):

View File

@ -46,7 +46,6 @@ from sqlalchemy import schema
from keystone.common import sql
from keystone.common.sql import migrate_repo
from keystone.common.sql import migration_helpers
from keystone.contrib import federation
from keystone.contrib import revoke
from keystone import exception
from keystone.tests import unit
@ -121,8 +120,7 @@ INITIAL_EXTENSION_TABLE_STRUCTURE = {
],
}
EXTENSIONS = {'federation': federation,
'revoke': revoke}
EXTENSIONS = {'revoke': revoke}
class SqlMigrateBase(unit.SQLDriverOverrides, unit.TestCase):
@ -567,6 +565,64 @@ class SqlUpgradeTests(SqlMigrateBase):
# that 081 did not create the table
self.assertTableDoesNotExist('policy_association')
def test_create_federation_tables(self):
self.identity_provider = 'identity_provider'
self.federation_protocol = 'federation_protocol'
self.service_provider = 'service_provider'
self.mapping = 'mapping'
self.remote_ids = 'idp_remote_ids'
self.assertTableDoesNotExist(self.identity_provider)
self.assertTableDoesNotExist(self.federation_protocol)
self.assertTableDoesNotExist(self.service_provider)
self.assertTableDoesNotExist(self.mapping)
self.assertTableDoesNotExist(self.remote_ids)
self.upgrade(82)
self.assertTableColumns(self.identity_provider,
['id', 'description', 'enabled'])
self.assertTableColumns(self.federation_protocol,
['id', 'idp_id', 'mapping_id'])
self.assertTableColumns(self.mapping,
['id', 'rules'])
self.assertTableColumns(self.service_provider,
['id', 'description', 'enabled', 'auth_url',
'relay_state_prefix', 'sp_url'])
self.assertTableColumns(self.remote_ids, ['idp_id', 'remote_id'])
federation_protocol = sqlalchemy.Table(self.federation_protocol,
self.metadata,
autoload=True)
self.assertFalse(federation_protocol.c.mapping_id.nullable)
sp_table = sqlalchemy.Table(self.service_provider,
self.metadata,
autoload=True)
self.assertFalse(sp_table.c.auth_url.nullable)
self.assertFalse(sp_table.c.sp_url.nullable)
@mock.patch.object(migration_helpers, 'get_db_version', return_value=8)
def test_federation_already_migrated(self, mock_federation):
# By setting the return value to 8, the migration has already been
# run, and there's no need to create the table again.
self.upgrade(82)
mock_federation.assert_any_call(extension='federation',
engine=mock.ANY)
# It won't exist because we are mocking it, but we can verify
# that 082 did not create the table.
self.assertTableDoesNotExist('identity_provider')
self.assertTableDoesNotExist('federation_protocol')
self.assertTableDoesNotExist('mapping')
self.assertTableDoesNotExist('service_provider')
self.assertTableDoesNotExist('idp_remote_ids')
def test_fixup_service_name_value_upgrade(self):
"""Update service name data from `extra` to empty string."""
def add_service(**extra_data):
@ -843,41 +899,6 @@ class VersionTests(SqlMigrateBase):
extension=name,
version=0)
def test_extension_federation_upgraded_values(self):
abs_path = migration_helpers.find_migrate_repo(federation)
migration.db_version_control(sql.get_engine(), abs_path)
migration.db_sync(sql.get_engine(), abs_path, version=6)
idp_table = sqlalchemy.Table("identity_provider",
self.metadata,
autoload=True)
idps = [{'id': uuid.uuid4().hex,
'enabled': True,
'description': uuid.uuid4().hex,
'remote_id': uuid.uuid4().hex},
{'id': uuid.uuid4().hex,
'enabled': True,
'description': uuid.uuid4().hex,
'remote_id': uuid.uuid4().hex}]
for idp in idps:
ins = idp_table.insert().values({'id': idp['id'],
'enabled': idp['enabled'],
'description': idp['description'],
'remote_id': idp['remote_id']})
self.engine.execute(ins)
migration.db_sync(sql.get_engine(), abs_path)
idp_remote_ids_table = sqlalchemy.Table("idp_remote_ids",
self.metadata,
autoload=True)
for idp in idps:
s = idp_remote_ids_table.select().where(
idp_remote_ids_table.c.idp_id == idp['id'])
remote = self.engine.execute(s).fetchone()
self.assertEqual(idp['remote_id'],
remote['remote_id'],
'remote_ids must be preserved during the '
'migration from identity_provider table to '
'idp_remote_ids table')
def test_unexpected_extension(self):
"""The version for a non-existent extension raises ImportError."""
extension_name = uuid.uuid4().hex