# Copyright 2016 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import os import fixtures from migrate.versioning import api as versioning_api from migrate.versioning import repository from oslo_db.sqlalchemy import test_base from oslo_db.sqlalchemy import test_migrations import sqlalchemy import testtools from keystone.common.sql import contract_repo from keystone.common.sql import data_migration_repo from keystone.common.sql import expand_repo from keystone.common.sql import migrate_repo from keystone.common.sql import upgrades class DBOperationNotAllowed(Exception): pass class BannedDBSchemaOperations(fixtures.Fixture): """Ban some operations for migrations.""" def __init__(self, banned_ops=None, migration_repo=migrate_repo.__file__): super(BannedDBSchemaOperations, self).__init__() self._banned_ops = banned_ops or {} self._migration_repo = migration_repo @staticmethod def _explode(resource_op, repo): # Extract the repo name prior to the trailing '/__init__.py' repo_name = repo.split('/')[-2] raise DBOperationNotAllowed( 'Operation %s() is not allowed in %s database migrations' % ( resource_op, repo_name)) def setUp(self): super(BannedDBSchemaOperations, self).setUp() explode_lambda = { 'Table.create': lambda *a, **k: self._explode( 'Table.create', self._migration_repo), 'Table.alter': lambda *a, **k: self._explode( 'Table.alter', self._migration_repo), 'Table.drop': lambda *a, **k: self._explode( 'Table.drop', self._migration_repo), 'Table.insert': lambda *a, **k: self._explode( 'Table.insert', self._migration_repo), 'Table.update': lambda *a, **k: self._explode( 'Table.update', self._migration_repo), 'Table.delete': lambda *a, **k: self._explode( 'Table.delete', self._migration_repo), 'Column.create': lambda *a, **k: self._explode( 'Column.create', self._migration_repo), 'Column.alter': lambda *a, **k: self._explode( 'Column.alter', self._migration_repo), 'Column.drop': lambda *a, **k: self._explode( 'Column.drop', self._migration_repo) } for resource in self._banned_ops: for op in self._banned_ops[resource]: resource_op = '%(resource)s.%(op)s' % { 'resource': resource, 'op': op} self.useFixture(fixtures.MonkeyPatch( 'sqlalchemy.%s' % resource_op, explode_lambda[resource_op])) class TestBannedDBSchemaOperations(testtools.TestCase): """Test the BannedDBSchemaOperations fixture.""" def test_column(self): """Test column operations raise DBOperationNotAllowed.""" column = sqlalchemy.Column() with BannedDBSchemaOperations( banned_ops={'Column': ['create', 'alter', 'drop']}): self.assertRaises(DBOperationNotAllowed, column.drop) self.assertRaises(DBOperationNotAllowed, column.alter) self.assertRaises(DBOperationNotAllowed, column.create) def test_table(self): """Test table operations raise DBOperationNotAllowed.""" table = sqlalchemy.Table() with BannedDBSchemaOperations( banned_ops={'Table': ['create', 'alter', 'drop', 'insert', 'update', 'delete']}): self.assertRaises(DBOperationNotAllowed, table.drop) self.assertRaises(DBOperationNotAllowed, table.alter) self.assertRaises(DBOperationNotAllowed, table.create) self.assertRaises(DBOperationNotAllowed, table.insert) self.assertRaises(DBOperationNotAllowed, table.update) self.assertRaises(DBOperationNotAllowed, table.delete) class KeystoneMigrationsCheckers(test_migrations.WalkVersionsMixin): """Walk over and test all sqlalchemy-migrate migrations.""" # NOTE(xek): We start requiring things be additive in Newton, so # ignore all migrations before the first version in Newton. migrate_file = migrate_repo.__file__ first_version = 101 # NOTE(henry-nash): We don't ban data modification in the legacy repo, # since there are already migrations that do this for Newton (and these # do not cause us issues, or are already worked around). banned_ops = {'Table': ['alter', 'drop'], 'Column': ['alter', 'drop']} exceptions = [ # NOTE(xek): Reviewers: DO NOT ALLOW THINGS TO BE ADDED HERE UNLESS # JUSTIFICATION CAN BE PROVIDED AS TO WHY THIS WILL NOT CAUSE # PROBLEMS FOR ROLLING UPGRADES. # Migration 102 drops the domain table in the Newton release. All # code that referenced the domain table was removed in the Mitaka # release, hence this migration will not cause problems when # running a mixture of Mitaka and Newton versions of keystone. 102, # Migration 106 simply allows the password column to be nullable. # This change would not impact a rolling upgrade. 106 ] @property def INIT_VERSION(self): return upgrades.get_init_version( abs_path=os.path.abspath(os.path.dirname(self.migrate_file))) @property def REPOSITORY(self): return repository.Repository( os.path.abspath(os.path.dirname(self.migrate_file)) ) @property def migration_api(self): temp = __import__('oslo_db.sqlalchemy.migration', globals(), locals(), ['versioning_api'], 0) return temp.versioning_api @property def migrate_engine(self): return self.engine def migrate_fully(self, repo_name): abs_path = os.path.abspath(os.path.dirname(repo_name)) init_version = upgrades.get_init_version(abs_path=abs_path) schema = versioning_api.ControlledSchema.create( self.migrate_engine, abs_path, init_version) max_version = schema.repository.version().version upgrade = True err = '' version = versioning_api._migrate_version( schema, max_version, upgrade, err) schema.upgrade(version) def migrate_up(self, version, with_data=False): """Check that migrations don't cause downtime. Schema migrations can be done online, allowing for rolling upgrades. """ # NOTE(xek): # self.exceptions contains a list of migrations where we allow the # banned operations. Only Migrations which don't cause # incompatibilities are allowed, for example dropping an index or # constraint. # # Please follow the guidelines outlined at: # https://docs.openstack.org/keystone/latest/contributor/database-migrations.html if version >= self.first_version and version not in self.exceptions: banned_ops = self.banned_ops else: banned_ops = None with BannedDBSchemaOperations(banned_ops, self.migrate_file): super(KeystoneMigrationsCheckers, self).migrate_up(version, with_data) snake_walk = False downgrade = False def test_walk_versions(self): self.walk_versions(self.snake_walk, self.downgrade) class TestKeystoneMigrationsMySQL( KeystoneMigrationsCheckers, test_base.MySQLOpportunisticTestCase): pass class TestKeystoneMigrationsPostgreSQL( KeystoneMigrationsCheckers, test_base.PostgreSQLOpportunisticTestCase): pass class TestKeystoneMigrationsSQLite( KeystoneMigrationsCheckers, test_base.DbTestCase): pass class TestKeystoneExpandSchemaMigrations( KeystoneMigrationsCheckers): migrate_file = expand_repo.__file__ first_version = 1 # TODO(henry-nash): we should include Table update here as well, but this # causes the update of the migration version to appear as a banned # operation! banned_ops = {'Table': ['alter', 'drop', 'insert', 'delete'], 'Column': ['alter', 'drop']} exceptions = [ # NOTE(xek, henry-nash): Reviewers: DO NOT ALLOW THINGS TO BE ADDED # HERE UNLESS JUSTIFICATION CAN BE PROVIDED AS TO WHY THIS WILL NOT # CAUSE PROBLEMS FOR ROLLING UPGRADES. # Migration 002 changes the column type, from datetime to timestamp in # the contract phase. Adding exception here to pass expand banned # tests, otherwise fails. 2, # NOTE(lbragstad): The expand 003 migration alters the credential table # to make `blob` nullable. This allows the triggers added in 003 to # catch writes when the `blob` attribute isn't populated. We do this so # that the triggers aren't aware of the encryption implementation. 3, # Migration 004 changes the password created_at column type, from # timestamp to datetime and updates the initial value in the contract # phase. Adding an exception here to pass expand banned tests, # otherwise fails. 4 ] def setUp(self): super(TestKeystoneExpandSchemaMigrations, self).setUp() self.migrate_fully(migrate_repo.__file__) class TestKeystoneExpandSchemaMigrationsMySQL( TestKeystoneExpandSchemaMigrations, test_base.MySQLOpportunisticTestCase): pass class TestKeystoneExpandSchemaMigrationsPostgreSQL( TestKeystoneExpandSchemaMigrations, test_base.PostgreSQLOpportunisticTestCase): pass class TestKeystoneDataMigrations( KeystoneMigrationsCheckers): migrate_file = data_migration_repo.__file__ first_version = 1 banned_ops = {'Table': ['create', 'alter', 'drop'], 'Column': ['create', 'alter', 'drop']} exceptions = [ # NOTE(xek, henry-nash): Reviewers: DO NOT ALLOW THINGS TO BE ADDED # HERE UNLESS JUSTIFICATION CAN BE PROVIDED AS TO WHY THIS WILL NOT # CAUSE PROBLEMS FOR ROLLING UPGRADES. # Migration 002 changes the column type, from datetime to timestamp in # the contract phase. Adding exception here to pass banned data # migration tests. Fails otherwise. 2, # Migration 004 changes the password created_at column type, from # timestamp to datetime and updates the initial value in the contract # phase. Adding an exception here to pass data migrations banned tests, # otherwise fails. 4 ] def setUp(self): super(TestKeystoneDataMigrations, self).setUp() self.migrate_fully(migrate_repo.__file__) self.migrate_fully(expand_repo.__file__) class TestKeystoneDataMigrationsMySQL( TestKeystoneDataMigrations, test_base.MySQLOpportunisticTestCase): pass class TestKeystoneDataMigrationsPostgreSQL( TestKeystoneDataMigrations, test_base.PostgreSQLOpportunisticTestCase): pass class TestKeystoneDataMigrationsSQLite( TestKeystoneDataMigrations, test_base.DbTestCase): pass class TestKeystoneContractSchemaMigrations( KeystoneMigrationsCheckers): migrate_file = contract_repo.__file__ first_version = 1 # TODO(henry-nash): we should include Table update here as well, but this # causes the update of the migration version to appear as a banned # operation! banned_ops = {'Table': ['create', 'insert', 'delete'], 'Column': ['create']} exceptions = [ # NOTE(xek, henry-nash): Reviewers: DO NOT ALLOW THINGS TO BE ADDED # HERE UNLESS JUSTIFICATION CAN BE PROVIDED AS TO WHY THIS WILL NOT # CAUSE PROBLEMS FOR ROLLING UPGRADES. # Migration 002 changes the column type, from datetime to timestamp. # To do this, the column is first dropped and recreated. This should # not have any negative impact on a rolling upgrade deployment. 2, # Migration 004 changes the password created_at column type, from # timestamp to datetime and updates the created_at value. This is # likely not going to impact a rolling upgrade as the contract repo is # executed once the code has been updated; thus the created_at column # would be populated for any password changes. That being said, there # could be a performance issue for existing large password tables, as # the migration is not batched. However, it's a compromise and not # likely going to be a problem for operators. 4, # Migration 013 updates a foreign key constraint at the federated_user # table. It is a composite key pointing to the procotol.id and # protocol.idp_id columns. Since we can't create a new foreign key # before dropping the old one and the operations happens in the same # upgrade phase, adding an exception here to pass the contract # banned tests. 13 ] def setUp(self): super(TestKeystoneContractSchemaMigrations, self).setUp() self.migrate_fully(migrate_repo.__file__) self.migrate_fully(expand_repo.__file__) self.migrate_fully(data_migration_repo.__file__) class TestKeystoneContractSchemaMigrationsMySQL( TestKeystoneContractSchemaMigrations, test_base.MySQLOpportunisticTestCase): pass class TestKeystoneContractSchemaMigrationsPostgreSQL( TestKeystoneContractSchemaMigrations, test_base.PostgreSQLOpportunisticTestCase): pass class TestKeystoneContractSchemaMigrationsSQLite( TestKeystoneContractSchemaMigrations, test_base.DbTestCase): # In Sqlite an alter will appear as a create, so if we check for creates # we will get false positives. def setUp(self): super(TestKeystoneContractSchemaMigrationsSQLite, self).setUp() self.banned_ops['Table'].remove('create')