openstack
/
keystone
Code Issues Proposed changes

Merge "sql: Squash newton migrations (part 2)"

This commit is contained in:
Zuul 2022-02-07 21:21:03 +00:00 committed by Gerrit Code Review
parent 2a86e0f8f9 4fe0f10b6e
commit af960f8c7f
14 changed files with 41 additions and 520 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
keystone/common/sql/contract_repo/versions/002_password_created_at_not_nullable.py
View File

@ -1,39 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
import sqlalchemy as sql
def upgrade(migrate_engine):
meta = sql.MetaData()
meta.bind = migrate_engine
password = sql.Table('password', meta, autoload=True)
# Because it's difficult to get a timestamp server default working among
# all of the supported databases and versions, I'm choosing to drop and
# then recreate the column as I think this is a more cleaner option. This
# will only impact operators that have already deployed the 105 migration;
# resetting the password created_at for security compliance features, if
# enabled.
password.c.created_at.drop()
# sqlite doesn't support server_default=sql.func.now(), so skipping.
if migrate_engine.name == 'sqlite':
created_at = sql.Column('created_at', sql.TIMESTAMP, nullable=True)
else:
# Changing type to timestamp as mysql 5.5 and older doesn't support
# datetime defaults.
created_at = sql.Column('created_at', sql.TIMESTAMP, nullable=False,
default=datetime.datetime.utcnow,
server_default=sql.func.now())
password.create_column(created_at)

60
keystone/common/sql/contract_repo/versions/003_remove_unencrypted_blob_column_from_credential.py
View File

@ -1,60 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from keystone.common.sql import upgrades
import sqlalchemy as sql
def upgrade(migrate_engine):
meta = sql.MetaData()
meta.bind = migrate_engine
credential_table = sql.Table('credential', meta, autoload=True)
credential_table.c.blob.drop()
if upgrades.USE_TRIGGERS:
if migrate_engine.name == 'postgresql':
drop_credential_update_trigger = (
'DROP TRIGGER credential_update_read_only on credential;'
)
drop_credential_insert_trigger = (
'DROP TRIGGER credential_insert_read_only on credential;'
)
elif migrate_engine.name == 'mysql':
drop_credential_update_trigger = (
'DROP TRIGGER credential_update_read_only;'
)
drop_credential_insert_trigger = (
'DROP TRIGGER credential_insert_read_only;'
)
else:
# NOTE(lbragstad, henry-nash): Apparently sqlalchemy and sqlite
# behave weird when using triggers, which is why we use the `IF
# EXISTS` conditional here. I think what is happening is that the
# credential_table.c.blob.drop() causes sqlalchemy to create a new
# credential table - but it doesn't copy the triggers over, which
# causes the DROP TRIGGER statement to fail without `IF EXISTS`
# because the trigger doesn't exist in the new table(?!).
drop_credential_update_trigger = (
'DROP TRIGGER IF EXISTS credential_update_read_only;'
)
drop_credential_insert_trigger = (
'DROP TRIGGER IF EXISTS credential_insert_read_only;'
)
migrate_engine.execute(drop_credential_update_trigger)
migrate_engine.execute(drop_credential_insert_trigger)
# NOTE(lbragstad): We close these so that they are not nullable because
# Newton code (and anything after) would always populate these values.
credential_table.c.encrypted_blob.alter(nullable=False)
credential_table.c.key_hash.alter(nullable=False)

0
keystone/common/sql/contract_repo/versions/001_contract_initial_migration.py → keystone/common/sql/contract_repo/versions/004_contract_initial_migration.py
View File

37
keystone/common/sql/contract_repo/versions/004_reset_password_created_at.py
View File

@ -1,37 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
import sqlalchemy as sql
import sqlalchemy.sql.expression as expression
def upgrade(migrate_engine):
meta = sql.MetaData()
meta.bind = migrate_engine
password = sql.Table('password', meta, autoload=True)
# reset created_at column
password.c.created_at.drop()
created_at = sql.Column('created_at', sql.DateTime(),
nullable=True,
default=datetime.datetime.utcnow)
password.create_column(created_at)
# update created_at value
now = datetime.datetime.utcnow()
values = {'created_at': now}
stmt = password.update().where(
password.c.created_at == expression.null()).values(values)
stmt.execute()
# set not nullable
password.c.created_at.alter(nullable=False)

15
keystone/common/sql/data_migration_repo/versions/002_password_created_at_not_nullable.py
View File

@ -1,15 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
def upgrade(migrate_engine):
pass

39
keystone/common/sql/data_migration_repo/versions/003_migrate_unencrypted_credentials.py
View File

@ -1,39 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sqlalchemy as sql
from keystone.credential.providers import fernet as credential_fernet
def upgrade(migrate_engine):
meta = sql.MetaData()
meta.bind = migrate_engine
session = sql.orm.sessionmaker(bind=migrate_engine)()
credential_table = sql.Table('credential', meta, autoload=True)
credentials = list(credential_table.select().execute())
for credential in credentials:
crypto, keys = credential_fernet.get_multi_fernet_keys()
primary_key_hash = credential_fernet.primary_key_hash(keys)
encrypted_blob = crypto.encrypt(credential['blob'].encode('utf-8'))
values = {
'encrypted_blob': encrypted_blob,
'key_hash': primary_key_hash
}
update = credential_table.update().where(
credential_table.c.id == credential.id
).values(values)
session.execute(update)
session.commit()
session.close()

0
keystone/common/sql/data_migration_repo/versions/001_data_initial_migration.py → keystone/common/sql/data_migration_repo/versions/004_data_initial_migration.py
View File

15
keystone/common/sql/data_migration_repo/versions/004_reset_password_created_at.py
View File

@ -1,15 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
def upgrade(migrate_engine):
pass

18
keystone/common/sql/expand_repo/versions/002_password_created_at_not_nullable.py
View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# A null initial migration to open this repo. Do not re-use replace this with
# a real migration, add additional ones in subsequent version scripts.
def upgrade(migrate_engine):
pass

129
keystone/common/sql/expand_repo/versions/003_add_key_hash_and_encrypted_blob_to_credential.py
View File

@ -1,129 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sqlalchemy as sql
from keystone.common import sql as ks_sql
from keystone.common.sql import upgrades
# NOTE(lbragstad): MySQL error state of 45000 is a generic unhandled exception.
# Keystone will return a 500 in this case.
MYSQL_INSERT_TRIGGER = """
CREATE TRIGGER credential_insert_read_only BEFORE INSERT ON credential
FOR EACH ROW
BEGIN
SIGNAL SQLSTATE '45000'
SET MESSAGE_TEXT = '%s';
END;
"""
MYSQL_UPDATE_TRIGGER = """
CREATE TRIGGER credential_update_read_only BEFORE UPDATE ON credential
FOR EACH ROW
BEGIN
IF NEW.encrypted_blob IS NULL THEN
SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s';
END IF;
IF NEW.encrypted_blob IS NOT NULL AND OLD.blob IS NULL THEN
SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s';
END IF;
END;
"""
SQLITE_INSERT_TRIGGER = """
CREATE TRIGGER credential_insert_read_only BEFORE INSERT ON credential
BEGIN
SELECT RAISE (ABORT, '%s');
END;
"""
SQLITE_UPDATE_TRIGGER = """
CREATE TRIGGER credential_update_read_only BEFORE UPDATE ON credential
WHEN NEW.encrypted_blob IS NULL
BEGIN
SELECT RAISE (ABORT, '%s');
END;
"""
POSTGRESQL_INSERT_TRIGGER = """
CREATE OR REPLACE FUNCTION keystone_read_only_insert()
RETURNS trigger AS
$BODY$
BEGIN
RAISE EXCEPTION '%s';
END
$BODY$ LANGUAGE plpgsql;
CREATE TRIGGER credential_insert_read_only BEFORE INSERT ON credential
FOR EACH ROW
EXECUTE PROCEDURE keystone_read_only_insert();
"""
POSTGRESQL_UPDATE_TRIGGER = """
CREATE OR REPLACE FUNCTION keystone_read_only_update()
RETURNS trigger AS
$BODY$
BEGIN
IF NEW.encrypted_blob IS NULL THEN
RAISE EXCEPTION '%s';
END IF;
IF NEW.encrypted_blob IS NOT NULL AND OLD.blob IS NULL THEN
RAISE EXCEPTION '%s';
END IF;
RETURN NEW;
END
$BODY$ LANGUAGE plpgsql;
CREATE TRIGGER credential_update_read_only BEFORE UPDATE ON credential
FOR EACH ROW
EXECUTE PROCEDURE keystone_read_only_update();
"""
def upgrade(migrate_engine):
meta = sql.MetaData()
meta.bind = migrate_engine
key_hash = sql.Column('key_hash', sql.String(64), nullable=True)
encrypted_blob = sql.Column(
'encrypted_blob',
ks_sql.Text,
nullable=True
)
credential_table = sql.Table('credential', meta, autoload=True)
credential_table.create_column(key_hash)
credential_table.create_column(encrypted_blob)
credential_table.c.blob.alter(nullable=True)
if not upgrades.USE_TRIGGERS:
# Skip managing triggers if we're doing an offline upgrade.
return
error_message = ('Credential migration in progress. Cannot perform '
'writes to credential table.')
if migrate_engine.name == 'postgresql':
credential_insert_trigger = POSTGRESQL_INSERT_TRIGGER % error_message
credential_update_trigger = POSTGRESQL_UPDATE_TRIGGER % (
error_message, error_message
)
elif migrate_engine.name == 'sqlite':
credential_insert_trigger = SQLITE_INSERT_TRIGGER % error_message
credential_update_trigger = SQLITE_UPDATE_TRIGGER % error_message
else:
credential_insert_trigger = MYSQL_INSERT_TRIGGER % error_message
credential_update_trigger = MYSQL_UPDATE_TRIGGER % (
error_message, error_message
)
migrate_engine.execute(credential_insert_trigger)
migrate_engine.execute(credential_update_trigger)

40
keystone/common/sql/expand_repo/versions/001_expand_initial_migration.py → keystone/common/sql/expand_repo/versions/004_expand_initial_migration.py
View File

@ -10,6 +10,8 @@
# License for the specific language governing permissions and limitations
# under the License.
import datetime
import textwrap
import migrate
from oslo_log import log
@ -83,9 +85,14 @@ def upgrade(migrate_engine):
sql.Column('id', sql.String(length=64), primary_key=True),
sql.Column('user_id', sql.String(length=64), nullable=False),
sql.Column('project_id', sql.String(length=64)),
sql.Column('blob', ks_sql.JsonBlob, nullable=False),
sql.Column('type', sql.String(length=255), nullable=False),
sql.Column('extra', ks_sql.JsonBlob.impl),
sql.Column('key_hash', sql.String(64), nullable=False),
sql.Column(
'encrypted_blob',
ks_sql.Text,
nullable=False,
),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
@ -249,7 +256,6 @@ def upgrade(migrate_engine):
nullable=False,
),
sql.Column('password', sql.String(128), nullable=True),
sql.Column('created_at', sql.DateTime(), nullable=True),
sql.Column('expires_at', sql.DateTime(), nullable=True),
sql.Column(
'self_service',
@ -258,6 +264,12 @@ def upgrade(migrate_engine):
server_default='0',
default=False,
),
sql.Column(
'created_at',
sql.DateTime(),
nullable=False,
default=datetime.datetime.utcnow,
),
)
policy = sql.Table(
@ -747,3 +759,27 @@ def upgrade(migrate_engine):
name=fkey.get('name'),
ondelete=fkey.get('ondelete'),
).create()
# TODO(stephenfin): Remove these procedures in a future contract migration
if migrate_engine.name == 'postgresql':
error_message = (
'Credential migration in progress. Cannot perform '
'writes to credential table.'
)
credential_update_trigger = textwrap.dedent(f"""
CREATE OR REPLACE FUNCTION keystone_read_only_update()
RETURNS trigger AS
$BODY$
BEGIN
IF NEW.encrypted_blob IS NULL THEN
RAISE EXCEPTION '{error_message}';
END IF;
IF NEW.encrypted_blob IS NOT NULL AND OLD.blob IS NULL THEN
RAISE EXCEPTION '{error_message}';
END IF;
RETURN NEW;
END
$BODY$ LANGUAGE plpgsql;
""")
migrate_engine.execute(credential_update_trigger)

15
keystone/common/sql/expand_repo/versions/004_reset_password_created_at.py
View File

@ -1,15 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
def upgrade(migrate_engine):
pass

2
keystone/common/sql/upgrades.py
View File

@ -29,7 +29,7 @@ from keystone.i18n import _
USE_TRIGGERS = True
INITIAL_VERSION = 0
INITIAL_VERSION = 3
EXPAND_REPO = 'expand_repo'
DATA_MIGRATION_REPO = 'data_migration_repo'
CONTRACT_REPO = 'contract_repo'

152
keystone/tests/unit/test_sql_upgrade.py
View File

@ -77,7 +77,8 @@ INITIAL_TABLE_STRUCTURE = {
'type', 'domain_id',
],
'credential': [
'id', 'user_id', 'project_id', 'blob', 'type', 'extra',
'id', 'user_id', 'project_id', 'type', 'extra', 'key_hash',
'encrypted_blob',
],
'endpoint': [
'id', 'legacy_endpoint_id', 'interface', 'region_id', 'service_id',
@ -602,151 +603,6 @@ class FullMigration(MigrateBase, unit.TestCase):
upgrades.INITIAL_VERSION + 2,
)
def test_migration_002_password_created_at_not_nullable(self):
# upgrade each repository to 001
self.expand(1)
self.migrate(1)
self.contract(1)
password = sqlalchemy.Table('password', self.metadata, autoload=True)
self.assertTrue(password.c.created_at.nullable)
# upgrade each repository to 002
self.expand(2)
self.migrate(2)
self.contract(2)
password = sqlalchemy.Table('password', self.metadata, autoload=True)
if self.engine.name != 'sqlite':
self.assertFalse(password.c.created_at.nullable)
def test_migration_003_migrate_unencrypted_credentials(self):
self.useFixture(
ksfixtures.KeyRepository(
self.config_fixture,
'credential',
credential_fernet.MAX_ACTIVE_KEYS
)
)
session = self.sessionmaker()
credential_table_name = 'credential'
# upgrade each repository to 002
self.expand(2)
self.migrate(2)
self.contract(2)
# populate the credential table with some sample credentials
credentials = list()
for i in range(5):
credential = {'id': uuid.uuid4().hex,
'blob': uuid.uuid4().hex,
'user_id': uuid.uuid4().hex,
'type': 'cert'}
credentials.append(credential)
self.insert_dict(session, credential_table_name, credential)
# verify the current schema
self.assertTableColumns(
credential_table_name,
['id', 'user_id', 'project_id', 'type', 'blob', 'extra']
)
# upgrade expand repo to 003 to add new columns
self.expand(3)
# verify encrypted_blob and key_hash columns have been added and verify
# the original blob column is still there
self.assertTableColumns(
credential_table_name,
['id', 'user_id', 'project_id', 'type', 'blob', 'extra',
'key_hash', 'encrypted_blob']
)
# verify triggers by making sure we can't write to the credential table
credential = {'id': uuid.uuid4().hex,
'blob': uuid.uuid4().hex,
'user_id': uuid.uuid4().hex,
'type': 'cert'}
self.assertRaises(db_exception.DBError,
self.insert_dict,
session,
credential_table_name,
credential)
# upgrade migrate repo to 003 to migrate existing credentials
self.migrate(3)
# make sure we've actually updated the credential with the
# encrypted blob and the corresponding key hash
credential_table = sqlalchemy.Table(
credential_table_name,
self.metadata,
autoload=True
)
for credential in credentials:
filter = credential_table.c.id == credential['id']
cols = [credential_table.c.key_hash, credential_table.c.blob,
credential_table.c.encrypted_blob]
q = sqlalchemy.select(cols).where(filter)
result = session.execute(q).fetchone()
self.assertIsNotNone(result.encrypted_blob)
self.assertIsNotNone(result.key_hash)
# verify the original blob column is still populated
self.assertEqual(result.blob, credential['blob'])
# verify we can't make any writes to the credential table
credential = {'id': uuid.uuid4().hex,
'blob': uuid.uuid4().hex,
'user_id': uuid.uuid4().hex,
'key_hash': uuid.uuid4().hex,
'type': 'cert'}
self.assertRaises(db_exception.DBError,
self.insert_dict,
session,
credential_table_name,
credential)
# upgrade contract repo to 003 to remove triggers and blob column
self.contract(3)
# verify the new schema doesn't have a blob column anymore
self.assertTableColumns(
credential_table_name,
['id', 'user_id', 'project_id', 'type', 'extra', 'key_hash',
'encrypted_blob']
)
# verify that the triggers are gone by writing to the database
credential = {'id': uuid.uuid4().hex,
'encrypted_blob': uuid.uuid4().hex,
'key_hash': uuid.uuid4().hex,
'user_id': uuid.uuid4().hex,
'type': 'cert'}
self.insert_dict(session, credential_table_name, credential)
def test_migration_004_reset_password_created_at(self):
# upgrade each repository to 003 and test
self.expand(3)
self.migrate(3)
self.contract(3)
password = sqlalchemy.Table('password', self.metadata, autoload=True)
# postgresql returns 'TIMESTAMP WITHOUT TIME ZONE'
self.assertTrue(
str(password.c.created_at.type).startswith('TIMESTAMP'))
# upgrade each repository to 004 and test
self.expand(4)
self.migrate(4)
self.contract(4)
password = sqlalchemy.Table('password', self.metadata, autoload=True)
# type would still be TIMESTAMP with postgresql
if self.engine.name == 'postgresql':
self.assertTrue(
str(password.c.created_at.type).startswith('TIMESTAMP'))
else:
self.assertEqual('DATETIME', str(password.c.created_at.type))
self.assertFalse(password.c.created_at.nullable)
def test_migration_010_add_revocation_event_indexes(self):
self.expand(9)
self.migrate(9)
@ -2325,10 +2181,6 @@ class FullMigration(MigrateBase, unit.TestCase):
class MySQLOpportunisticFullMigration(FullMigration):
FIXTURE = db_fixtures.MySQLOpportunisticFixture
def test_migration_003_migrate_unencrypted_credentials(self):
self.skip_test_overrides('skipped to update u-c for PyMySql version'
'to 0.10.0')
def test_migration_012_add_domain_id_to_idp(self):
self.skip_test_overrides('skipped to update u-c for PyMySql version'
'to 0.10.0')