From c8ed5b05e9e8644d7418f8c5c46bee64a9835500 Mon Sep 17 00:00:00 2001 From: Takashi Kajinami Date: Sun, 11 Feb 2024 16:50:31 +0900 Subject: [PATCH] db: Fix charset updates in migrations We are failing py311 jobs with our migration checks; and the reason is that we have some 'implied' sql defaults that might be inconsistent with the server versions - one of them is the charset. We recently [1] fixed a bunch of un-annotated tables with explicit charset declarations. This is a continuation of that attempt. Unfortunately, we can't change history. On existing deployments, changes to these migrations will be ignored, and must be done manually if there are any inconsistencies to be fixed. While we're at it, this commit also cleans up separate steps to set the charset and engine to pre-created tables. [1] I7cd6fa0cc8e054af112493746e753fef2024000f Closes-Bug: #2053211 Change-Id: I16345f63a279549e81cbde7b60fed29521d48a02 Signed-off-by: Goutham Pacha Ravi --- .../versions/162a3e673105_manila_init.py | 83 +++++++------------ ...621e5a_change_volume_type_to_share_type.py | 9 +- ...2171410f_add_drivers_private_data_table.py | 1 + ..._security_service_update_control_fields.py | 1 + .../4a482571410f_add_backends_info_table.py | 1 + ...78_remove_host_from_driver_private_data.py | 1 + 6 files changed, 42 insertions(+), 54 deletions(-) diff --git a/manila/db/migrations/alembic/versions/162a3e673105_manila_init.py b/manila/db/migrations/alembic/versions/162a3e673105_manila_init.py index b19f939505..4c8b2ce1b6 100644 --- a/manila/db/migrations/alembic/versions/162a3e673105_manila_init.py +++ b/manila/db/migrations/alembic/versions/162a3e673105_manila_init.py @@ -26,7 +26,7 @@ down_revision = None from alembic import op from oslo_log import log -from sqlalchemy import Boolean, Column, DateTime, ForeignKey +from sqlalchemy import Boolean, Column, DateTime, ForeignKeyConstraint from sqlalchemy import Integer, MetaData, String, Table, UniqueConstraint LOG = log.getLogger(__name__) @@ -129,10 +129,7 @@ def upgrade(): unicode_error=None, _warn_on_bytestring=False), nullable=False), - Column('usage_id', - Integer(), - ForeignKey('quota_usages.id'), - nullable=False), + Column('usage_id', Integer(), nullable=False), Column('project_id', String(length=255, convert_unicode=True, unicode_error=None, @@ -144,6 +141,7 @@ def upgrade(): _warn_on_bytestring=False)), Column('delta', Integer(), nullable=False), Column('expire', DateTime(timezone=False)), + ForeignKeyConstraint(['usage_id'], ['quota_usages.id']), mysql_engine='InnoDB', mysql_charset='utf8', ) @@ -183,17 +181,13 @@ def upgrade(): Column('display_name', String(length=255)), Column('display_description', String(length=255)), Column('snapshot_id', String(length=36)), - Column('share_network_id', - String(length=36), - ForeignKey('share_networks.id'), - nullable=True), - Column('share_server_id', - String(length=36), - ForeignKey('share_servers.id'), - nullable=True), + Column('share_network_id', String(length=36), nullable=True), + Column('share_server_id', String(length=36), nullable=True), Column('share_proto', String(255)), Column('export_location', String(255)), Column('volume_type_id', String(length=36)), + ForeignKeyConstraint(['share_network_id'], ['share_networks.id']), + ForeignKeyConstraint(['share_server_id'], ['share_servers.id']), mysql_engine='InnoDB', mysql_charset='utf8' ) @@ -205,11 +199,11 @@ def upgrade(): Column('deleted_at', DateTime), Column('deleted', String(length=36), default='False'), Column('id', String(length=36), primary_key=True, nullable=False), - Column('share_id', String(36), ForeignKey('shares.id'), - nullable=False), + Column('share_id', String(36), nullable=False), Column('access_type', String(255)), Column('access_to', String(255)), Column('state', String(255)), + ForeignKeyConstraint(['share_id'], ['shares.id']), mysql_engine='InnoDB', mysql_charset='utf8' ) @@ -223,8 +217,7 @@ def upgrade(): Column('id', String(length=36), primary_key=True, nullable=False), Column('user_id', String(length=255)), Column('project_id', String(length=255)), - Column('share_id', String(36), ForeignKey('shares.id'), - nullable=False), + Column('share_id', String(36), nullable=False), Column('size', Integer), Column('status', String(length=255)), Column('progress', String(length=255)), @@ -233,6 +226,7 @@ def upgrade(): Column('share_size', Integer), Column('share_proto', String(length=255)), Column('export_location', String(255)), + ForeignKeyConstraint(['share_id'], ['shares.id']), mysql_engine='InnoDB', mysql_charset='utf8' ) @@ -244,10 +238,10 @@ def upgrade(): Column('deleted_at', DateTime), Column('deleted', Integer, default=0), Column('id', Integer, primary_key=True, nullable=False), - Column('share_id', String(length=36), ForeignKey('shares.id'), - nullable=False), + Column('share_id', String(length=36), nullable=False), Column('key', String(length=255), nullable=False), Column('value', String(length=1023), nullable=False), + ForeignKeyConstraint(['share_id'], ['shares.id']), mysql_engine='InnoDB', mysql_charset='utf8' ) @@ -301,10 +295,10 @@ def upgrade(): Column('deleted_at', DateTime), Column('deleted', String(length=36), default='False'), Column('id', String(length=36), primary_key=True, nullable=False), - Column('share_network_id', String(length=36), - ForeignKey('share_networks.id'), nullable=True), + Column('share_network_id', String(length=36), nullable=True), Column('host', String(length=255), nullable=True), Column('status', String(length=32)), + ForeignKeyConstraint(['share_network_id'], ['share_networks.id']), mysql_engine='InnoDB', mysql_charset='utf8', ) @@ -316,11 +310,10 @@ def upgrade(): Column('deleted_at', DateTime), Column('deleted', String(length=36), default=0), Column('id', Integer, primary_key=True, nullable=False), - Column('share_server_id', String(length=36), - ForeignKey('share_servers.id'), - nullable=False), + Column('share_server_id', String(length=36), nullable=False), Column('key', String(length=255), nullable=False), Column('value', String(length=1023), nullable=False), + ForeignKeyConstraint(['share_server_id'], ['share_servers.id']), mysql_engine='InnoDB', mysql_charset='utf8' ) @@ -334,9 +327,9 @@ def upgrade(): Column('id', String(length=36), primary_key=True, nullable=False), Column('ip_address', String(length=64), nullable=True), Column('mac_address', String(length=32), nullable=True), - Column('share_server_id', String(length=36), - ForeignKey('share_servers.id'), nullable=False), + Column('share_server_id', String(length=36), nullable=False), Column('status', String(length=32)), + ForeignKeyConstraint(['share_server_id'], ['share_servers.id']), mysql_engine='InnoDB', mysql_charset='utf8', ) @@ -348,10 +341,11 @@ def upgrade(): Column('deleted_at', DateTime), Column('deleted', Integer, default=0), Column('id', Integer, primary_key=True, nullable=False), - Column('share_network_id', String(length=36), - ForeignKey('share_networks.id'), nullable=False), - Column('security_service_id', String(length=36), - ForeignKey('security_services.id'), nullable=False), + Column('share_network_id', String(length=36), nullable=False), + Column('security_service_id', String(length=36), nullable=False), + ForeignKeyConstraint(['share_network_id'], ['share_networks.id']), + ForeignKeyConstraint(['security_service_id'], + ['security_services.id']), mysql_engine='InnoDB', mysql_charset='utf8', ) @@ -365,7 +359,8 @@ def upgrade(): Column('id', String(length=36), primary_key=True, nullable=False), Column('name', String(length=255)), UniqueConstraint('name', 'deleted', name='vt_name_uc'), - mysql_engine='InnoDB' + mysql_engine='InnoDB', + mysql_charset='utf8' ) volume_type_extra_specs = Table( @@ -375,11 +370,12 @@ def upgrade(): Column('deleted_at', DateTime), Column('deleted', Boolean), Column('id', Integer, primary_key=True, nullable=False), - Column('volume_type_id', String(length=36), - ForeignKey('volume_types.id'), nullable=False), + Column('volume_type_id', String(length=36), nullable=False), Column('key', String(length=255)), Column('value', String(length=255)), - mysql_engine='InnoDB' + ForeignKeyConstraint(['volume_type_id'], ['volume_types.id']), + mysql_engine='InnoDB', + mysql_charset='utf8' ) # create all tables @@ -400,25 +396,6 @@ def upgrade(): LOG.exception('Exception while creating table.') raise - if migrate_engine.name == "mysql": - tables = ["quotas", "services", "quota_classes", "quota_usages", - "reservations", "project_user_quotas", "share_access_map", - "share_snapshots", "share_metadata", "security_services", - "share_networks", "network_allocations", "shares", - "share_servers", - "share_network_security_service_association", "volume_types", - "volume_type_extra_specs", "share_server_backend_details"] - - migrate_engine.execute("SET foreign_key_checks = 0") - for table in tables: - migrate_engine.execute( - "ALTER TABLE %s CONVERT TO CHARACTER SET utf8" % table) - migrate_engine.execute("SET foreign_key_checks = 1") - migrate_engine.execute( - "ALTER DATABASE %s DEFAULT CHARACTER SET utf8" % - migrate_engine.url.database) - migrate_engine.execute("ALTER TABLE %s Engine=InnoDB" % table) - def downgrade(): raise NotImplementedError('Downgrade from initial Manila install is not' diff --git a/manila/db/migrations/alembic/versions/38e632621e5a_change_volume_type_to_share_type.py b/manila/db/migrations/alembic/versions/38e632621e5a_change_volume_type_to_share_type.py index 223b40c1ef..82b9806e51 100644 --- a/manila/db/migrations/alembic/versions/38e632621e5a_change_volume_type_to_share_type.py +++ b/manila/db/migrations/alembic/versions/38e632621e5a_change_volume_type_to_share_type.py @@ -80,9 +80,10 @@ def downgrade(): sa.Column('deleted', sa.Boolean), sa.Column('id', sa.Integer, primary_key=True, nullable=False), sa.Column('volume_type_id', sa.String(length=36), - sa.ForeignKey('share_types.id'), nullable=False), + nullable=False), sa.Column('key', sa.String(length=255)), sa.Column('value', sa.String(length=255)), + mysql_charset='utf8', mysql_engine='InnoDB') LOG.info("Migrating share_type_extra_specs to " @@ -97,6 +98,12 @@ def downgrade(): op.create_unique_constraint('vt_name_uc', 'share_types', ['name', 'deleted']) op.rename_table("share_types", "volume_types") + op.create_foreign_key( + "volume_type_extra_specs_ibfk_1", + "volume_type_extra_specs", + "volume_types", + ["volume_type_id"], ["id"] + ) LOG.info("Renaming column name shares.share_type_id to " "shares.volume_type.id") diff --git a/manila/db/migrations/alembic/versions/3a482171410f_add_drivers_private_data_table.py b/manila/db/migrations/alembic/versions/3a482171410f_add_drivers_private_data_table.py index df39c303b0..1ff9f3ac0b 100644 --- a/manila/db/migrations/alembic/versions/3a482171410f_add_drivers_private_data_table.py +++ b/manila/db/migrations/alembic/versions/3a482171410f_add_drivers_private_data_table.py @@ -50,6 +50,7 @@ def upgrade(): nullable=False, primary_key=True), sql.Column('value', sql.String(1023), nullable=False), mysql_engine='InnoDB', + mysql_charset='utf8', ) except Exception: LOG.error("Table |%s| not created!", diff --git a/manila/db/migrations/alembic/versions/478c445d8d3e_add_security_service_update_control_fields.py b/manila/db/migrations/alembic/versions/478c445d8d3e_add_security_service_update_control_fields.py index fb40aab07f..cd4a2c2e8b 100644 --- a/manila/db/migrations/alembic/versions/478c445d8d3e_add_security_service_update_control_fields.py +++ b/manila/db/migrations/alembic/versions/478c445d8d3e_add_security_service_update_control_fields.py @@ -51,6 +51,7 @@ def upgrade(): nullable=False, primary_key=True), sa.Column('value', sa.String(1023), nullable=False), mysql_engine='InnoDB', + mysql_charset='utf8', ) op.add_column( SHARE_SERVERS_TABLE, diff --git a/manila/db/migrations/alembic/versions/4a482571410f_add_backends_info_table.py b/manila/db/migrations/alembic/versions/4a482571410f_add_backends_info_table.py index 26eaf6e268..122ef16f3a 100644 --- a/manila/db/migrations/alembic/versions/4a482571410f_add_backends_info_table.py +++ b/manila/db/migrations/alembic/versions/4a482571410f_add_backends_info_table.py @@ -47,6 +47,7 @@ def upgrade(): sql.Column('info_hash', sql.String(255), nullable=False), mysql_engine='InnoDB', + mysql_charset='utf8', ) except Exception: LOG.error("Table |%s| not created!", diff --git a/manila/db/migrations/alembic/versions/e8ea58723178_remove_host_from_driver_private_data.py b/manila/db/migrations/alembic/versions/e8ea58723178_remove_host_from_driver_private_data.py index 48f2c03b05..88b4c54ecb 100644 --- a/manila/db/migrations/alembic/versions/e8ea58723178_remove_host_from_driver_private_data.py +++ b/manila/db/migrations/alembic/versions/e8ea58723178_remove_host_from_driver_private_data.py @@ -80,6 +80,7 @@ def downgrade(): nullable=False, primary_key=True), sql.Column('value', sql.String(1023), nullable=False), mysql_engine='InnoDB', + mysql_charset='utf8', ) LOG.info("Copying data from %(from_table)s to the migration "