From abf2e577aa1b795330adff6d2f24b042a24ac79b Mon Sep 17 00:00:00 2001 From: Riccardo Pittau Date: Fri, 23 Jan 2026 15:59:50 +0100 Subject: [PATCH] Migrate database charset from UTF8MB3 to UTF8MB4 UTF8MB3 (3-byte UTF-8) character encoding is deprecated in MySQL and its support will be removed in future versions. This change migrates Ironic's database schema to use UTF8MB4 (4-byte UTF-8) which provides full Unicode support including supplementary characters. This is a breaking change that requires MySQL 8.0+ or MariaDB 10.3+, which use DYNAMIC row format by default and support the required index key prefix lengths for UTF8MB4 columns. Changes include: - Update models.py to use utf8mb4 charset for new tables - Add Alembic migration to convert existing tables to UTF8MB4 - Update ironic-status upgrade check to verify UTF8MB4 encoding - Update documentation with new database requirements - Update test setup to use utf8mb4 - Add test to verify the migration Assisted-By: Claude 4.5 Opus High Change-Id: Icbcf5f061c8dad239425448204733aa0d78a741f Signed-off-by: Riccardo Pittau Closes-Bug: 2130359 --- .../install/include/common-prerequisites.inc | 12 +- ironic/command/status.py | 16 +- .../c1fd28861bb9_migrate_to_utf8mb4.py | 194 ++++++++++++++++++ ironic/db/sqlalchemy/models.py | 2 +- ironic/tests/unit/command/test_status.py | 52 +++-- .../utf8mb4-migration-4cbde1fe756875f5.yaml | 45 ++++ tools/test-setup.sh | 2 +- 7 files changed, 294 insertions(+), 29 deletions(-) create mode 100644 ironic/db/sqlalchemy/alembic/versions/c1fd28861bb9_migrate_to_utf8mb4.py create mode 100644 releasenotes/notes/utf8mb4-migration-4cbde1fe756875f5.yaml diff --git a/doc/source/install/include/common-prerequisites.inc b/doc/source/install/include/common-prerequisites.inc index 64c12cd5c82..34b8df4d8e1 100644 --- a/doc/source/install/include/common-prerequisites.inc +++ b/doc/source/install/include/common-prerequisites.inc @@ -22,19 +22,17 @@ In MySQL, create an ``ironic`` database that is accessible by the .. code-block:: console # mysql -u root -p - mysql> CREATE DATABASE ironic CHARACTER SET utf8mb3; + mysql> CREATE DATABASE ironic CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; mysql> GRANT ALL PRIVILEGES ON ironic.* TO 'ironic'@'localhost' \ IDENTIFIED BY 'IRONIC_DBPASSWORD'; mysql> GRANT ALL PRIVILEGES ON ironic.* TO 'ironic'@'%' \ IDENTIFIED BY 'IRONIC_DBPASSWORD'; .. note:: - When creating the database to house Ironic, specifically on MySQL/MariaDB, - the character set *cannot* be 4 byte Unicode characters. This is due to - an internal structural constraint. UTF8, in these database platforms, - has traditionally meant ``utf8mb3``, short for "UTF-8, 3 byte encoding", - however the platforms are expected to move to ``utf8mb4`` which is - incompatible with Ironic. + Ironic requires ``utf8mb4`` character encoding (4-byte UTF-8) for full + Unicode support. This requires **MySQL 8.0+** or **MariaDB 10.3+**, which + use the DYNAMIC row format by default and support the required index key + lengths. Older MySQL/MariaDB versions are not supported. Running on SQLite ^^^^^^^^^^^^^^^^^ diff --git a/ironic/command/status.py b/ironic/command/status.py index a15c1f2edb8..b882660391a 100644 --- a/ironic/command/status.py +++ b/ironic/command/status.py @@ -107,12 +107,14 @@ class Checks(upgradecheck.UpgradeCommands): res = conn.execute( sqlalchemy.text("show create table allocations")) results = str(res.all()).lower() - if 'utf8' not in results: - msg = ('The Allocations table is is not using UTF8 encoding. ' - 'This is corrected in later versions of Ironic, where ' - 'the table character set schema is automatically ' - 'migrated. Continued use of a non-UTF8 character ' - 'set may produce unexpected results.') + # Check for utf8mb4 (4-byte UTF-8) which is the required encoding. + # Note: 'utf8mb4' will not match 'utf8mb3' or legacy 'utf8' aliases. + if 'utf8mb4' not in results: + msg = ('The Allocations table is not using UTF8MB4 encoding. ' + 'Ironic requires UTF8MB4 (4-byte UTF-8) character ' + 'encoding for full Unicode support. Please run ' + '"ironic-dbsync upgrade" to migrate to UTF8MB4. ' + 'This requires MySQL 8.0+ or MariaDB 10.3+.') if 'innodb' not in results: warning = ('The engine used by MySQL for the allocations ' @@ -231,7 +233,7 @@ class Checks(upgradecheck.UpgradeCommands): _upgrade_checks = ( (_('Object versions'), _check_obj_versions), (_('Database Index Status'), _check_db_indexes), - (_('Allocations Name Field Length Check'), + (_('MySQL UTF8MB4 Encoding Check'), _check_allocations_table), # Victoria -> Wallaby migration (_('Policy File JSON to YAML Migration'), diff --git a/ironic/db/sqlalchemy/alembic/versions/c1fd28861bb9_migrate_to_utf8mb4.py b/ironic/db/sqlalchemy/alembic/versions/c1fd28861bb9_migrate_to_utf8mb4.py new file mode 100644 index 00000000000..508a799791a --- /dev/null +++ b/ironic/db/sqlalchemy/alembic/versions/c1fd28861bb9_migrate_to_utf8mb4.py @@ -0,0 +1,194 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Migrate to UTF8MB4 character encoding + +Revision ID: c1fd28861bb9 +Revises: 9c0446cb6bc3 +Create Date: 2026-01-23 10:00:00.000000 + +""" + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'c1fd28861bb9' +down_revision = '9c0446cb6bc3' + + +# List of all tables that need to be converted to UTF8MB4 +TABLES = [ + 'chassis', + 'conductors', + 'conductor_hardware_interfaces', + 'nodes', + 'ports', + 'portgroups', + 'node_tags', + 'volume_connectors', + 'volume_targets', + 'node_traits', + 'bios_settings', + 'allocations', + 'deploy_templates', + 'deploy_template_steps', + 'node_history', + 'node_inventory', + 'firmware_information', + 'runbooks', + 'runbook_steps', + 'inspection_rules', +] + +# When MySQL converts tables to UTF8MB4 using CONVERT TO CHARACTER SET, +# it automatically promotes TEXT columns to MEDIUMTEXT to preserve the same +# character capacity (since UTF8MB4 uses 4 bytes per character vs 3 for +# UTF8MB3). We need to explicitly restore these columns to TEXT type. +# This includes both plain Text columns and oslo.db's JsonEncodedDict/List +# types which are stored as TEXT in MySQL. +# Note: Columns with mysql_as_long=True (LONGTEXT) are not affected. +# Format: {table_name: [(column_name, nullable), ...]} +TEXT_COLUMNS = { + 'allocations': [ + ('last_error', True), + ('traits', True), + ('candidate_nodes', True), + ('extra', True), + ], + 'bios_settings': [ + ('value', True), + ('allowable_values', True), + ], + 'chassis': [ + ('extra', True), + ], + 'conductors': [ + ('drivers', True), + ], + 'deploy_template_steps': [ + ('args', False), + ], + 'deploy_templates': [ + ('extra', True), + ], + 'node_history': [ + ('event', True), + ], + 'nodes': [ + ('last_error', True), + ('maintenance_reason', True), + ('protected_reason', True), + ('description', True), + ('retired_reason', True), + ('properties', True), + ('driver_info', True), + ('driver_internal_info', True), + ('clean_step', True), + ('deploy_step', True), + ('raid_config', True), + ('target_raid_config', True), + ('extra', True), + ('network_data', True), + ('service_step', True), + ], + 'portgroups': [ + ('extra', True), + ('internal_info', True), + ('properties', True), + ], + 'ports': [ + ('extra', True), + ('local_link_connection', True), + ('internal_info', True), + ], + 'runbook_steps': [ + ('args', False), + ], + 'runbooks': [ + ('extra', True), + ], + 'volume_connectors': [ + ('extra', True), + ], + 'volume_targets': [ + ('properties', True), + ('extra', True), + ], +} + + +def _verify_utf8mb4_conversion(connection): + """Verify all tables have been converted to utf8mb4. + + Queries information_schema to check that all Ironic tables are using + utf8mb4 character set. Raises an exception if any table is not converted. + """ + # Get the database name from the connection + db_name = connection.execute( + sa.text("SELECT DATABASE()") + ).scalar() + + # Check table character sets + result = connection.execute(sa.text( + "SELECT TABLE_NAME, TABLE_COLLATION " + "FROM information_schema.TABLES " + "WHERE TABLE_SCHEMA = :db_name " + "AND TABLE_NAME IN :tables " + "AND (TABLE_COLLATION IS NULL OR TABLE_COLLATION NOT LIKE 'utf8mb4%')" + ), {"db_name": db_name, "tables": tuple(TABLES)}) + + failed_tables = result.fetchall() + if failed_tables: + table_list = ", ".join(f"{t[0]} ({t[1]})" for t in failed_tables) + raise Exception( + f"UTF8MB4 migration verification failed. " + f"Tables not using utf8mb4: {table_list}" + ) + + +def upgrade(): + # This migration only applies to MySQL/MariaDB databases. + # For other databases (SQLite, PostgreSQL), this is a no-op. + connection = op.get_bind() + if connection.dialect.name != 'mysql': + return + + # Convert each table to UTF8MB4 character set with unicode collation. + # This requires MySQL 8.0+ or MariaDB 10.3+ which use DYNAMIC row format + # by default, allowing index key prefixes up to 3072 bytes. + for table in TABLES: + op.execute( + sa.text( + f"ALTER TABLE {table} CONVERT TO CHARACTER SET utf8mb4 " + "COLLATE utf8mb4_unicode_ci" + ) + ) + + # Restore TEXT columns that were promoted to MEDIUMTEXT during conversion. + # This is necessary to keep the schema in sync with the SQLAlchemy models. + # We batch all column modifications per table into a single ALTER TABLE + # statement for efficiency (one table rebuild instead of many). + for table, columns in TEXT_COLUMNS.items(): + modifications = [] + for column, nullable in columns: + null_str = "NULL" if nullable else "NOT NULL" + modifications.append( + f"MODIFY COLUMN {column} TEXT " + f"CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci {null_str}" + ) + op.execute(sa.text( + f"ALTER TABLE {table} " + ", ".join(modifications) + )) + + # Verify the conversion was successful + _verify_utf8mb4_conversion(connection) diff --git a/ironic/db/sqlalchemy/models.py b/ironic/db/sqlalchemy/models.py index c666e49bbc2..c7c93a19af1 100644 --- a/ironic/db/sqlalchemy/models.py +++ b/ironic/db/sqlalchemy/models.py @@ -47,7 +47,7 @@ def table_args(): engine_name = urlparse.urlparse(CONF.database.connection).scheme if engine_name == 'mysql': return {'mysql_engine': CONF.database.mysql_engine, - 'mysql_charset': "utf8"} + 'mysql_charset': "utf8mb4"} return None diff --git a/ironic/tests/unit/command/test_status.py b/ironic/tests/unit/command/test_status.py index 80e7bac1d22..db549c15a08 100644 --- a/ironic/tests/unit/command/test_status.py +++ b/ironic/tests/unit/command/test_status.py @@ -75,12 +75,38 @@ class TestUpgradeChecks(db_base.DbTestCase): check_result = self.cmd._check_allocations_table() self.assertEqual(Code.WARNING, check_result.code) - expected_msg = ('The Allocations table is is not using UTF8 ' - 'encoding. This is corrected in later versions ' - 'of Ironic, where the table character set schema ' - 'is automatically migrated. Continued use of a ' - 'non-UTF8 character set may produce unexpected ' - 'results.') + expected_msg = ('The Allocations table is not using UTF8MB4 ' + 'encoding. Ironic requires UTF8MB4 (4-byte UTF-8) ' + 'character encoding for full Unicode support. ' + 'Please run "ironic-dbsync upgrade" to migrate to ' + 'UTF8MB4. This requires MySQL 8.0+ or MariaDB 10.3+.') + self.assertEqual(expected_msg, check_result.details) + + @mock.patch.object(sqlalchemy.enginefacade.reader, + 'get_engine', autospec=True) + def test__check_allocations_table_utf8mb3(self, mock_reader): + """Test that legacy utf8 (utf8mb3) triggers a warning.""" + mock_engine = mock.Mock() + mock_res = mock.Mock() + mock_engine.url = sa_url.make_url( + 'mysql+pymysql://ironic:pass@192.0.2.10/ironic') + mock_res.all.return_value = ( + '... ENGINE=InnoDB DEFAULT CHARSET=utf8', + ) + mock_conn = self._create_mock_context_manager(True) + mock_trans = self._create_mock_context_manager(False) + mock_engine.connect.return_value = mock_conn + mock_conn.execute.return_value = mock_res + mock_conn.begin.return_value = mock_trans + mock_reader.return_value = mock_engine + check_result = self.cmd._check_allocations_table() + self.assertEqual(Code.WARNING, + check_result.code) + expected_msg = ('The Allocations table is not using UTF8MB4 ' + 'encoding. Ironic requires UTF8MB4 (4-byte UTF-8) ' + 'character encoding for full Unicode support. ' + 'Please run "ironic-dbsync upgrade" to migrate to ' + 'UTF8MB4. This requires MySQL 8.0+ or MariaDB 10.3+.') self.assertEqual(expected_msg, check_result.details) @mock.patch.object(sqlalchemy.enginefacade.reader, @@ -91,7 +117,7 @@ class TestUpgradeChecks(db_base.DbTestCase): mock_engine.url = sa_url.make_url( 'mysql+pymysql://ironic:pass@192.0.2.10/ironic') mock_res.all.return_value = ( - '... ENGINE=MyIASM DEFAULT CHARSET=utf8', + '... ENGINE=MyIASM DEFAULT CHARSET=utf8mb4', ) mock_conn = self._create_mock_context_manager(True) mock_trans = self._create_mock_context_manager(False) @@ -131,12 +157,12 @@ class TestUpgradeChecks(db_base.DbTestCase): check_result = self.cmd._check_allocations_table() self.assertEqual(Code.WARNING, check_result.code) - expected_msg = ('The Allocations table is is not using UTF8 ' - 'encoding. This is corrected in later versions ' - 'of Ironic, where the table character set schema ' - 'is automatically migrated. Continued use of a ' - 'non-UTF8 character set may produce unexpected ' - 'results. Additionally: ' + expected_msg = ('The Allocations table is not using UTF8MB4 ' + 'encoding. Ironic requires UTF8MB4 (4-byte UTF-8) ' + 'character encoding for full Unicode support. ' + 'Please run "ironic-dbsync upgrade" to migrate to ' + 'UTF8MB4. This requires MySQL 8.0+ or MariaDB 10.3+. ' + 'Additionally: ' 'The engine used by MySQL for the allocations ' 'table is not the intended engine for the Ironic ' 'database tables to use. This may have been a ' diff --git a/releasenotes/notes/utf8mb4-migration-4cbde1fe756875f5.yaml b/releasenotes/notes/utf8mb4-migration-4cbde1fe756875f5.yaml new file mode 100644 index 00000000000..7473fadb8bb --- /dev/null +++ b/releasenotes/notes/utf8mb4-migration-4cbde1fe756875f5.yaml @@ -0,0 +1,45 @@ +--- +upgrade: + - | + Ironic now requires ``UTF8MB4`` (4-byte UTF-8) character encoding for + MySQL/MariaDB databases. This is a **breaking change** that requires + **MySQL 8.0+** or **MariaDB 10.3+**. + + The database migration will automatically convert all existing tables + from ``UTF8MB3`` (3-byte UTF-8) to ``UTF8MB4``. During this conversion, + each table is locked while the ``ALTER TABLE`` operation rewrites the + table data. The duration of these locks depends on the amount of data + in each table. Smaller deployments with fewer nodes will experience + brief locks, while larger deployments with many nodes, ports, and + historical data may experience longer lock times per table. + + Operators should plan for downtime during the database migration and + consider performing test upgrades on a copy of production data to + estimate the migration duration for their environment. + + Benefits of UTF8MB4: + + * Full Unicode support including supplementary characters (emojis, etc.) + * Compliance with modern MySQL/MariaDB defaults + * Removal of dependency on deprecated UTF8MB3 character set + + For new installations, create the database with: + + .. code-block:: sql + + CREATE DATABASE ironic CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; + + For existing installations, the migration is handled automatically by + running ``ironic-dbsync upgrade``. +deprecations: + - | + Support for MySQL versions prior to 8.0 and MariaDB versions prior to + 10.3 has been removed. These older versions do not support the required + ``UTF8MB4`` character encoding with DYNAMIC row format by default. +features: + - | + Ironic now uses ``UTF8MB4`` character encoding for all database tables + when using MySQL/MariaDB. This enables full Unicode support including + emojis and other supplementary characters that were previously not + supported with the legacy ``UTF8MB3`` (3-byte UTF-8) encoding. + diff --git a/tools/test-setup.sh b/tools/test-setup.sh index 3f4c44edce4..cd62f4abf26 100755 --- a/tools/test-setup.sh +++ b/tools/test-setup.sh @@ -33,5 +33,5 @@ sudo -H mysql -u root -p$DB_ROOT_PW -h localhost -e " mysql -u $DB_USER -p$DB_PW -h 127.0.0.1 -e " SET default_storage_engine=MYISAM; DROP DATABASE IF EXISTS openstack_citest; - CREATE DATABASE openstack_citest CHARACTER SET utf8;" + CREATE DATABASE openstack_citest CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;"