openstack/nova
Code Issues Proposed changes

db: Compact Train database migrations

Browse Source 
Compact Train database migrations into a single migration,
'402_train.py'.

Users will now need to update to Train before updating to Ussuri or
later.

Specific changes include:

- Add 'cross_cell_move' column to 'migrations' table (397)
- Add 'vpmems' column to 'instance_extra' table (398)
- Add 'hidden' column to 'instances' table (399)
- Add 'user_id' and 'project_id' columns to 'migrations' table (401)
- Add 'resources' column to 'instance_extra' table (402)

As there are boolean tables here, we're once again hitting the issue
whereby we need to explicitly disable contraints for boolean fields to
keep the schemas the same.

When testing, the previous base version was 391. It is now 402.

Change-Id: Iecb531728a83194117e0e7e5f4b6189073541f68
Signed-off-by: Stephen Finucane <stephenfin@redhat.com>
This commit is contained in:
Stephen Finucane 2020-10-21 12:01:31 +01:00
parent f0175a346a
commit 5c9c81e0f9
15 changed files with 17 additions and 364 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
nova/db/sqlalchemy/migrate_repo/versions/392_placeholder.py
View File

@ -1,22 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for backports.
# Do not use this number for new work. New work starts after
# all the placeholders.
#
# See this for more information:
# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html
def upgrade(migrate_engine):
pass

22
nova/db/sqlalchemy/migrate_repo/versions/393_placeholder.py
View File

@ -1,22 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for backports.
# Do not use this number for new work. New work starts after
# all the placeholders.
#
# See this for more information:
# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html
def upgrade(migrate_engine):
pass

22
nova/db/sqlalchemy/migrate_repo/versions/394_placeholder.py
View File

@ -1,22 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for backports.
# Do not use this number for new work. New work starts after
# all the placeholders.
#
# See this for more information:
# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html
def upgrade(migrate_engine):
pass

22
nova/db/sqlalchemy/migrate_repo/versions/395_placeholder.py
View File

@ -1,22 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for backports.
# Do not use this number for new work. New work starts after
# all the placeholders.
#
# See this for more information:
# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html
def upgrade(migrate_engine):
pass

22
nova/db/sqlalchemy/migrate_repo/versions/396_placeholder.py
View File

@ -1,22 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is a placeholder for backports.
# Do not use this number for new work. New work starts after
# all the placeholders.
#
# See this for more information:
# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html
def upgrade(migrate_engine):
pass

24
nova/db/sqlalchemy/migrate_repo/versions/397_migrations_cross_cell_move.py
View File

@ -1,24 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from sqlalchemy import MetaData, Column, Table
from sqlalchemy import Boolean
def upgrade(migrate_engine):
meta = MetaData(bind=migrate_engine)
for prefix in ('', 'shadow_'):
migrations = Table('%smigrations' % prefix, meta, autoload=True)
if not hasattr(migrations.c, 'cross_cell_move'):
cross_cell_move = Column('cross_cell_move', Boolean, default=False)
migrations.create_column(cross_cell_move)

31
nova/db/sqlalchemy/migrate_repo/versions/398_add_vpmems.py
View File

@ -1,31 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from sqlalchemy import Column
from sqlalchemy import MetaData
from sqlalchemy import Table
from sqlalchemy import Text
BASE_TABLE_NAME = 'instance_extra'
NEW_COLUMN_NAME = 'vpmems'
def upgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
for prefix in ('', 'shadow_'):
table = Table(prefix + BASE_TABLE_NAME, meta, autoload=True)
new_column = Column(NEW_COLUMN_NAME, Text, nullable=True)
if not hasattr(table.c, NEW_COLUMN_NAME):
table.create_column(new_column)

28
nova/db/sqlalchemy/migrate_repo/versions/399_add_instances_hidden.py
View File

@ -1,28 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from sqlalchemy import MetaData, Column, Table
from sqlalchemy import Boolean
def upgrade(migrate_engine):
meta = MetaData(bind=migrate_engine)
for prefix in ('', 'shadow_'):
instances = Table('%sinstances' % prefix, meta, autoload=True)
if not hasattr(instances.c, 'hidden'):
# NOTE(danms): This column originally included default=False. We
# discovered in bug #1862205 that this will attempt to rewrite
# the entire instances table with that value, which can time out
# for large data sets (and does not even abort).
hidden = Column('hidden', Boolean)
instances.create_column(hidden)

33
nova/db/sqlalchemy/migrate_repo/versions/400_enforce_service_uuid.py
View File

@ -1,33 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from sqlalchemy import MetaData, Table, func, null, select
from sqlalchemy.sql import and_
from nova import exception
from nova.i18n import _
def upgrade(migrate_engine):
meta = MetaData(migrate_engine)
services = Table('services', meta, autoload=True)
# Count non-deleted services where uuid is null.
count = select([func.count()]).select_from(services).where(and_(
services.c.deleted == 0,
services.c.uuid == null())).execute().scalar()
if count > 0:
msg = _('There are still %(count)i unmigrated records in '
'the services table. Migration cannot continue '
'until all records have been migrated. Run the '
'"nova-manage db online_data_migrations" routine.') % {
'count': count}
raise exception.ValidationError(detail=msg)

27
nova/db/sqlalchemy/migrate_repo/versions/401_add_user_id_and_project_id_to_migrations.py
View File

@ -1,27 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from sqlalchemy import MetaData, Column, Table, String
NEW_COLUMNS_NAME = ['user_id', 'project_id']
BASE_TABLE_NAME = 'migrations'
def upgrade(migrate_engine):
meta = MetaData(bind=migrate_engine)
for prefix in ('', 'shadow_'):
table = Table(prefix + BASE_TABLE_NAME, meta, autoload=True)
for new_column_name in NEW_COLUMNS_NAME:
new_column = Column(new_column_name, String(255), nullable=True)
if not hasattr(table.c, new_column_name):
table.create_column(new_column)

31
nova/db/sqlalchemy/migrate_repo/versions/402_add_resources.py
View File

@ -1,31 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from sqlalchemy import Column
from sqlalchemy import MetaData
from sqlalchemy import Table
from sqlalchemy import Text
BASE_TABLE_NAME = 'instance_extra'
NEW_COLUMN_NAME = 'resources'
def upgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
for prefix in ('', 'shadow_'):
table = Table(prefix + BASE_TABLE_NAME, meta, autoload=True)
new_column = Column(NEW_COLUMN_NAME, Text, nullable=True)
if not hasattr(table.c, NEW_COLUMN_NAME):
table.create_column(new_column)

16
nova/db/sqlalchemy/migrate_repo/versions/391_stein.py → nova/db/sqlalchemy/migrate_repo/versions/402_train.py
View File

@ -675,6 +675,13 @@ def upgrade(migrate_engine):
'locked_by', Enum('owner', 'admin', name='instances0locked_by')),
Column('cleaned', Integer, default=0),
Column('ephemeral_key_uuid', String(36)),
# NOTE(danms): This column originally included default=False. We
# discovered in bug #1862205 that this will attempt to rewrite
# the entire instances table with that value, which can time out
# for large data sets (and does not even abort).
# NOTE(stephenfin): This was originally added by sqlalchemy-migrate
# which did not generate the constraints
Column('hidden', Boolean(create_constraint=False)),
Index('uuid', 'uuid', unique=True),
UniqueConstraint('uuid', name='uniq_instances0uuid'),
mysql_engine='InnoDB',
@ -732,6 +739,8 @@ def upgrade(migrate_engine):
Column('keypairs', Text, nullable=True),
Column('device_metadata', Text, nullable=True),
Column('trusted_certs', Text, nullable=True),
Column('vpmems', Text, nullable=True),
Column('resources', Text, nullable=True),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
@ -811,6 +820,13 @@ def upgrade(migrate_engine):
Column('disk_processed', BigInteger, nullable=True),
Column('disk_remaining', BigInteger, nullable=True),
Column('uuid', String(36)),
# NOTE(stephenfin): This was originally added by sqlalchemy-migrate
# which did not generate the constraints
Column(
'cross_cell_move', Boolean(create_constraint=False),
default=False),
Column('user_id', String(255), nullable=True),
Column('project_id', String(255), nullable=True),
Index('migrations_uuid', 'uuid', unique=True),
mysql_engine='InnoDB',
mysql_charset='utf8'

2
nova/db/sqlalchemy/migration.py
View File

@ -29,7 +29,7 @@ from nova import exception
from nova.i18n import _
INIT_VERSION = {}
INIT_VERSION['main'] = 390
INIT_VERSION['main'] = 401
INIT_VERSION['api'] = 0
_REPOSITORY = {}

32
nova/tests/unit/db/test_migrations.py
View File

@ -162,13 +162,11 @@ class NovaMigrationsCheckers(test_migrations.ModelsMigrationsSync,
self.INIT_VERSION + 1,
]
stein_placeholders = list(range(392, 397))
train_placeholders = list(range(403, 408))
ussuri_placeholders = list(range(408, 413))
victoria_placeholders = list(range(413, 418))
return (special +
stein_placeholders +
train_placeholders +
ussuri_placeholders +
victoria_placeholders)
@ -226,36 +224,6 @@ class NovaMigrationsCheckers(test_migrations.ModelsMigrationsSync,
def test_walk_versions(self):
self.walk_versions(snake_walk=False, downgrade=False)
def _check_397(self, engine, data):
for prefix in ('', 'shadow_'):
self.assertColumnExists(
engine, '%smigrations' % prefix, 'cross_cell_move')
def _check_398(self, engine, data):
self.assertColumnExists(engine, 'instance_extra', 'vpmems')
self.assertColumnExists(engine, 'shadow_instance_extra', 'vpmems')
def _check_399(self, engine, data):
for prefix in ('', 'shadow_'):
self.assertColumnExists(
engine, '%sinstances' % prefix, 'hidden')
def _check_400(self, engine, data):
# NOTE(mriedem): This is a dummy migration that just does a consistency
# check. The actual test for 400 is in TestServicesUUIDCheck.
pass
def _check_401(self, engine, data):
for prefix in ('', 'shadow_'):
self.assertColumnExists(
engine, '%smigrations' % prefix, 'user_id')
self.assertColumnExists(
engine, '%smigrations' % prefix, 'project_id')
def _check_402(self, engine, data):
self.assertColumnExists(engine, 'instance_extra', 'resources')
self.assertColumnExists(engine, 'shadow_instance_extra', 'resources')
class TestNovaMigrationsSQLite(NovaMigrationsCheckers,
test_fixtures.OpportunisticDBTestMixin,

47
nova/tests/unit/db/test_sqlalchemy_migration.py
View File

@ -324,50 +324,3 @@ class TestNewtonCellsCheck(test.NoDBTestCase):
def test_upgrade_new_deploy(self):
self.migration.upgrade(self.engine)
class TestServicesUUIDCheck(test.TestCase):
"""Tests the 400_enforce_service_uuid blocker migration."""
def setUp(self):
super(TestServicesUUIDCheck, self).setUp()
self.useFixture(nova_fixtures.Database(version=398))
self.context = context.get_admin_context()
self.migration = importlib.import_module(
'nova.db.sqlalchemy.migrate_repo.versions.'
'400_enforce_service_uuid')
self.engine = db_api.get_engine()
def test_upgrade_unmigrated_deleted_service(self):
"""Tests to make sure the 400 migration filters out deleted services"""
services = db_utils.get_table(self.engine, 'services')
service = {
'host': 'fake-host',
'binary': 'nova-compute',
'topic': 'compute',
'report_count': 514,
'version': 16,
'uuid': None,
'deleted': 1
}
services.insert().execute(service)
self.migration.upgrade(self.engine)
def test_upgrade_unmigrated_service_validation_error(self):
"""Tests that the migration raises ValidationError when an unmigrated
non-deleted service record is found.
"""
services = db_utils.get_table(self.engine, 'services')
service = {
'host': 'fake-host',
'binary': 'nova-compute',
'topic': 'compute',
'report_count': 514,
'version': 16,
'uuid': None,
'deleted': 0
}
services.insert().execute(service)
ex = self.assertRaises(exception.ValidationError,
self.migration.upgrade, self.engine)
self.assertIn('There are still 1 unmigrated records in the '
'services table.', str(ex))