Merge "db: Compact Train database migrations"
This commit is contained in:
Zuul
Gerrit Code Review
commit
5c276fb180
|
|
@ -1,22 +0,0 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# This is a placeholder for backports.
|
||||
# Do not use this number for new work. New work starts after
|
||||
# all the placeholders.
|
||||
#
|
||||
# See this for more information:
|
||||
# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html
|
||||
|
||||
|
||||
def upgrade(migrate_engine):
|
||||
pass
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# This is a placeholder for backports.
|
||||
# Do not use this number for new work. New work starts after
|
||||
# all the placeholders.
|
||||
#
|
||||
# See this for more information:
|
||||
# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html
|
||||
|
||||
|
||||
def upgrade(migrate_engine):
|
||||
pass
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# This is a placeholder for backports.
|
||||
# Do not use this number for new work. New work starts after
|
||||
# all the placeholders.
|
||||
#
|
||||
# See this for more information:
|
||||
# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html
|
||||
|
||||
|
||||
def upgrade(migrate_engine):
|
||||
pass
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# This is a placeholder for backports.
|
||||
# Do not use this number for new work. New work starts after
|
||||
# all the placeholders.
|
||||
#
|
||||
# See this for more information:
|
||||
# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html
|
||||
|
||||
|
||||
def upgrade(migrate_engine):
|
||||
pass
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# This is a placeholder for backports.
|
||||
# Do not use this number for new work. New work starts after
|
||||
# all the placeholders.
|
||||
#
|
||||
# See this for more information:
|
||||
# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html
|
||||
|
||||
|
||||
def upgrade(migrate_engine):
|
||||
pass
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from sqlalchemy import MetaData, Column, Table
|
||||
from sqlalchemy import Boolean
|
||||
|
||||
|
||||
def upgrade(migrate_engine):
|
||||
meta = MetaData(bind=migrate_engine)
|
||||
|
||||
for prefix in ('', 'shadow_'):
|
||||
migrations = Table('%smigrations' % prefix, meta, autoload=True)
|
||||
if not hasattr(migrations.c, 'cross_cell_move'):
|
||||
cross_cell_move = Column('cross_cell_move', Boolean, default=False)
|
||||
migrations.create_column(cross_cell_move)
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from sqlalchemy import Column
|
||||
from sqlalchemy import MetaData
|
||||
from sqlalchemy import Table
|
||||
from sqlalchemy import Text
|
||||
|
||||
|
||||
BASE_TABLE_NAME = 'instance_extra'
|
||||
NEW_COLUMN_NAME = 'vpmems'
|
||||
|
||||
|
||||
def upgrade(migrate_engine):
|
||||
meta = MetaData()
|
||||
meta.bind = migrate_engine
|
||||
|
||||
for prefix in ('', 'shadow_'):
|
||||
table = Table(prefix + BASE_TABLE_NAME, meta, autoload=True)
|
||||
new_column = Column(NEW_COLUMN_NAME, Text, nullable=True)
|
||||
if not hasattr(table.c, NEW_COLUMN_NAME):
|
||||
table.create_column(new_column)
|
||||
|
|
@ -1,28 +0,0 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from sqlalchemy import MetaData, Column, Table
|
||||
from sqlalchemy import Boolean
|
||||
|
||||
|
||||
def upgrade(migrate_engine):
|
||||
meta = MetaData(bind=migrate_engine)
|
||||
|
||||
for prefix in ('', 'shadow_'):
|
||||
instances = Table('%sinstances' % prefix, meta, autoload=True)
|
||||
if not hasattr(instances.c, 'hidden'):
|
||||
# NOTE(danms): This column originally included default=False. We
|
||||
# discovered in bug #1862205 that this will attempt to rewrite
|
||||
# the entire instances table with that value, which can time out
|
||||
# for large data sets (and does not even abort).
|
||||
hidden = Column('hidden', Boolean)
|
||||
instances.create_column(hidden)
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from sqlalchemy import MetaData, Table, func, null, select
|
||||
from sqlalchemy.sql import and_
|
||||
|
||||
from nova import exception
|
||||
from nova.i18n import _
|
||||
|
||||
|
||||
def upgrade(migrate_engine):
|
||||
meta = MetaData(migrate_engine)
|
||||
services = Table('services', meta, autoload=True)
|
||||
# Count non-deleted services where uuid is null.
|
||||
count = select([func.count()]).select_from(services).where(and_(
|
||||
services.c.deleted == 0,
|
||||
services.c.uuid == null())).execute().scalar()
|
||||
if count > 0:
|
||||
msg = _('There are still %(count)i unmigrated records in '
|
||||
'the services table. Migration cannot continue '
|
||||
'until all records have been migrated. Run the '
|
||||
'"nova-manage db online_data_migrations" routine.') % {
|
||||
'count': count}
|
||||
raise exception.ValidationError(detail=msg)
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from sqlalchemy import MetaData, Column, Table, String
|
||||
|
||||
NEW_COLUMNS_NAME = ['user_id', 'project_id']
|
||||
BASE_TABLE_NAME = 'migrations'
|
||||
|
||||
|
||||
def upgrade(migrate_engine):
|
||||
meta = MetaData(bind=migrate_engine)
|
||||
|
||||
for prefix in ('', 'shadow_'):
|
||||
table = Table(prefix + BASE_TABLE_NAME, meta, autoload=True)
|
||||
for new_column_name in NEW_COLUMNS_NAME:
|
||||
new_column = Column(new_column_name, String(255), nullable=True)
|
||||
if not hasattr(table.c, new_column_name):
|
||||
table.create_column(new_column)
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from sqlalchemy import Column
|
||||
from sqlalchemy import MetaData
|
||||
from sqlalchemy import Table
|
||||
from sqlalchemy import Text
|
||||
|
||||
|
||||
BASE_TABLE_NAME = 'instance_extra'
|
||||
NEW_COLUMN_NAME = 'resources'
|
||||
|
||||
|
||||
def upgrade(migrate_engine):
|
||||
meta = MetaData()
|
||||
meta.bind = migrate_engine
|
||||
|
||||
for prefix in ('', 'shadow_'):
|
||||
table = Table(prefix + BASE_TABLE_NAME, meta, autoload=True)
|
||||
new_column = Column(NEW_COLUMN_NAME, Text, nullable=True)
|
||||
if not hasattr(table.c, NEW_COLUMN_NAME):
|
||||
table.create_column(new_column)
|
||||
|
|
@ -675,6 +675,13 @@ def upgrade(migrate_engine):
|
|||
'locked_by', Enum('owner', 'admin', name='instances0locked_by')),
|
||||
Column('cleaned', Integer, default=0),
|
||||
Column('ephemeral_key_uuid', String(36)),
|
||||
# NOTE(danms): This column originally included default=False. We
|
||||
# discovered in bug #1862205 that this will attempt to rewrite
|
||||
# the entire instances table with that value, which can time out
|
||||
# for large data sets (and does not even abort).
|
||||
# NOTE(stephenfin): This was originally added by sqlalchemy-migrate
|
||||
# which did not generate the constraints
|
||||
Column('hidden', Boolean(create_constraint=False)),
|
||||
Index('uuid', 'uuid', unique=True),
|
||||
UniqueConstraint('uuid', name='uniq_instances0uuid'),
|
||||
mysql_engine='InnoDB',
|
||||
|
|
@ -732,6 +739,8 @@ def upgrade(migrate_engine):
|
|||
Column('keypairs', Text, nullable=True),
|
||||
Column('device_metadata', Text, nullable=True),
|
||||
Column('trusted_certs', Text, nullable=True),
|
||||
Column('vpmems', Text, nullable=True),
|
||||
Column('resources', Text, nullable=True),
|
||||
mysql_engine='InnoDB',
|
||||
mysql_charset='utf8',
|
||||
)
|
||||
|
|
@ -811,6 +820,13 @@ def upgrade(migrate_engine):
|
|||
Column('disk_processed', BigInteger, nullable=True),
|
||||
Column('disk_remaining', BigInteger, nullable=True),
|
||||
Column('uuid', String(36)),
|
||||
# NOTE(stephenfin): This was originally added by sqlalchemy-migrate
|
||||
# which did not generate the constraints
|
||||
Column(
|
||||
'cross_cell_move', Boolean(create_constraint=False),
|
||||
default=False),
|
||||
Column('user_id', String(255), nullable=True),
|
||||
Column('project_id', String(255), nullable=True),
|
||||
Index('migrations_uuid', 'uuid', unique=True),
|
||||
mysql_engine='InnoDB',
|
||||
mysql_charset='utf8'
|
||||
|
|
@ -29,7 +29,7 @@ from nova import exception
|
|||
from nova.i18n import _
|
||||
|
||||
INIT_VERSION = {}
|
||||
INIT_VERSION['main'] = 390
|
||||
INIT_VERSION['main'] = 401
|
||||
INIT_VERSION['api'] = 0
|
||||
_REPOSITORY = {}
|
||||
|
||||
|
|
|
|||
|
|
@ -162,13 +162,11 @@ class NovaMigrationsCheckers(test_migrations.ModelsMigrationsSync,
|
|||
self.INIT_VERSION + 1,
|
||||
]
|
||||
|
||||
stein_placeholders = list(range(392, 397))
|
||||
train_placeholders = list(range(403, 408))
|
||||
ussuri_placeholders = list(range(408, 413))
|
||||
victoria_placeholders = list(range(413, 418))
|
||||
|
||||
return (special +
|
||||
stein_placeholders +
|
||||
train_placeholders +
|
||||
ussuri_placeholders +
|
||||
victoria_placeholders)
|
||||
|
|
@ -226,36 +224,6 @@ class NovaMigrationsCheckers(test_migrations.ModelsMigrationsSync,
|
|||
def test_walk_versions(self):
|
||||
self.walk_versions(snake_walk=False, downgrade=False)
|
||||
|
||||
def _check_397(self, engine, data):
|
||||
for prefix in ('', 'shadow_'):
|
||||
self.assertColumnExists(
|
||||
engine, '%smigrations' % prefix, 'cross_cell_move')
|
||||
|
||||
def _check_398(self, engine, data):
|
||||
self.assertColumnExists(engine, 'instance_extra', 'vpmems')
|
||||
self.assertColumnExists(engine, 'shadow_instance_extra', 'vpmems')
|
||||
|
||||
def _check_399(self, engine, data):
|
||||
for prefix in ('', 'shadow_'):
|
||||
self.assertColumnExists(
|
||||
engine, '%sinstances' % prefix, 'hidden')
|
||||
|
||||
def _check_400(self, engine, data):
|
||||
# NOTE(mriedem): This is a dummy migration that just does a consistency
|
||||
# check. The actual test for 400 is in TestServicesUUIDCheck.
|
||||
pass
|
||||
|
||||
def _check_401(self, engine, data):
|
||||
for prefix in ('', 'shadow_'):
|
||||
self.assertColumnExists(
|
||||
engine, '%smigrations' % prefix, 'user_id')
|
||||
self.assertColumnExists(
|
||||
engine, '%smigrations' % prefix, 'project_id')
|
||||
|
||||
def _check_402(self, engine, data):
|
||||
self.assertColumnExists(engine, 'instance_extra', 'resources')
|
||||
self.assertColumnExists(engine, 'shadow_instance_extra', 'resources')
|
||||
|
||||
|
||||
class TestNovaMigrationsSQLite(NovaMigrationsCheckers,
|
||||
test_fixtures.OpportunisticDBTestMixin,
|
||||
|
|
|
|||
|
|
@ -324,50 +324,3 @@ class TestNewtonCellsCheck(test.NoDBTestCase):
|
|||
|
||||
def test_upgrade_new_deploy(self):
|
||||
self.migration.upgrade(self.engine)
|
||||
|
||||
|
||||
class TestServicesUUIDCheck(test.TestCase):
|
||||
"""Tests the 400_enforce_service_uuid blocker migration."""
|
||||
def setUp(self):
|
||||
super(TestServicesUUIDCheck, self).setUp()
|
||||
self.useFixture(nova_fixtures.Database(version=398))
|
||||
self.context = context.get_admin_context()
|
||||
self.migration = importlib.import_module(
|
||||
'nova.db.sqlalchemy.migrate_repo.versions.'
|
||||
'400_enforce_service_uuid')
|
||||
self.engine = db_api.get_engine()
|
||||
|
||||
def test_upgrade_unmigrated_deleted_service(self):
|
||||
"""Tests to make sure the 400 migration filters out deleted services"""
|
||||
services = db_utils.get_table(self.engine, 'services')
|
||||
service = {
|
||||
'host': 'fake-host',
|
||||
'binary': 'nova-compute',
|
||||
'topic': 'compute',
|
||||
'report_count': 514,
|
||||
'version': 16,
|
||||
'uuid': None,
|
||||
'deleted': 1
|
||||
}
|
||||
services.insert().execute(service)
|
||||
self.migration.upgrade(self.engine)
|
||||
|
||||
def test_upgrade_unmigrated_service_validation_error(self):
|
||||
"""Tests that the migration raises ValidationError when an unmigrated
|
||||
non-deleted service record is found.
|
||||
"""
|
||||
services = db_utils.get_table(self.engine, 'services')
|
||||
service = {
|
||||
'host': 'fake-host',
|
||||
'binary': 'nova-compute',
|
||||
'topic': 'compute',
|
||||
'report_count': 514,
|
||||
'version': 16,
|
||||
'uuid': None,
|
||||
'deleted': 0
|
||||
}
|
||||
services.insert().execute(service)
|
||||
ex = self.assertRaises(exception.ValidationError,
|
||||
self.migration.upgrade, self.engine)
|
||||
self.assertIn('There are still 1 unmigrated records in the '
|
||||
'services table.', str(ex))
|
||||
|
|
|
|||
Loading…
Reference in New Issue