From 407e16b863bac1dfbf4e954837009abf9c17f018 Mon Sep 17 00:00:00 2001 From: John Griffith Date: Thu, 12 Apr 2012 20:36:48 -0600 Subject: [PATCH] Convert Volume and Snapshot IDs to use UUID * Three migrations 1. create id mappings 2. convert volume_id and snapshot_id from int to string 3. change volume/snapshot id's from int to uuid * DB migration for Volume and Related tables * Addition of new volume id mapping tables * Added methods in ec2utils * Minor tweaks to unit tests * Other changes to migration to ensure consistency in id's * Fixed bug in the block-device-mapping table (wasn't setting autoinc) Change-Id: Ic6c3646e0f01c26467a4a3c20e13eebaa2baa97e --- nova/api/ec2/cloud.py | 18 +- nova/api/ec2/ec2utils.py | 70 +++++- nova/db/api.py | 23 +- nova/db/sqlalchemy/api.py | 102 ++++++++ .../versions/089_add_volume_id_mappings.py | 116 +++++++++ .../versions/090_modify_volume_id_datatype.py | 236 ++++++++++++++++++ .../versions/090_sqlite_downgrade.sql | 226 +++++++++++++++++ .../versions/090_sqlite_upgrade.sql | 226 +++++++++++++++++ .../091_convert_volume_ids_to_uuid.py | 145 +++++++++++ nova/db/sqlalchemy/models.py | 35 ++- nova/tests/api/ec2/test_cloud.py | 23 +- nova/tests/api/ec2/test_ec2_validate.py | 17 -- nova/tests/integrated/test_volumes.py | 12 +- nova/tests/test_bdm.py | 25 +- nova/tests/test_compute.py | 45 ++-- nova/tests/test_volume.py | 15 ++ 16 files changed, 1251 insertions(+), 83 deletions(-) create mode 100644 nova/db/sqlalchemy/migrate_repo/versions/089_add_volume_id_mappings.py create mode 100644 nova/db/sqlalchemy/migrate_repo/versions/090_modify_volume_id_datatype.py create mode 100644 nova/db/sqlalchemy/migrate_repo/versions/090_sqlite_downgrade.sql create mode 100644 nova/db/sqlalchemy/migrate_repo/versions/090_sqlite_upgrade.sql create mode 100644 nova/db/sqlalchemy/migrate_repo/versions/091_convert_volume_ids_to_uuid.py diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 8c6a1fdc335a..26cc4a70cad9 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -131,7 +131,7 @@ def _parse_block_device_mapping(bdm): if ebs: ec2_id = ebs.pop('snapshot_id', None) if ec2_id: - id = ec2utils.ec2_id_to_id(ec2_id) + id = ec2utils.ec2_vol_id_to_uuid(ec2_id) if ec2_id.startswith('snap-'): bdm['snapshot_id'] = id elif ec2_id.startswith('vol-'): @@ -310,7 +310,7 @@ class CloudController(object): if snapshot_id: snapshots = [] for ec2_id in snapshot_id: - internal_id = ec2utils.ec2_id_to_id(ec2_id) + internal_id = ec2utils.ec2_snap_id_to_uuid(ec2_id) snapshot = self.volume_api.get_snapshot( context, snapshot_id=internal_id) @@ -336,7 +336,7 @@ class CloudController(object): validate_ec2_id(volume_id) LOG.audit(_("Create snapshot of volume %s"), volume_id, context=context) - volume_id = ec2utils.ec2_id_to_id(volume_id) + volume_id = ec2utils.ec2_vol_id_to_uuid(volume_id) volume = self.volume_api.get(context, volume_id) snapshot = self.volume_api.create_snapshot( context, @@ -346,7 +346,7 @@ class CloudController(object): return self._format_snapshot(context, snapshot) def delete_snapshot(self, context, snapshot_id, **kwargs): - snapshot_id = ec2utils.ec2_id_to_id(snapshot_id) + snapshot_id = ec2utils.ec2_snap_id_to_uuid(snapshot_id) snapshot = self.volume_api.get_snapshot(context, snapshot_id) self.volume_api.delete_snapshot(context, snapshot) return True @@ -853,7 +853,7 @@ class CloudController(object): volumes = [] for ec2_id in volume_id: validate_ec2_id(ec2_id) - internal_id = ec2utils.ec2_id_to_id(ec2_id) + internal_id = ec2utils.ec2_vol_id_to_uuid(ec2_id) volume = self.volume_api.get(context, internal_id) volumes.append(volume) else: @@ -901,7 +901,7 @@ class CloudController(object): def create_volume(self, context, **kwargs): size = kwargs.get('size') if kwargs.get('snapshot_id') is not None: - snapshot_id = ec2utils.ec2_id_to_id(kwargs['snapshot_id']) + snapshot_id = ec2utils.ec2_snap_id_to_uuid(kwargs['snapshot_id']) snapshot = self.volume_api.get_snapshot(context, snapshot_id) LOG.audit(_("Create volume from snapshot %s"), snapshot_id, context=context) @@ -924,7 +924,7 @@ class CloudController(object): def delete_volume(self, context, volume_id, **kwargs): validate_ec2_id(volume_id) - volume_id = ec2utils.ec2_id_to_id(volume_id) + volume_id = ec2utils.ec2_vol_id_to_uuid(volume_id) try: volume = self.volume_api.get(context, volume_id) @@ -937,7 +937,7 @@ class CloudController(object): def attach_volume(self, context, volume_id, instance_id, device, **kwargs): validate_ec2_id(instance_id) validate_ec2_id(volume_id) - volume_id = ec2utils.ec2_id_to_id(volume_id) + volume_id = ec2utils.ec2_vol_id_to_uuid(volume_id) instance_id = ec2utils.ec2_id_to_id(instance_id) instance = self.compute_api.get(context, instance_id) msg = _("Attach volume %(volume_id)s to instance %(instance_id)s" @@ -960,7 +960,7 @@ class CloudController(object): def detach_volume(self, context, volume_id, **kwargs): validate_ec2_id(volume_id) - volume_id = ec2utils.ec2_id_to_id(volume_id) + volume_id = ec2utils.ec2_vol_id_to_uuid(volume_id) LOG.audit(_("Detach volume %s"), volume_id, context=context) volume = self.volume_api.get(context, volume_id) diff --git a/nova/api/ec2/ec2utils.py b/nova/api/ec2/ec2utils.py index 1a6bb96bdedb..045ee12d376d 100644 --- a/nova/api/ec2/ec2utils.py +++ b/nova/api/ec2/ec2utils.py @@ -18,12 +18,13 @@ import re +from nova import context from nova import db from nova import exception from nova import flags from nova import log as logging -from nova import network from nova.network import model as network_model +from nova import utils FLAGS = flags.FLAGS @@ -132,15 +133,68 @@ def id_to_ec2_id(instance_id, template='i-%08x'): return template % int(instance_id) -def id_to_ec2_snap_id(instance_id): - """Convert an snapshot ID (int) to an ec2 snapshot ID - (snap-[base 16 number])""" - return id_to_ec2_id(instance_id, 'snap-%08x') +def id_to_ec2_snap_id(snapshot_id): + """Get or create an ec2 volume ID (vol-[base 16 number]) from uuid.""" + if utils.is_uuid_like(snapshot_id): + ctxt = context.get_admin_context() + int_id = get_int_id_from_snapshot_uuid(ctxt, snapshot_id) + return id_to_ec2_id(int_id) + else: + return id_to_ec2_id(snapshot_id, 'snap-%08x') -def id_to_ec2_vol_id(instance_id): - """Convert an volume ID (int) to an ec2 volume ID (vol-[base 16 number])""" - return id_to_ec2_id(instance_id, 'vol-%08x') +def id_to_ec2_vol_id(volume_id): + """Get or create an ec2 volume ID (vol-[base 16 number]) from uuid.""" + if utils.is_uuid_like(volume_id): + ctxt = context.get_admin_context() + int_id = get_int_id_from_volume_uuid(ctxt, volume_id) + return id_to_ec2_id(int_id) + else: + return id_to_ec2_id(volume_id, 'vol-%08x') + + +def ec2_vol_id_to_uuid(ec2_id): + """Get the cooresponding UUID for the given ec2-id.""" + ctxt = context.get_admin_context() + + # NOTE(jgriffith) first strip prefix to get just the numeric + int_id = ec2_id_to_id(ec2_id) + return get_volume_uuid_from_int_id(ctxt, int_id) + + +def get_int_id_from_volume_uuid(context, volume_uuid): + if volume_uuid is None: + return + try: + return db.get_ec2_volume_id_by_uuid(context, volume_uuid) + except exception.NotFound: + raise exception.VolumeNotFound() + + +def get_volume_uuid_from_int_id(context, int_id): + return db.get_volume_uuid_by_ec2_id(context, int_id) + + +def ec2_snap_id_to_uuid(ec2_id): + """Get the cooresponding UUID for the given ec2-id.""" + ctxt = context.get_admin_context() + + # NOTE(jgriffith) first strip prefix to get just the numeric + int_id = ec2_id_to_id(ec2_id) + return get_snapshot_uuid_from_int_id(ctxt, int_id) + + +def get_int_id_from_snapshot_uuid(context, snapshot_uuid): + if snapshot_uuid is None: + return + try: + return db.get_ec2_snapshot_id_by_uuid(context, snapshot_uuid) + except exception.NotFound: + raise exception.SnapshotNotFound() + + +def get_snapshot_uuid_from_int_id(context, int_id): + return db.get_snapshot_uuid_by_ec2_id(context, int_id) _c2u = re.compile('(((?<=[a-z])[A-Z])|([A-Z](?![A-Z]|$)))') diff --git a/nova/db/api.py b/nova/db/api.py index 5de921667e5d..d6eb43146098 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -60,10 +60,10 @@ db_opts = [ default='instance-%08x', help='Template string to be used to generate instance names'), cfg.StrOpt('volume_name_template', - default='volume-%08x', + default='volume-%s', help='Template string to be used to generate instance names'), cfg.StrOpt('snapshot_name_template', - default='snapshot-%08x', + default='snapshot-%s', help='Template string to be used to generate snapshot names'), ] @@ -1025,6 +1025,25 @@ def volume_update(context, volume_id, values): return IMPL.volume_update(context, volume_id, values) +def get_ec2_volume_id_by_uuid(context, volume_id): + return IMPL.get_ec2_volume_id_by_uuid(context, volume_id) + + +def get_volume_uuid_by_ec2_id(context, ec2_id): + return IMPL.get_volume_uuid_by_ec2_id(context, ec2_id) + + +def ec2_volume_create(context, volume_id, forced_id=None): + return IMPL.ec2_volume_create(context, volume_id, forced_id) + + +def get_snapshot_uuid_by_ec2_id(context, ec2_id): + return IMPL.get_snapshot_uuid_by_ec2_id(context, ec2_id) + + +def get_ec2_snapshot_id_by_uuid(context, snapshot_id): + return IMPL.get_ec2_snapshot_id_by_uuid(context, snapshot_id) + #################### diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 1d7509aefef3..15a843306158 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -2171,6 +2171,7 @@ def iscsi_target_count_by_host(context, host): @require_admin_context def iscsi_target_create_safe(context, values): iscsi_target_ref = models.IscsiTarget() + for (key, value) in values.iteritems(): iscsi_target_ref[key] = value try: @@ -2409,11 +2410,15 @@ def volume_create(context, values): values['volume_metadata'] = _metadata_refs(values.get('metadata'), models.VolumeMetadata) volume_ref = models.Volume() + if not values.get('id'): + values['id'] = str(utils.gen_uuid()) volume_ref.update(values) session = get_session() with session.begin(): volume_ref.save(session=session) + + ec2_volume_create(context, volume_ref['id']) return volume_ref @@ -2470,6 +2475,18 @@ def _volume_get_query(context, session=None, project_only=False): options(joinedload('volume_type')) +@require_context +def _ec2_volume_get_query(context, session=None, project_only=False): + return model_query(context, models.VolumeIdMapping, session=session, + project_only=project_only) + + +@require_context +def _ec2_snapshot_get_query(context, session=None, project_only=False): + return model_query(context, models.SnapshotIdMapping, session=session, + project_only=project_only) + + @require_context def volume_get(context, volume_id, session=None): result = _volume_get_query(context, session=session, project_only=True).\ @@ -2549,6 +2566,88 @@ def volume_update(context, volume_id, values): volume_ref.save(session=session) +@require_context +def ec2_volume_create(context, volume_uuid, id=None): + """Create ec2 compatable volume by provided uuid""" + ec2_volume_ref = models.VolumeIdMapping() + ec2_volume_ref.update({'uuid': volume_uuid}) + if id is not None: + ec2_volume_ref.update({'id': id}) + + ec2_volume_ref.save() + + return ec2_volume_ref + + +@require_context +def get_ec2_volume_id_by_uuid(context, volume_id, session=None): + result = _ec2_volume_get_query(context, + session=session, + project_only=True).\ + filter_by(uuid=volume_id).\ + first() + + if not result: + raise exception.VolumeNotFound(uuid=volume_id) + + return result['id'] + + +@require_context +def get_volume_uuid_by_ec2_id(context, ec2_id, session=None): + result = _ec2_volume_get_query(context, + session=session, + project_only=True).\ + filter_by(id=ec2_id).\ + first() + + if not result: + raise exception.VolumeNotFound(ec2_id=ec2_id) + + return result['uuid'] + + +@require_context +def ec2_snapshot_create(context, snapshot_uuid, id=None): + """Create ec2 compatable snapshot by provided uuid""" + ec2_snapshot_ref = models.SnapshotIdMapping() + ec2_snapshot_ref.update({'uuid': snapshot_uuid}) + if id is not None: + ec2_snapshot_ref.update({'id': id}) + + ec2_snapshot_ref.save() + + return ec2_snapshot_ref + + +@require_context +def get_ec2_snapshot_id_by_uuid(context, snapshot_id, session=None): + result = _ec2_snapshot_get_query(context, + session=session, + project_only=True).\ + filter_by(uuid=snapshot_id).\ + first() + + if not result: + raise exception.SnapshotNotFound(uuid=snapshot_id) + + return result['id'] + + +@require_context +def get_snapshot_uuid_by_ec2_id(context, ec2_id, session=None): + result = _ec2_snapshot_get_query(context, + session=session, + project_only=True).\ + filter_by(id=ec2_id).\ + first() + + if not result: + raise exception.SnapshotNotFound(ec2_id=ec2_id) + + return result['uuid'] + + #################### def _volume_metadata_get_query(context, volume_id, session=None): @@ -2633,11 +2732,14 @@ def volume_metadata_update(context, volume_id, metadata, delete): @require_context def snapshot_create(context, values): snapshot_ref = models.Snapshot() + if not values.get('id'): + values['id'] = str(utils.gen_uuid()) snapshot_ref.update(values) session = get_session() with session.begin(): snapshot_ref.save(session=session) + ec2_snapshot_create(context, snapshot_ref['id']) return snapshot_ref diff --git a/nova/db/sqlalchemy/migrate_repo/versions/089_add_volume_id_mappings.py b/nova/db/sqlalchemy/migrate_repo/versions/089_add_volume_id_mappings.py new file mode 100644 index 000000000000..d3a705abccd6 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/089_add_volume_id_mappings.py @@ -0,0 +1,116 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import Boolean, Column, DateTime, Integer +from sqlalchemy import MetaData, String, Table +from nova import log as logging +from nova import utils + +LOG = logging.getLogger(__name__) + + +def upgrade(migrate_engine): + """Build mapping tables for our volume uuid migration. + + These mapping tables serve two purposes: + 1. Provide a method for downgrade after UUID conversion + 2. Provide a uuid to associate with existing volumes and snapshots + when we do the actual datatype migration from int to uuid + + """ + meta = MetaData() + meta.bind = migrate_engine + + volume_id_mappings = Table('volume_id_mappings', meta, + Column('created_at', + DateTime(timezone=False)), + Column('updated_at', + DateTime(timezone=False)), + Column('deleted_at', + DateTime(timezone=False)), + Column('deleted', + Boolean(create_constraint=True, name=None)), + Column('id', Integer(), + primary_key=True, + nullable=False, + autoincrement=True), + Column('uuid', String(36), + nullable=False)) + try: + volume_id_mappings.create() + except Exception: + LOG.exception("Exception while creating table 'volume_id_mappings'") + meta.drop_all(tables=[volume_id_mappings]) + raise + + snapshot_id_mappings = Table('snapshot_id_mappings', meta, + Column('created_at', + DateTime(timezone=False)), + Column('updated_at', + DateTime(timezone=False)), + Column('deleted_at', + DateTime(timezone=False)), + Column('deleted', + Boolean(create_constraint=True, name=None)), + Column('id', Integer(), + primary_key=True, + nullable=False, + autoincrement=True), + Column('uuid', String(36), + nullable=False)) + try: + snapshot_id_mappings.create() + except Exception: + LOG.exception("Exception while creating table 'snapshot_id_mappings'") + meta.drop_all(tables=[snapshot_id_mappings]) + raise + + if migrate_engine.name == "mysql": + migrate_engine.execute("ALTER TABLE volume_id_mappings Engine=InnoDB") + migrate_engine.execute("ALTER TABLE snapshot_id_mappings "\ + "Engine=InnoDB") + + volumes = Table('volumes', meta, autoload=True) + snapshots = Table('snapshots', meta, autoload=True) + volume_id_mappings = Table('volume_id_mappings', meta, autoload=True) + snapshot_id_mappings = Table('snapshot_id_mappings', meta, autoload=True) + + volume_list = list(volumes.select().execute()) + for v in volume_list: + old_id = v['id'] + new_id = utils.gen_uuid() + row = volume_id_mappings.insert() + row.execute({'id': old_id, + 'uuid': str(new_id)}) + + snapshot_list = list(snapshots.select().execute()) + for s in snapshot_list: + old_id = s['id'] + new_id = utils.gen_uuid() + row = snapshot_id_mappings.insert() + row.execute({'id': old_id, + 'uuid': str(new_id)}) + + +def downgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + volume_id_mappings = Table('volume_id_mappings', meta, autoload=True) + volume_id_mappings.drop() + + snapshot_id_mappings = Table('snapshot_id_mappings', meta, autoload=True) + snapshot_id_mappings.drop() diff --git a/nova/db/sqlalchemy/migrate_repo/versions/090_modify_volume_id_datatype.py b/nova/db/sqlalchemy/migrate_repo/versions/090_modify_volume_id_datatype.py new file mode 100644 index 000000000000..36a6ba200bae --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/090_modify_volume_id_datatype.py @@ -0,0 +1,236 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import Integer +from sqlalchemy import MetaData, String, Table +from migrate import ForeignKeyConstraint +from nova import log as logging + +LOG = logging.getLogger(__name__) + + +def upgrade(migrate_engine): + """Convert volume and snapshot id columns from int to varchar.""" + meta = MetaData() + meta.bind = migrate_engine + dialect = migrate_engine.url.get_dialect().name + + volumes = Table('volumes', meta, autoload=True) + snapshots = Table('snapshots', meta, autoload=True) + iscsi_targets = Table('iscsi_targets', meta, autoload=True) + volume_metadata = Table('volume_metadata', meta, autoload=True) + sm_volume = Table('sm_volume', meta, autoload=True) + block_device_mapping = Table('block_device_mapping', meta, autoload=True) + + try: + fkeys = list(snapshots.c.volume_id.foreign_keys) + if fkeys: + fkey_name = fkeys[0].constraint.name + ForeignKeyConstraint(columns=[snapshots.c.volume_id], + refcolumns=[volumes.c.id], + name=fkey_name).drop() + + fkeys = list(iscsi_targets.c.volume_id.foreign_keys) + if fkeys: + fkey_name = fkeys[0].constraint.name + ForeignKeyConstraint(columns=[iscsi_targets.c.volume_id], + refcolumns=[volumes.c.id], + name=fkey_name).drop() + + fkeys = list(volume_metadata.c.volume_id.foreign_keys) + if fkeys: + fkey_name = fkeys[0].constraint.name + ForeignKeyConstraint(columns=[volume_metadata.c.volume_id], + refcolumns=[volumes.c.id], + name=fkey_name).drop() + + fkeys = list(sm_volume.c.id.foreign_keys) + if fkeys: + fkey_name = fkeys[0].constraint.name + ForeignKeyConstraint(columns=[sm_volume.c.id], + refcolumns=[volumes.c.id], + name=fkey_name).drop() + + fkeys = list(block_device_mapping.c.volume_id.foreign_keys) + if fkeys: + fkey_name = fkeys[0].constraint.name + ForeignKeyConstraint(columns=[block_device_mapping.c.volume_id], + refcolumns=[volumes.c.id], + name=fkey_name).drop() + + fkeys = list(block_device_mapping.c.snapshot_id.foreign_keys) + if fkeys: + fkey_name = fkeys[0].constraint.name + ForeignKeyConstraint(columns=[block_device_mapping.c.snapshot_id], + refcolumns=[snapshots.c.id], + name=fkey_name).drop() + + except Exception: + LOG.error(_("Foreign Key constraint couldn't be removed")) + raise + + volumes.c.id.alter(String(36), primary_key=True) + volumes.c.snapshot_id.alter(String(36)) + volume_metadata.c.volume_id.alter(String(36), nullable=False) + snapshots.c.id.alter(String(36), primary_key=True) + snapshots.c.volume_id.alter(String(36)) + sm_volume.c.id.alter(String(36)) + block_device_mapping.c.volume_id.alter(String(36)) + block_device_mapping.c.snapshot_id.alter(String(36)) + iscsi_targets.c.volume_id.alter(String(36), nullable=True) + + try: + fkeys = list(snapshots.c.volume_id.foreign_keys) + if fkeys: + fkey_name = fkeys[0].constraint.name + ForeignKeyConstraint(columns=[snapshots.c.volume_id], + refcolumns=[volumes.c.id], + name=fkey_name).create() + + fkeys = list(iscsi_targets.c.volume_id.foreign_keys) + if fkeys: + fkey_name = fkeys[0].constraint.name + ForeignKeyConstraint(columns=[iscsi_targets.c.volume_id], + refcolumns=[volumes.c.id], + name=fkey_name).create() + + fkeys = list(volume_metadata.c.volume_id.foreign_keys) + if fkeys: + fkey_name = fkeys[0].constraint.name + ForeignKeyConstraint(columns=[volume_metadata.c.volume_id], + refcolumns=[volumes.c.id], + name=fkey_name).create() + + fkeys = list(sm_volume.c.id.foreign_keys) + if fkeys: + fkey_name = fkeys[0].constraint.name + ForeignKeyConstraint(columns=[sm_volume.c.id], + refcolumns=[volumes.c.id], + name=fkey_name).create() + # NOTE(jdg) We're intentionally leaving off FK's on BDM + + except Exception: + LOG.error(_("Foreign Key constraint couldn't be removed")) + raise + + +def downgrade(migrate_engine): + """Convert volume and snapshot id columns back to int.""" + meta = MetaData() + meta.bind = migrate_engine + dialect = migrate_engine.url.get_dialect().name + + if dialect.startswith('sqlite'): + return + + volumes = Table('volumes', meta, autoload=True) + snapshots = Table('snapshots', meta, autoload=True) + iscsi_targets = Table('iscsi_targets', meta, autoload=True) + volume_metadata = Table('volume_metadata', meta, autoload=True) + sm_volume = Table('sm_volume', meta, autoload=True) + block_device_mapping = Table('block_device_mapping', meta, autoload=True) + + try: + fkeys = list(snapshots.c.volume_id.foreign_keys) + if fkeys: + fkey_name = fkeys[0].constraint.name + ForeignKeyConstraint(columns=[snapshots.c.volume_id], + refcolumns=[volumes.c.id], + name=fkey_name).drop() + + fkeys = list(iscsi_targets.c.volume_id.foreign_keys) + if fkeys: + fkey_name = fkeys[0].constraint.name + ForeignKeyConstraint(columns=[iscsi_targets.c.volume_id], + refcolumns=[volumes.c.id], + name=fkey_name).drop() + + fkeys = list(volume_metadata.c.volume_id.foreign_keys) + if fkeys: + fkey_name = fkeys[0].constraint.name + ForeignKeyConstraint(columns=[volume_metadata.c.volume_id], + refcolumns=[volumes.c.id], + name=fkey_name).drop() + + fkeys = list(sm_volume.c.id.foreign_keys) + if fkeys: + fkey_name = fkeys[0].constraint.name + ForeignKeyConstraint(columns=[sm_volume.c.id], + refcolumns=[volumes.c.id], + name=fkey_name).drop() + + except Exception: + LOG.error(_("Foreign Key constraint couldn't be removed")) + raise + + volumes.c.id.alter(Integer, primary_key=True, autoincrement=True) + volumes.c.snapshot_id.alter(Integer) + volume_metadata.c.volume_id.alter(Integer, nullable=False) + snapshots.c.id.alter(Integer, primary_key=True, autoincrement=True) + snapshots.c.volume_id.alter(Integer) + sm_volume.c.id.alter(Integer) + block_device_mapping.c.volume_id.alter(Integer) + block_device_mapping.c.snapshot_id.alter(Integer) + iscsi_targets.c.volume_id.alter(Integer, nullable=True) + + try: + fkeys = list(snapshots.c.volume_id.foreign_keys) + if fkeys: + fkey_name = fkeys[0].constraint.name + ForeignKeyConstraint(columns=[snapshots.c.volume_id], + refcolumns=[volumes.c.id], + name=fkey_name).create() + + fkeys = list(iscsi_targets.c.volume_id.foreign_keys) + if fkeys: + fkey_name = fkeys[0].constraint.name + ForeignKeyConstraint(columns=[iscsi_targets.c.volume_id], + refcolumns=[volumes.c.id], + name=fkey_name).create() + + fkeys = list(volume_metadata.c.volume_id.foreign_keys) + if fkeys: + fkey_name = fkeys[0].constraint.name + ForeignKeyConstraint(columns=[volume_metadata.c.volume_id], + refcolumns=[volumes.c.id], + name=fkey_name).create() + + fkeys = list(sm_volume.c.id.foreign_keys) + if fkeys: + fkey_name = fkeys[0].constraint.name + ForeignKeyConstraint(columns=[sm_volume.c.id], + refcolumns=[volumes.c.id], + name=fkey_name).create() + + # NOTE(jdg) Put the BDM foreign keys back in place + fkeys = list(block_device_mapping.c.volume_id.foreign_keys) + if fkeys: + fkey_name = fkeys[0].constraint.name + ForeignKeyConstraint(columns=[block_device_mapping.c.volume_id], + refcolumns=[volumes.c.id], + name=fkey_name).drop() + + fkeys = list(block_device_mapping.c.snapshot_id.foreign_keys) + if fkeys: + fkey_name = fkeys[0].constraint.name + ForeignKeyConstraint(columns=[block_device_mapping.c.snapshot_id], + refcolumns=[snapshots.c.id], + name=fkey_name).drop() + + except Exception: + LOG.error(_("Foreign Key constraint couldn't be removed")) + raise diff --git a/nova/db/sqlalchemy/migrate_repo/versions/090_sqlite_downgrade.sql b/nova/db/sqlalchemy/migrate_repo/versions/090_sqlite_downgrade.sql new file mode 100644 index 000000000000..7d89da247b3f --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/090_sqlite_downgrade.sql @@ -0,0 +1,226 @@ +BEGIN TRANSACTION; + + -- change id and snapshot_id datatypes in volumes table + CREATE TABLE volumes_backup( + created_at DATETIME, + updated_at DATETIME, + deleted_at DATETIME, + deleted BOOLEAN, + id INTEGER NOT NULL, + ec2_id INTEGER, + user_id VARCHAR(255), + project_id VARCHAR(255), + snapshot_id VARCHAR(255), + host VARCHAR(255), + size INTEGER, + availability_zone VARCHAR(255), + instance_id INTEGER, + mountpoint VARCHAR(255), + attach_time VARCHAR(255), + status VARCHAR(255), + attach_status VARCHAR(255), + scheduled_at DATETIME, + launched_at DATETIME, + terminated_at DATETIME, + display_name VARCHAR(255), + display_description VARCHAR(255), + provider_location VARCHAR(255), + provider_auth VARCHAR(255), + volume_type_id INTEGER, + PRIMARY KEY (id), + FOREIGN KEY(instance_id) REFERENCES instances (id), + UNIQUE (id), + CHECK (deleted IN (0, 1)) + ); + + INSERT INTO volumes_backup SELECT + created_at, + updated_at, + deleted_at, + deleted, + id, + ec2_id, + user_id, + project_id, + snapshot_id, + host, + size, + availability_zone, + instance_id, + mountpoint, + attach_time, + status, + attach_status, + scheduled_at, + launched_at, + terminated_at, + display_name, + display_description, + provider_location, + provider_auth, + volume_type_id + FROM volumes; + DROP TABLE volumes; + ALTER TABLE volumes_backup RENAME TO volumes; + + -- change id and volume_id datatypes in snapshots table + CREATE TABLE snapshots_backup ( + created_at DATETIME, + updated_at DATETIME, + deleted_at DATETIME, + deleted BOOLEAN, + id INTEGER NOT NULL, + user_id VARCHAR(255), + project_id VARCHAR(255), + volume_id INTEGER, + status VARCHAR(255), + progress VARCHAR(255), + volume_size INTEGER, + display_name VARCHAR(255), + display_description VARCHAR(255), + PRIMARY KEY (id), + UNIQUE (id), + CHECK (deleted IN (0, 1)) + ); + INSERT INTO snapshots_backup SELECT + created_at, + updated_at, + deleted_at, + deleted, + id, + user_id, + project_id, + volume_id, + status, + progress, + volume_size, + display_name, + display_description + FROM snapshots; + DROP TABLE snapshots; + ALTER TABLE snapshots_backup RENAME TO snapshots; + + -- change id and volume_id datatypes in iscsi_targets table + CREATE TABLE iscsi_targets_backup ( + created_at DATETIME, + updated_at DATETIME, + deleted_at DATETIME, + deleted BOOLEAN, + id INTEGER NOT NULL, + target_num INTEGER, + host VARCHAR(255), + volume_id INTEGER, + PRIMARY KEY (id), + FOREIGN KEY(volume_id) REFERENCES volumes(id), + UNIQUE (id), + CHECK (deleted IN (0, 1)) + ); + INSERT INTO iscsi_targets_backup SELECT + created_at, + updated_at, + deleted_at, + deleted, + id, + target_num, + host, + volume_id + FROM iscsi_targets; + DROP TABLE iscsi_targets; + ALTER TABLE iscsi_targets_backup RENAME TO iscsi_targets; + + CREATE TABLE volume_metadata_backup ( + created_at DATETIME, + updated_at DATETIME, + deleted_at DATETIME, + deleted BOOLEAN, + id INTEGER NOT NULL, + key VARCHAR(255), + value VARCHAR(255), + volume_id INTEGER, + PRIMARY KEY (id), + FOREIGN KEY(volume_id) REFERENCES volumes(id), + UNIQUE (id), + CHECK (deleted IN (0, 1)) + ); + INSERT INTO volume_metadata_backup SELECT + created_at, + updated_at, + deleted_at, + deleted, + id, + key, + value, + volume_id + FROM volume_metadata; + DROP TABLE volume_metadata; + ALTER TABLE volume_metadata_backup RENAME TO volume_metadata; + + -- change volume_id and snapshot_id datatypes in bdm table + CREATE TABLE block_device_mapping_backup ( + created_at DATETIME, + updated_at DATETIME, + deleted_at DATETIME, + deleted BOOLEAN, + id INTEGER NOT NULL, + instance_uuid VARCHAR(36) NOT NULL, + device_name VARCHAR(255), + delete_on_termination BOOLEAN, + virtual_name VARCHAR(255), + snapshot_id INTEGER, + volume_id INTEGER, + volume_size INTEGER, + no_device BOOLEAN, + connection_info VARCHAR(255), + FOREIGN KEY(instance_uuid) REFERENCES instances(id), + FOREIGN KEY(volume_id) REFERENCES volumes(id), + FOREIGN KEY(snapshot_id) REFERENCES snapshots(id), + PRIMARY KEY (id), + UNIQUE (id), + CHECK (deleted IN (0, 1)) + ); + INSERT INTO block_device_mapping_backup SELECT + created_at, + updated_at, + deleted_at, + deleted, + id, + instance_uuid, + device_name, + delete_on_termination, + virtual_name, + snapshot_id, + volume_id, + volume_size, + no_device, + connection_info + FROM block_device_mapping; + DROP TABLE block_device_mapping; + ALTER TABLE block_device_mapping_backup RENAME TO block_device_mapping; + + -- change volume_id and sm_volume_table + CREATE TABLE sm_volume_backup ( + created_at DATETIME, + updated_at DATETIME, + deleted_at DATETIME, + deleted BOOLEAN, + id INTEGER NOT NULL, + backend_id INTEGER NOT NULL, + vdi_uuid VARCHAR(255), + PRIMARY KEY (id), + FOREIGN KEY(id) REFERENCES volumes(id), + UNIQUE (id), + CHECK (deleted IN (0,1)) + ); + INSERT INTO sm_volume_backup SELECT + created_at, + updated_at, + deleted_at, + deleted, + id, + backend_id, + vdi_uuid + FROM sm_volume; + DROP TABLE sm_volume; + ALTER TABLE sm_volume_backup RENAME TO sm_volume; + +COMMIT; diff --git a/nova/db/sqlalchemy/migrate_repo/versions/090_sqlite_upgrade.sql b/nova/db/sqlalchemy/migrate_repo/versions/090_sqlite_upgrade.sql new file mode 100644 index 000000000000..53fbc69f6e63 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/090_sqlite_upgrade.sql @@ -0,0 +1,226 @@ +BEGIN TRANSACTION; + + -- change id and snapshot_id datatypes in volumes table + CREATE TABLE volumes_backup( + created_at DATETIME, + updated_at DATETIME, + deleted_at DATETIME, + deleted BOOLEAN, + id VARCHAR(36) NOT NULL, + ec2_id INTEGER, + user_id VARCHAR(255), + project_id VARCHAR(255), + snapshot_id VARCHAR(36), + host VARCHAR(255), + size INTEGER, + availability_zone VARCHAR(255), + instance_id INTEGER, + mountpoint VARCHAR(255), + attach_time VARCHAR(255), + status VARCHAR(255), + attach_status VARCHAR(255), + scheduled_at DATETIME, + launched_at DATETIME, + terminated_at DATETIME, + display_name VARCHAR(255), + display_description VARCHAR(255), + provider_location VARCHAR(255), + provider_auth VARCHAR(255), + volume_type_id INTEGER, + PRIMARY KEY (id), + FOREIGN KEY(instance_id) REFERENCES instances (id), + UNIQUE (id), + CHECK (deleted IN (0, 1)) + ); + + INSERT INTO volumes_backup SELECT + created_at, + updated_at, + deleted_at, + deleted, + id, + ec2_id, + user_id, + project_id, + snapshot_id, + host, + size, + availability_zone, + instance_id, + mountpoint, + attach_time, + status, + attach_status, + scheduled_at, + launched_at, + terminated_at, + display_name, + display_description, + provider_location, + provider_auth, + volume_type_id + FROM volumes; + DROP TABLE volumes; + ALTER TABLE volumes_backup RENAME TO volumes; + + -- change id and volume_id datatypes in snapshots table + CREATE TABLE snapshots_backup ( + created_at DATETIME, + updated_at DATETIME, + deleted_at DATETIME, + deleted BOOLEAN, + id VARCHAR(36) NOT NULL, + user_id VARCHAR(255), + project_id VARCHAR(255), + volume_id VARCHAR(36), + status VARCHAR(255), + progress VARCHAR(255), + volume_size INTEGER, + display_name VARCHAR(255), + display_description VARCHAR(255), + PRIMARY KEY (id), + UNIQUE (id), + CHECK (deleted IN (0, 1)) + ); + INSERT INTO snapshots_backup SELECT + created_at, + updated_at, + deleted_at, + deleted, + id, + user_id, + project_id, + volume_id, + status, + progress, + volume_size, + display_name, + display_description + FROM snapshots; + DROP TABLE snapshots; + ALTER TABLE snapshots_backup RENAME TO snapshots; + + -- change id and volume_id datatypes in iscsi_targets table + CREATE TABLE iscsi_targets_backup ( + created_at DATETIME, + updated_at DATETIME, + deleted_at DATETIME, + deleted BOOLEAN, + id INTEGER NOT NULL, + target_num INTEGER, + host VARCHAR(255), + volume_id VARCHAR(36), + PRIMARY KEY (id), + FOREIGN KEY(volume_id) REFERENCES volumes(id), + UNIQUE (id), + CHECK (deleted IN (0, 1)) + ); + INSERT INTO iscsi_targets_backup SELECT + created_at, + updated_at, + deleted_at, + deleted, + id, + target_num, + host, + volume_id + FROM iscsi_targets; + DROP TABLE iscsi_targets; + ALTER TABLE iscsi_targets_backup RENAME TO iscsi_targets; + + CREATE TABLE volume_metadata_backup ( + created_at DATETIME, + updated_at DATETIME, + deleted_at DATETIME, + deleted BOOLEAN, + id INTEGER NOT NULL, + key VARCHAR(255), + value VARCHAR(255), + volume_id VARCHAR(36), + PRIMARY KEY (id), + FOREIGN KEY(volume_id) REFERENCES volumes(id), + UNIQUE (id), + CHECK (deleted IN (0, 1)) + ); + INSERT INTO volume_metadata_backup SELECT + created_at, + updated_at, + deleted_at, + deleted, + id, + key, + value, + volume_id + FROM volume_metadata; + DROP TABLE volume_metadata; + ALTER TABLE volume_metadata_backup RENAME TO volume_metadata; + + -- change volume_id and snapshot_id datatypes in bdm table + CREATE TABLE block_device_mapping_backup ( + created_at DATETIME, + updated_at DATETIME, + deleted_at DATETIME, + deleted BOOLEAN, + id INTEGER NOT NULL, + instance_uuid VARCHAR(36) NOT NULL, + device_name VARCHAR(255), + delete_on_termination BOOLEAN, + virtual_name VARCHAR(255), + snapshot_id VARCHAR(36), + volume_id VARCHAR(36), + volume_size INTEGER, + no_device BOOLEAN, + connection_info VARCHAR(255), + FOREIGN KEY(instance_uuid) REFERENCES instances(id), + FOREIGN KEY(volume_id) REFERENCES volumes(id), + FOREIGN KEY(snapshot_id) REFERENCES snapshots(id), + PRIMARY KEY (id), + UNIQUE (id), + CHECK (deleted IN (0, 1)) + ); + INSERT INTO block_device_mapping_backup SELECT + created_at, + updated_at, + deleted_at, + deleted, + id, + instance_uuid, + device_name, + delete_on_termination, + virtual_name, + snapshot_id, + volume_id, + volume_size, + no_device, + connection_info + FROM block_device_mapping; + DROP TABLE block_device_mapping; + ALTER TABLE block_device_mapping_backup RENAME TO block_device_mapping; + + -- change volume_id and sm_volume_table + CREATE TABLE sm_volume_backup ( + created_at DATETIME, + updated_at DATETIME, + deleted_at DATETIME, + deleted BOOLEAN, + id VARCHAR(36) NOT NULL, + backend_id INTEGER NOT NULL, + vdi_uuid VARCHAR(255), + PRIMARY KEY (id), + FOREIGN KEY(id) REFERENCES volumes(id), + UNIQUE (id), + CHECK (deleted IN (0,1)) + ); + INSERT INTO sm_volume_backup SELECT + created_at, + updated_at, + deleted_at, + deleted, + id, + backend_id, + vdi_uuid + FROM sm_volume; + DROP TABLE sm_volume; + ALTER TABLE sm_volume_backup RENAME TO sm_volume; + +COMMIT; diff --git a/nova/db/sqlalchemy/migrate_repo/versions/091_convert_volume_ids_to_uuid.py b/nova/db/sqlalchemy/migrate_repo/versions/091_convert_volume_ids_to_uuid.py new file mode 100644 index 000000000000..5d0440ffef44 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/091_convert_volume_ids_to_uuid.py @@ -0,0 +1,145 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import MetaData, select, Table +from nova import log as logging + +LOG = logging.getLogger(__name__) + + +def upgrade(migrate_engine): + """Convert volume and snapshot id columns from int to varchar.""" + meta = MetaData() + meta.bind = migrate_engine + + volumes = Table('volumes', meta, autoload=True) + snapshots = Table('snapshots', meta, autoload=True) + iscsi_targets = Table('iscsi_targets', meta, autoload=True) + volume_metadata = Table('volume_metadata', meta, autoload=True) + block_device_mapping = Table('block_device_mapping', meta, autoload=True) + sm_volumes = Table('sm_volume', meta, autoload=True) + + volume_mappings = Table('volume_id_mappings', meta, autoload=True) + snapshot_mappings = Table('snapshot_id_mappings', meta, autoload=True) + + volume_list = list(volumes.select().execute()) + for v in volume_list: + new_id = select([volume_mappings.c.uuid], + volume_mappings.c.id == v['id']) + + volumes.update().\ + where(volumes.c.id == v['id']).\ + values(id=new_id).execute() + + sm_volumes.update().\ + where(sm_volumes.c.id == v['id']).\ + values(id=new_id).execute() + + snapshots.update().\ + where(snapshots.c.volume_id == v['id']).\ + values(volume_id=new_id).execute() + + iscsi_targets.update().\ + where(iscsi_targets.c.volume_id == v['id']).\ + values(volume_id=new_id).execute() + + volume_metadata.update().\ + where(volume_metadata.c.volume_id == v['id']).\ + values(volume_id=new_id).execute() + + block_device_mapping.update().\ + where(block_device_mapping.c.volume_id == v['id']).\ + values(volume_id=new_id).execute() + + snapshot_list = list(snapshots.select().execute()) + for s in snapshot_list: + new_id = select([snapshot_mappings.c.uuid], + volume_mappings.c.id == s['id']) + + volumes.update().\ + where(volumes.c.snapshot_id == s['id']).\ + values(snapshot_id=new_id).execute() + + snapshots.update().\ + where(snapshots.c.id == s['id']).\ + values(volume_id=new_id).execute() + + block_device_mapping.update().\ + where(block_device_mapping.c.snapshot_id == s['id']).\ + values(snapshot_id=new_id).execute() + + +def downgrade(migrate_engine): + """Convert volume and snapshot id columns back to int.""" + meta = MetaData() + meta.bind = migrate_engine + + volumes = Table('volumes', meta, autoload=True) + snapshots = Table('snapshots', meta, autoload=True) + iscsi_targets = Table('iscsi_targets', meta, autoload=True) + volume_metadata = Table('volume_metadata', meta, autoload=True) + block_device_mapping = Table('block_device_mapping', meta, autoload=True) + sm_volumes = Table('sm_volume', meta, autoload=True) + + volume_mappings = Table('volume_id_mappings', meta, autoload=True) + snapshot_mappings = Table('snapshot_id_mappings', meta, autoload=True) + + volume_list = list(volumes.select().execute()) + for v in volume_list: + new_id = select([volume_mappings.c.id], + volume_mappings.c.uuid == v['id']) + + volumes.update().\ + where(volumes.c.id == v['id']).\ + values(id=new_id).execute() + + sm_volumes.update().\ + where(sm_volumes.c.id == v['id']).\ + values(id=new_id).execute() + + snapshots.update().\ + where(snapshots.c.volume_id == v['id']).\ + values(volume_id=new_id).execute() + + iscsi_targets.update().\ + where(iscsi_targets.c.volume_id == v['id']).\ + values(volume_id=new_id).execute() + + volume_metadata.update().\ + where(volume_metadata.c.volume_id == v['id']).\ + values(volume_id=new_id).execute() + + block_device_mapping.update().\ + where(block_device_mapping.c.volume_id == v['id']).\ + values(volume_id=new_id).execute() + + snapshot_list = list(snapshots.select().execute()) + for s in snapshot_list: + new_id = select([snapshot_mappings.c.id], + volume_mappings.c.uuid == s['id']) + + volumes.update().\ + where(volumes.c.snapshot_id == s['id']).\ + values(snapshot_id=new_id).execute() + + snapshots.update().\ + where(snapshots.c.id == s['id']).\ + values(volume_id=new_id).execute() + + block_device_mapping.update().\ + where(block_device_mapping.c.snapshot_id == s['id']).\ + values(snapshot_id=new_id).execute() diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 1544629ff011..e4e47c882e80 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -335,16 +335,17 @@ class InstanceTypes(BASE, NovaBase): class Volume(BASE, NovaBase): """Represents a block storage device that can be attached to a vm.""" __tablename__ = 'volumes' - id = Column(Integer, primary_key=True, autoincrement=True) + id = Column(String(36), primary_key=True) @property def name(self): return FLAGS.volume_name_template % self.id + ec2_id = Column(Integer) user_id = Column(String(255)) project_id = Column(String(255)) - snapshot_id = Column(String(255)) + snapshot_id = Column(String(36)) host = Column(String(255)) # , ForeignKey('hosts.id')) size = Column(Integer) @@ -379,7 +380,7 @@ class VolumeMetadata(BASE, NovaBase): id = Column(Integer, primary_key=True) key = Column(String(255)) value = Column(String(255)) - volume_id = Column(Integer, ForeignKey('volumes.id'), nullable=False) + volume_id = Column(String(36), ForeignKey('volumes.id'), nullable=False) volume = relationship(Volume, backref="volume_metadata", foreign_keys=volume_id, primaryjoin='and_(' @@ -455,7 +456,7 @@ class QuotaClass(BASE, NovaBase): class Snapshot(BASE, NovaBase): """Represents a block storage device that can be attached to a vm.""" __tablename__ = 'snapshots' - id = Column(Integer, primary_key=True, autoincrement=True) + id = Column(String(36), primary_key=True) @property def name(self): @@ -468,7 +469,7 @@ class Snapshot(BASE, NovaBase): user_id = Column(String(255)) project_id = Column(String(255)) - volume_id = Column(Integer) + volume_id = Column(String(36)) status = Column(String(255)) progress = Column(String(255)) volume_size = Column(Integer) @@ -504,12 +505,12 @@ class BlockDeviceMapping(BASE, NovaBase): virtual_name = Column(String(255), nullable=True) # for snapshot or volume - snapshot_id = Column(Integer, ForeignKey('snapshots.id'), nullable=True) + snapshot_id = Column(String(36), ForeignKey('snapshots.id')) # outer join snapshot = relationship(Snapshot, foreign_keys=snapshot_id) - volume_id = Column(Integer, ForeignKey('volumes.id'), nullable=True) + volume_id = Column(String(36), ForeignKey('volumes.id'), nullable=True) volume = relationship(Volume, foreign_keys=volume_id) volume_size = Column(Integer, nullable=True) @@ -528,7 +529,7 @@ class IscsiTarget(BASE, NovaBase): id = Column(Integer, primary_key=True) target_num = Column(Integer) host = Column(String(255)) - volume_id = Column(Integer, ForeignKey('volumes.id'), nullable=True) + volume_id = Column(String(36), ForeignKey('volumes.id'), nullable=True) volume = relationship(Volume, backref=backref('iscsi_target', uselist=False), foreign_keys=volume_id, @@ -962,6 +963,20 @@ class S3Image(BASE, NovaBase): uuid = Column(String(36), nullable=False) +class VolumeIdMapping(BASE, NovaBase): + """Compatability layer for the EC2 volume service""" + __tablename__ = 'volume_id_mappings' + id = Column(Integer, primary_key=True, nullable=False, autoincrement=True) + uuid = Column(String(36), nullable=False) + + +class SnapshotIdMapping(BASE, NovaBase): + """Compatability layer for the EC2 snapshot service""" + __tablename__ = 'snapshot_id_mappings' + id = Column(Integer, primary_key=True, nullable=False, autoincrement=True) + uuid = Column(String(36), nullable=False) + + class SMFlavors(BASE, NovaBase): """Represents a flavor for SM volumes.""" __tablename__ = 'sm_flavors' @@ -982,7 +997,7 @@ class SMBackendConf(BASE, NovaBase): class SMVolume(BASE, NovaBase): __tablename__ = 'sm_volume' - id = Column(Integer(), ForeignKey(Volume.id), primary_key=True) + id = Column(String(36), ForeignKey(Volume.id), primary_key=True) backend_id = Column(Integer, ForeignKey('sm_backend_config.id'), nullable=False) vdi_uuid = Column(String(255)) @@ -1040,6 +1055,8 @@ def register_models(): VolumeMetadata, VolumeTypeExtraSpecs, VolumeTypes, + VolumeIdMapping, + SnapshotIdMapping, ) engine = create_engine(FLAGS.sql_connection, echo=False) for model in models: diff --git a/nova/tests/api/ec2/test_cloud.py b/nova/tests/api/ec2/test_cloud.py index 7b6fb34c28fc..52d0a5a6e25a 100644 --- a/nova/tests/api/ec2/test_cloud.py +++ b/nova/tests/api/ec2/test_cloud.py @@ -596,7 +596,8 @@ class CloudTestCase(test.TestCase): volume_id=[volume_id]) self.assertEqual(len(result['volumeSet']), 1) self.assertEqual( - ec2utils.ec2_id_to_id(result['volumeSet'][0]['volumeId']), + ec2utils.ec2_vol_id_to_uuid( + result['volumeSet'][0]['volumeId']), vol2['id']) db.volume_destroy(self.context, vol1['id']) db.volume_destroy(self.context, vol2['id']) @@ -669,7 +670,8 @@ class CloudTestCase(test.TestCase): snapshot_id=[snapshot_id]) self.assertEqual(len(result['snapshotSet']), 1) self.assertEqual( - ec2utils.ec2_id_to_id(result['snapshotSet'][0]['snapshotId']), + ec2utils.ec2_snap_id_to_uuid( + result['snapshotSet'][0]['snapshotId']), snap2['id']) db.snapshot_destroy(self.context, snap1['id']) db.snapshot_destroy(self.context, snap2['id']) @@ -998,6 +1000,7 @@ class CloudTestCase(test.TestCase): db.instance_destroy(self.context, inst2['id']) db.instance_destroy(self.context, inst1['id']) + # NOTE(jdg) Modified expected volume_id's to string _expected_instance_bdm1 = { 'instanceId': 'i-00000001', 'rootDeviceName': '/dev/sdb1', @@ -1007,32 +1010,32 @@ class CloudTestCase(test.TestCase): {'deviceName': '/dev/sdb1', 'ebs': {'status': 'in-use', 'deleteOnTermination': False, - 'volumeId': 2, + 'volumeId': '2', }}, {'deviceName': '/dev/sdb2', 'ebs': {'status': 'in-use', 'deleteOnTermination': False, - 'volumeId': 3, + 'volumeId': '3', }}, {'deviceName': '/dev/sdb3', 'ebs': {'status': 'in-use', 'deleteOnTermination': True, - 'volumeId': 5, + 'volumeId': '5', }}, {'deviceName': '/dev/sdb4', 'ebs': {'status': 'in-use', 'deleteOnTermination': False, - 'volumeId': 7, + 'volumeId': '7', }}, {'deviceName': '/dev/sdb5', 'ebs': {'status': 'in-use', 'deleteOnTermination': False, - 'volumeId': 9, + 'volumeId': '9', }}, {'deviceName': '/dev/sdb6', 'ebs': {'status': 'in-use', 'deleteOnTermination': False, - 'volumeId': 11, }}] + 'volumeId': '11', }}] # NOTE(yamahata): swap/ephemeral device case isn't supported yet. _expected_instance_bdm2 = { @@ -2030,9 +2033,9 @@ class CloudTestCase(test.TestCase): ec2_volume_id = ec2utils.id_to_ec2_vol_id(vol['id']) ec2_snapshot1_id = self._create_snapshot(ec2_volume_id) - snapshot1_id = ec2utils.ec2_id_to_id(ec2_snapshot1_id) + snapshot1_id = ec2utils.ec2_snap_id_to_uuid(ec2_snapshot1_id) ec2_snapshot2_id = self._create_snapshot(ec2_volume_id) - snapshot2_id = ec2utils.ec2_id_to_id(ec2_snapshot2_id) + snapshot2_id = ec2utils.ec2_snap_id_to_uuid(ec2_snapshot2_id) kwargs = {'image_id': 'ami-1', 'instance_type': FLAGS.default_instance_type, diff --git a/nova/tests/api/ec2/test_ec2_validate.py b/nova/tests/api/ec2/test_ec2_validate.py index 3765c9425a06..ae494ccde8c4 100644 --- a/nova/tests/api/ec2/test_ec2_validate.py +++ b/nova/tests/api/ec2/test_ec2_validate.py @@ -107,23 +107,6 @@ class EC2ValidateTestCase(test.TestCase): context=self.context, instance_id=[ec2_id]) - def test_attach_volume(self): - for ec2_id, e in self.ec2_id_exception_map: - self.assertRaises(e, - self.cloud.attach_volume, - context=self.context, - volume_id='i-1234', - instance_id=ec2_id, - device='/dev/vdc') - #missing instance error gets priority - for ec2_id, e in self.ec2_id_exception_map: - self.assertRaises(e, - self.cloud.attach_volume, - context=self.context, - volume_id=ec2_id, - instance_id='i-1234', - device='/dev/vdc') - def test_describe_instance_attribute(self): for ec2_id, e in self.ec2_id_exception_map: self.assertRaises(e, diff --git a/nova/tests/integrated/test_volumes.py b/nova/tests/integrated/test_volumes.py index 8ad2e2bef862..5e61f759f8a9 100644 --- a/nova/tests/integrated/test_volumes.py +++ b/nova/tests/integrated/test_volumes.py @@ -118,29 +118,29 @@ class VolumesTest(integrated_helpers._IntegratedTestBase): create_actions = driver.LoggingVolumeDriver.logs_like( 'create_volume', - id=int(created_volume_id)) + id=created_volume_id) LOG.debug("Create_Actions: %s" % create_actions) self.assertEquals(1, len(create_actions)) create_action = create_actions[0] - self.assertEquals(create_action['id'], int(created_volume_id)) + self.assertEquals(create_action['id'], created_volume_id) self.assertEquals(create_action['availability_zone'], 'nova') self.assertEquals(create_action['size'], 1) export_actions = driver.LoggingVolumeDriver.logs_like( 'create_export', - id=int(created_volume_id)) + id=created_volume_id) self.assertEquals(1, len(export_actions)) export_action = export_actions[0] - self.assertEquals(export_action['id'], int(created_volume_id)) + self.assertEquals(export_action['id'], created_volume_id) self.assertEquals(export_action['availability_zone'], 'nova') delete_actions = driver.LoggingVolumeDriver.logs_like( 'delete_volume', - id=int(created_volume_id)) + id=created_volume_id) self.assertEquals(1, len(delete_actions)) delete_action = export_actions[0] - self.assertEquals(delete_action['id'], int(created_volume_id)) + self.assertEquals(delete_action['id'], created_volume_id) def test_create_volume_with_metadata(self): """Creates a volume with metadata.""" diff --git a/nova/tests/test_bdm.py b/nova/tests/test_bdm.py index eec412d21d41..381ed807012f 100644 --- a/nova/tests/test_bdm.py +++ b/nova/tests/test_bdm.py @@ -20,23 +20,40 @@ Tests for Block Device Mapping Code. """ from nova.api.ec2 import cloud +from nova.api.ec2 import ec2utils from nova import test class BlockDeviceMappingEc2CloudTestCase(test.TestCase): """Test Case for Block Device Mapping""" + def fake_ec2_vol_id_to_uuid(obj, ec2_id): + if ec2_id == 'snap-12345678': + return '00000000-1111-2222-3333-444444444444' + elif ec2_id == 'snap-23456789': + return '11111111-2222-3333-4444-555555555555' + elif ec2_id == 'vol-87654321': + return '22222222-3333-4444-5555-666666666666' + elif ec2_id == 'vol-98765432': + return '77777777-8888-9999-0000-aaaaaaaaaaaa' + else: + return 'OhNoooo' + def _assertApply(self, action, bdm_list): for bdm, expected_result in bdm_list: self.assertDictMatch(action(bdm), expected_result) def test_parse_block_device_mapping(self): + self.stubs.Set(ec2utils, + 'ec2_vol_id_to_uuid', + self.fake_ec2_vol_id_to_uuid) + bdm_list = [ ({'device_name': '/dev/fake0', 'ebs': {'snapshot_id': 'snap-12345678', 'volume_size': 1}}, {'device_name': '/dev/fake0', - 'snapshot_id': 0x12345678, + 'snapshot_id': '00000000-1111-2222-3333-444444444444', 'volume_size': 1, 'delete_on_termination': True}), @@ -44,14 +61,14 @@ class BlockDeviceMappingEc2CloudTestCase(test.TestCase): 'ebs': {'snapshot_id': 'snap-23456789', 'delete_on_termination': False}}, {'device_name': '/dev/fake1', - 'snapshot_id': 0x23456789, + 'snapshot_id': '11111111-2222-3333-4444-555555555555', 'delete_on_termination': False}), ({'device_name': '/dev/fake2', 'ebs': {'snapshot_id': 'vol-87654321', 'volume_size': 2}}, {'device_name': '/dev/fake2', - 'volume_id': 0x87654321, + 'volume_id': '22222222-3333-4444-5555-666666666666', 'volume_size': 2, 'delete_on_termination': True}), @@ -59,7 +76,7 @@ class BlockDeviceMappingEc2CloudTestCase(test.TestCase): 'ebs': {'snapshot_id': 'vol-98765432', 'delete_on_termination': False}}, {'device_name': '/dev/fake3', - 'volume_id': 0x98765432, + 'volume_id': '77777777-8888-9999-0000-aaaaaaaaaaaa', 'delete_on_termination': False}), ({'device_name': '/dev/fake4', diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index ca159c891963..b6b2271689ac 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -1406,9 +1406,9 @@ class ComputeTestCase(BaseTestCase): c = context.get_admin_context() topic = db.queue_get_for(c, FLAGS.compute_topic, inst_ref['host']) + # creating volume testdata - volume_id = 1 - db.volume_create(c, {'id': volume_id}) + volume_id = db.volume_create(c, {'size': 1})['id'] values = {'instance_uuid': inst_ref['uuid'], 'device_name': '/dev/vdc', 'delete_on_termination': False, 'volume_id': volume_id} db.block_device_mapping_create(c, values) @@ -1428,6 +1428,7 @@ class ComputeTestCase(BaseTestCase): 'block_migration': True, 'disk': None} }).AndRaise(rpc.common.RemoteError('', '', '')) + # mocks for rollback rpc.call(c, 'network', {'method': 'setup_networks_on_host', 'args': {'instance_id': inst_ref['id'], @@ -3063,36 +3064,36 @@ class ComputeAPITestCase(BaseTestCase): block_device_mapping = [ # root {'device_name': '/dev/sda1', - 'snapshot_id': 0x12345678, + 'snapshot_id': '00000000-aaaa-bbbb-cccc-000000000000', 'delete_on_termination': False}, # overwrite swap {'device_name': '/dev/sdb2', - 'snapshot_id': 0x23456789, + 'snapshot_id': '11111111-aaaa-bbbb-cccc-111111111111', 'delete_on_termination': False}, {'device_name': '/dev/sdb3', - 'snapshot_id': 0x3456789A}, + 'snapshot_id': '22222222-aaaa-bbbb-cccc-222222222222'}, {'device_name': '/dev/sdb4', 'no_device': True}, # overwrite ephemeral {'device_name': '/dev/sdc2', - 'snapshot_id': 0x456789AB, + 'snapshot_id': '33333333-aaaa-bbbb-cccc-333333333333', 'delete_on_termination': False}, {'device_name': '/dev/sdc3', - 'snapshot_id': 0x56789ABC}, + 'snapshot_id': '44444444-aaaa-bbbb-cccc-444444444444'}, {'device_name': '/dev/sdc4', 'no_device': True}, # volume {'device_name': '/dev/sdd1', - 'snapshot_id': 0x87654321, + 'snapshot_id': '55555555-aaaa-bbbb-cccc-555555555555', 'delete_on_termination': False}, {'device_name': '/dev/sdd2', - 'snapshot_id': 0x98765432}, + 'snapshot_id': '66666666-aaaa-bbbb-cccc-666666666666'}, {'device_name': '/dev/sdd3', - 'snapshot_id': 0xA9875463}, + 'snapshot_id': '77777777-aaaa-bbbb-cccc-777777777777'}, {'device_name': '/dev/sdd4', 'no_device': True}] @@ -3123,22 +3124,30 @@ class ComputeAPITestCase(BaseTestCase): for bdm_ref in db.block_device_mapping_get_all_by_instance( self.context, instance['uuid'])] expected_result = [ - {'snapshot_id': 0x12345678, 'device_name': '/dev/sda1'}, + {'snapshot_id': '00000000-aaaa-bbbb-cccc-000000000000', + 'device_name': '/dev/sda1'}, {'virtual_name': 'swap', 'device_name': '/dev/sdb1', 'volume_size': swap_size}, - {'snapshot_id': 0x23456789, 'device_name': '/dev/sdb2'}, - {'snapshot_id': 0x3456789A, 'device_name': '/dev/sdb3'}, + {'snapshot_id': '11111111-aaaa-bbbb-cccc-111111111111', + 'device_name': '/dev/sdb2'}, + {'snapshot_id': '22222222-aaaa-bbbb-cccc-222222222222', + 'device_name': '/dev/sdb3'}, {'no_device': True, 'device_name': '/dev/sdb4'}, {'virtual_name': 'ephemeral0', 'device_name': '/dev/sdc1'}, - {'snapshot_id': 0x456789AB, 'device_name': '/dev/sdc2'}, - {'snapshot_id': 0x56789ABC, 'device_name': '/dev/sdc3'}, + {'snapshot_id': '33333333-aaaa-bbbb-cccc-333333333333', + 'device_name': '/dev/sdc2'}, + {'snapshot_id': '44444444-aaaa-bbbb-cccc-444444444444', + 'device_name': '/dev/sdc3'}, {'no_device': True, 'device_name': '/dev/sdc4'}, - {'snapshot_id': 0x87654321, 'device_name': '/dev/sdd1'}, - {'snapshot_id': 0x98765432, 'device_name': '/dev/sdd2'}, - {'snapshot_id': 0xA9875463, 'device_name': '/dev/sdd3'}, + {'snapshot_id': '55555555-aaaa-bbbb-cccc-555555555555', + 'device_name': '/dev/sdd1'}, + {'snapshot_id': '66666666-aaaa-bbbb-cccc-666666666666', + 'device_name': '/dev/sdd2'}, + {'snapshot_id': '77777777-aaaa-bbbb-cccc-777777777777', + 'device_name': '/dev/sdd3'}, {'no_device': True, 'device_name': '/dev/sdd4'}] bdms.sort() expected_result.sort() diff --git a/nova/tests/test_volume.py b/nova/tests/test_volume.py index af3ba9f64b9c..ece04f89f945 100644 --- a/nova/tests/test_volume.py +++ b/nova/tests/test_volume.py @@ -67,6 +67,21 @@ class VolumeTestCase(test.TestCase): vol['attach_status'] = "detached" return db.volume_create(context.get_admin_context(), vol) + def test_ec2_uuid_mapping(self): + ec2_vol = db.ec2_volume_create(context.get_admin_context(), + 'aaaaaaaa-bbbb-bbbb-bbbb-aaaaaaaaaaaa', 5) + self.assertEqual(5, ec2_vol['id']) + self.assertEqual('aaaaaaaa-bbbb-bbbb-bbbb-aaaaaaaaaaaa', + db.get_volume_uuid_by_ec2_id(context.get_admin_context(), 5)) + + ec2_vol = db.ec2_volume_create(context.get_admin_context(), + 'aaaaaaaa-bbbb-bbbb-bbbb-aaaaaaaaaaaa', 1) + self.assertEqual(1, ec2_vol['id']) + + ec2_vol = db.ec2_volume_create(context.get_admin_context(), + 'aaaaaaaa-bbbb-bbbb-bbbb-aaaaaaaaazzz') + self.assertEqual(6, ec2_vol['id']) + def test_create_delete_volume(self): """Test volume can be created and deleted.""" volume = self._create_volume()