openstack/nova
Code Issues Proposed changes

Make snapshot_volume_backed use new-world objects

Browse Source 
This patch makes snapshotting of a volume backed instance code path use
new-world block device objects, and thus transitions snapshotting to use
block device mapping v2 data format.

This means that all images that are created by snapshotting volume
backed instances after this change will have new block device mapping
format in their properties. To be able to distinguish between those, an
additional field (bdm_v2) was added to the image properties. This flag
will be taken into account when booting an instance and all necessary
conversions will be done so that both formats are supported. The legacy
block device format will continue to be supported in images, even when
we deprecate it in the API (after v2).

Another noteworthy point is that block device mapping data added to the
image will no longer contain the device name, as it will be dependent on
the configuration when booting a new instance. The mapping will however
keep the boot order.

The code in snapshot will also take care that all of the mappings
present in the image properties are handled and not re-created on
booting an instance from the snapshot image.

Part of the blueprint: icehouse-objects
Part of the blueprint: clean-up-legacy-block-device-mapping

Change-Id: I441cf8f2df0b92a9cc4096e77a90c37a06270eb5
This commit is contained in:
Nikola Dipanov
2013-11-28 14:59:57 +01:00
parent 15e91d7a61
commit 33e3d4c6b9
7 changed files with 249 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
nova/block_device.py
View File

@@ -223,6 +223,14 @@ class BlockDeviceDict(dict):
return legacy_block_device
def get_image_mapping(self):
drop_fields = (set(['connection_info', 'device_name']) |
self._db_only_fields)
mapping_dict = dict(self)
for fld in drop_fields:
mapping_dict.pop(fld, None)
return mapping_dict
def is_safe_for_update(block_device_dict):
"""Determine if passed dict is a safe subset for update.
@@ -252,6 +260,18 @@ def create_image_bdm(image_ref, boot_index=0):
'destination_type': 'local'})
def snapshot_from_bdm(snapshot_id, template):
"""Create a basic volume snapshot BDM from a given template bdm."""
copy_from_template = ['disk_bus', 'device_type', 'boot_index']
snapshot_dict = {'source_type': 'snapshot',
'destination_type': 'volume',
'snapshot_id': snapshot_id}
for key in copy_from_template:
snapshot_dict[key] = template.get(key)
return BlockDeviceDict(snapshot_dict)
def legacy_mapping(block_device_mapping):
"""Transform a list of block devices of an instance back to the
legacy data format.
@@ -277,11 +297,19 @@ def legacy_mapping(block_device_mapping):
def from_legacy_mapping(legacy_block_device_mapping, image_uuid='',
root_device_name=None):
root_device_name=None, no_root=False):
"""Transform a legacy list of block devices to the new data format."""
new_bdms = [BlockDeviceDict.from_legacy(legacy_bdm)
for legacy_bdm in legacy_block_device_mapping]
# NOTE (ndipanov): We will not decide which device is root here - we assume
# that it will be supplied later. This is useful for having the root device
# as part of the image defined mappings that are already in the v2 format.
if no_root:
for bdm in new_bdms:
bdm['boot_index'] = -1
return new_bdms
image_bdm = None
volume_backed = False

91
nova/compute/api.py
View File

@@ -626,15 +626,40 @@ class API(base.Base):
# Get the block device mappings defined by the image.
image_defined_bdms = \
image_meta.get('properties', {}).get('block_device_mapping', [])
legacy_image_defined = not image_meta.get(
'properties', {}).get('bdm_v2', False)
if not legacy_image_defined:
image_defined_bdms = map(block_device.BlockDeviceDict,
image_defined_bdms)
if legacy_bdm:
if legacy_image_defined:
block_device_mapping += image_defined_bdms
block_device_mapping = block_device.from_legacy_mapping(
block_device_mapping, image_ref, root_device_name)
else:
root_in_image_bdms = block_device.get_root_bdm(
image_defined_bdms) is not None
block_device_mapping = block_device.from_legacy_mapping(
block_device_mapping, image_ref, root_device_name,
no_root=root_in_image_bdms) + image_defined_bdms
else:
# NOTE (ndipanov): client will insert an image mapping into the v2
# block_device_mapping, but if there is a bootable device in image
# mappings - we need to get rid of the inserted image.
if legacy_image_defined:
image_defined_bdms = block_device.from_legacy_mapping(
image_defined_bdms, None, root_device_name)
root_in_image_bdms = block_device.get_root_bdm(
image_defined_bdms) is not None
if image_ref and root_in_image_bdms:
block_device_mapping = [bdm for bdm in block_device_mapping
if not (
bdm.get('source_type') == 'image'
and bdm.get('boot_index') == 0)]
block_device_mapping += image_defined_bdms
block_device_mapping = block_device.from_legacy_mapping(
block_device_mapping, image_ref, root_device_name)
elif image_defined_bdms:
# NOTE (ndipanov): For now assume that image mapping is legacy
block_device_mapping += block_device.from_legacy_mapping(
image_defined_bdms, None, root_device_name)
if min_count > 1 or max_count > 1:
if any(map(lambda bdm: bdm['source_type'] == 'volume',
@@ -1963,57 +1988,41 @@ class API(base.Base):
properties['root_device_name'] = instance['root_device_name']
properties.update(extra_properties or {})
# TODO(xqueralt): Use new style BDM in volume snapshots
bdms = self.get_instance_bdms(context, instance)
bdms = block_device_obj.BlockDeviceMappingList.get_by_instance_uuid(
context, instance['uuid'])
mapping = []
for bdm in bdms:
if bdm['no_device']:
if bdm.no_device:
continue
# Clean the BDM of the database related fields to prevent
# duplicates in the future (e.g. the id was being preserved)
for field in block_device.BlockDeviceDict._db_only_fields:
bdm.pop(field, None)
volume_id = bdm.get('volume_id')
if volume_id:
if bdm.is_volume:
# create snapshot based on volume_id
volume = self.volume_api.get(context, volume_id)
volume = self.volume_api.get(context, bdm.volume_id)
# NOTE(yamahata): Should we wait for snapshot creation?
# Linux LVM snapshot creation completes in
# short time, it doesn't matter for now.
name = _('snapshot for %s') % image_meta['name']
snapshot = self.volume_api.create_snapshot_force(
context, volume['id'], name, volume['display_description'])
bdm['snapshot_id'] = snapshot['id']
mapping_dict = block_device.snapshot_from_bdm(snapshot['id'],
bdm)
mapping_dict = mapping_dict.get_image_mapping()
else:
mapping_dict = bdm.get_image_mapping()
# Clean the extra volume related fields that will be generated
# when booting from the new snapshot.
bdm.pop('volume_id')
bdm.pop('connection_info')
mapping.append(bdm)
for m in block_device.mappings_prepend_dev(properties.get('mappings',
[])):
virtual_name = m['virtual']
if virtual_name in ('ami', 'root'):
continue
assert block_device.is_swap_or_ephemeral(virtual_name)
device_name = m['device']
if device_name in [b['device_name'] for b in mapping
if not b.get('no_device', False)]:
continue
# NOTE(yamahata): swap and ephemeral devices are specified in
# AMI, but disabled for this instance by user.
# So disable those device by no_device.
mapping.append({'device_name': device_name, 'no_device': True})
mapping.append(mapping_dict)
# NOTE (ndipanov): Remove swap/ephemerals from mappings as they will be
# in the block_device_mapping for the new image.
image_mappings = properties.get('mappings')
if image_mappings:
properties['mappings'] = [m for m in image_mappings
if not block_device.is_swap_or_ephemeral(
m['virtual'])]
if mapping:
properties['block_device_mapping'] = mapping
properties['bdm_v2'] = True
for attr in ('status', 'location', 'id'):
image_meta.pop(attr, None)

8
nova/tests/api/openstack/compute/plugins/v3/test_server_actions.py
View File

@@ -900,10 +900,16 @@ class ServerActionsControllerTest(test.TestCase):
self.assertEqual(properties['kernel_id'], _fake_id('b'))
self.assertEqual(properties['ramdisk_id'], _fake_id('c'))
self.assertEqual(properties['root_device_name'], '/dev/vda')
self.assertEqual(properties['bdm_v2'], True)
bdms = properties['block_device_mapping']
self.assertEqual(len(bdms), 1)
self.assertEqual(bdms[0]['device_name'], 'vda')
self.assertEqual(bdms[0]['boot_index'], 0)
self.assertEqual(bdms[0]['source_type'], 'snapshot')
self.assertEqual(bdms[0]['destination_type'], 'volume')
self.assertEqual(bdms[0]['snapshot_id'], snapshot['id'])
for fld in ('connection_info', 'id',
'instance_uuid', 'device_name'):
self.assertTrue(fld not in bdms[0])
for k in extra_properties.keys():
self.assertEqual(properties[k], extra_properties[k])

8
nova/tests/api/openstack/compute/test_server_actions.py
View File

@@ -1066,10 +1066,16 @@ class ServerActionsControllerTest(test.TestCase):
self.assertEqual(properties['kernel_id'], _fake_id('b'))
self.assertEqual(properties['ramdisk_id'], _fake_id('c'))
self.assertEqual(properties['root_device_name'], '/dev/vda')
self.assertEqual(properties['bdm_v2'], True)
bdms = properties['block_device_mapping']
self.assertEqual(len(bdms), 1)
self.assertEqual(bdms[0]['device_name'], 'vda')
self.assertEqual(bdms[0]['boot_index'], 0)
self.assertEqual(bdms[0]['source_type'], 'snapshot')
self.assertEqual(bdms[0]['destination_type'], 'volume')
self.assertEqual(bdms[0]['snapshot_id'], snapshot['id'])
for fld in ('connection_info', 'id',
'instance_uuid', 'device_name'):
self.assertTrue(fld not in bdms[0])
for k in extra_properties.keys():
self.assertEqual(properties[k], extra_properties[k])

101
nova/tests/compute/test_compute.py
View File

@@ -7560,7 +7560,106 @@ class ComputeAPITestCase(BaseTestCase):
self.compute.terminate_instance(self.context,
self._objectify(instance), [], [])
def test_check_and_transform_bdm(self):
def _test_check_and_transform_bdm(self, bdms, expected_bdms,
image_bdms=None, base_options=None,
legacy_bdms=False,
legacy_image_bdms=False):
image_bdms = image_bdms or []
image_meta = {}
if image_bdms:
image_meta = {'properties': {'block_device_mapping': image_bdms}}
if not legacy_image_bdms:
image_meta['properties']['bdm_v2'] = True
base_options = base_options or {'root_device_name': 'vda',
'image_ref': FAKE_IMAGE_REF}
transformed_bdm = self.compute_api._check_and_transform_bdm(
base_options, image_meta, 1, 1, bdms, legacy_bdms)
self.assertThat(expected_bdms,
matchers.DictListMatches(transformed_bdm))
def test_check_and_transform_legacy_bdm_no_image_bdms(self):
legacy_bdms = [
{'device_name': '/dev/vda',
'volume_id': '33333333-aaaa-bbbb-cccc-333333333333',
'delete_on_termination': False}]
expected_bdms = [block_device.BlockDeviceDict.from_legacy(
legacy_bdms[0])]
expected_bdms[0]['boot_index'] = 0
self._test_check_and_transform_bdm(legacy_bdms, expected_bdms,
legacy_bdms=True)
def test_check_and_transform_legacy_bdm_legacy_image_bdms(self):
image_bdms = [
{'device_name': '/dev/vda',
'volume_id': '33333333-aaaa-bbbb-cccc-333333333333',
'delete_on_termination': False}]
legacy_bdms = [
{'device_name': '/dev/vdb',
'volume_id': '33333333-aaaa-bbbb-cccc-444444444444',
'delete_on_termination': False}]
expected_bdms = [
block_device.BlockDeviceDict.from_legacy(legacy_bdms[0]),
block_device.BlockDeviceDict.from_legacy(image_bdms[0])]
expected_bdms[0]['boot_index'] = -1
expected_bdms[1]['boot_index'] = 0
self._test_check_and_transform_bdm(legacy_bdms, expected_bdms,
image_bdms=image_bdms,
legacy_bdms=True,
legacy_image_bdms=True)
def test_check_and_transform_legacy_bdm_image_bdms(self):
legacy_bdms = [
{'device_name': '/dev/vdb',
'volume_id': '33333333-aaaa-bbbb-cccc-444444444444',
'delete_on_termination': False}]
image_bdms = [block_device.BlockDeviceDict(
{'source_type': 'volume', 'destination_type': 'volume',
'volume_id': '33333333-aaaa-bbbb-cccc-444444444444',
'boot_index': 0})]
expected_bdms = [
block_device.BlockDeviceDict.from_legacy(legacy_bdms[0]),
image_bdms[0]]
expected_bdms[0]['boot_index'] = -1
self._test_check_and_transform_bdm(legacy_bdms, expected_bdms,
image_bdms=image_bdms,
legacy_bdms=True)
def test_check_and_transform_bdm_no_image_bdms(self):
bdms = [block_device.BlockDeviceDict({'source_type': 'image',
'destination_type': 'local',
'image_id': FAKE_IMAGE_REF,
'boot_index': 0})]
expected_bdms = bdms
self._test_check_and_transform_bdm(bdms, expected_bdms)
def test_check_and_transform_bdm_image_bdms(self):
bdms = [block_device.BlockDeviceDict({'source_type': 'image',
'destination_type': 'local',
'image_id': FAKE_IMAGE_REF,
'boot_index': 0})]
image_bdms = [block_device.BlockDeviceDict(
{'source_type': 'volume', 'destination_type': 'volume',
'volume_id': '33333333-aaaa-bbbb-cccc-444444444444'})]
expected_bdms = bdms + image_bdms
self._test_check_and_transform_bdm(bdms, expected_bdms,
image_bdms=image_bdms)
def test_check_and_transform_bdm_legacy_image_bdms(self):
bdms = [block_device.BlockDeviceDict({'source_type': 'image',
'destination_type': 'local',
'image_id': FAKE_IMAGE_REF,
'boot_index': 0})]
image_bdms = [{'device_name': '/dev/vda',
'volume_id': '33333333-aaaa-bbbb-cccc-333333333333',
'delete_on_termination': False}]
expected_bdms = [block_device.BlockDeviceDict.from_legacy(
image_bdms[0])]
expected_bdms[0]['boot_index'] = 0
self._test_check_and_transform_bdm(bdms, expected_bdms,
image_bdms=image_bdms,
legacy_image_bdms=True)
def test_check_and_transform_image(self):
base_options = {'root_device_name': 'vdb',
'image_ref': FAKE_IMAGE_REF}
fake_legacy_bdms = [

37
nova/tests/compute/test_compute_api.py
View File

@@ -18,7 +18,6 @@ import datetime
import iso8601
import mox
from nova import block_device
from nova.compute import api as compute_api
from nova.compute import cells_api as compute_cells_api
from nova.compute import flavors
@@ -1449,12 +1448,13 @@ class _ComputeAPIUnitTestMixIn(object):
expect_meta = {
'name': 'test-snapshot',
'properties': {'root_device_name': 'vda', 'mappings': 'DONTCARE'},
'properties': {'root_device_name': 'vda',
'mappings': 'DONTCARE'},
'size': 0,
'is_public': False
}
def fake_get_instance_bdms(context, instance):
def fake_get_all_by_instance(context, instance):
return copy.deepcopy(instance_bdms)
def fake_image_create(context, image_meta, data):
@@ -1466,8 +1466,8 @@ class _ComputeAPIUnitTestMixIn(object):
def fake_volume_create_snapshot(context, volume_id, name, description):
return {'id': '%s-snapshot' % volume_id}
self.stubs.Set(self.compute_api, 'get_instance_bdms',
fake_get_instance_bdms)
self.stubs.Set(db, 'block_device_mapping_get_all_by_instance',
fake_get_all_by_instance)
self.stubs.Set(self.compute_api.image_service, 'create',
fake_image_create)
self.stubs.Set(self.compute_api.volume_api, 'get',
@@ -1479,33 +1479,34 @@ class _ComputeAPIUnitTestMixIn(object):
self.compute_api.snapshot_volume_backed(
self.context, instance, copy.deepcopy(image_meta), 'test-snapshot')
bdm = {'no_device': False, 'volume_id': '1',
'connection_info': 'inf', 'device_name': '/dev/vda'}
for key in block_device.BlockDeviceDict._db_only_fields:
bdm[key] = 'MUST DELETE'
bdm = fake_block_device.FakeDbBlockDeviceDict(
{'no_device': False, 'volume_id': '1', 'boot_index': 0,
'connection_info': 'inf', 'device_name': '/dev/vda',
'source_type': 'volume', 'destination_type': 'volume'})
instance_bdms.append(bdm)
expect_meta['properties']['bdm_v2'] = True
expect_meta['properties']['block_device_mapping'] = []
expect_meta['properties']['block_device_mapping'].append(
{'no_device': False, 'snapshot_id': '1-snapshot',
'device_name': '/dev/vda'})
{'guest_format': None, 'boot_index': 0, 'no_device': None,
'image_id': None, 'volume_id': None, 'disk_bus': None,
'volume_size': None, 'source_type': 'snapshot',
'device_type': None, 'snapshot_id': '1-snapshot',
'destination_type': 'volume', 'delete_on_termination': None})
# All the db_only fields and the volume ones are removed
self.compute_api.snapshot_volume_backed(
self.context, instance, copy.deepcopy(image_meta), 'test-snapshot')
image_mappings = [{'device': 'vda', 'virtual': 'ephemeral0'},
image_mappings = [{'virtual': 'ami', 'device': 'vda'},
{'device': 'vda', 'virtual': 'ephemeral0'},
{'device': 'vdb', 'virtual': 'swap'},
{'device': 'vdc', 'virtual': 'ephemeral1'}]
image_meta['properties']['mappings'] = image_mappings
expect_meta['properties']['block_device_mapping'].extend([
{'no_device': True, 'device_name': '/dev/vdb'},
{'no_device': True, 'device_name': '/dev/vdc'}])
expect_meta['properties']['mappings'] = [
{'virtual': 'ami', 'device': 'vda'}]
# Check that the mappgins from the image properties are included
self.compute_api.snapshot_volume_backed(

37
nova/tests/test_block_device.py
View File

@@ -21,7 +21,9 @@ Tests for Block Device utility functions.
from nova import block_device
from nova import exception
from nova.objects import block_device as block_device_obj
from nova import test
from nova.tests import fake_block_device
from nova.tests import matchers
@@ -383,6 +385,11 @@ class TestBlockDeviceDict(test.NoDBTestCase):
self.assertEqual(boot_bdms[0]['boot_index'], 0)
self.assertEqual(boot_bdms[0]['source_type'], 'volume')
new_no_root = block_device.from_legacy_mapping(
self.legacy_mapping, 'fake_image_ref', 'sda1', no_root=True)
self.assertEqual(len(_get_image_bdms(new_no_root)), 0)
self.assertEqual(len(_get_bootable_bdms(new_no_root)), 0)
def test_from_api(self):
for api, new in zip(self.api_mapping, self.new_mapping):
new['connection_info'] = None
@@ -417,3 +424,33 @@ class TestBlockDeviceDict(test.NoDBTestCase):
for legacy, expected in zip(got_legacy, self.legacy_mapping):
self.assertThat(expected, matchers.IsSubDictOf(legacy))
def test_image_mapping(self):
removed_fields = ['id', 'instance_uuid', 'connection_info',
'device_name', 'created_at', 'updated_at',
'deleted_at', 'deleted']
for bdm in self.new_mapping:
mapping_bdm = fake_block_device.FakeDbBlockDeviceDict(
bdm).get_image_mapping()
for fld in removed_fields:
self.assertTrue(fld not in mapping_bdm)
def _test_snapshot_from_bdm(self, template):
snapshot = block_device.snapshot_from_bdm('new-snapshot-id', template)
self.assertEqual(snapshot['snapshot_id'], 'new-snapshot-id')
self.assertEqual(snapshot['source_type'], 'snapshot')
self.assertEqual(snapshot['destination_type'], 'volume')
for key in ['disk_bus', 'device_type', 'boot_index']:
self.assertEqual(snapshot[key], template[key])
def test_snapshot_from_bdm(self):
for bdm in self.new_mapping:
self._test_snapshot_from_bdm(bdm)
def test_snapshot_from_object(self):
for bdm in self.new_mapping[:-1]:
obj = block_device_obj.BlockDeviceMapping()
obj = block_device_obj.BlockDeviceMapping._from_db_object(
None, obj, fake_block_device.FakeDbBlockDeviceDict(
bdm))
self._test_snapshot_from_bdm(obj)