openstack/ec2-api
Code Issues Proposed changes

Support bdm v2 for describe image operations

Browse Source 
Currently image describe operation doesn't support some bdm v2 related
features of block device mappings. Like virtual (ephemeral and swap)
devices in block_device_mapping property, image source for a volume and
so on.

This patch uses previously introduced get_os_image_mappings to get a
list of image mappings in bdm v2 format and formats it.

Also rootDeviceType image attribute is always placed to the image.
deleteOnTermination attribute is always displayed as well.

Change-Id: Ie96b0dbf69926ff54b2581ad5cb6fe2636af2717
This commit is contained in:
Feodor Tersin
2015-07-16 11:40:33 +03:00
parent 9be22b8ac8
commit 5c562dcbb2
3 changed files with 237 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

191
ec2api/api/image.py
View File

@@ -17,7 +17,6 @@ import binascii
import itertools
import json
import os
import re
import shutil
import tarfile
import tempfile
@@ -92,6 +91,7 @@ CONTAINER_TO_KIND = {'aki': 'aki',
IMAGE_TYPES = {'aki': 'kernel',
'ari': 'ramdisk',
'ami': 'machine'}
EPHEMERAL_PREFIX_LEN = len('ephemeral')
# TODO(yamahata): race condition
@@ -328,7 +328,10 @@ def describe_images(context, executable_by=None, image_id=None,
def describe_image_attribute(context, image_id, attribute):
def _block_device_mapping_attribute(os_image, image, result):
_cloud_format_mappings(context, os_image.properties, result)
properties = ec2utils.deserialize_os_image_properties(os_image)
mappings = _format_mappings(context, properties)
if mappings:
result['blockDeviceMapping'] = mappings
def _description_attribute(os_image, image, result):
result['description'] = {'value': image.get('description')}
@@ -354,8 +357,9 @@ def describe_image_attribute(context, image_id, attribute):
# NOTE(ft): Openstack extension, AWS-incompability
def _root_device_name_attribute(os_image, image, result):
properties = ec2utils.deserialize_os_image_properties(os_image)
result['rootDeviceName'] = (
_block_device_properties_root_device_name(os_image.properties))
_block_device_properties_root_device_name(properties))
supported_attributes = {
'blockDeviceMapping': _block_device_mapping_attribute,
@@ -373,7 +377,6 @@ def describe_image_attribute(context, image_id, attribute):
os_image = ec2utils.get_os_image(context, image_id)
_check_owner(context, os_image)
_prepare_mappings(os_image)
image = ec2utils.get_db_item(context, image_id)
result = {'imageId': image_id}
@@ -423,8 +426,9 @@ def modify_image_attribute(context, image_id, attribute=None,
attribute = 'productCodes'
if attribute in ['kernel', 'ramdisk', 'productCodes',
'blockDeviceMapping']:
raise exception.InvalidParameter(_('Parameter %s is invalid. '
'The attribute is not supported.') % attribute)
raise exception.InvalidParameter(
_('Parameter %s is invalid. '
'The attribute is not supported.') % attribute)
raise exception.InvalidParameterCombination('No attributes specified.')
if len(attributes) > 1:
raise exception.InvalidParameterCombination(
@@ -523,39 +527,110 @@ def _format_image(context, image, os_image, images_dict, ids_dict,
else:
ec2_image['name'] = name
_prepare_mappings(os_image)
properties = os_image.properties
properties = ec2utils.deserialize_os_image_properties(os_image)
root_device_name = _block_device_properties_root_device_name(properties)
mappings = _format_mappings(context, properties, root_device_name,
snapshot_ids, os_image.owner)
if mappings:
ec2_image['blockDeviceMapping'] = mappings
root_device_type = 'instance-store'
if root_device_name:
ec2_image['rootDeviceName'] = root_device_name
root_device_type = 'instance-store'
short_root_device_name = ec2utils.block_device_strip_dev(
root_device_name)
for bdm in properties.get('block_device_mapping', []):
if (('snapshot_id' in bdm or 'volume_id' in bdm) and
not bdm.get('no_device') and
(bdm.get('boot_index') == 0 or
short_root_device_name ==
ec2utils.block_device_strip_dev(
bdm.get('device_name')))):
root_device_type = 'ebs'
break
ec2_image['rootDeviceType'] = root_device_type
_cloud_format_mappings(context, properties, ec2_image,
root_device_name, snapshot_ids, os_image.owner)
if any((short_root_device_name ==
ec2utils.block_device_strip_dev(bdm.get('deviceName')))
for bdm in mappings):
root_device_type = 'ebs'
ec2_image['rootDeviceType'] = root_device_type
return ec2_image
def _prepare_mappings(os_image):
def prepare_property(property_name):
if property_name in os_image.properties:
os_image.properties[property_name] = json.loads(
os_image.properties[property_name])
prepare_property('mappings')
prepare_property('block_device_mapping')
def _format_mappings(context, os_image_properties, root_device_name=None,
snapshot_ids=None, project_id=None):
formatted_mappings = []
bdms = ec2utils.get_os_image_mappings(os_image_properties)
ephemeral_numbers = _ephemeral_free_number_generator(bdms)
for bdm in bdms:
# NOTE(yamahata): trim ebs.no_device == true. Is this necessary?
# TODO(ft): figure out AWS and Nova behaviors
if bdm.get('no_device'):
continue
item = {}
if bdm.get('boot_index') == 0 and root_device_name:
item['deviceName'] = root_device_name
elif 'device_name' in bdm:
item['deviceName'] = bdm['device_name']
if bdm.get('destination_type') == 'volume':
ebs = _format_volume_mapping(
context, bdm, snapshot_ids=snapshot_ids, project_id=project_id)
if not ebs:
# TODO(ft): what to do with the wrong bdm?
continue
item['ebs'] = ebs
elif bdm.get('destination_type') == 'local':
virtual_name = _format_virtual_name(bdm, ephemeral_numbers)
if not virtual_name:
# TODO(ft): what to do with the wrong bdm?
continue
item['virtualName'] = virtual_name
else:
# TODO(ft): what to do with the wrong bdm?
continue
formatted_mappings.append(item)
return formatted_mappings
def _format_volume_mapping(context, bdm, snapshot_ids=None, project_id=None):
ebs = {'deleteOnTermination': bdm['delete_on_termination']}
# TODO(ft): set default volumeSize from the source
if bdm.get('volume_size') is not None:
ebs['volumeSize'] = bdm['volume_size']
if bdm.get('source_type') == 'snapshot':
if bdm.get('snapshot_id'):
ebs['snapshotId'] = ec2utils.os_id_to_ec2_id(
context, 'snap', bdm['snapshot_id'],
ids_by_os_id=snapshot_ids, project_id=project_id)
# NOTE(ft): Openstack extension, AWS-incompability
elif bdm.get('source_type') == 'volume':
if bdm.get('volume_id'):
ebs['snapshotId'] = ec2utils.os_id_to_ec2_id(
context, 'vol', bdm['volume_id'], project_id=project_id)
# NOTE(ft): extension, AWS-incompability
elif bdm.get('source_type') == 'image':
if bdm.get('image_id'):
ebs['snapshotId'] = ec2utils.os_id_to_ec2_id(
context, 'ami', bdm['image_id'], project_id=project_id)
if ebs.get('snapshotId') or bdm.get('source_type') == 'blank':
return ebs
def _format_virtual_name(bdm, ephemeral_numbers):
if bdm.get('source_type') == 'blank':
if bdm.get('guest_format') == 'swap':
return 'swap'
else:
return (bdm.get('virtual_name') or
'ephemeral%s' % next(ephemeral_numbers))
def _ephemeral_free_number_generator(bdms):
named_ephemeral_nums = set(
int(bdm['virtual_name'][EPHEMERAL_PREFIX_LEN:])
for bdm in bdms
if (bdm.get('destination_type') == 'local' and
bdm.get('source_type') == 'blank' and
bdm.get('guest_format') != 'swap' and
bdm.get('virtual_name')))
ephemeral_free_num = 0
while True:
if ephemeral_free_num not in named_ephemeral_nums:
yield ephemeral_free_num
ephemeral_free_num += 1
def _get_os_image_kind(os_image):
@@ -576,64 +651,6 @@ ec2utils.register_auto_create_db_item_extension(
# NOTE(ft): following functions are copied from various parts of Nova
_ephemeral = re.compile('^ephemeral(\d|[1-9]\d+)$')
def _cloud_format_mappings(context, properties, result, root_device_name=None,
snapshot_ids=None, project_id=None):
"""Format multiple BlockDeviceMappingItemType."""
mappings = [
{'virtualName': m['virtual'],
'deviceName': ec2utils.block_device_prepend_dev(m['device'])}
for m in properties.get('mappings', [])
if (m['virtual'] and
(m['virtual'] == 'swap' or _ephemeral.match(m['virtual'])))]
for bdm in properties.get('block_device_mapping', []):
formatted_bdm = _cloud_format_block_device_mapping(
context, bdm, root_device_name, snapshot_ids, project_id)
# NOTE(yamahata): overwrite mappings with block_device_mapping
for i in range(len(mappings)):
if (formatted_bdm.get('deviceName')
== mappings[i].get('deviceName')):
del mappings[i]
break
mappings.append(formatted_bdm)
# NOTE(yamahata): trim ebs.no_device == true. Is this necessary?
mappings = [bdm for bdm in mappings if not (bdm.get('noDevice', False))]
if mappings:
result['blockDeviceMapping'] = mappings
def _cloud_format_block_device_mapping(context, bdm, root_device_name=None,
snapshot_ids=None, project_id=None):
"""Construct BlockDeviceMappingItemType."""
keys = (('deviceName', 'device_name'),
('virtualName', 'virtual_name'))
item = {name: bdm[k] for name, k in keys if k in bdm}
if bdm.get('no_device'):
item['noDevice'] = True
if bdm.get('boot_index') == 0 and root_device_name:
item['deviceName'] = root_device_name
if ('snapshot_id' in bdm) or ('volume_id' in bdm):
ebs_keys = (('volumeSize', 'volume_size'),
('deleteOnTermination', 'delete_on_termination'))
ebs = {name: bdm[k] for name, k in ebs_keys if bdm.get(k) is not None}
if bdm.get('snapshot_id'):
ebs['snapshotId'] = ec2utils.os_id_to_ec2_id(
context, 'snap', bdm['snapshot_id'], ids_by_os_id=snapshot_ids,
project_id=project_id)
# NOTE(ft): Openstack extension, AWS-incompability
elif bdm.get('volume_id'):
ebs['snapshotId'] = ec2utils.os_id_to_ec2_id(
context, 'vol', bdm['volume_id'], project_id=project_id)
assert 'snapshotId' in ebs
item['ebs'] = ebs
return item
def _block_device_properties_root_device_name(properties):
"""get root device name from image meta data.

18
ec2api/tests/unit/fakes.py
View File

@@ -1284,17 +1284,21 @@ EC2_IMAGE_1 = {
'virtualName': 'ephemeral0'},
{'deviceName': '/dev/sdb1',
'ebs': {'snapshotId': ID_EC2_SNAPSHOT_1,
'volumeSize': 22}},
'volumeSize': 22,
'deleteOnTermination': False}},
{'deviceName': '/dev/sdb2',
'ebs': {'snapshotId': ID_EC2_VOLUME_1}},
'ebs': {'snapshotId': ID_EC2_VOLUME_1,
'deleteOnTermination': False}},
{'deviceName': '/dev/sdb3',
'virtualName': 'ephemeral5'},
{'deviceName': '/dev/sdc0',
'virtualName': 'swap'},
{'deviceName': '/dev/sdc1',
'ebs': {'snapshotId': ID_EC2_SNAPSHOT_2}},
'ebs': {'snapshotId': ID_EC2_SNAPSHOT_2,
'deleteOnTermination': False}},
{'deviceName': '/dev/sdc2',
'ebs': {'snapshotId': ID_EC2_VOLUME_2}},
'ebs': {'snapshotId': ID_EC2_VOLUME_2,
'deleteOnTermination': False}},
{'deviceName': '/dev/sdc3',
'virtualName': 'ephemeral6'}],
}
@@ -1314,7 +1318,8 @@ EC2_IMAGE_2 = {
'architecture': 'x86_64',
'blockDeviceMapping': [
{'deviceName': '/dev/sdb1',
'ebs': {'snapshotId': ID_EC2_SNAPSHOT_1}}],
'ebs': {'snapshotId': ID_EC2_SNAPSHOT_1,
'deleteOnTermination': True}}],
}
@@ -1398,7 +1403,8 @@ OS_IMAGE_2 = {
'virtual': 'root'}],
'block_device_mapping': [
{'device_name': '/dev/sdb1',
'snapshot_id': ID_OS_SNAPSHOT_1}],
'snapshot_id': ID_OS_SNAPSHOT_1,
'delete_on_termination': True}],
}
}
OS_IMAGE_AKI_1 = {

157
ec2api/tests/unit/test_image.py
View File

@@ -256,9 +256,12 @@ class ImageTestCase(base.ApiTestCase):
self._setup_model()
resp = self.execute('DescribeImages', {})
self.assertThat(resp, matchers.DictMatches(
{'imagesSet': [fakes.EC2_IMAGE_1, fakes.EC2_IMAGE_2]},
orderless_lists=True))
self.assertThat(
resp,
matchers.DictMatches(
{'imagesSet': [fakes.EC2_IMAGE_1, fakes.EC2_IMAGE_2]},
orderless_lists=True),
verbose=True)
self.db_api.get_items.assert_any_call(mock.ANY, 'ami')
self.db_api.get_items.assert_any_call(mock.ANY, 'aki')
@@ -269,9 +272,10 @@ class ImageTestCase(base.ApiTestCase):
resp = self.execute('DescribeImages',
{'ImageId.1': fakes.ID_EC2_IMAGE_1})
self.assertThat(resp, matchers.DictMatches(
{'imagesSet': [fakes.EC2_IMAGE_1]},
orderless_lists=True))
self.assertThat(resp,
matchers.DictMatches(
{'imagesSet': [fakes.EC2_IMAGE_1]},
orderless_lists=True))
self.db_api.get_items_by_ids.assert_any_call(
mock.ANY, set([fakes.ID_EC2_IMAGE_1]))
@@ -316,8 +320,10 @@ class ImageTestCase(base.ApiTestCase):
{'ImageId': ec2_image_id,
'Attribute': attr})
response['imageId'] = ec2_image_id
self.assertThat(resp, matchers.DictMatches(response,
orderless_lists=True))
self.assertThat(resp,
matchers.DictMatches(response,
orderless_lists=True),
verbose=True)
do_check('launchPermission',
fakes.ID_EC2_IMAGE_2,
@@ -388,8 +394,9 @@ class ImagePrivateTestCase(test_base.BaseTestCase):
image_ids = {fakes.ID_OS_IMAGE_1: fakes.ID_EC2_IMAGE_1,
fakes.ID_OS_IMAGE_AKI_1: fakes.ID_EC2_IMAGE_AKI_1,
fakes.ID_OS_IMAGE_ARI_1: fakes.ID_EC2_IMAGE_ARI_1}
os_image = copy.deepcopy(fakes.OS_IMAGE_1)
# check name and location attributes for an unnamed image
os_image['properties'] = {'image_location': 'location'}
os_image['name'] = None
@@ -400,6 +407,7 @@ class ImagePrivateTestCase(test_base.BaseTestCase):
self.assertEqual('location', image['imageLocation'])
self.assertEqual('location', image['name'])
# check name and location attributes for complete image
os_image['properties'] = {}
os_image['name'] = 'fake_name'
@@ -410,12 +418,15 @@ class ImagePrivateTestCase(test_base.BaseTestCase):
self.assertEqual('None (fake_name)', image['imageLocation'])
self.assertEqual('fake_name', image['name'])
# check ebs image type for bdm_v2 mapping type
os_image['properties'] = {
'bdm_v2': True,
'root_device_name': '/dev/vda',
'block_device_mapping': [
{'boot_index': 0,
'snapshot_id': fakes.ID_OS_SNAPSHOT_2}]}
'snapshot_id': fakes.ID_OS_SNAPSHOT_2,
'source_type': 'snapshot',
'destination_type': 'volume'}]}
image = image_api._format_image(
'fake_context', fakes.DB_IMAGE_1, fakes.OSImage(os_image),
@@ -424,7 +435,18 @@ class ImagePrivateTestCase(test_base.BaseTestCase):
self.assertEqual('ebs', image['rootDeviceType'])
def test_cloud_format_mappings(self):
# check instance-store image attributes with no any device mappings
os_image['properties'] = {'root_device_name': '/dev/vda'}
image = image_api._format_image(
'fake_context', fakes.DB_IMAGE_1, fakes.OSImage(os_image),
None, None)
self.assertEqual('instance-store', image['rootDeviceType'])
self.assertNotIn('blockDeviceMapping', image)
@mock.patch('ec2api.db.api.IMPL')
def test_format_mappings(self, db_api):
# check virtual mapping formatting
properties = {
'mappings': [
{'virtual': 'ami', 'device': '/dev/sda'},
@@ -436,37 +458,100 @@ class ImagePrivateTestCase(test_base.BaseTestCase):
{'virtual': 'ephemeral', 'device': 'sdf'},
{'virtual': '/dev/sdf1', 'device': 'root'}],
}
expected = {
'blockDeviceMapping': [
{'virtualName': 'ephemeral0', 'deviceName': '/dev/sdb'},
{'virtualName': 'swap', 'deviceName': '/dev/sdc'},
{'virtualName': 'ephemeral1', 'deviceName': '/dev/sdd'},
{'virtualName': 'ephemeral2', 'deviceName': '/dev/sde'},
]
}
expected = [
{'virtualName': 'ephemeral0', 'deviceName': '/dev/sdb'},
{'virtualName': 'swap', 'deviceName': '/dev/sdc'},
{'virtualName': 'ephemeral1', 'deviceName': '/dev/sdd'},
{'virtualName': 'ephemeral2', 'deviceName': '/dev/sde'},
]
result = {}
image_api._cloud_format_mappings('fake_context', properties, result)
self.assertThat(result,
matchers.DictMatches(expected, orderless_lists=True))
result = image_api._format_mappings('fake_context', properties)
self.assertEqual(expected, result)
# check bdm v2 formatting
db_api.get_items_ids.side_effect = (
tools.get_db_api_get_items_ids(fakes.DB_IMAGE_2,
fakes.DB_VOLUME_3))
properties = {
'block_device_mapping':
[{'boot_index': 0,
'snapshot_id': fakes.ID_OS_SNAPSHOT_1},
{'boot_index': None,
'snapshot_id': fakes.ID_OS_SNAPSHOT_2}],
'bdm_v2': True,
'block_device_mapping': [
{'boot_index': 0,
'snapshot_id': fakes.ID_OS_SNAPSHOT_1,
'source_type': 'snapshot',
'destination_type': 'volume'},
{'boot_index': None,
'snapshot_id': fakes.ID_OS_SNAPSHOT_2,
'source_type': 'snapshot',
'destination_type': 'volume'},
{'device_name': 'vdi',
'boot_index': -1,
'image_id': fakes.ID_OS_IMAGE_2,
'source_type': 'image',
'destination_type': 'volume',
'volume_size': 20},
{'device_name': 'vdv',
'boot_index': -1,
'volume_id': fakes.ID_OS_VOLUME_3,
'source_type': 'volume',
'destination_type': 'volume'},
{'device_name': 'vdb',
'boot_index': -1,
'source_type': 'blank',
'destination_type': 'volume',
'volume_size': 100,
'delete_on_termination': True},
],
}
result = {}
image_api._cloud_format_mappings('fake_context', properties, result,
root_device_name='vdx',
expected = [
{'deviceName': 'vdx',
'ebs': {'snapshotId': fakes.ID_EC2_SNAPSHOT_1,
'deleteOnTermination': False}},
{'ebs': {'snapshotId': fakes.ID_EC2_SNAPSHOT_2,
'deleteOnTermination': False}},
{'deviceName': 'vdi',
'ebs': {'snapshotId': fakes.ID_EC2_IMAGE_2,
'volumeSize': 20,
'deleteOnTermination': False}},
{'deviceName': 'vdv',
'ebs': {'snapshotId': fakes.ID_EC2_VOLUME_3,
'deleteOnTermination': False}},
{'deviceName': 'vdb',
'ebs': {'volumeSize': 100,
'deleteOnTermination': True}},
]
result = image_api._format_mappings(
'fake_context', properties, root_device_name='vdx',
snapshot_ids={fakes.ID_OS_SNAPSHOT_1: fakes.ID_EC2_SNAPSHOT_1,
fakes.ID_OS_SNAPSHOT_2: fakes.ID_EC2_SNAPSHOT_2})
expected = {'blockDeviceMapping':
[{'deviceName': 'vdx',
'ebs': {'snapshotId': fakes.ID_EC2_SNAPSHOT_1}},
{'ebs': {'snapshotId': fakes.ID_EC2_SNAPSHOT_2}}]}
self.assertEqual(expected, result)
# check inheritance and generation of virtual name
properties = {
'mappings': [
{'device': 'vdd', 'virtual': 'ephemeral1'},
],
'bdm_v2': True,
'block_device_mapping': [
{'device_name': '/dev/vdb',
'source_type': 'blank',
'destination_type': 'local',
'guest_format': 'swap'},
{'device_name': 'vdc',
'source_type': 'blank',
'destination_type': 'local',
'volume_size': 5},
{'device_name': 'vde',
'source_type': 'blank',
'destination_type': 'local'},
],
}
expected = [
{'deviceName': '/dev/vdd', 'virtualName': 'ephemeral1'},
{'deviceName': '/dev/vdb', 'virtualName': 'swap'},
{'deviceName': 'vdc', 'virtualName': 'ephemeral0'},
{'deviceName': 'vde', 'virtualName': 'ephemeral2'},
]
result = image_api._format_mappings('fake_context', properties)
self.assertEqual(expected, result)
@mock.patch('ec2api.db.api.IMPL')