Port S3 utils tests
Change-Id: I392e03f91a3809cdfe029c9a26969ecd74615ed6
This commit is contained in:
Feodor Tersin
parent
b8067571ed
commit
855113ef2f
|
|
@ -523,8 +523,6 @@ def _block_device_properties_root_device_name(properties):
|
|||
|
||||
def _s3_create(context, metadata):
|
||||
"""Gets a manifest from s3 and makes an image."""
|
||||
image_path = tempfile.mkdtemp(dir=CONF.image_decryption_dir)
|
||||
|
||||
image_location = metadata['properties']['image_location'].lstrip('/')
|
||||
bucket_name = image_location.split('/')[0]
|
||||
manifest_path = image_location[len(bucket_name) + 1:]
|
||||
|
|
@ -532,7 +530,21 @@ def _s3_create(context, metadata):
|
|||
key = bucket.get_key(manifest_path)
|
||||
manifest = key.get_contents_as_string()
|
||||
|
||||
manifest, image = _s3_parse_manifest(context, metadata, manifest)
|
||||
(image_metadata, image_parts,
|
||||
encrypted_key, encrypted_iv) = _s3_parse_manifest(context, manifest)
|
||||
properties = metadata['properties']
|
||||
properties.update(image_metadata['properties'])
|
||||
properties['image_state'] = 'pending'
|
||||
metadata.update(image_metadata)
|
||||
metadata.update({'properties': properties,
|
||||
'is_public': False})
|
||||
|
||||
# TODO(bcwaldon): right now, this removes user-defined ids
|
||||
# We need to re-enable this.
|
||||
metadata.pop('id', None)
|
||||
|
||||
glance = clients.glance(context)
|
||||
image = glance.images.create(**metadata)
|
||||
|
||||
def _update_image_state(image_state):
|
||||
image.update(properties={'image_state': image_state})
|
||||
|
|
@ -540,16 +552,14 @@ def _s3_create(context, metadata):
|
|||
def delayed_create():
|
||||
"""This handles the fetching and decrypting of the part files."""
|
||||
context.update_store()
|
||||
|
||||
try:
|
||||
_update_image_state('downloading')
|
||||
image_path = tempfile.mkdtemp(dir=CONF.image_decryption_dir)
|
||||
|
||||
_update_image_state('downloading')
|
||||
try:
|
||||
parts = []
|
||||
elements = manifest.find('image').getiterator('filename')
|
||||
for fn_element in elements:
|
||||
part = _s3_download_file(bucket, fn_element.text,
|
||||
image_path)
|
||||
for part_name in image_parts:
|
||||
part = _s3_download_file(bucket, part_name, image_path)
|
||||
parts.append(part)
|
||||
|
||||
# NOTE(vish): this may be suboptimal, should we use cat?
|
||||
|
|
@ -564,13 +574,7 @@ def _s3_create(context, metadata):
|
|||
return
|
||||
|
||||
_update_image_state('decrypting')
|
||||
|
||||
try:
|
||||
hex_key = manifest.find('image/ec2_encrypted_key').text
|
||||
encrypted_key = binascii.a2b_hex(hex_key)
|
||||
hex_iv = manifest.find('image/ec2_encrypted_iv').text
|
||||
encrypted_iv = binascii.a2b_hex(hex_iv)
|
||||
|
||||
dec_filename = os.path.join(image_path, 'image.tar.gz')
|
||||
_s3_decrypt_image(context, enc_filename, encrypted_key,
|
||||
encrypted_iv, dec_filename)
|
||||
|
|
@ -579,7 +583,6 @@ def _s3_create(context, metadata):
|
|||
return
|
||||
|
||||
_update_image_state('untarring')
|
||||
|
||||
try:
|
||||
unz_filename = _s3_untarzip_image(image_path, dec_filename)
|
||||
except Exception:
|
||||
|
|
@ -598,6 +601,10 @@ def _s3_create(context, metadata):
|
|||
|
||||
shutil.rmtree(image_path)
|
||||
except glance_exception.HTTPNotFound:
|
||||
# TODO(ft): the image was deleted underneath us, add logging
|
||||
return
|
||||
except Exception:
|
||||
# TODO(ft): add logging
|
||||
return
|
||||
|
||||
eventlet.spawn_n(delayed_create)
|
||||
|
|
@ -605,42 +612,16 @@ def _s3_create(context, metadata):
|
|||
return image
|
||||
|
||||
|
||||
def _s3_parse_manifest(context, metadata, manifest):
|
||||
def _s3_parse_manifest(context, manifest):
|
||||
manifest = etree.fromstring(manifest)
|
||||
image_format = 'ami'
|
||||
|
||||
try:
|
||||
kernel_id = manifest.find('machine_configuration/kernel_id').text
|
||||
if kernel_id == 'true':
|
||||
image_format = 'aki'
|
||||
kernel_id = None
|
||||
except Exception:
|
||||
kernel_id = None
|
||||
|
||||
try:
|
||||
ramdisk_id = manifest.find('machine_configuration/ramdisk_id').text
|
||||
if ramdisk_id == 'true':
|
||||
image_format = 'ari'
|
||||
ramdisk_id = None
|
||||
except Exception:
|
||||
ramdisk_id = None
|
||||
|
||||
try:
|
||||
arch = manifest.find('machine_configuration/architecture').text
|
||||
except Exception:
|
||||
arch = 'x86_64'
|
||||
|
||||
# NOTE(yamahata):
|
||||
# EC2 ec2-budlne-image --block-device-mapping accepts
|
||||
# <virtual name>=<device name> where
|
||||
# virtual name = {ami, root, swap, ephemeral<N>}
|
||||
# where N is no negative integer
|
||||
# device name = the device name seen by guest kernel.
|
||||
# They are converted into
|
||||
# block_device_mapping/mapping/{virtual, device}
|
||||
#
|
||||
# Do NOT confuse this with ec2-register's block device mapping
|
||||
# argument.
|
||||
properties = {'architecture': arch}
|
||||
|
||||
mappings = []
|
||||
try:
|
||||
block_device_mapping = manifest.findall('machine_configuration/'
|
||||
|
|
@ -652,36 +633,39 @@ def _s3_parse_manifest(context, metadata, manifest):
|
|||
except Exception:
|
||||
mappings = []
|
||||
|
||||
properties = metadata['properties']
|
||||
properties['architecture'] = arch
|
||||
|
||||
def _translate_dependent_image_id(image_key, image_id):
|
||||
image_uuid = ec2utils.ec2_id_to_glance_id(context, image_id)
|
||||
properties[image_key] = image_uuid
|
||||
|
||||
if kernel_id:
|
||||
_translate_dependent_image_id('kernel_id', kernel_id)
|
||||
|
||||
if ramdisk_id:
|
||||
_translate_dependent_image_id('ramdisk_id', ramdisk_id)
|
||||
|
||||
if mappings:
|
||||
properties['mappings'] = mappings
|
||||
|
||||
metadata.update({'disk_format': image_format,
|
||||
'container_format': image_format,
|
||||
'is_public': False,
|
||||
'properties': properties})
|
||||
metadata['properties']['image_state'] = 'pending'
|
||||
image_format = 'ami'
|
||||
|
||||
# TODO(bcwaldon): right now, this removes user-defined ids
|
||||
# We need to re-enable this.
|
||||
metadata.pop('id', None)
|
||||
def set_dependent_image_id(image_key, kind):
|
||||
try:
|
||||
image_key_path = ('machine_configuration/%(image_key)s' %
|
||||
{'image_key': image_key})
|
||||
image_id = manifest.find(image_key_path).text
|
||||
except Exception:
|
||||
return
|
||||
if image_id == 'true':
|
||||
image_format = kind
|
||||
else:
|
||||
images = db_api.get_public_items(context, kind, (image_id,))
|
||||
image = (images[0] if len(images) else
|
||||
ec2utils.get_db_item(context, kind, image_id))
|
||||
properties[image_key] = image['os_id']
|
||||
|
||||
glance = clients.glance(context)
|
||||
image = glance.images.create(**metadata)
|
||||
set_dependent_image_id('kernel_id', 'aki')
|
||||
set_dependent_image_id('ramdisk_id', 'ari')
|
||||
|
||||
return manifest, image
|
||||
metadata = {'disk_format': image_format,
|
||||
'container_format': image_format,
|
||||
'properties': properties}
|
||||
image_parts = [
|
||||
fn_element.text
|
||||
for fn_element in manifest.find('image').getiterator('filename')]
|
||||
encrypted_key = manifest.find('image/ec2_encrypted_key').text
|
||||
encrypted_iv = manifest.find('image/ec2_encrypted_iv').text
|
||||
|
||||
return metadata, image_parts, encrypted_key, encrypted_iv
|
||||
|
||||
|
||||
def _s3_download_file(bucket, filename, local_dir):
|
||||
|
|
@ -693,6 +677,8 @@ def _s3_download_file(bucket, filename, local_dir):
|
|||
|
||||
def _s3_decrypt_image(context, encrypted_filename, encrypted_key,
|
||||
encrypted_iv, decrypted_filename):
|
||||
encrypted_key = binascii.a2b_hex(encrypted_key)
|
||||
encrypted_iv = binascii.a2b_hex(encrypted_iv)
|
||||
cert_client = clients.nova_cert(context)
|
||||
try:
|
||||
key = cert_client.decrypt_text(base64.b64encode(encrypted_key))
|
||||
|
|
|
|||
Binary file not shown.
Binary file not shown.
|
|
@ -13,17 +13,83 @@
|
|||
# limitations under the License.
|
||||
|
||||
import copy
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
import eventlet
|
||||
import mock
|
||||
from oslo.config import cfg
|
||||
from oslotest import base as test_base
|
||||
|
||||
from ec2api.api import image as image_api
|
||||
from ec2api import exception
|
||||
from ec2api.tests import base
|
||||
from ec2api.tests import fakes
|
||||
from ec2api.tests import matchers
|
||||
from ec2api.tests import tools
|
||||
|
||||
|
||||
AMI_MANIFEST_XML = """<?xml version="1.0" ?>
|
||||
<manifest>
|
||||
<version>2011-06-17</version>
|
||||
<bundler>
|
||||
<name>test-s3</name>
|
||||
<version>0</version>
|
||||
<release>0</release>
|
||||
</bundler>
|
||||
<machine_configuration>
|
||||
<architecture>x86_64</architecture>
|
||||
<block_device_mapping>
|
||||
<mapping>
|
||||
<virtual>ami</virtual>
|
||||
<device>sda1</device>
|
||||
</mapping>
|
||||
<mapping>
|
||||
<virtual>root</virtual>
|
||||
<device>/dev/sda1</device>
|
||||
</mapping>
|
||||
<mapping>
|
||||
<virtual>ephemeral0</virtual>
|
||||
<device>sda2</device>
|
||||
</mapping>
|
||||
<mapping>
|
||||
<virtual>swap</virtual>
|
||||
<device>sda3</device>
|
||||
</mapping>
|
||||
</block_device_mapping>
|
||||
<kernel_id>%(aki-id)s</kernel_id>
|
||||
<ramdisk_id>%(ari-id)s</ramdisk_id>
|
||||
</machine_configuration>
|
||||
<image>
|
||||
<ec2_encrypted_key>foo</ec2_encrypted_key>
|
||||
<user_encrypted_key>foo</user_encrypted_key>
|
||||
<ec2_encrypted_iv>foo</ec2_encrypted_iv>
|
||||
<parts count="1">
|
||||
<part index="0">
|
||||
<filename>foo</filename>
|
||||
</part>
|
||||
</parts>
|
||||
</image>
|
||||
</manifest>
|
||||
""" % {'aki-id': fakes.ID_EC2_IMAGE_AKI_1,
|
||||
'ari-id': fakes.ID_EC2_IMAGE_ARI_1}
|
||||
|
||||
FILE_MANIFEST_XML = """<?xml version="1.0" ?>
|
||||
<manifest>
|
||||
<image>
|
||||
<ec2_encrypted_key>foo</ec2_encrypted_key>
|
||||
<user_encrypted_key>foo</user_encrypted_key>
|
||||
<ec2_encrypted_iv>foo</ec2_encrypted_iv>
|
||||
<parts count="1">
|
||||
<part index="0">
|
||||
<filename>foo</filename>
|
||||
</part>
|
||||
</parts>
|
||||
</image>
|
||||
</manifest>
|
||||
"""
|
||||
|
||||
|
||||
class ImageTestCase(base.ApiTestCase):
|
||||
|
||||
@mock.patch('ec2api.api.image._s3_create')
|
||||
|
|
@ -298,3 +364,150 @@ class ImagePrivateTestCase(test_base.BaseTestCase):
|
|||
self.assertEqual(
|
||||
image_api._block_device_properties_root_device_name(properties1),
|
||||
root_device1)
|
||||
|
||||
|
||||
class S3TestCase(base.ApiTestCase):
|
||||
# TODO(ft): 'execute' feature isn't used here, but some mocks and
|
||||
# fake context are. ApiTestCase should be split to some classes to use
|
||||
# its feature optimally
|
||||
|
||||
def test_s3_parse_manifest(self):
|
||||
self.db_api.get_public_items.side_effect = (
|
||||
fakes.get_db_api_get_items({
|
||||
'aki': ({'id': fakes.ID_EC2_IMAGE_AKI_1,
|
||||
'os_id': fakes.ID_OS_IMAGE_AKI_1},),
|
||||
'ari': ({'id': fakes.ID_EC2_IMAGE_ARI_1,
|
||||
'os_id': fakes.ID_OS_IMAGE_ARI_1},)}))
|
||||
self.db_api.get_item_by_id.return_value = None
|
||||
|
||||
fake_context = self._create_context()
|
||||
metadata, image_parts, key, iv = image_api._s3_parse_manifest(
|
||||
fake_context, AMI_MANIFEST_XML)
|
||||
|
||||
expected_metadata = {
|
||||
'disk_format': 'ami',
|
||||
'container_format': 'ami',
|
||||
'properties': {'architecture': 'x86_64',
|
||||
'kernel_id': fakes.ID_OS_IMAGE_AKI_1,
|
||||
'ramdisk_id': fakes.ID_OS_IMAGE_ARI_1,
|
||||
'mappings': [
|
||||
{"device": "sda1", "virtual": "ami"},
|
||||
{"device": "/dev/sda1", "virtual": "root"},
|
||||
{"device": "sda2", "virtual": "ephemeral0"},
|
||||
{"device": "sda3", "virtual": "swap"}]}}
|
||||
self.assertThat(metadata,
|
||||
matchers.DictMatches(expected_metadata,
|
||||
orderless_lists=True))
|
||||
self.assertThat(image_parts,
|
||||
matchers.ListMatches(['foo']))
|
||||
self.assertEqual('foo', key)
|
||||
self.assertEqual('foo', iv)
|
||||
self.db_api.get_public_items.assert_any_call(
|
||||
mock.ANY, 'aki', (fakes.ID_EC2_IMAGE_AKI_1,))
|
||||
self.db_api.get_public_items.assert_any_call(
|
||||
mock.ANY, 'ari', (fakes.ID_EC2_IMAGE_ARI_1,))
|
||||
|
||||
@mock.patch.object(fakes.OSImage, 'update', autospec=True)
|
||||
def test_s3_create_image_locations(self, osimage_update):
|
||||
conf = cfg.CONF
|
||||
conf.set_override('image_decryption_dir', None)
|
||||
self.addCleanup(conf.reset)
|
||||
_handle, tempf = tempfile.mkstemp()
|
||||
fake_context = self._create_context()
|
||||
with mock.patch(
|
||||
'ec2api.api.image._s3_conn') as s3_conn, mock.patch(
|
||||
'ec2api.api.image._s3_download_file'
|
||||
) as s3_download_file, mock.patch(
|
||||
'ec2api.api.image._s3_decrypt_image'
|
||||
) as s3_decrypt_image, mock.patch(
|
||||
'ec2api.api.image._s3_untarzip_image'
|
||||
) as s3_untarzip_image:
|
||||
|
||||
(s3_conn.return_value.
|
||||
get_bucket.return_value.
|
||||
get_key.return_value.
|
||||
get_contents_as_string.return_value) = FILE_MANIFEST_XML
|
||||
s3_download_file.return_value = tempf
|
||||
s3_untarzip_image.return_value = tempf
|
||||
(self.glance.images.create.return_value) = (
|
||||
fakes.OSImage({'id': fakes.random_os_id(),
|
||||
'owner': fakes.ID_OS_PROJECT,
|
||||
'is_public': False,
|
||||
'status': 'queued',
|
||||
'container_format': 'ami',
|
||||
'name': 'fake_name',
|
||||
'properties': {}}))
|
||||
|
||||
data = [
|
||||
({'properties': {
|
||||
'image_location': 'testbucket_1/test.img.manifest.xml'}},
|
||||
'testbucket_1', 'test.img.manifest.xml'),
|
||||
({'properties': {
|
||||
'image_location': '/testbucket_2/test.img.manifest.xml'}},
|
||||
'testbucket_2', 'test.img.manifest.xml')]
|
||||
for mdata, bucket, manifest in data:
|
||||
image = image_api._s3_create(fake_context, mdata)
|
||||
eventlet.sleep()
|
||||
osimage_update.assert_called_with(
|
||||
image, properties={'image_state': 'available'})
|
||||
osimage_update.assert_any_call(
|
||||
image, data=mock.ANY)
|
||||
s3_conn.return_value.get_bucket.assert_called_with(bucket)
|
||||
(s3_conn.return_value.get_bucket.return_value.
|
||||
get_key.assert_called_with(manifest))
|
||||
(s3_conn.return_value.get_bucket.return_value.
|
||||
get_key.return_value.
|
||||
get_contents_as_string.assert_called_with())
|
||||
s3_download_file.assert_called_with(
|
||||
s3_conn.return_value.get_bucket.return_value,
|
||||
'foo', mock.ANY)
|
||||
s3_decrypt_image.assert_called_with(
|
||||
fake_context, mock.ANY, 'foo', 'foo', mock.ANY)
|
||||
s3_untarzip_image.assert_called_with(mock.ANY, mock.ANY)
|
||||
|
||||
@mock.patch('ec2api.api.image.eventlet.spawn_n')
|
||||
def test_s3_create_bdm(self, spawn_n):
|
||||
metadata = {'properties': {
|
||||
'image_location': 'fake_bucket/fake_manifest',
|
||||
'root_device_name': '/dev/sda1',
|
||||
'block_device_mapping': [
|
||||
{'device_name': '/dev/sda1',
|
||||
'snapshot_id': fakes.ID_OS_SNAPSHOT_1,
|
||||
'delete_on_termination': True},
|
||||
{'device_name': '/dev/sda2',
|
||||
'virtual_name': 'ephemeral0'},
|
||||
{'device_name': '/dev/sdb0',
|
||||
'no_device': True}]}}
|
||||
fake_context = self._create_context()
|
||||
with mock.patch(
|
||||
'ec2api.api.image._s3_conn') as s3_conn:
|
||||
|
||||
(s3_conn.return_value.
|
||||
get_bucket.return_value.
|
||||
get_key.return_value.
|
||||
get_contents_as_string.return_value) = FILE_MANIFEST_XML
|
||||
|
||||
image_api._s3_create(fake_context, metadata)
|
||||
|
||||
self.glance.images.create.assert_called_once_with(
|
||||
disk_format='ami', container_format='ami', is_public=False,
|
||||
properties={'architecture': 'x86_64',
|
||||
'image_state': 'pending',
|
||||
'root_device_name': '/dev/sda1',
|
||||
'block_device_mapping': [
|
||||
{'device_name': '/dev/sda1',
|
||||
'snapshot_id': fakes.ID_OS_SNAPSHOT_1,
|
||||
'delete_on_termination': True},
|
||||
{'device_name': '/dev/sda2',
|
||||
'virtual_name': 'ephemeral0'},
|
||||
{'device_name': '/dev/sdb0',
|
||||
'no_device': True}],
|
||||
'image_location': 'fake_bucket/fake_manifest'})
|
||||
|
||||
def test_s3_malicious_tarballs(self):
|
||||
self.assertRaises(exception.Invalid,
|
||||
image_api._s3_test_for_malicious_tarball,
|
||||
"/unused", os.path.join(os.path.dirname(__file__), 'abs.tar.gz'))
|
||||
self.assertRaises(exception.Invalid,
|
||||
image_api._s3_test_for_malicious_tarball,
|
||||
"/unused", os.path.join(os.path.dirname(__file__), 'rel.tar.gz'))
|
||||
|
|
|
|||
Loading…
Reference in New Issue