1101 lines
44 KiB
Python
1101 lines
44 KiB
Python
# Copyright 2014
|
|
# The Cloudscaling Group, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import binascii
|
|
import os
|
|
import shutil
|
|
import tarfile
|
|
import tempfile
|
|
import time
|
|
|
|
import botocore.client
|
|
import botocore.config
|
|
import botocore.session
|
|
from cinderclient import exceptions as cinder_exception
|
|
from cryptography.hazmat import backends
|
|
from cryptography.hazmat.primitives.asymmetric import padding
|
|
from cryptography.hazmat.primitives import serialization
|
|
import eventlet
|
|
from glanceclient.common import exceptions as glance_exception
|
|
from lxml import etree
|
|
from oslo_concurrency import processutils
|
|
from oslo_config import cfg
|
|
from oslo_log import log as logging
|
|
from oslo_serialization import jsonutils
|
|
import six
|
|
|
|
from ec2api.api import common
|
|
from ec2api.api import ec2utils
|
|
from ec2api.api import instance as instance_api
|
|
from ec2api import clients
|
|
from ec2api.db import api as db_api
|
|
from ec2api import exception
|
|
from ec2api.i18n import _
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
s3_opts = [
|
|
cfg.StrOpt('image_decryption_dir',
|
|
default='/tmp',
|
|
help='Parent directory for tempdir used for image decryption'),
|
|
cfg.StrOpt('s3_url',
|
|
default='http://$my_ip:3334',
|
|
help='URL to S3 server'),
|
|
# TODO(andrey-mp): this should be reworked with all region`s logic
|
|
cfg.StrOpt('s3_region',
|
|
default='RegionOne',
|
|
help='Region of S3 server'),
|
|
cfg.StrOpt('x509_root_private_key',
|
|
help='Path to ca private key file'),
|
|
]
|
|
|
|
CONF = cfg.CONF
|
|
CONF.register_opts(s3_opts)
|
|
|
|
rpcapi_opts = [
|
|
cfg.StrOpt('cert_topic',
|
|
default='cert',
|
|
deprecated_for_removal=True,
|
|
deprecated_reason='"nova_cert" service is removed',
|
|
help='The topic cert nodes listen on'),
|
|
]
|
|
|
|
CONF.register_opts(rpcapi_opts)
|
|
|
|
|
|
"""Images related API implementation
|
|
"""
|
|
|
|
|
|
Validator = common.Validator
|
|
|
|
|
|
CONTAINER_TO_KIND = {'aki': 'aki',
|
|
'ari': 'ari',
|
|
'ami': 'ami',
|
|
# NOTE(ft): this mappings are ported from legacy Nova EC2
|
|
# There is no idea about its actuality
|
|
'kernel': 'aki',
|
|
'ramdisk': 'ari'}
|
|
IMAGE_TYPES = {'aki': 'kernel',
|
|
'ari': 'ramdisk',
|
|
'ami': 'machine'}
|
|
GLANCE_STATUS_TO_EC2 = {'queued': 'pending',
|
|
'saving': 'pending',
|
|
'active': 'available',
|
|
'killed': 'deregistered',
|
|
'pending_delete': 'deregistered',
|
|
'deleted': 'deregistered',
|
|
'deactivated': 'invalid'}
|
|
EPHEMERAL_PREFIX_LEN = len('ephemeral')
|
|
|
|
|
|
# TODO(yamahata): race condition
|
|
# At the moment there is no way to prevent others from
|
|
# manipulating instances/volumes/snapshots.
|
|
# As other code doesn't take it into consideration, here we don't
|
|
# care of it for now. Ostrich algorithm
|
|
def create_image(context, instance_id, name=None, description=None,
|
|
no_reboot=False, block_device_mapping=None):
|
|
instance = ec2utils.get_db_item(context, instance_id)
|
|
|
|
if not instance_api._is_ebs_instance(context, instance['os_id']):
|
|
msg = _('Instance does not have a volume attached at root (null).')
|
|
raise exception.InvalidParameterValue(value=instance_id,
|
|
parameter='InstanceId',
|
|
reason=msg)
|
|
|
|
nova = clients.nova(context)
|
|
os_instance = nova.servers.get(instance['os_id'])
|
|
restart_instance = False
|
|
if not no_reboot and os_instance.status != 'SHUTOFF':
|
|
if os_instance.status != 'ACTIVE':
|
|
# TODO(ft): Change the error code and message with the real AWS
|
|
# ones
|
|
msg = _('Instance must be run or stopped')
|
|
raise exception.IncorrectState(reason=msg)
|
|
|
|
restart_instance = True
|
|
|
|
# meaningful image name
|
|
name_map = dict(instance=instance['os_id'], now=ec2utils.isotime())
|
|
name = name or _('image of %(instance)s at %(now)s') % name_map
|
|
|
|
def delayed_create(context, image, name, os_instance):
|
|
try:
|
|
os_instance.stop()
|
|
|
|
# wait instance for really stopped
|
|
start_time = time.time()
|
|
while os_instance.status != 'SHUTOFF':
|
|
time.sleep(1)
|
|
os_instance.get()
|
|
# NOTE(yamahata): timeout and error. 1 hour for now for safety.
|
|
# Is it too short/long?
|
|
# Or is there any better way?
|
|
timeout = 1 * 60 * 60
|
|
if time.time() > start_time + timeout:
|
|
err = (_("Couldn't stop instance within %d sec") % timeout)
|
|
raise exception.EC2Exception(message=err)
|
|
|
|
# NOTE(ft): create an image with ec2_id metadata to let other code
|
|
# link os and db objects in race conditions
|
|
os_image_id = os_instance.create_image(
|
|
name, metadata={'ec2_id': image['id']})
|
|
image['os_id'] = os_image_id
|
|
db_api.update_item(context, image)
|
|
except Exception:
|
|
LOG.exception('Failed to complete image %s creation', image.id)
|
|
try:
|
|
image['state'] = 'failed'
|
|
db_api.update_item(context, image)
|
|
except Exception:
|
|
LOG.warning("Couldn't set 'failed' state for db image %s",
|
|
image.id, exc_info=True)
|
|
|
|
try:
|
|
os_instance.start()
|
|
except Exception:
|
|
LOG.warning('Failed to start instance %(i_id)s after '
|
|
'completed creation of image %(image_id)s',
|
|
{'i_id': instance['id'],
|
|
'image_id': image['id']},
|
|
exc_info=True)
|
|
|
|
image = {'is_public': False,
|
|
'description': description}
|
|
if restart_instance:
|
|
# NOTE(ft): image type is hardcoded, because we don't know it now,
|
|
# but cannot change it later. But Nova doesn't specify container format
|
|
# for snapshots of volume backed instances, so that it is 'ami' in fact
|
|
image = db_api.add_item(context, 'ami', image)
|
|
eventlet.spawn_n(delayed_create, context, image, name, os_instance)
|
|
else:
|
|
glance = clients.glance(context)
|
|
with common.OnCrashCleaner() as cleaner:
|
|
os_image_id = os_instance.create_image(name)
|
|
cleaner.addCleanup(glance.images.delete, os_image_id)
|
|
# TODO(andrey-mp): snapshot and volume also must be deleted in case
|
|
# of error
|
|
os_image = glance.images.get(os_image_id)
|
|
image['os_id'] = os_image_id
|
|
image = db_api.add_item(context, _get_os_image_kind(os_image),
|
|
image)
|
|
return {'imageId': image['id']}
|
|
|
|
|
|
def register_image(context, name=None, image_location=None,
|
|
description=None, architecture=None,
|
|
root_device_name=None, block_device_mapping=None,
|
|
virtualization_type=None, kernel_id=None,
|
|
ramdisk_id=None, sriov_net_support=None):
|
|
|
|
# Setup default flags
|
|
is_s3_import = False
|
|
is_url_import = False
|
|
|
|
# Process the input arguments
|
|
if not image_location and not root_device_name:
|
|
# NOTE(ft): for backward compatibility with a hypothetical code
|
|
# which uses name as image_location
|
|
image_location = name
|
|
if not image_location and not root_device_name:
|
|
msg = _("Either imageLocation or rootDeviceName must be set.")
|
|
raise exception.InvalidParameterCombination(msg)
|
|
if not image_location and not name:
|
|
msg = _('The request must contain the parameter name')
|
|
raise exception.MissingParameter(msg)
|
|
|
|
# TODO(ft): check parameters
|
|
metadata = {}
|
|
if name:
|
|
# TODO(ft): check the name is unique (at least for EBS image case)
|
|
metadata['name'] = name
|
|
if image_location:
|
|
|
|
# Resolve the import type
|
|
metadata['image_location'] = image_location
|
|
parsed_url = six.moves.urllib.parse.urlparse(image_location)
|
|
is_s3_import = (parsed_url.scheme == '') or (parsed_url.scheme == 's3')
|
|
is_url_import = not is_s3_import
|
|
|
|
# Check if the name is in the metadata
|
|
if 'name' not in metadata:
|
|
# NOTE(ft): it's needed for backward compatibility
|
|
metadata['name'] = image_location
|
|
if root_device_name:
|
|
metadata['root_device_name'] = root_device_name
|
|
cinder = clients.cinder(context)
|
|
if block_device_mapping:
|
|
mappings = instance_api._parse_block_device_mapping(
|
|
context, block_device_mapping)
|
|
# TODO(ft): merge with image manifets's virtual device mappings
|
|
short_root_device_name = (
|
|
ec2utils.block_device_strip_dev(root_device_name))
|
|
for bdm in mappings:
|
|
instance_api._populate_parsed_bdm_parameter(
|
|
bdm, short_root_device_name)
|
|
if 'volume_size' in bdm:
|
|
continue
|
|
try:
|
|
if bdm['source_type'] == 'snapshot':
|
|
snapshot = cinder.volume_snapshots.get(bdm['snapshot_id'])
|
|
bdm['volume_size'] = snapshot.size
|
|
elif bdm['source_type'] == 'volume':
|
|
volume = cinder.volumes.get(bdm['volume_id'])
|
|
bdm['volume_size'] = volume.size
|
|
except cinder_exception.NotFound:
|
|
pass
|
|
metadata['bdm_v2'] = 'True'
|
|
metadata['block_device_mapping'] = jsonutils.dumps(mappings)
|
|
if architecture is not None:
|
|
metadata['architecture'] = architecture
|
|
if kernel_id:
|
|
metadata['kernel_id'] = ec2utils.get_os_image(context,
|
|
kernel_id).id
|
|
if ramdisk_id:
|
|
metadata['ramdisk_id'] = ec2utils.get_os_image(context,
|
|
ramdisk_id).id
|
|
|
|
# Begin the import/registration process
|
|
with common.OnCrashCleaner() as cleaner:
|
|
|
|
# Setup the glance client
|
|
glance = clients.glance(context)
|
|
|
|
# Check if this is an S3 import
|
|
if is_s3_import:
|
|
os_image = _s3_create(context, metadata)
|
|
|
|
# Condition for all non-S3 imports
|
|
else:
|
|
|
|
# Create the image in glance
|
|
metadata.update({'visibility': 'private',
|
|
'container_format': 'bare',
|
|
'disk_format': 'raw'})
|
|
os_image = glance.images.create(**metadata)
|
|
|
|
# Kick-off the URL image import if from URL
|
|
if is_url_import:
|
|
glance.images.image_import(os_image.id, method='web-download',
|
|
uri=metadata['image_location'])
|
|
|
|
# Otherwise, use the default method
|
|
else:
|
|
glance.images.upload(os_image.id, '', image_size=0)
|
|
|
|
# Add cleanups and complete the registration process
|
|
cleaner.addCleanup(glance.images.delete, os_image.id)
|
|
kind = _get_os_image_kind(os_image)
|
|
image = db_api.add_item(context, kind, {'os_id': os_image.id,
|
|
'is_public': False,
|
|
'description': description})
|
|
|
|
# Return the image ID for the registration process
|
|
return {'imageId': image['id']}
|
|
|
|
|
|
def deregister_image(context, image_id):
|
|
os_image = ec2utils.get_os_image(context, image_id)
|
|
if not os_image:
|
|
image = db_api.get_item_by_id(context, image_id)
|
|
if image.get('state') != 'failed':
|
|
# TODO(ft): figure out corresponding AWS error
|
|
raise exception.IncorrectState(
|
|
reason='Image is still being created')
|
|
else:
|
|
_check_owner(context, os_image)
|
|
|
|
glance = clients.glance(context)
|
|
try:
|
|
glance.images.delete(os_image.id)
|
|
except glance_exception.HTTPNotFound:
|
|
pass
|
|
db_api.delete_item(context, image_id)
|
|
return True
|
|
|
|
|
|
class ImageDescriber(common.TaggableItemsDescriber):
|
|
|
|
KIND = 'ami'
|
|
FILTER_MAP = {'architecture': 'architecture',
|
|
'block-device-mapping.device-name': ['blockDeviceMapping',
|
|
'deviceName'],
|
|
'block-device-mapping.snapshot-id': ['blockDeviceMapping',
|
|
('ebs', 'snapshotId')],
|
|
'block-device-mapping.volume-size': ['blockDeviceMapping',
|
|
('ebs', 'volumeSize')],
|
|
'description': 'description',
|
|
'image-id': 'imageId',
|
|
'image-type': 'imageType',
|
|
'is-public': 'isPublic',
|
|
'kernel_id': 'kernelId',
|
|
'name': 'name',
|
|
'owner-id': 'imageOwnerId',
|
|
'ramdisk-id': 'ramdiskId',
|
|
'root-device-name': 'rootDeviceName',
|
|
'root-device-type': 'rootDeviceType',
|
|
'state': 'imageState',
|
|
}
|
|
|
|
def format(self, image, os_image):
|
|
return _format_image(self.context, image, os_image, self.items_dict,
|
|
self.ids_dict, self.snapshot_ids)
|
|
|
|
def get_db_items(self):
|
|
# TODO(ft): we can't get all images from DB per one request due
|
|
# different kinds. It's need to refactor DB API and ec2utils functions
|
|
# to work with kind smarter
|
|
if self.ids:
|
|
local_images = db_api.get_items_by_ids(self.context, self.ids)
|
|
else:
|
|
local_images = sum((db_api.get_items(self.context, kind)
|
|
for kind in ('ami', 'ari', 'aki')), [])
|
|
public_images = sum((db_api.get_public_items(self.context, kind,
|
|
self.ids)
|
|
for kind in ('ami', 'ari', 'aki')), [])
|
|
|
|
mapped_ids = []
|
|
if self.ids:
|
|
mapped_ids = [{'id': item_id,
|
|
'os_id': os_id}
|
|
for kind in ('ami', 'ari', 'aki')
|
|
for item_id, os_id in db_api.get_items_ids(
|
|
self.context, kind, item_ids=self.ids)]
|
|
|
|
# NOTE(ft): mapped_ids must be the first to let complete items from
|
|
# next lists to override mappings, which do not have item body data
|
|
images = sum((mapped_ids, local_images, public_images), [])
|
|
if self.ids:
|
|
# NOTE(ft): public images, owned by a current user, appear in both
|
|
# local and public lists of images. Therefore it's not enough to
|
|
# just compare length of requested and retrieved lists to make sure
|
|
# that all requested images are retrieved.
|
|
images_ids = set(i['id'] for i in images)
|
|
if len(images_ids) < len(self.ids):
|
|
missed_ids = self.ids - images_ids
|
|
raise exception.InvalidAMIIDNotFound(id=next(iter(missed_ids)))
|
|
self.pending_images = {i['id']: i for i in local_images
|
|
if not i['os_id']}
|
|
self.snapshot_ids = dict(
|
|
(s['os_id'], s['id'])
|
|
for s in db_api.get_items(self.context, 'snap'))
|
|
self.local_images_os_ids = set(i['os_id'] for i in local_images)
|
|
self.ids_dict = {}
|
|
return images
|
|
|
|
def get_os_items(self):
|
|
os_images = list(clients.glance(self.context).images.list())
|
|
self.ec2_created_os_images = {
|
|
os_image.ec2_id: os_image
|
|
for os_image in os_images
|
|
if (hasattr(os_image, 'ec2_id') and
|
|
self.context.project_id == os_image.owner)}
|
|
return os_images
|
|
|
|
def auto_update_db(self, image, os_image):
|
|
if not image:
|
|
kind = _get_os_image_kind(os_image)
|
|
if self.context.project_id == os_image.owner:
|
|
if getattr(os_image, 'ec2_id', None) in self.pending_images:
|
|
# NOTE(ft): the image is being creating, Glance had created
|
|
# image, but creating thread doesn't yet update db item
|
|
image = self.pending_images[os_image.ec2_id]
|
|
image['os_id'] = os_image.id
|
|
image['is_public'] = os_image.visibility == 'public'
|
|
db_api.update_item(self.context, image)
|
|
else:
|
|
image = ec2utils.get_db_item_by_os_id(
|
|
self.context, kind, os_image.id, self.items_dict,
|
|
os_image=os_image)
|
|
else:
|
|
image_id = ec2utils.os_id_to_ec2_id(
|
|
self.context, kind, os_image.id,
|
|
items_by_os_id=self.items_dict, ids_by_os_id=self.ids_dict)
|
|
image = {'id': image_id,
|
|
'os_id': os_image.id}
|
|
elif (self.context.project_id == os_image.owner and
|
|
image.get('is_public') != os_image.visibility == 'public'):
|
|
image['is_public'] = os_image.visibility == 'public'
|
|
if image['id'] in self.local_images_os_ids:
|
|
db_api.update_item(self.context, image)
|
|
else:
|
|
# TODO(ft): currently update_item can not update id mapping,
|
|
# because its project_id is None. Instead of upgrade db_api,
|
|
# we use add_item. But its execution leads to a needless
|
|
# DB call. This should be reworked in the future.
|
|
kind = ec2utils.get_ec2_id_kind(image['id'])
|
|
db_api.add_item(self.context, kind, image)
|
|
return image
|
|
|
|
def get_name(self, os_item):
|
|
return ''
|
|
|
|
def delete_obsolete_item(self, image):
|
|
if image['os_id'] in self.local_images_os_ids:
|
|
db_api.delete_item(self.context, image['id'])
|
|
|
|
def get_tags(self):
|
|
return db_api.get_tags(self.context, ('ami', 'ari', 'aki'), self.ids)
|
|
|
|
def handle_unpaired_item(self, item):
|
|
if item['os_id']:
|
|
return super(ImageDescriber, self).handle_unpaired_item(item)
|
|
|
|
if 'is_public' not in item:
|
|
return None
|
|
|
|
# NOTE(ft): process creating images, ignoring ids mapping
|
|
# NOTE(ft): the image is being creating, Glance had created
|
|
# image, but creating thread doesn't yet update db item
|
|
os_image = self.ec2_created_os_images.get(item['id'])
|
|
if os_image:
|
|
item['os_id'] = os_image.id
|
|
item['is_public'] = os_image.visibility == 'public'
|
|
db_api.update_item(self.context, item)
|
|
image = self.format(item, os_image)
|
|
else:
|
|
# NOTE(ft): Glance image is not yet created, but DB item
|
|
# exists. So that we adds EC2 image to output results
|
|
# with all data we have.
|
|
# TODO(ft): check if euca2ools can process such result
|
|
image = {'imageId': item['id'],
|
|
'imageOwnerId': self.context.project_id,
|
|
'imageType': IMAGE_TYPES[
|
|
ec2utils.get_ec2_id_kind(item['id'])],
|
|
'isPublic': item['is_public']}
|
|
if 'description' in item:
|
|
image['description'] = item['description']
|
|
image['imageState'] = item.get('state', 'pending')
|
|
return image
|
|
|
|
|
|
def describe_images(context, executable_by=None, image_id=None,
|
|
owner=None, filter=None):
|
|
formatted_images = ImageDescriber().describe(
|
|
context, ids=image_id, filter=filter)
|
|
return {'imagesSet': formatted_images}
|
|
|
|
|
|
def describe_image_attribute(context, image_id, attribute):
|
|
def _block_device_mapping_attribute(os_image, image, result):
|
|
properties = ec2utils.deserialize_os_image_properties(os_image)
|
|
mappings = _format_mappings(context, properties)
|
|
if mappings:
|
|
result['blockDeviceMapping'] = mappings
|
|
|
|
def _description_attribute(os_image, image, result):
|
|
result['description'] = {'value': image.get('description')}
|
|
|
|
def _launch_permission_attribute(os_image, image, result):
|
|
result['launchPermission'] = []
|
|
if os_image.visibility == 'public':
|
|
result['launchPermission'].append({'group': 'all'})
|
|
|
|
def _kernel_attribute(os_image, image, result):
|
|
kernel_id = getattr(os_image, 'kernel_id', None)
|
|
if kernel_id:
|
|
result['kernel'] = {
|
|
'value': ec2utils.os_id_to_ec2_id(context, 'aki', kernel_id)
|
|
}
|
|
|
|
def _ramdisk_attribute(os_image, image, result):
|
|
ramdisk_id = getattr(os_image, 'ramdisk_id', None)
|
|
if ramdisk_id:
|
|
result['ramdisk'] = {
|
|
'value': ec2utils.os_id_to_ec2_id(context, 'ari', ramdisk_id)
|
|
}
|
|
|
|
# NOTE(ft): Openstack extension, AWS-incompability
|
|
def _root_device_name_attribute(os_image, image, result):
|
|
properties = ec2utils.deserialize_os_image_properties(os_image)
|
|
result['rootDeviceName'] = (
|
|
ec2utils.block_device_properties_root_device_name(properties))
|
|
|
|
supported_attributes = {
|
|
'blockDeviceMapping': _block_device_mapping_attribute,
|
|
'description': _description_attribute,
|
|
'launchPermission': _launch_permission_attribute,
|
|
'kernel': _kernel_attribute,
|
|
'ramdisk': _ramdisk_attribute,
|
|
# NOTE(ft): Openstack extension, AWS-incompability
|
|
'rootDeviceName': _root_device_name_attribute,
|
|
}
|
|
|
|
fn = supported_attributes.get(attribute)
|
|
if fn is None:
|
|
raise exception.InvalidRequest()
|
|
|
|
os_image = ec2utils.get_os_image(context, image_id)
|
|
if not os_image:
|
|
# TODO(ft): figure out corresponding AWS error
|
|
raise exception.IncorrectState(
|
|
reason='Image is still being created or failed')
|
|
_check_owner(context, os_image)
|
|
image = ec2utils.get_db_item(context, image_id)
|
|
|
|
result = {'imageId': image_id}
|
|
fn(os_image, image, result)
|
|
return result
|
|
|
|
|
|
def modify_image_attribute(context, image_id, attribute=None,
|
|
user_group=None, operation_type=None,
|
|
description=None, launch_permission=None,
|
|
product_code=None, user_id=None, value=None):
|
|
os_image = ec2utils.get_os_image(context, image_id)
|
|
if not os_image:
|
|
# TODO(ft): figure out corresponding AWS error
|
|
raise exception.IncorrectState(
|
|
reason='Image is still being created or failed')
|
|
|
|
attributes = set()
|
|
|
|
# NOTE(andrey-mp): launchPermission structure is converted here
|
|
# to plain parameters: attribute, user_group, operation_type, user_id
|
|
if launch_permission is not None:
|
|
attributes.add('launchPermission')
|
|
user_group = list()
|
|
user_id = list()
|
|
if len(launch_permission) == 0:
|
|
msg = _('No operation specified for launchPermission attribute.')
|
|
raise exception.InvalidParameterCombination(msg)
|
|
if len(launch_permission) > 1:
|
|
msg = _('Only one operation can be specified.')
|
|
raise exception.InvalidParameterCombination(msg)
|
|
operation_type, permissions = launch_permission.popitem()
|
|
for index_key in permissions:
|
|
permission = permissions[index_key]
|
|
if 'group' in permission:
|
|
user_group.append(permission['group'])
|
|
if 'user_id' in permission:
|
|
user_id.append(permission['user_id'])
|
|
if attribute == 'launchPermission':
|
|
attributes.add('launchPermission')
|
|
|
|
if description is not None:
|
|
attributes.add('description')
|
|
value = description
|
|
if attribute == 'description':
|
|
attributes.add('description')
|
|
|
|
# check attributes
|
|
if len(attributes) == 0:
|
|
if product_code is not None:
|
|
attribute = 'productCodes'
|
|
if attribute in ['kernel', 'ramdisk', 'productCodes',
|
|
'blockDeviceMapping']:
|
|
raise exception.InvalidParameter(
|
|
_('Parameter %s is invalid. '
|
|
'The attribute is not supported.') % attribute)
|
|
raise exception.InvalidParameterCombination('No attributes specified.')
|
|
if len(attributes) > 1:
|
|
raise exception.InvalidParameterCombination(
|
|
_('Fields for multiple attribute types specified: %s')
|
|
% str(attributes))
|
|
|
|
if 'launchPermission' in attributes:
|
|
if not user_group:
|
|
msg = _('No operation specified for launchPermission attribute.')
|
|
raise exception.InvalidParameterCombination(msg)
|
|
if len(user_group) != 1 and user_group[0] != 'all':
|
|
msg = _('only group "all" is supported')
|
|
raise exception.InvalidParameterValue(parameter='UserGroup',
|
|
value=user_group,
|
|
reason=msg)
|
|
if operation_type not in ['add', 'remove']:
|
|
msg = _('operation_type must be add or remove')
|
|
raise exception.InvalidParameterValue(parameter='OperationType',
|
|
value='operation_type',
|
|
reason=msg)
|
|
|
|
_check_owner(context, os_image)
|
|
glance = clients.glance(context)
|
|
visibility = 'public' if operation_type == 'add' else 'private'
|
|
glance.images.update(os_image.id, visibility=visibility)
|
|
return True
|
|
|
|
if 'description' in attributes:
|
|
if not value:
|
|
raise exception.MissingParameter(
|
|
'The request must contain the parameter description')
|
|
|
|
_check_owner(context, os_image)
|
|
image = ec2utils.get_db_item(context, image_id)
|
|
image['description'] = value
|
|
db_api.update_item(context, image)
|
|
return True
|
|
|
|
|
|
def reset_image_attribute(context, image_id, attribute):
|
|
if attribute != 'launchPermission':
|
|
raise exception.InvalidRequest()
|
|
|
|
os_image = ec2utils.get_os_image(context, image_id)
|
|
_check_owner(context, os_image)
|
|
glance = clients.glance(context)
|
|
glance.images.update(os_image.id, visibility='private')
|
|
return True
|
|
|
|
|
|
def _check_owner(context, os_image):
|
|
if os_image.owner != context.project_id:
|
|
raise exception.AuthFailure(_('Not authorized for image:%s')
|
|
% os_image.id)
|
|
|
|
|
|
def _format_image(context, image, os_image, images_dict, ids_dict,
|
|
snapshot_ids=None):
|
|
ec2_image = {'imageId': image['id'],
|
|
'imageOwnerId': os_image.owner,
|
|
'imageType': IMAGE_TYPES[
|
|
ec2utils.get_ec2_id_kind(image['id'])],
|
|
'isPublic': os_image.visibility == 'public',
|
|
'architecture': getattr(os_image, 'architecture', None),
|
|
'creationDate': os_image.created_at
|
|
}
|
|
if 'description' in image:
|
|
ec2_image['description'] = image['description']
|
|
if 'state' in image:
|
|
state = image['state']
|
|
else:
|
|
state = GLANCE_STATUS_TO_EC2.get(os_image.status, 'error')
|
|
if state in ('available', 'pending'):
|
|
state = _s3_image_state_map.get(getattr(os_image, 'image_state', None),
|
|
state)
|
|
ec2_image['imageState'] = state
|
|
|
|
kernel_id = getattr(os_image, 'kernel_id', None)
|
|
if kernel_id:
|
|
ec2_image['kernelId'] = ec2utils.os_id_to_ec2_id(
|
|
context, 'aki', kernel_id,
|
|
items_by_os_id=images_dict, ids_by_os_id=ids_dict)
|
|
ramdisk_id = getattr(os_image, 'ramdisk_id', None)
|
|
if ramdisk_id:
|
|
ec2_image['ramdiskId'] = ec2utils.os_id_to_ec2_id(
|
|
context, 'ari', ramdisk_id,
|
|
items_by_os_id=images_dict, ids_by_os_id=ids_dict)
|
|
|
|
name = os_image.name
|
|
img_loc = getattr(os_image, 'image_location', None)
|
|
if img_loc:
|
|
ec2_image['imageLocation'] = img_loc
|
|
else:
|
|
ec2_image['imageLocation'] = "%s (%s)" % (img_loc, name)
|
|
if not name and img_loc:
|
|
# This should only occur for images registered with ec2 api
|
|
# prior to that api populating the glance name
|
|
ec2_image['name'] = img_loc
|
|
else:
|
|
ec2_image['name'] = name
|
|
|
|
properties = ec2utils.deserialize_os_image_properties(os_image)
|
|
root_device_name = (
|
|
ec2utils.block_device_properties_root_device_name(properties))
|
|
mappings = _format_mappings(context, properties, root_device_name,
|
|
snapshot_ids, os_image.owner)
|
|
if mappings:
|
|
ec2_image['blockDeviceMapping'] = mappings
|
|
|
|
root_device_type = 'instance-store'
|
|
if root_device_name:
|
|
ec2_image['rootDeviceName'] = root_device_name
|
|
|
|
short_root_device_name = ec2utils.block_device_strip_dev(
|
|
root_device_name)
|
|
if any((short_root_device_name ==
|
|
ec2utils.block_device_strip_dev(bdm.get('deviceName')))
|
|
for bdm in mappings):
|
|
root_device_type = 'ebs'
|
|
ec2_image['rootDeviceType'] = root_device_type
|
|
|
|
return ec2_image
|
|
|
|
|
|
def _format_mappings(context, os_image_properties, root_device_name=None,
|
|
snapshot_ids=None, project_id=None):
|
|
formatted_mappings = []
|
|
bdms = ec2utils.get_os_image_mappings(os_image_properties)
|
|
ephemeral_numbers = _ephemeral_free_number_generator(bdms)
|
|
for bdm in bdms:
|
|
# NOTE(yamahata): trim ebs.no_device == true. Is this necessary?
|
|
# TODO(ft): figure out AWS and Nova behaviors
|
|
if bdm.get('no_device'):
|
|
continue
|
|
item = {}
|
|
if bdm.get('boot_index') == 0 and root_device_name:
|
|
item['deviceName'] = root_device_name
|
|
elif 'device_name' in bdm:
|
|
item['deviceName'] = bdm['device_name']
|
|
if bdm.get('destination_type') == 'volume':
|
|
ebs = _format_volume_mapping(
|
|
context, bdm, snapshot_ids=snapshot_ids, project_id=project_id)
|
|
if not ebs:
|
|
# TODO(ft): what to do with the wrong bdm?
|
|
continue
|
|
item['ebs'] = ebs
|
|
elif bdm.get('destination_type') == 'local':
|
|
virtual_name = _format_virtual_name(bdm, ephemeral_numbers)
|
|
if not virtual_name:
|
|
# TODO(ft): what to do with the wrong bdm?
|
|
continue
|
|
item['virtualName'] = virtual_name
|
|
else:
|
|
# TODO(ft): what to do with the wrong bdm?
|
|
continue
|
|
formatted_mappings.append(item)
|
|
|
|
return formatted_mappings
|
|
|
|
|
|
def _format_volume_mapping(context, bdm, snapshot_ids=None, project_id=None):
|
|
ebs = {'deleteOnTermination': bdm['delete_on_termination']}
|
|
# TODO(ft): set default volumeSize from the source
|
|
if bdm.get('volume_size') is not None:
|
|
ebs['volumeSize'] = bdm['volume_size']
|
|
if bdm.get('source_type') == 'snapshot':
|
|
if bdm.get('snapshot_id'):
|
|
ebs['snapshotId'] = ec2utils.os_id_to_ec2_id(
|
|
context, 'snap', bdm['snapshot_id'],
|
|
ids_by_os_id=snapshot_ids, project_id=project_id)
|
|
# NOTE(ft): Openstack extension, AWS-incompability
|
|
elif bdm.get('source_type') == 'volume':
|
|
if bdm.get('volume_id'):
|
|
ebs['snapshotId'] = ec2utils.os_id_to_ec2_id(
|
|
context, 'vol', bdm['volume_id'], project_id=project_id)
|
|
# NOTE(ft): extension, AWS-incompability
|
|
elif bdm.get('source_type') == 'image':
|
|
if bdm.get('image_id'):
|
|
ebs['snapshotId'] = ec2utils.os_id_to_ec2_id(
|
|
context, 'ami', bdm['image_id'])
|
|
if ebs.get('snapshotId') or bdm.get('source_type') == 'blank':
|
|
return ebs
|
|
|
|
|
|
def _format_virtual_name(bdm, ephemeral_numbers):
|
|
if bdm.get('source_type') == 'blank':
|
|
if bdm.get('guest_format') == 'swap':
|
|
return 'swap'
|
|
else:
|
|
return (bdm.get('virtual_name') or
|
|
'ephemeral%s' % next(ephemeral_numbers))
|
|
|
|
|
|
def _ephemeral_free_number_generator(bdms):
|
|
named_ephemeral_nums = set(
|
|
int(bdm['virtual_name'][EPHEMERAL_PREFIX_LEN:])
|
|
for bdm in bdms
|
|
if (bdm.get('destination_type') == 'local' and
|
|
bdm.get('source_type') == 'blank' and
|
|
bdm.get('guest_format') != 'swap' and
|
|
bdm.get('virtual_name')))
|
|
ephemeral_free_num = 0
|
|
while True:
|
|
if ephemeral_free_num not in named_ephemeral_nums:
|
|
yield ephemeral_free_num
|
|
ephemeral_free_num += 1
|
|
|
|
|
|
def _get_os_image_kind(os_image):
|
|
# NOTE(ft): for 'get' operation Glance image doesn't have an attribute
|
|
# if it isn't sent by Glance. But Glance doesn't send null-value
|
|
# attributes, and the attributes above are null for volume-backed images.
|
|
if not hasattr(os_image, 'container_format'):
|
|
return 'ami'
|
|
return CONTAINER_TO_KIND.get(os_image.container_format, 'ami')
|
|
|
|
|
|
def _auto_create_image_extension(context, image, os_image):
|
|
image['is_public'] = os_image.visibility == 'public'
|
|
|
|
|
|
ec2utils.register_auto_create_db_item_extension(
|
|
'ami', _auto_create_image_extension)
|
|
ec2utils.register_auto_create_db_item_extension(
|
|
'ari', _auto_create_image_extension)
|
|
ec2utils.register_auto_create_db_item_extension(
|
|
'aki', _auto_create_image_extension)
|
|
|
|
|
|
# NOTE(ft): following functions are copied from various parts of Nova
|
|
|
|
# translate our internal state to states valid by the EC2 API documentation
|
|
_s3_image_state_map = {'downloading': 'pending',
|
|
'failed_download': 'failed',
|
|
'decrypting': 'pending',
|
|
'failed_decrypt': 'failed',
|
|
'untarring': 'pending',
|
|
'failed_untar': 'failed',
|
|
'uploading': 'pending',
|
|
'failed_upload': 'failed',
|
|
'available': 'available'}
|
|
|
|
|
|
def _s3_create(context, metadata):
|
|
"""Gets a manifest from s3 and makes an image."""
|
|
|
|
# Parse the metadata into bucket and manifest path
|
|
parsed_url = six.moves.urllib.parse.urlparse(metadata['image_location'])
|
|
if parsed_url.hostname is not None:
|
|
# Handle s3://<BUCKET_NAME>/<KEY_PATH> case
|
|
bucket_name = parsed_url.hostname
|
|
manifest_path = parsed_url.path[1:]
|
|
else:
|
|
# Handle <BUCKET_NAME>/<KEY_PATH> case
|
|
bucket_name = parsed_url.path.split('/')[0]
|
|
manifest_path = '/'.join(parsed_url.path.split('/')[1:])
|
|
|
|
# Continue with S3 import
|
|
s3_client = _s3_conn(context)
|
|
image_location = '/'.join([bucket_name, manifest_path])
|
|
key = s3_client.get_object(Bucket=bucket_name, Key=manifest_path)
|
|
body = key['Body']
|
|
if isinstance(body, six.string_types):
|
|
manifest = body
|
|
else:
|
|
# TODO(andrey-mp): check big objects
|
|
manifest = body.read()
|
|
|
|
(image_metadata, image_parts,
|
|
encrypted_key, encrypted_iv) = _s3_parse_manifest(context, manifest)
|
|
metadata.update(image_metadata)
|
|
metadata.update({'image_state': 'pending',
|
|
'visibility': 'private'})
|
|
|
|
# TODO(bcwaldon): right now, this removes user-defined ids
|
|
# We need to re-enable this.
|
|
metadata.pop('id', None)
|
|
|
|
glance = clients.glance(context)
|
|
image = glance.images.create(**metadata)
|
|
|
|
def _update_image_state(image_state):
|
|
glance.images.update(image.id, image_state=image_state)
|
|
|
|
def delayed_create():
|
|
"""This handles the fetching and decrypting of the part files."""
|
|
context.update_store()
|
|
try:
|
|
image_path = tempfile.mkdtemp(dir=CONF.image_decryption_dir)
|
|
log_vars = {'image_location': image_location,
|
|
'image_path': image_path}
|
|
|
|
_update_image_state('downloading')
|
|
try:
|
|
parts = []
|
|
for part_name in image_parts:
|
|
part = _s3_download_file(s3_client, bucket_name,
|
|
part_name, image_path)
|
|
parts.append(part)
|
|
|
|
# NOTE(vish): this may be suboptimal, should we use cat?
|
|
enc_filename = os.path.join(image_path, 'image.encrypted')
|
|
with open(enc_filename, 'wb') as combined:
|
|
for filename in parts:
|
|
with open(filename, "rb") as part:
|
|
combined.write(part.read())
|
|
|
|
except Exception:
|
|
LOG.exception('Failed to download %(image_location)s '
|
|
'to %(image_path)s', log_vars)
|
|
_update_image_state('failed_download')
|
|
return
|
|
|
|
_update_image_state('decrypting')
|
|
try:
|
|
dec_filename = os.path.join(image_path, 'image.tar.gz')
|
|
_s3_decrypt_image(context, enc_filename, encrypted_key,
|
|
encrypted_iv, dec_filename)
|
|
except Exception:
|
|
LOG.exception('Failed to decrypt %(image_location)s '
|
|
'to %(image_path)s', log_vars)
|
|
_update_image_state('failed_decrypt')
|
|
return
|
|
|
|
_update_image_state('untarring')
|
|
try:
|
|
unz_filename = _s3_untarzip_image(image_path, dec_filename)
|
|
except Exception:
|
|
LOG.exception('Failed to untar %(image_location)s '
|
|
'to %(image_path)s', log_vars)
|
|
_update_image_state('failed_untar')
|
|
return
|
|
|
|
_update_image_state('uploading')
|
|
try:
|
|
with open(unz_filename, "rb") as image_file:
|
|
glance.images.upload(image.id, image_file)
|
|
except Exception:
|
|
LOG.exception('Failed to upload %(image_location)s '
|
|
'to %(image_path)s', log_vars)
|
|
_update_image_state('failed_upload')
|
|
return
|
|
|
|
_update_image_state('available')
|
|
|
|
shutil.rmtree(image_path)
|
|
except glance_exception.HTTPNotFound:
|
|
LOG.info('Image %swas deleted underneath us', image.id)
|
|
except Exception:
|
|
LOG.exception('Failed to complete image %s creation', image.id)
|
|
|
|
eventlet.spawn_n(delayed_create)
|
|
|
|
return image
|
|
|
|
|
|
def _s3_parse_manifest(context, manifest):
|
|
manifest = etree.fromstring(manifest)
|
|
|
|
try:
|
|
arch = manifest.find('machine_configuration/architecture').text
|
|
except Exception:
|
|
arch = 'x86_64'
|
|
|
|
metadata = {'architecture': arch}
|
|
|
|
mappings = []
|
|
try:
|
|
block_device_mapping = manifest.findall('machine_configuration/'
|
|
'block_device_mapping/'
|
|
'mapping')
|
|
for bdm in block_device_mapping:
|
|
mappings.append({'virtual': bdm.find('virtual').text,
|
|
'device': bdm.find('device').text})
|
|
except Exception:
|
|
mappings = []
|
|
|
|
if mappings:
|
|
metadata['mappings'] = mappings
|
|
|
|
def set_dependent_image_id(image_key):
|
|
try:
|
|
image_key_path = ('machine_configuration/%(image_key)s' %
|
|
{'image_key': image_key})
|
|
image_id = manifest.find(image_key_path).text
|
|
except Exception:
|
|
return
|
|
if image_id == 'true':
|
|
return True
|
|
os_image = ec2utils.get_os_image(context, image_id)
|
|
metadata[image_key] = os_image.id
|
|
|
|
image_format = 'ami'
|
|
if set_dependent_image_id('kernel_id'):
|
|
image_format = 'aki'
|
|
if set_dependent_image_id('ramdisk_id'):
|
|
image_format = 'ari'
|
|
|
|
metadata.update({'disk_format': image_format,
|
|
'container_format': image_format})
|
|
image_parts = [
|
|
fn_element.text
|
|
for fn_element in manifest.find('image').getiterator('filename')]
|
|
encrypted_key = manifest.find('image/ec2_encrypted_key').text
|
|
encrypted_iv = manifest.find('image/ec2_encrypted_iv').text
|
|
|
|
return metadata, image_parts, encrypted_key, encrypted_iv
|
|
|
|
|
|
def _s3_download_file(s3_client, bucket_name, filename, local_dir):
|
|
s3_object = s3_client.get_object(Bucket=bucket_name, Key=filename)
|
|
local_filename = os.path.join(local_dir, os.path.basename(filename))
|
|
body = s3_object['Body']
|
|
with open(local_filename, 'wb') as f:
|
|
f.write(body.read())
|
|
return local_filename
|
|
|
|
|
|
def _s3_decrypt_image(context, encrypted_filename, encrypted_key,
|
|
encrypted_iv, decrypted_filename):
|
|
encrypted_key = binascii.a2b_hex(encrypted_key)
|
|
encrypted_iv = binascii.a2b_hex(encrypted_iv)
|
|
try:
|
|
key = _decrypt_text(encrypted_key).decode()
|
|
except Exception as exc:
|
|
msg = _('Failed to decrypt private key: %s') % exc
|
|
raise exception.EC2Exception(msg)
|
|
try:
|
|
iv = _decrypt_text(encrypted_iv).decode()
|
|
except Exception as exc:
|
|
msg = _('Failed to decrypt initialization vector: %s') % exc
|
|
raise exception.EC2Exception(msg)
|
|
|
|
try:
|
|
processutils.execute('openssl', 'enc',
|
|
'-d', '-aes-128-cbc',
|
|
'-in', '%s' % (encrypted_filename,),
|
|
'-K', '%s' % (key,),
|
|
'-iv', '%s' % (iv,),
|
|
'-out', '%s' % (decrypted_filename,))
|
|
except processutils.ProcessExecutionError as exc:
|
|
raise exception.EC2Exception(_('Failed to decrypt image file '
|
|
'%(image_file)s: %(err)s') %
|
|
{'image_file': encrypted_filename,
|
|
'err': exc.stdout})
|
|
|
|
|
|
def _s3_untarzip_image(path, filename):
|
|
_s3_test_for_malicious_tarball(path, filename)
|
|
tar_file = tarfile.open(filename, 'r|gz')
|
|
tar_file.extractall(path)
|
|
image_file = tar_file.getnames()[0]
|
|
tar_file.close()
|
|
return os.path.join(path, image_file)
|
|
|
|
|
|
def _s3_test_for_malicious_tarball(path, filename):
|
|
"""Raises exception if extracting tarball would escape extract path."""
|
|
tar_file = tarfile.open(filename, 'r|gz')
|
|
for n in tar_file.getnames():
|
|
if not os.path.abspath(os.path.join(path, n)).startswith(path):
|
|
tar_file.close()
|
|
# TODO(ft): figure out actual AWS exception
|
|
raise exception.EC2InvalidException(_('Unsafe filenames in image'))
|
|
tar_file.close()
|
|
|
|
|
|
def _s3_conn(context):
|
|
region = CONF.s3_region
|
|
ec2_creds = clients.keystone(context).ec2.list(context.user_id)
|
|
|
|
# Here we a) disable user's default config to let ec2api works independetly
|
|
# of user's local settings;
|
|
# b) specify region to be used by botocore;
|
|
# c) do not change standard botocore keys to get these settings
|
|
# from environment
|
|
connection_data = {
|
|
'config_file': (None, 'AWS_CONFIG_FILE', None, None),
|
|
'region': ('region', 'AWS_DEFAULT_REGION', region, None),
|
|
}
|
|
session = botocore.session.get_session(connection_data)
|
|
return session.create_client(
|
|
's3', region_name=region, endpoint_url=CONF.s3_url,
|
|
aws_access_key_id=ec2_creds[0].access,
|
|
aws_secret_access_key=ec2_creds[0].secret,
|
|
config=botocore.config.Config(signature_version='s3v4'))
|
|
|
|
|
|
def _decrypt_text(text):
|
|
private_key_file = CONF.x509_root_private_key
|
|
if not private_key_file:
|
|
msg = _("Path to ca private key isn't configured")
|
|
raise exception.EC2Exception(msg)
|
|
with open(private_key_file, 'rb') as f:
|
|
data = f.read()
|
|
priv_key = serialization.load_pem_private_key(
|
|
data, None, backends.default_backend())
|
|
return priv_key.decrypt(text, padding.PKCS1v15())
|