diff --git a/glance/store/common/__init__.py b/glance/store/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/glance/store/common/config.py b/glance/store/common/config.py new file mode 100644 index 00000000..3177b073 --- /dev/null +++ b/glance/store/common/config.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python + +# Copyright 2011 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Routines for configuring Glance +""" + +import logging +import logging.config +import logging.handlers +import os + +from oslo.config import cfg +from paste import deploy + +from glance.version import version_info as version + +paste_deploy_opts = [ + cfg.StrOpt('flavor', + help=_('Partial name of a pipeline in your paste configuration ' + 'file with the service name removed. For example, if ' + 'your paste section name is ' + '[pipeline:glance-api-keystone] use the value ' + '"keystone"')), + cfg.StrOpt('config_file', + help=_('Name of the paste configuration file.')), +] +image_format_opts = [ + cfg.ListOpt('container_formats', + default=['ami', 'ari', 'aki', 'bare', 'ovf'], + help=_("Supported values for the 'container_format' " + "image attribute"), + deprecated_opts=[cfg.DeprecatedOpt('container_formats', + group='DEFAULT')]), + cfg.ListOpt('disk_formats', + default=['ami', 'ari', 'aki', 'vhd', 'vmdk', 'raw', 'qcow2', + 'vdi', 'iso'], + help=_("Supported values for the 'disk_format' " + "image attribute"), + deprecated_opts=[cfg.DeprecatedOpt('disk_formats', + group='DEFAULT')]), +] +task_opts = [ + cfg.IntOpt('task_time_to_live', + default=48, + help=_("Time in hours for which a task lives after, either " + "succeeding or failing"), + deprecated_opts=[cfg.DeprecatedOpt('task_time_to_live', + group='DEFAULT')]), +] +common_opts = [ + cfg.BoolOpt('allow_additional_image_properties', default=True, + help=_('Whether to allow users to specify image properties ' + 'beyond what the image schema provides')), + cfg.IntOpt('image_member_quota', default=128, + help=_('Maximum number of image members per image. ' + 'Negative values evaluate to unlimited.')), + cfg.IntOpt('image_property_quota', default=128, + help=_('Maximum number of properties allowed on an image. ' + 'Negative values evaluate to unlimited.')), + cfg.IntOpt('image_tag_quota', default=128, + help=_('Maximum number of tags allowed on an image. ' + 'Negative values evaluate to unlimited.')), + cfg.IntOpt('image_location_quota', default=10, + help=_('Maximum number of locations allowed on an image. ' + 'Negative values evaluate to unlimited.')), + cfg.StrOpt('data_api', default='glance.db.sqlalchemy.api', + help=_('Python module path of data access API')), + cfg.IntOpt('limit_param_default', default=25, + help=_('Default value for the number of items returned by a ' + 'request if not specified explicitly in the request')), + cfg.IntOpt('api_limit_max', default=1000, + help=_('Maximum permissible number of items that could be ' + 'returned by a request')), + cfg.BoolOpt('show_image_direct_url', default=False, + help=_('Whether to include the backend image storage location ' + 'in image properties. Revealing storage location can ' + 'be a security risk, so use this setting with ' + 'caution!')), + cfg.BoolOpt('show_multiple_locations', default=False, + help=_('Whether to include the backend image locations ' + 'in image properties. Revealing storage location can ' + 'be a security risk, so use this setting with ' + 'caution! The overrides show_image_direct_url.')), + cfg.IntOpt('image_size_cap', default=1099511627776, + help=_("Maximum size of image a user can upload in bytes. " + "Defaults to 1099511627776 bytes (1 TB).")), + cfg.IntOpt('user_storage_quota', default=0, + help=_("Set a system wide quota for every user. This value is " + "the total number of bytes that a user can use across " + "all storage systems. A value of 0 means unlimited.")), + cfg.BoolOpt('enable_v1_api', default=True, + help=_("Deploy the v1 OpenStack Images API.")), + cfg.BoolOpt('enable_v2_api', default=True, + help=_("Deploy the v2 OpenStack Images API.")), + cfg.StrOpt('pydev_worker_debug_host', default=None, + help=_('The hostname/IP of the pydev process listening for ' + 'debug connections')), + cfg.IntOpt('pydev_worker_debug_port', default=5678, + help=_('The port on which a pydev process is listening for ' + 'connections.')), + cfg.StrOpt('metadata_encryption_key', secret=True, + help=_('Key used for encrypting sensitive metadata while ' + 'talking to the registry or database.')), +] + +CONF = cfg.CONF +CONF.register_opts(paste_deploy_opts, group='paste_deploy') +CONF.register_opts(image_format_opts, group='image_format') +CONF.register_opts(task_opts, group='task') +CONF.register_opts(common_opts) + + +def parse_args(args=None, usage=None, default_config_files=None): + CONF(args=args, + project='glance', + version=version.cached_version_string(), + usage=usage, + default_config_files=default_config_files) + + +def parse_cache_args(args=None): + config_files = cfg.find_config_files(project='glance', prog='glance-cache') + parse_args(args=args, default_config_files=config_files) + + +def _get_deployment_flavor(flavor=None): + """ + Retrieve the paste_deploy.flavor config item, formatted appropriately + for appending to the application name. + + :param flavor: if specified, use this setting rather than the + paste_deploy.flavor configuration setting + """ + if not flavor: + flavor = CONF.paste_deploy.flavor + return '' if not flavor else ('-' + flavor) + + +def _get_paste_config_path(): + paste_suffix = '-paste.ini' + conf_suffix = '.conf' + if CONF.config_file: + # Assume paste config is in a paste.ini file corresponding + # to the last config file + path = CONF.config_file[-1].replace(conf_suffix, paste_suffix) + else: + path = CONF.prog + paste_suffix + return CONF.find_file(os.path.basename(path)) + + +def _get_deployment_config_file(): + """ + Retrieve the deployment_config_file config item, formatted as an + absolute pathname. + """ + path = CONF.paste_deploy.config_file + if not path: + path = _get_paste_config_path() + if not path: + msg = "Unable to locate paste config file for %s." % CONF.prog + raise RuntimeError(msg) + return os.path.abspath(path) + + +def load_paste_app(app_name, flavor=None, conf_file=None): + """ + Builds and returns a WSGI app from a paste config file. + + We assume the last config file specified in the supplied ConfigOpts + object is the paste config file, if conf_file is None. + + :param app_name: name of the application to load + :param flavor: name of the variant of the application to load + :param conf_file: path to the paste config file + + :raises RuntimeError when config file cannot be located or application + cannot be loaded from config file + """ + # append the deployment flavor to the application name, + # in order to identify the appropriate paste pipeline + app_name += _get_deployment_flavor(flavor) + + if not conf_file: + conf_file = _get_deployment_config_file() + + try: + logger = logging.getLogger(__name__) + logger.debug(_("Loading %(app_name)s from %(conf_file)s"), + {'conf_file': conf_file, 'app_name': app_name}) + + app = deploy.loadapp("config:%s" % conf_file, name=app_name) + + # Log the options used when starting if we're in debug mode... + if CONF.debug: + CONF.log_opt_values(logger, logging.DEBUG) + + return app + except (LookupError, ImportError) as e: + msg = (_("Unable to load %(app_name)s from " + "configuration file %(conf_file)s." + "\nGot: %(e)r") % {'app_name': app_name, + 'conf_file': conf_file, + 'e': e}) + logger.error(msg) + raise RuntimeError(msg) diff --git a/glance/store/common/exception.py b/glance/store/common/exception.py new file mode 100644 index 00000000..bcffe1ab --- /dev/null +++ b/glance/store/common/exception.py @@ -0,0 +1,222 @@ +# Copyright (c) 2014 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Glance Store exception subclasses""" + + +class GlanceStoreException(Exception): + """ + Base Glance Store Exception + + To correctly use this class, inherit from it and define + a 'message' property. That message will get printf'd + with the keyword arguments provided to the constructor. + """ + message = '' + + def __init__(self, **kwargs): + msg = self.message.format(**kwargs) + super(Exception, self).__init__(msg) + + +class MissingCredentialError(GlanceStoreException): + message = _("Missing required credential: %(required)s") + + +class BadAuthStrategy(GlanceStoreException): + message = _("Incorrect auth strategy, expected \"%(expected)s\" but " + "received \"%(received)s\"") + + +class NotFound(GlanceStoreException): + message = _("An object with the specified identifier was not found.") + + +class UnknownScheme(GlanceStoreException): + message = _("Unknown scheme '%(scheme)s' found in URI") + + +class BadStoreUri(GlanceStoreException): + message = _("The Store URI was malformed.") + + +class Duplicate(GlanceStoreException): + message = _("An object with the same identifier already exists.") + + +class Conflict(GlanceStoreException): + message = _("An object with the same identifier is currently being " + "operated on.") + + +class StorageFull(GlanceStoreException): + message = _("There is not enough disk space on the image storage media.") + + +class StorageQuotaFull(GlanceStoreException): + message = _("The size of the data %(image_size)s will exceed the limit. " + "%(remaining)s bytes remaining.") + + +class StorageWriteDenied(GlanceStoreException): + message = _("Permission to write image storage media denied.") + + +class AuthBadRequest(GlanceStoreException): + message = _("Connect error/bad request to Auth service at URL %(url)s.") + + +class AuthUrlNotFound(GlanceStoreException): + message = _("Auth service at URL %(url)s not found.") + + +class AuthorizationFailure(GlanceStoreException): + message = _("Authorization failed.") + + +class NotAuthenticated(GlanceStoreException): + message = _("You are not authenticated.") + + +class Forbidden(GlanceStoreException): + message = _("You are not authorized to complete this action.") + + +class ForbiddenPublicImage(Forbidden): + message = _("You are not authorized to complete this action.") + + +class ProtectedImageDelete(Forbidden): + message = _("Image %(image_id)s is protected and cannot be deleted.") + + +class Invalid(GlanceStoreException): + message = _("Data supplied was not valid.") + + +class InvalidSortKey(Invalid): + message = _("Sort key supplied was not valid.") + + +class InvalidPropertyProtectionConfiguration(Invalid): + message = _("Invalid configuration in property protection file.") + + +class InvalidFilterRangeValue(Invalid): + message = _("Unable to filter using the specified range.") + + +class ReadonlyProperty(Forbidden): + message = _("Attribute '%(property)s' is read-only.") + + +class ReservedProperty(Forbidden): + message = _("Attribute '%(property)s' is reserved.") + + +class AuthorizationRedirect(GlanceStoreException): + message = _("Redirecting to %(uri)s for authorization.") + + +class DatabaseMigrationError(GlanceStoreException): + message = _("There was an error migrating the database.") + + +class ClientConnectionError(GlanceStoreException): + message = _("There was an error connecting to a server") + + +class ClientConfigurationError(GlanceStoreException): + message = _("There was an error configuring the client.") + + +class BadStoreConfiguration(GlanceStoreException): + message = _("Store %(store_name)s could not be configured correctly. " + "Reason: %(reason)s") + + +class BadDriverConfiguration(GlanceStoreException): + message = _("Driver %(driver_name)s could not be configured correctly. " + "Reason: %(reason)s") + + +class StoreDeleteNotSupported(GlanceStoreException): + message = _("Deleting images from this store is not supported.") + + +class StoreGetNotSupported(GlanceStoreException): + message = _("Getting images from this store is not supported.") + + +class StoreAddNotSupported(GlanceStoreException): + message = _("Adding images to this store is not supported.") + + +class StoreAddDisabled(GlanceStoreException): + message = _("Configuration for store failed. Adding images to this " + "store is disabled.") + + +class MaxRedirectsExceeded(GlanceStoreException): + message = _("Maximum redirects (%(redirects)s) was exceeded.") + + +class InvalidRedirect(GlanceStoreException): + message = _("Received invalid HTTP redirect.") + + +class NoServiceEndpoint(GlanceStoreException): + message = _("Response from Keystone does not contain a Glance endpoint.") + + +class RegionAmbiguity(GlanceStoreException): + message = _("Multiple 'image' service matches for region %(region)s. This " + "generally means that a region is required and you have not " + "supplied one.") + + +class WorkerCreationFailure(GlanceStoreException): + message = _("Server worker creation failed: %(reason)s.") + + +class SchemaLoadError(GlanceStoreException): + message = _("Unable to load schema: %(reason)s") + + +class InvalidObject(GlanceStoreException): + message = _("Provided object does not match schema " + "'%(schema)s': %(reason)s") + + +class UnsupportedHeaderFeature(GlanceStoreException): + message = _("Provided header feature is unsupported: %(feature)s") + + +class InUseByStore(GlanceStoreException): + message = _("The image cannot be deleted because it is in use through " + "the backend store outside of Glance.") + +class ImageDataNotFound(NotFound): + message = _("No image data could be found") + + +class InvalidParameterValue(Invalid): + message = _("Invalid value '%(value)s' for parameter '%(param)s': " + "%(extra_msg)s") + + +class InvalidImageStatusTransition(Invalid): + message = _("Image status transition from %(cur_status)s to" + " %(new_status)s is not allowed") diff --git a/glance/store/common/location_strategy/__init__.py b/glance/store/common/location_strategy/__init__.py new file mode 100644 index 00000000..e926d436 --- /dev/null +++ b/glance/store/common/location_strategy/__init__.py @@ -0,0 +1,108 @@ +# Copyright 2014 IBM Corp. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy + +from oslo.config import cfg +import stevedore + +import glance.openstack.common.log as logging + +location_strategy_opts = [ + cfg.StrOpt('location_strategy', default='location_order', + help=_("This value sets what strategy will be used to " + "determine the image location order. Currently " + "two strategies are packaged with Glance " + "'location_order' and 'store_type'.")) +] + +CONF = cfg.CONF +CONF.register_opts(location_strategy_opts) + +LOG = logging.getLogger(__name__) + + +def _load_strategies(): + """Load all strategy modules.""" + modules = {} + namespace = "glance.common.image_location_strategy.modules" + ex = stevedore.extension.ExtensionManager(namespace) + for module_name in ex.names(): + try: + mgr = stevedore.driver.DriverManager( + namespace=namespace, + name=module_name, + invoke_on_load=False) + + # Obtain module name + strategy_name = str(mgr.driver.get_strategy_name()) + if strategy_name in modules: + msg = (_('%(strategy)s is registered as a module twice. ' + '%(module)s is not being used.') % + {'strategy': strategy_name, 'module': module_name}) + LOG.warn(msg) + else: + # Initialize strategy module + mgr.driver.init() + modules[strategy_name] = mgr.driver + except Exception as e: + LOG.error(_("Failed to load location strategy module " + "%(module)s: %(e)s") % {'module': module_name, 'e': e}) + return modules + + +_available_strategies = _load_strategies() + + +def verify_location_strategy(conf=None, strategies=_available_strategies): + """Validate user configured 'location_strategy' option value.""" + if not conf: + conf = CONF.location_strategy + if conf not in strategies: + msg = (_('Invalid location_strategy option: %(name)s. ' + 'The valid strategy option(s) is(are): %(strategies)s') % + {'name': conf, 'strategies': ", ".join(strategies.keys())}) + LOG.error(msg) + raise RuntimeError(msg) + + +def get_ordered_locations(locations, **kwargs): + """ + Order image location list by configured strategy. + + :param locations: The original image location list. + :param kwargs: Strategy-specific arguments for under layer strategy module. + :return: The image location list with strategy-specific order. + """ + if not locations: + return [] + strategy_module = _available_strategies[CONF.location_strategy] + return strategy_module.get_ordered_locations(copy.deepcopy(locations), + **kwargs) + + +def choose_best_location(locations, **kwargs): + """ + Choose best location from image location list by configured strategy. + + :param locations: The original image location list. + :param kwargs: Strategy-specific arguments for under layer strategy module. + :return: The best location from image location list. + """ + locations = get_ordered_locations(locations, **kwargs) + if locations: + return locations[0] + else: + return None diff --git a/glance/store/common/location_strategy/location_order.py b/glance/store/common/location_strategy/location_order.py new file mode 100644 index 00000000..022fe762 --- /dev/null +++ b/glance/store/common/location_strategy/location_order.py @@ -0,0 +1,36 @@ +# Copyright 2014 IBM Corp. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Image location order based location strategy module""" + + +def get_strategy_name(): + """Return strategy module name.""" + return 'location_order' + + +def init(): + """Initialize strategy module.""" + pass + + +def get_ordered_locations(locations, **kwargs): + """ + Order image location list. + + :param locations: The original image location list. + :return: The image location list with original natural order. + """ + return locations diff --git a/glance/store/common/location_strategy/store_type.py b/glance/store/common/location_strategy/store_type.py new file mode 100644 index 00000000..93d187aa --- /dev/null +++ b/glance/store/common/location_strategy/store_type.py @@ -0,0 +1,116 @@ +# Copyright 2014 IBM Corp. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Storage preference based location strategy module""" + +import urlparse + +from oslo.config import cfg +import six + +store_type_opts = [ + cfg.ListOpt("store_type_preference", + default=[], + help=_("The store names to use to get store preference order. " + "The name must be registered by one of the stores " + "defined by the 'known_stores' config option. " + "This option will be applied when you using " + "'store_type' option as image location strategy " + "defined by the 'location_strategy' config option.")) +] + +CONF = cfg.CONF +CONF.register_opts(store_type_opts, group='store_type_location_strategy') + +_STORE_TO_SCHEME_MAP = {} + + +def get_strategy_name(): + """Return strategy module name.""" + return 'store_type' + + +def init(): + """Initialize strategy module.""" + # NOTE(zhiyan): We have a plan to do a reusable glance client library for + # all clients like Nova and Cinder in near period, it would be able to + # contains common code to provide uniform image service interface for them, + # just like Brick in Cinder, this code can be moved to there and shared + # between Glance and client both side. So this implementation as far as + # possible to prevent make relationships with Glance(server)-specific code, + # for example: using functions within store module to validate + # 'store_type_preference' option. + mapping = {'filesystem': ['file', 'filesystem'], + 'http': ['http', 'https'], + 'rbd': ['rbd'], + 's3': ['s3', 's3+http', 's3+https'], + 'swift': ['swift', 'swift+https', 'swift+http'], + 'gridfs': ['gridfs'], + 'sheepdog': ['sheepdog'], + 'cinder': ['cinder']} + _STORE_TO_SCHEME_MAP.clear() + _STORE_TO_SCHEME_MAP.update(mapping) + + +def get_ordered_locations(locations, uri_key='url', **kwargs): + """ + Order image location list. + + :param locations: The original image location list. + :param uri_key: The key name for location URI in image location dictionary. + :return: The image location list with preferred store type order. + """ + def _foreach_store_type_preference(): + store_types = CONF.store_type_location_strategy.store_type_preference + for prefered_store in store_types: + prefered_store = str(prefered_store).strip() + if not prefered_store: + continue + yield prefered_store + + if not locations: + return locations + + preferences = {} + others = [] + for prefered_store in _foreach_store_type_preference(): + preferences[prefered_store] = [] + + for location in locations: + uri = location.get(uri_key) + if not uri: + continue + pieces = urlparse.urlparse(uri.strip()) + + store_name = None + for store, schemes in six.iteritems(_STORE_TO_SCHEME_MAP): + if pieces.scheme.strip() in schemes: + store_name = store + break + + if store_name in preferences: + preferences[store_name].append(location) + else: + others.append(location) + + ret = [] + # NOTE(zhiyan): While configuration again since py26 does not support + # ordereddict container. + for prefered_store in _foreach_store_type_preference(): + ret.extend(preferences[prefered_store]) + + ret.extend(others) + + return ret diff --git a/glance/store/common/utils.py b/glance/store/common/utils.py new file mode 100644 index 00000000..7c5fab2b --- /dev/null +++ b/glance/store/common/utils.py @@ -0,0 +1,559 @@ +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +System-level utilities and helper functions. +""" + +import errno + +try: + from eventlet import sleep +except ImportError: + from time import sleep +from eventlet.green import socket + +import functools +import os +import platform +import subprocess +import sys +import uuid + +from OpenSSL import crypto +from oslo.config import cfg +from webob import exc + +from glance.common import exception +import glance.openstack.common.log as logging +from glance.openstack.common import strutils + +CONF = cfg.CONF + +LOG = logging.getLogger(__name__) + +FEATURE_BLACKLIST = ['content-length', 'content-type', 'x-image-meta-size'] + +# Whitelist of v1 API headers of form x-image-meta-xxx +IMAGE_META_HEADERS = ['x-image-meta-location', 'x-image-meta-size', + 'x-image-meta-is_public', 'x-image-meta-disk_format', + 'x-image-meta-container_format', 'x-image-meta-name', + 'x-image-meta-status', 'x-image-meta-copy_from', + 'x-image-meta-uri', 'x-image-meta-checksum', + 'x-image-meta-created_at', 'x-image-meta-updated_at', + 'x-image-meta-deleted_at', 'x-image-meta-min_ram', + 'x-image-meta-min_disk', 'x-image-meta-owner', + 'x-image-meta-store', 'x-image-meta-id', + 'x-image-meta-protected', 'x-image-meta-deleted'] + +GLANCE_TEST_SOCKET_FD_STR = 'GLANCE_TEST_SOCKET_FD' + + +def chunkreadable(iter, chunk_size=65536): + """ + Wrap a readable iterator with a reader yielding chunks of + a preferred size, otherwise leave iterator unchanged. + + :param iter: an iter which may also be readable + :param chunk_size: maximum size of chunk + """ + return chunkiter(iter, chunk_size) if hasattr(iter, 'read') else iter + + +def chunkiter(fp, chunk_size=65536): + """ + Return an iterator to a file-like obj which yields fixed size chunks + + :param fp: a file-like object + :param chunk_size: maximum size of chunk + """ + while True: + chunk = fp.read(chunk_size) + if chunk: + yield chunk + else: + break + + +def cooperative_iter(iter): + """ + Return an iterator which schedules after each + iteration. This can prevent eventlet thread starvation. + + :param iter: an iterator to wrap + """ + try: + for chunk in iter: + sleep(0) + yield chunk + except Exception as err: + msg = _("Error: cooperative_iter exception %s") % err + LOG.error(msg) + raise + + +def cooperative_read(fd): + """ + Wrap a file descriptor's read with a partial function which schedules + after each read. This can prevent eventlet thread starvation. + + :param fd: a file descriptor to wrap + """ + def readfn(*args): + result = fd.read(*args) + sleep(0) + return result + return readfn + + +class CooperativeReader(object): + """ + An eventlet thread friendly class for reading in image data. + + When accessing data either through the iterator or the read method + we perform a sleep to allow a co-operative yield. When there is more than + one image being uploaded/downloaded this prevents eventlet thread + starvation, ie allows all threads to be scheduled periodically rather than + having the same thread be continuously active. + """ + def __init__(self, fd): + """ + :param fd: Underlying image file object + """ + self.fd = fd + self.iterator = None + # NOTE(markwash): if the underlying supports read(), overwrite the + # default iterator-based implementation with cooperative_read which + # is more straightforward + if hasattr(fd, 'read'): + self.read = cooperative_read(fd) + + def read(self, length=None): + """Return the next chunk of the underlying iterator. + + This is replaced with cooperative_read in __init__ if the underlying + fd already supports read(). + """ + if self.iterator is None: + self.iterator = self.__iter__() + try: + return self.iterator.next() + except StopIteration: + return '' + + def __iter__(self): + return cooperative_iter(self.fd.__iter__()) + + +class LimitingReader(object): + """ + Reader designed to fail when reading image data past the configured + allowable amount. + """ + def __init__(self, data, limit): + """ + :param data: Underlying image data object + :param limit: maximum number of bytes the reader should allow + """ + self.data = data + self.limit = limit + self.bytes_read = 0 + + def __iter__(self): + for chunk in self.data: + self.bytes_read += len(chunk) + if self.bytes_read > self.limit: + raise exception.ImageSizeLimitExceeded() + else: + yield chunk + + def read(self, i): + result = self.data.read(i) + self.bytes_read += len(result) + if self.bytes_read > self.limit: + raise exception.ImageSizeLimitExceeded() + return result + + +def image_meta_to_http_headers(image_meta): + """ + Returns a set of image metadata into a dict + of HTTP headers that can be fed to either a Webob + Request object or an httplib.HTTP(S)Connection object + + :param image_meta: Mapping of image metadata + """ + headers = {} + for k, v in image_meta.items(): + if v is not None: + if k == 'properties': + for pk, pv in v.items(): + if pv is not None: + headers["x-image-meta-property-%s" + % pk.lower()] = unicode(pv) + else: + headers["x-image-meta-%s" % k.lower()] = unicode(v) + return headers + + +def add_features_to_http_headers(features, headers): + """ + Adds additional headers representing glance features to be enabled. + + :param headers: Base set of headers + :param features: Map of enabled features + """ + if features: + for k, v in features.items(): + if k.lower() in FEATURE_BLACKLIST: + raise exception.UnsupportedHeaderFeature(feature=k) + if v is not None: + headers[k.lower()] = unicode(v) + + +def get_image_meta_from_headers(response): + """ + Processes HTTP headers from a supplied response that + match the x-image-meta and x-image-meta-property and + returns a mapping of image metadata and properties + + :param response: Response to process + """ + result = {} + properties = {} + + if hasattr(response, 'getheaders'): # httplib.HTTPResponse + headers = response.getheaders() + else: # webob.Response + headers = response.headers.items() + + for key, value in headers: + key = str(key.lower()) + if key.startswith('x-image-meta-property-'): + field_name = key[len('x-image-meta-property-'):].replace('-', '_') + properties[field_name] = value or None + elif key.startswith('x-image-meta-'): + field_name = key[len('x-image-meta-'):].replace('-', '_') + if 'x-image-meta-' + field_name not in IMAGE_META_HEADERS: + msg = _("Bad header: %(header_name)s") % {'header_name': key} + raise exc.HTTPBadRequest(msg, content_type="text/plain") + result[field_name] = value or None + result['properties'] = properties + + for key in ('size', 'min_disk', 'min_ram'): + if key in result: + try: + result[key] = int(result[key]) + except ValueError: + extra = (_("Cannot convert image %(key)s '%(value)s' " + "to an integer.") + % {'key': key, 'value': result[key]}) + raise exception.InvalidParameterValue(value=result[key], + param=key, + extra_msg=extra) + if result[key] < 0: + extra = (_("Image %(key)s must be >= 0 " + "('%(value)s' specified).") + % {'key': key, 'value': result[key]}) + raise exception.InvalidParameterValue(value=result[key], + param=key, + extra_msg=extra) + + for key in ('is_public', 'deleted', 'protected'): + if key in result: + result[key] = strutils.bool_from_string(result[key]) + return result + + +def safe_mkdirs(path): + try: + os.makedirs(path) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + +def safe_remove(path): + try: + os.remove(path) + except OSError as e: + if e.errno != errno.ENOENT: + raise + + +class PrettyTable(object): + """Creates an ASCII art table for use in bin/glance + + Example: + + ID Name Size Hits + --- ----------------- ------------ ----- + 122 image 22 0 + """ + def __init__(self): + self.columns = [] + + def add_column(self, width, label="", just='l'): + """Add a column to the table + + :param width: number of characters wide the column should be + :param label: column heading + :param just: justification for the column, 'l' for left, + 'r' for right + """ + self.columns.append((width, label, just)) + + def make_header(self): + label_parts = [] + break_parts = [] + for width, label, _ in self.columns: + # NOTE(sirp): headers are always left justified + label_part = self._clip_and_justify(label, width, 'l') + label_parts.append(label_part) + + break_part = '-' * width + break_parts.append(break_part) + + label_line = ' '.join(label_parts) + break_line = ' '.join(break_parts) + return '\n'.join([label_line, break_line]) + + def make_row(self, *args): + row = args + row_parts = [] + for data, (width, _, just) in zip(row, self.columns): + row_part = self._clip_and_justify(data, width, just) + row_parts.append(row_part) + + row_line = ' '.join(row_parts) + return row_line + + @staticmethod + def _clip_and_justify(data, width, just): + # clip field to column width + clipped_data = str(data)[:width] + + if just == 'r': + # right justify + justified = clipped_data.rjust(width) + else: + # left justify + justified = clipped_data.ljust(width) + + return justified + + +def get_terminal_size(): + + def _get_terminal_size_posix(): + import fcntl + import struct + import termios + + height_width = None + + try: + height_width = struct.unpack('hh', fcntl.ioctl(sys.stderr.fileno(), + termios.TIOCGWINSZ, + struct.pack('HH', 0, 0))) + except Exception: + pass + + if not height_width: + try: + p = subprocess.Popen(['stty', 'size'], + shell=False, + stdout=subprocess.PIPE, + stderr=open(os.devnull, 'w')) + result = p.communicate() + if p.returncode == 0: + return tuple(int(x) for x in result[0].split()) + except Exception: + pass + + return height_width + + def _get_terminal_size_win32(): + try: + from ctypes import windll, create_string_buffer + handle = windll.kernel32.GetStdHandle(-12) + csbi = create_string_buffer(22) + res = windll.kernel32.GetConsoleScreenBufferInfo(handle, csbi) + except Exception: + return None + if res: + import struct + unpack_tmp = struct.unpack("hhhhHhhhhhh", csbi.raw) + (bufx, bufy, curx, cury, wattr, + left, top, right, bottom, maxx, maxy) = unpack_tmp + height = bottom - top + 1 + width = right - left + 1 + return (height, width) + else: + return None + + def _get_terminal_size_unknownOS(): + raise NotImplementedError + + func = {'posix': _get_terminal_size_posix, + 'win32': _get_terminal_size_win32} + + height_width = func.get(platform.os.name, _get_terminal_size_unknownOS)() + + if height_width is None: + raise exception.Invalid() + + for i in height_width: + if not isinstance(i, int) or i <= 0: + raise exception.Invalid() + + return height_width[0], height_width[1] + + +def mutating(func): + """Decorator to enforce read-only logic""" + @functools.wraps(func) + def wrapped(self, req, *args, **kwargs): + if req.context.read_only: + msg = _("Read-only access") + LOG.debug(msg) + raise exc.HTTPForbidden(msg, request=req, + content_type="text/plain") + return func(self, req, *args, **kwargs) + return wrapped + + +def setup_remote_pydev_debug(host, port): + error_msg = ('Error setting up the debug environment. Verify that the' + ' option pydev_worker_debug_port is pointing to a valid ' + 'hostname or IP on which a pydev server is listening on' + ' the port indicated by pydev_worker_debug_port.') + + try: + try: + from pydev import pydevd + except ImportError: + import pydevd + + pydevd.settrace(host, + port=port, + stdoutToServer=True, + stderrToServer=True) + return True + except Exception: + LOG.exception(error_msg) + raise + + +class LazyPluggable(object): + """A pluggable backend loaded lazily based on some value.""" + + def __init__(self, pivot, config_group=None, **backends): + self.__backends = backends + self.__pivot = pivot + self.__backend = None + self.__config_group = config_group + + def __get_backend(self): + if not self.__backend: + if self.__config_group is None: + backend_name = CONF[self.__pivot] + else: + backend_name = CONF[self.__config_group][self.__pivot] + if backend_name not in self.__backends: + msg = _('Invalid backend: %s') % backend_name + raise exception.GlanceException(msg) + + backend = self.__backends[backend_name] + if isinstance(backend, tuple): + name = backend[0] + fromlist = backend[1] + else: + name = backend + fromlist = backend + + self.__backend = __import__(name, None, None, fromlist) + return self.__backend + + def __getattr__(self, key): + backend = self.__get_backend() + return getattr(backend, key) + + +def validate_key_cert(key_file, cert_file): + try: + error_key_name = "private key" + error_filename = key_file + key_str = open(key_file, "r").read() + key = crypto.load_privatekey(crypto.FILETYPE_PEM, key_str) + + error_key_name = "certficate" + error_filename = cert_file + cert_str = open(cert_file, "r").read() + cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_str) + except IOError as ioe: + raise RuntimeError(_("There is a problem with your %(error_key_name)s " + "%(error_filename)s. Please verify it." + " Error: %(ioe)s") % + {'error_key_name': error_key_name, + 'error_filename': error_filename, + 'ioe': ioe}) + except crypto.Error as ce: + raise RuntimeError(_("There is a problem with your %(error_key_name)s " + "%(error_filename)s. Please verify it. OpenSSL" + " error: %(ce)s") % + {'error_key_name': error_key_name, + 'error_filename': error_filename, + 'ce': ce}) + + try: + data = str(uuid.uuid4()) + digest = "sha1" + + out = crypto.sign(key, data, digest) + crypto.verify(cert, out, data, digest) + except crypto.Error as ce: + raise RuntimeError(_("There is a problem with your key pair. " + "Please verify that cert %(cert_file)s and " + "key %(key_file)s belong together. OpenSSL " + "error %(ce)s") % {'cert_file': cert_file, + 'key_file': key_file, + 'ce': ce}) + + +def get_test_suite_socket(): + global GLANCE_TEST_SOCKET_FD_STR + if GLANCE_TEST_SOCKET_FD_STR in os.environ: + fd = int(os.environ[GLANCE_TEST_SOCKET_FD_STR]) + sock = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM) + sock = socket.SocketType(_sock=sock) + sock.listen(CONF.backlog) + del os.environ[GLANCE_TEST_SOCKET_FD_STR] + os.close(fd) + return sock + return None + + +def is_uuid_like(val): + """Returns validation of a value as a UUID. + + For our purposes, a UUID is a canonical form string: + aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa + """ + try: + return str(uuid.UUID(val)) == val + except (TypeError, ValueError, AttributeError): + return False