Add glance.store common

This commit is contained in:
Flavio Percoco 2014-01-30 11:21:45 +01:00
parent 28d0258873
commit 365e4f58f4
7 changed files with 1261 additions and 0 deletions

View File

View File

@ -0,0 +1,220 @@
#!/usr/bin/env python
# Copyright 2011 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Routines for configuring Glance
"""
import logging
import logging.config
import logging.handlers
import os
from oslo.config import cfg
from paste import deploy
from glance.version import version_info as version
paste_deploy_opts = [
cfg.StrOpt('flavor',
help=_('Partial name of a pipeline in your paste configuration '
'file with the service name removed. For example, if '
'your paste section name is '
'[pipeline:glance-api-keystone] use the value '
'"keystone"')),
cfg.StrOpt('config_file',
help=_('Name of the paste configuration file.')),
]
image_format_opts = [
cfg.ListOpt('container_formats',
default=['ami', 'ari', 'aki', 'bare', 'ovf'],
help=_("Supported values for the 'container_format' "
"image attribute"),
deprecated_opts=[cfg.DeprecatedOpt('container_formats',
group='DEFAULT')]),
cfg.ListOpt('disk_formats',
default=['ami', 'ari', 'aki', 'vhd', 'vmdk', 'raw', 'qcow2',
'vdi', 'iso'],
help=_("Supported values for the 'disk_format' "
"image attribute"),
deprecated_opts=[cfg.DeprecatedOpt('disk_formats',
group='DEFAULT')]),
]
task_opts = [
cfg.IntOpt('task_time_to_live',
default=48,
help=_("Time in hours for which a task lives after, either "
"succeeding or failing"),
deprecated_opts=[cfg.DeprecatedOpt('task_time_to_live',
group='DEFAULT')]),
]
common_opts = [
cfg.BoolOpt('allow_additional_image_properties', default=True,
help=_('Whether to allow users to specify image properties '
'beyond what the image schema provides')),
cfg.IntOpt('image_member_quota', default=128,
help=_('Maximum number of image members per image. '
'Negative values evaluate to unlimited.')),
cfg.IntOpt('image_property_quota', default=128,
help=_('Maximum number of properties allowed on an image. '
'Negative values evaluate to unlimited.')),
cfg.IntOpt('image_tag_quota', default=128,
help=_('Maximum number of tags allowed on an image. '
'Negative values evaluate to unlimited.')),
cfg.IntOpt('image_location_quota', default=10,
help=_('Maximum number of locations allowed on an image. '
'Negative values evaluate to unlimited.')),
cfg.StrOpt('data_api', default='glance.db.sqlalchemy.api',
help=_('Python module path of data access API')),
cfg.IntOpt('limit_param_default', default=25,
help=_('Default value for the number of items returned by a '
'request if not specified explicitly in the request')),
cfg.IntOpt('api_limit_max', default=1000,
help=_('Maximum permissible number of items that could be '
'returned by a request')),
cfg.BoolOpt('show_image_direct_url', default=False,
help=_('Whether to include the backend image storage location '
'in image properties. Revealing storage location can '
'be a security risk, so use this setting with '
'caution!')),
cfg.BoolOpt('show_multiple_locations', default=False,
help=_('Whether to include the backend image locations '
'in image properties. Revealing storage location can '
'be a security risk, so use this setting with '
'caution! The overrides show_image_direct_url.')),
cfg.IntOpt('image_size_cap', default=1099511627776,
help=_("Maximum size of image a user can upload in bytes. "
"Defaults to 1099511627776 bytes (1 TB).")),
cfg.IntOpt('user_storage_quota', default=0,
help=_("Set a system wide quota for every user. This value is "
"the total number of bytes that a user can use across "
"all storage systems. A value of 0 means unlimited.")),
cfg.BoolOpt('enable_v1_api', default=True,
help=_("Deploy the v1 OpenStack Images API.")),
cfg.BoolOpt('enable_v2_api', default=True,
help=_("Deploy the v2 OpenStack Images API.")),
cfg.StrOpt('pydev_worker_debug_host', default=None,
help=_('The hostname/IP of the pydev process listening for '
'debug connections')),
cfg.IntOpt('pydev_worker_debug_port', default=5678,
help=_('The port on which a pydev process is listening for '
'connections.')),
cfg.StrOpt('metadata_encryption_key', secret=True,
help=_('Key used for encrypting sensitive metadata while '
'talking to the registry or database.')),
]
CONF = cfg.CONF
CONF.register_opts(paste_deploy_opts, group='paste_deploy')
CONF.register_opts(image_format_opts, group='image_format')
CONF.register_opts(task_opts, group='task')
CONF.register_opts(common_opts)
def parse_args(args=None, usage=None, default_config_files=None):
CONF(args=args,
project='glance',
version=version.cached_version_string(),
usage=usage,
default_config_files=default_config_files)
def parse_cache_args(args=None):
config_files = cfg.find_config_files(project='glance', prog='glance-cache')
parse_args(args=args, default_config_files=config_files)
def _get_deployment_flavor(flavor=None):
"""
Retrieve the paste_deploy.flavor config item, formatted appropriately
for appending to the application name.
:param flavor: if specified, use this setting rather than the
paste_deploy.flavor configuration setting
"""
if not flavor:
flavor = CONF.paste_deploy.flavor
return '' if not flavor else ('-' + flavor)
def _get_paste_config_path():
paste_suffix = '-paste.ini'
conf_suffix = '.conf'
if CONF.config_file:
# Assume paste config is in a paste.ini file corresponding
# to the last config file
path = CONF.config_file[-1].replace(conf_suffix, paste_suffix)
else:
path = CONF.prog + paste_suffix
return CONF.find_file(os.path.basename(path))
def _get_deployment_config_file():
"""
Retrieve the deployment_config_file config item, formatted as an
absolute pathname.
"""
path = CONF.paste_deploy.config_file
if not path:
path = _get_paste_config_path()
if not path:
msg = "Unable to locate paste config file for %s." % CONF.prog
raise RuntimeError(msg)
return os.path.abspath(path)
def load_paste_app(app_name, flavor=None, conf_file=None):
"""
Builds and returns a WSGI app from a paste config file.
We assume the last config file specified in the supplied ConfigOpts
object is the paste config file, if conf_file is None.
:param app_name: name of the application to load
:param flavor: name of the variant of the application to load
:param conf_file: path to the paste config file
:raises RuntimeError when config file cannot be located or application
cannot be loaded from config file
"""
# append the deployment flavor to the application name,
# in order to identify the appropriate paste pipeline
app_name += _get_deployment_flavor(flavor)
if not conf_file:
conf_file = _get_deployment_config_file()
try:
logger = logging.getLogger(__name__)
logger.debug(_("Loading %(app_name)s from %(conf_file)s"),
{'conf_file': conf_file, 'app_name': app_name})
app = deploy.loadapp("config:%s" % conf_file, name=app_name)
# Log the options used when starting if we're in debug mode...
if CONF.debug:
CONF.log_opt_values(logger, logging.DEBUG)
return app
except (LookupError, ImportError) as e:
msg = (_("Unable to load %(app_name)s from "
"configuration file %(conf_file)s."
"\nGot: %(e)r") % {'app_name': app_name,
'conf_file': conf_file,
'e': e})
logger.error(msg)
raise RuntimeError(msg)

View File

@ -0,0 +1,222 @@
# Copyright (c) 2014 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Glance Store exception subclasses"""
class GlanceStoreException(Exception):
"""
Base Glance Store Exception
To correctly use this class, inherit from it and define
a 'message' property. That message will get printf'd
with the keyword arguments provided to the constructor.
"""
message = ''
def __init__(self, **kwargs):
msg = self.message.format(**kwargs)
super(Exception, self).__init__(msg)
class MissingCredentialError(GlanceStoreException):
message = _("Missing required credential: %(required)s")
class BadAuthStrategy(GlanceStoreException):
message = _("Incorrect auth strategy, expected \"%(expected)s\" but "
"received \"%(received)s\"")
class NotFound(GlanceStoreException):
message = _("An object with the specified identifier was not found.")
class UnknownScheme(GlanceStoreException):
message = _("Unknown scheme '%(scheme)s' found in URI")
class BadStoreUri(GlanceStoreException):
message = _("The Store URI was malformed.")
class Duplicate(GlanceStoreException):
message = _("An object with the same identifier already exists.")
class Conflict(GlanceStoreException):
message = _("An object with the same identifier is currently being "
"operated on.")
class StorageFull(GlanceStoreException):
message = _("There is not enough disk space on the image storage media.")
class StorageQuotaFull(GlanceStoreException):
message = _("The size of the data %(image_size)s will exceed the limit. "
"%(remaining)s bytes remaining.")
class StorageWriteDenied(GlanceStoreException):
message = _("Permission to write image storage media denied.")
class AuthBadRequest(GlanceStoreException):
message = _("Connect error/bad request to Auth service at URL %(url)s.")
class AuthUrlNotFound(GlanceStoreException):
message = _("Auth service at URL %(url)s not found.")
class AuthorizationFailure(GlanceStoreException):
message = _("Authorization failed.")
class NotAuthenticated(GlanceStoreException):
message = _("You are not authenticated.")
class Forbidden(GlanceStoreException):
message = _("You are not authorized to complete this action.")
class ForbiddenPublicImage(Forbidden):
message = _("You are not authorized to complete this action.")
class ProtectedImageDelete(Forbidden):
message = _("Image %(image_id)s is protected and cannot be deleted.")
class Invalid(GlanceStoreException):
message = _("Data supplied was not valid.")
class InvalidSortKey(Invalid):
message = _("Sort key supplied was not valid.")
class InvalidPropertyProtectionConfiguration(Invalid):
message = _("Invalid configuration in property protection file.")
class InvalidFilterRangeValue(Invalid):
message = _("Unable to filter using the specified range.")
class ReadonlyProperty(Forbidden):
message = _("Attribute '%(property)s' is read-only.")
class ReservedProperty(Forbidden):
message = _("Attribute '%(property)s' is reserved.")
class AuthorizationRedirect(GlanceStoreException):
message = _("Redirecting to %(uri)s for authorization.")
class DatabaseMigrationError(GlanceStoreException):
message = _("There was an error migrating the database.")
class ClientConnectionError(GlanceStoreException):
message = _("There was an error connecting to a server")
class ClientConfigurationError(GlanceStoreException):
message = _("There was an error configuring the client.")
class BadStoreConfiguration(GlanceStoreException):
message = _("Store %(store_name)s could not be configured correctly. "
"Reason: %(reason)s")
class BadDriverConfiguration(GlanceStoreException):
message = _("Driver %(driver_name)s could not be configured correctly. "
"Reason: %(reason)s")
class StoreDeleteNotSupported(GlanceStoreException):
message = _("Deleting images from this store is not supported.")
class StoreGetNotSupported(GlanceStoreException):
message = _("Getting images from this store is not supported.")
class StoreAddNotSupported(GlanceStoreException):
message = _("Adding images to this store is not supported.")
class StoreAddDisabled(GlanceStoreException):
message = _("Configuration for store failed. Adding images to this "
"store is disabled.")
class MaxRedirectsExceeded(GlanceStoreException):
message = _("Maximum redirects (%(redirects)s) was exceeded.")
class InvalidRedirect(GlanceStoreException):
message = _("Received invalid HTTP redirect.")
class NoServiceEndpoint(GlanceStoreException):
message = _("Response from Keystone does not contain a Glance endpoint.")
class RegionAmbiguity(GlanceStoreException):
message = _("Multiple 'image' service matches for region %(region)s. This "
"generally means that a region is required and you have not "
"supplied one.")
class WorkerCreationFailure(GlanceStoreException):
message = _("Server worker creation failed: %(reason)s.")
class SchemaLoadError(GlanceStoreException):
message = _("Unable to load schema: %(reason)s")
class InvalidObject(GlanceStoreException):
message = _("Provided object does not match schema "
"'%(schema)s': %(reason)s")
class UnsupportedHeaderFeature(GlanceStoreException):
message = _("Provided header feature is unsupported: %(feature)s")
class InUseByStore(GlanceStoreException):
message = _("The image cannot be deleted because it is in use through "
"the backend store outside of Glance.")
class ImageDataNotFound(NotFound):
message = _("No image data could be found")
class InvalidParameterValue(Invalid):
message = _("Invalid value '%(value)s' for parameter '%(param)s': "
"%(extra_msg)s")
class InvalidImageStatusTransition(Invalid):
message = _("Image status transition from %(cur_status)s to"
" %(new_status)s is not allowed")

View File

@ -0,0 +1,108 @@
# Copyright 2014 IBM Corp.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import copy
from oslo.config import cfg
import stevedore
import glance.openstack.common.log as logging
location_strategy_opts = [
cfg.StrOpt('location_strategy', default='location_order',
help=_("This value sets what strategy will be used to "
"determine the image location order. Currently "
"two strategies are packaged with Glance "
"'location_order' and 'store_type'."))
]
CONF = cfg.CONF
CONF.register_opts(location_strategy_opts)
LOG = logging.getLogger(__name__)
def _load_strategies():
"""Load all strategy modules."""
modules = {}
namespace = "glance.common.image_location_strategy.modules"
ex = stevedore.extension.ExtensionManager(namespace)
for module_name in ex.names():
try:
mgr = stevedore.driver.DriverManager(
namespace=namespace,
name=module_name,
invoke_on_load=False)
# Obtain module name
strategy_name = str(mgr.driver.get_strategy_name())
if strategy_name in modules:
msg = (_('%(strategy)s is registered as a module twice. '
'%(module)s is not being used.') %
{'strategy': strategy_name, 'module': module_name})
LOG.warn(msg)
else:
# Initialize strategy module
mgr.driver.init()
modules[strategy_name] = mgr.driver
except Exception as e:
LOG.error(_("Failed to load location strategy module "
"%(module)s: %(e)s") % {'module': module_name, 'e': e})
return modules
_available_strategies = _load_strategies()
def verify_location_strategy(conf=None, strategies=_available_strategies):
"""Validate user configured 'location_strategy' option value."""
if not conf:
conf = CONF.location_strategy
if conf not in strategies:
msg = (_('Invalid location_strategy option: %(name)s. '
'The valid strategy option(s) is(are): %(strategies)s') %
{'name': conf, 'strategies': ", ".join(strategies.keys())})
LOG.error(msg)
raise RuntimeError(msg)
def get_ordered_locations(locations, **kwargs):
"""
Order image location list by configured strategy.
:param locations: The original image location list.
:param kwargs: Strategy-specific arguments for under layer strategy module.
:return: The image location list with strategy-specific order.
"""
if not locations:
return []
strategy_module = _available_strategies[CONF.location_strategy]
return strategy_module.get_ordered_locations(copy.deepcopy(locations),
**kwargs)
def choose_best_location(locations, **kwargs):
"""
Choose best location from image location list by configured strategy.
:param locations: The original image location list.
:param kwargs: Strategy-specific arguments for under layer strategy module.
:return: The best location from image location list.
"""
locations = get_ordered_locations(locations, **kwargs)
if locations:
return locations[0]
else:
return None

View File

@ -0,0 +1,36 @@
# Copyright 2014 IBM Corp.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Image location order based location strategy module"""
def get_strategy_name():
"""Return strategy module name."""
return 'location_order'
def init():
"""Initialize strategy module."""
pass
def get_ordered_locations(locations, **kwargs):
"""
Order image location list.
:param locations: The original image location list.
:return: The image location list with original natural order.
"""
return locations

View File

@ -0,0 +1,116 @@
# Copyright 2014 IBM Corp.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Storage preference based location strategy module"""
import urlparse
from oslo.config import cfg
import six
store_type_opts = [
cfg.ListOpt("store_type_preference",
default=[],
help=_("The store names to use to get store preference order. "
"The name must be registered by one of the stores "
"defined by the 'known_stores' config option. "
"This option will be applied when you using "
"'store_type' option as image location strategy "
"defined by the 'location_strategy' config option."))
]
CONF = cfg.CONF
CONF.register_opts(store_type_opts, group='store_type_location_strategy')
_STORE_TO_SCHEME_MAP = {}
def get_strategy_name():
"""Return strategy module name."""
return 'store_type'
def init():
"""Initialize strategy module."""
# NOTE(zhiyan): We have a plan to do a reusable glance client library for
# all clients like Nova and Cinder in near period, it would be able to
# contains common code to provide uniform image service interface for them,
# just like Brick in Cinder, this code can be moved to there and shared
# between Glance and client both side. So this implementation as far as
# possible to prevent make relationships with Glance(server)-specific code,
# for example: using functions within store module to validate
# 'store_type_preference' option.
mapping = {'filesystem': ['file', 'filesystem'],
'http': ['http', 'https'],
'rbd': ['rbd'],
's3': ['s3', 's3+http', 's3+https'],
'swift': ['swift', 'swift+https', 'swift+http'],
'gridfs': ['gridfs'],
'sheepdog': ['sheepdog'],
'cinder': ['cinder']}
_STORE_TO_SCHEME_MAP.clear()
_STORE_TO_SCHEME_MAP.update(mapping)
def get_ordered_locations(locations, uri_key='url', **kwargs):
"""
Order image location list.
:param locations: The original image location list.
:param uri_key: The key name for location URI in image location dictionary.
:return: The image location list with preferred store type order.
"""
def _foreach_store_type_preference():
store_types = CONF.store_type_location_strategy.store_type_preference
for prefered_store in store_types:
prefered_store = str(prefered_store).strip()
if not prefered_store:
continue
yield prefered_store
if not locations:
return locations
preferences = {}
others = []
for prefered_store in _foreach_store_type_preference():
preferences[prefered_store] = []
for location in locations:
uri = location.get(uri_key)
if not uri:
continue
pieces = urlparse.urlparse(uri.strip())
store_name = None
for store, schemes in six.iteritems(_STORE_TO_SCHEME_MAP):
if pieces.scheme.strip() in schemes:
store_name = store
break
if store_name in preferences:
preferences[store_name].append(location)
else:
others.append(location)
ret = []
# NOTE(zhiyan): While configuration again since py26 does not support
# ordereddict container.
for prefered_store in _foreach_store_type_preference():
ret.extend(preferences[prefered_store])
ret.extend(others)
return ret

View File

@ -0,0 +1,559 @@
# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
System-level utilities and helper functions.
"""
import errno
try:
from eventlet import sleep
except ImportError:
from time import sleep
from eventlet.green import socket
import functools
import os
import platform
import subprocess
import sys
import uuid
from OpenSSL import crypto
from oslo.config import cfg
from webob import exc
from glance.common import exception
import glance.openstack.common.log as logging
from glance.openstack.common import strutils
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
FEATURE_BLACKLIST = ['content-length', 'content-type', 'x-image-meta-size']
# Whitelist of v1 API headers of form x-image-meta-xxx
IMAGE_META_HEADERS = ['x-image-meta-location', 'x-image-meta-size',
'x-image-meta-is_public', 'x-image-meta-disk_format',
'x-image-meta-container_format', 'x-image-meta-name',
'x-image-meta-status', 'x-image-meta-copy_from',
'x-image-meta-uri', 'x-image-meta-checksum',
'x-image-meta-created_at', 'x-image-meta-updated_at',
'x-image-meta-deleted_at', 'x-image-meta-min_ram',
'x-image-meta-min_disk', 'x-image-meta-owner',
'x-image-meta-store', 'x-image-meta-id',
'x-image-meta-protected', 'x-image-meta-deleted']
GLANCE_TEST_SOCKET_FD_STR = 'GLANCE_TEST_SOCKET_FD'
def chunkreadable(iter, chunk_size=65536):
"""
Wrap a readable iterator with a reader yielding chunks of
a preferred size, otherwise leave iterator unchanged.
:param iter: an iter which may also be readable
:param chunk_size: maximum size of chunk
"""
return chunkiter(iter, chunk_size) if hasattr(iter, 'read') else iter
def chunkiter(fp, chunk_size=65536):
"""
Return an iterator to a file-like obj which yields fixed size chunks
:param fp: a file-like object
:param chunk_size: maximum size of chunk
"""
while True:
chunk = fp.read(chunk_size)
if chunk:
yield chunk
else:
break
def cooperative_iter(iter):
"""
Return an iterator which schedules after each
iteration. This can prevent eventlet thread starvation.
:param iter: an iterator to wrap
"""
try:
for chunk in iter:
sleep(0)
yield chunk
except Exception as err:
msg = _("Error: cooperative_iter exception %s") % err
LOG.error(msg)
raise
def cooperative_read(fd):
"""
Wrap a file descriptor's read with a partial function which schedules
after each read. This can prevent eventlet thread starvation.
:param fd: a file descriptor to wrap
"""
def readfn(*args):
result = fd.read(*args)
sleep(0)
return result
return readfn
class CooperativeReader(object):
"""
An eventlet thread friendly class for reading in image data.
When accessing data either through the iterator or the read method
we perform a sleep to allow a co-operative yield. When there is more than
one image being uploaded/downloaded this prevents eventlet thread
starvation, ie allows all threads to be scheduled periodically rather than
having the same thread be continuously active.
"""
def __init__(self, fd):
"""
:param fd: Underlying image file object
"""
self.fd = fd
self.iterator = None
# NOTE(markwash): if the underlying supports read(), overwrite the
# default iterator-based implementation with cooperative_read which
# is more straightforward
if hasattr(fd, 'read'):
self.read = cooperative_read(fd)
def read(self, length=None):
"""Return the next chunk of the underlying iterator.
This is replaced with cooperative_read in __init__ if the underlying
fd already supports read().
"""
if self.iterator is None:
self.iterator = self.__iter__()
try:
return self.iterator.next()
except StopIteration:
return ''
def __iter__(self):
return cooperative_iter(self.fd.__iter__())
class LimitingReader(object):
"""
Reader designed to fail when reading image data past the configured
allowable amount.
"""
def __init__(self, data, limit):
"""
:param data: Underlying image data object
:param limit: maximum number of bytes the reader should allow
"""
self.data = data
self.limit = limit
self.bytes_read = 0
def __iter__(self):
for chunk in self.data:
self.bytes_read += len(chunk)
if self.bytes_read > self.limit:
raise exception.ImageSizeLimitExceeded()
else:
yield chunk
def read(self, i):
result = self.data.read(i)
self.bytes_read += len(result)
if self.bytes_read > self.limit:
raise exception.ImageSizeLimitExceeded()
return result
def image_meta_to_http_headers(image_meta):
"""
Returns a set of image metadata into a dict
of HTTP headers that can be fed to either a Webob
Request object or an httplib.HTTP(S)Connection object
:param image_meta: Mapping of image metadata
"""
headers = {}
for k, v in image_meta.items():
if v is not None:
if k == 'properties':
for pk, pv in v.items():
if pv is not None:
headers["x-image-meta-property-%s"
% pk.lower()] = unicode(pv)
else:
headers["x-image-meta-%s" % k.lower()] = unicode(v)
return headers
def add_features_to_http_headers(features, headers):
"""
Adds additional headers representing glance features to be enabled.
:param headers: Base set of headers
:param features: Map of enabled features
"""
if features:
for k, v in features.items():
if k.lower() in FEATURE_BLACKLIST:
raise exception.UnsupportedHeaderFeature(feature=k)
if v is not None:
headers[k.lower()] = unicode(v)
def get_image_meta_from_headers(response):
"""
Processes HTTP headers from a supplied response that
match the x-image-meta and x-image-meta-property and
returns a mapping of image metadata and properties
:param response: Response to process
"""
result = {}
properties = {}
if hasattr(response, 'getheaders'): # httplib.HTTPResponse
headers = response.getheaders()
else: # webob.Response
headers = response.headers.items()
for key, value in headers:
key = str(key.lower())
if key.startswith('x-image-meta-property-'):
field_name = key[len('x-image-meta-property-'):].replace('-', '_')
properties[field_name] = value or None
elif key.startswith('x-image-meta-'):
field_name = key[len('x-image-meta-'):].replace('-', '_')
if 'x-image-meta-' + field_name not in IMAGE_META_HEADERS:
msg = _("Bad header: %(header_name)s") % {'header_name': key}
raise exc.HTTPBadRequest(msg, content_type="text/plain")
result[field_name] = value or None
result['properties'] = properties
for key in ('size', 'min_disk', 'min_ram'):
if key in result:
try:
result[key] = int(result[key])
except ValueError:
extra = (_("Cannot convert image %(key)s '%(value)s' "
"to an integer.")
% {'key': key, 'value': result[key]})
raise exception.InvalidParameterValue(value=result[key],
param=key,
extra_msg=extra)
if result[key] < 0:
extra = (_("Image %(key)s must be >= 0 "
"('%(value)s' specified).")
% {'key': key, 'value': result[key]})
raise exception.InvalidParameterValue(value=result[key],
param=key,
extra_msg=extra)
for key in ('is_public', 'deleted', 'protected'):
if key in result:
result[key] = strutils.bool_from_string(result[key])
return result
def safe_mkdirs(path):
try:
os.makedirs(path)
except OSError as e:
if e.errno != errno.EEXIST:
raise
def safe_remove(path):
try:
os.remove(path)
except OSError as e:
if e.errno != errno.ENOENT:
raise
class PrettyTable(object):
"""Creates an ASCII art table for use in bin/glance
Example:
ID Name Size Hits
--- ----------------- ------------ -----
122 image 22 0
"""
def __init__(self):
self.columns = []
def add_column(self, width, label="", just='l'):
"""Add a column to the table
:param width: number of characters wide the column should be
:param label: column heading
:param just: justification for the column, 'l' for left,
'r' for right
"""
self.columns.append((width, label, just))
def make_header(self):
label_parts = []
break_parts = []
for width, label, _ in self.columns:
# NOTE(sirp): headers are always left justified
label_part = self._clip_and_justify(label, width, 'l')
label_parts.append(label_part)
break_part = '-' * width
break_parts.append(break_part)
label_line = ' '.join(label_parts)
break_line = ' '.join(break_parts)
return '\n'.join([label_line, break_line])
def make_row(self, *args):
row = args
row_parts = []
for data, (width, _, just) in zip(row, self.columns):
row_part = self._clip_and_justify(data, width, just)
row_parts.append(row_part)
row_line = ' '.join(row_parts)
return row_line
@staticmethod
def _clip_and_justify(data, width, just):
# clip field to column width
clipped_data = str(data)[:width]
if just == 'r':
# right justify
justified = clipped_data.rjust(width)
else:
# left justify
justified = clipped_data.ljust(width)
return justified
def get_terminal_size():
def _get_terminal_size_posix():
import fcntl
import struct
import termios
height_width = None
try:
height_width = struct.unpack('hh', fcntl.ioctl(sys.stderr.fileno(),
termios.TIOCGWINSZ,
struct.pack('HH', 0, 0)))
except Exception:
pass
if not height_width:
try:
p = subprocess.Popen(['stty', 'size'],
shell=False,
stdout=subprocess.PIPE,
stderr=open(os.devnull, 'w'))
result = p.communicate()
if p.returncode == 0:
return tuple(int(x) for x in result[0].split())
except Exception:
pass
return height_width
def _get_terminal_size_win32():
try:
from ctypes import windll, create_string_buffer
handle = windll.kernel32.GetStdHandle(-12)
csbi = create_string_buffer(22)
res = windll.kernel32.GetConsoleScreenBufferInfo(handle, csbi)
except Exception:
return None
if res:
import struct
unpack_tmp = struct.unpack("hhhhHhhhhhh", csbi.raw)
(bufx, bufy, curx, cury, wattr,
left, top, right, bottom, maxx, maxy) = unpack_tmp
height = bottom - top + 1
width = right - left + 1
return (height, width)
else:
return None
def _get_terminal_size_unknownOS():
raise NotImplementedError
func = {'posix': _get_terminal_size_posix,
'win32': _get_terminal_size_win32}
height_width = func.get(platform.os.name, _get_terminal_size_unknownOS)()
if height_width is None:
raise exception.Invalid()
for i in height_width:
if not isinstance(i, int) or i <= 0:
raise exception.Invalid()
return height_width[0], height_width[1]
def mutating(func):
"""Decorator to enforce read-only logic"""
@functools.wraps(func)
def wrapped(self, req, *args, **kwargs):
if req.context.read_only:
msg = _("Read-only access")
LOG.debug(msg)
raise exc.HTTPForbidden(msg, request=req,
content_type="text/plain")
return func(self, req, *args, **kwargs)
return wrapped
def setup_remote_pydev_debug(host, port):
error_msg = ('Error setting up the debug environment. Verify that the'
' option pydev_worker_debug_port is pointing to a valid '
'hostname or IP on which a pydev server is listening on'
' the port indicated by pydev_worker_debug_port.')
try:
try:
from pydev import pydevd
except ImportError:
import pydevd
pydevd.settrace(host,
port=port,
stdoutToServer=True,
stderrToServer=True)
return True
except Exception:
LOG.exception(error_msg)
raise
class LazyPluggable(object):
"""A pluggable backend loaded lazily based on some value."""
def __init__(self, pivot, config_group=None, **backends):
self.__backends = backends
self.__pivot = pivot
self.__backend = None
self.__config_group = config_group
def __get_backend(self):
if not self.__backend:
if self.__config_group is None:
backend_name = CONF[self.__pivot]
else:
backend_name = CONF[self.__config_group][self.__pivot]
if backend_name not in self.__backends:
msg = _('Invalid backend: %s') % backend_name
raise exception.GlanceException(msg)
backend = self.__backends[backend_name]
if isinstance(backend, tuple):
name = backend[0]
fromlist = backend[1]
else:
name = backend
fromlist = backend
self.__backend = __import__(name, None, None, fromlist)
return self.__backend
def __getattr__(self, key):
backend = self.__get_backend()
return getattr(backend, key)
def validate_key_cert(key_file, cert_file):
try:
error_key_name = "private key"
error_filename = key_file
key_str = open(key_file, "r").read()
key = crypto.load_privatekey(crypto.FILETYPE_PEM, key_str)
error_key_name = "certficate"
error_filename = cert_file
cert_str = open(cert_file, "r").read()
cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_str)
except IOError as ioe:
raise RuntimeError(_("There is a problem with your %(error_key_name)s "
"%(error_filename)s. Please verify it."
" Error: %(ioe)s") %
{'error_key_name': error_key_name,
'error_filename': error_filename,
'ioe': ioe})
except crypto.Error as ce:
raise RuntimeError(_("There is a problem with your %(error_key_name)s "
"%(error_filename)s. Please verify it. OpenSSL"
" error: %(ce)s") %
{'error_key_name': error_key_name,
'error_filename': error_filename,
'ce': ce})
try:
data = str(uuid.uuid4())
digest = "sha1"
out = crypto.sign(key, data, digest)
crypto.verify(cert, out, data, digest)
except crypto.Error as ce:
raise RuntimeError(_("There is a problem with your key pair. "
"Please verify that cert %(cert_file)s and "
"key %(key_file)s belong together. OpenSSL "
"error %(ce)s") % {'cert_file': cert_file,
'key_file': key_file,
'ce': ce})
def get_test_suite_socket():
global GLANCE_TEST_SOCKET_FD_STR
if GLANCE_TEST_SOCKET_FD_STR in os.environ:
fd = int(os.environ[GLANCE_TEST_SOCKET_FD_STR])
sock = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM)
sock = socket.SocketType(_sock=sock)
sock.listen(CONF.backlog)
del os.environ[GLANCE_TEST_SOCKET_FD_STR]
os.close(fd)
return sock
return None
def is_uuid_like(val):
"""Returns validation of a value as a UUID.
For our purposes, a UUID is a canonical form string:
aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa
"""
try:
return str(uuid.UUID(val)) == val
except (TypeError, ValueError, AttributeError):
return False