openstack/glance_store
Code Issues Proposed changes

Port swift store

Browse Source 
Change-Id: Ie070eb95b22729e30e7ced359e890e69e590d04c
This commit is contained in:
Flavio Percoco 2014-07-18 15:07:33 +02:00
parent 0937a9ae6e
commit 219eaa22c2
13 changed files with 2567 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
glance/store/_drivers/swift/__init__.py Normal file
View File

@ -0,0 +1,17 @@
# Copyright 2014 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from glance.store._drivers.swift import utils # noqa
from glance.store._drivers.swift.store import * # noqa

837
glance/store/_drivers/swift/store.py Normal file
View File

@ -0,0 +1,837 @@
# Copyright 2010-2011 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Storage backend for SWIFT"""
import hashlib
import httplib
import logging
import math
from oslo.config import cfg
import six.moves.urllib.parse as urlparse
import swiftclient
import urllib
import glance.store
from glance.store._drivers.swift import utils as sutils
from glance.store.common import auth
from glance.store import driver
from glance.store import exceptions
from glance.store import i18n
from glance.store import location
from glance.store.openstack.common import excutils
_ = i18n._
LOG = logging.getLogger(__name__)
_LI = i18n._LI
DEFAULT_CONTAINER = 'glance'
DEFAULT_LARGE_OBJECT_SIZE = 5 * 1024 # 5GB
DEFAULT_LARGE_OBJECT_CHUNK_SIZE = 200 # 200M
ONE_MB = 1000 * 1024
_SWIFT_OPTS = [
cfg.BoolOpt('swift_enable_snet', default=False,
help=_('Whether to use ServiceNET to communicate with the '
'Swift storage servers.')),
cfg.StrOpt('swift_store_auth_version', default='2',
help=_('Version of the authentication service to use. '
'Valid versions are 2 for keystone and 1 for swauth '
'and rackspace. (deprecated)')),
cfg.BoolOpt('swift_store_auth_insecure', default=False,
help=_('If True, swiftclient won\'t check for a valid SSL '
'certificate when authenticating.')),
cfg.StrOpt('swift_store_region',
help=_('The region of the swift endpoint to be used for '
'single tenant. This setting is only necessary if the '
'tenant has multiple swift endpoints.')),
cfg.StrOpt('swift_store_endpoint_type', default='publicURL',
help=_('A string giving the endpoint type of the swift '
'service to use (publicURL, adminURL or internalURL). '
'This setting is only used if swift_store_auth_version '
'is 2.')),
cfg.StrOpt('swift_store_service_type', default='object-store',
help=_('A string giving the service type of the swift service '
'to use. This setting is only used if '
'swift_store_auth_version is 2.')),
cfg.StrOpt('swift_store_container',
default=DEFAULT_CONTAINER,
help=_('Container within the account that the account should '
'use for storing images in Swift.')),
cfg.IntOpt('swift_store_large_object_size',
default=DEFAULT_LARGE_OBJECT_SIZE,
help=_('The size, in MB, that Glance will start chunking image '
'files and do a large object manifest in Swift.')),
cfg.IntOpt('swift_store_large_object_chunk_size',
default=DEFAULT_LARGE_OBJECT_CHUNK_SIZE,
help=_('The amount of data written to a temporary '
'disk buffer during the process of chunking '
'the image file.')),
cfg.BoolOpt('swift_store_create_container_on_put', default=False,
help=_('A boolean value that determines if we create the '
'container if it does not exist.')),
cfg.BoolOpt('swift_store_multi_tenant', default=False,
help=_('If set to True, enables multi-tenant storage '
'mode which causes Glance images to be stored in '
'tenant specific Swift accounts.')),
cfg.ListOpt('swift_store_admin_tenants', default=[],
help=_('A list of tenants that will be granted read/write '
'access on all Swift containers created by Glance in '
'multi-tenant mode.')),
cfg.BoolOpt('swift_store_ssl_compression', default=True,
help=_('If set to False, disables SSL layer compression of '
'https swift requests. Setting to False may improve '
'performance for images which are already in a '
'compressed format, eg qcow2.')),
cfg.IntOpt('swift_store_retry_get_count', default=0,
help=_('The number of times a Swift download will be retried '
'before the request fails.'))
]
CONF = cfg.CONF
SWIFT_STORE_REF_PARAMS = sutils.SwiftParams().params
def swift_retry_iter(resp_iter, length, store, location, context):
length = length if length else (resp_iter.len
if hasattr(resp_iter, 'len') else 0)
retries = 0
bytes_read = 0
while retries <= store.conf.glance_store.swift_store_retry_get_count:
try:
for chunk in resp_iter:
yield chunk
bytes_read += len(chunk)
except swiftclient.ClientException as e:
LOG.warn(_(u"Swift exception raised %s") % unicode(e))
if bytes_read != length:
if retries == store.conf.glance_store.swift_store_retry_get_count:
# terminate silently and let higher level decide
LOG.error(_("Stopping Swift retries after %d "
"attempts") % retries)
break
else:
retries += 1
glance_conf = store.conf.glance_store
retry_count = glance_conf.swift_store_retry_get_count
LOG.info(_("Retrying Swift connection "
"(%(retries)d/%(max_retries)d) with "
"range=%(start)d-%(end)d") %
{'retries': retries,
'max_retries': retry_count,
'start': bytes_read,
'end': length})
(resp_headers, resp_iter) = store._get_object(location, None,
bytes_read,
context=context)
else:
break
class StoreLocation(location.StoreLocation):
"""
Class describing a Swift URI. A Swift URI can look like any of
the following:
swift://user:pass@authurl.com/container/obj-id
swift://account:user:pass@authurl.com/container/obj-id
swift+http://user:pass@authurl.com/container/obj-id
swift+https://user:pass@authurl.com/container/obj-id
When using multi-tenant a URI might look like this (a storage URL):
swift+https://example.com/container/obj-id
The swift+http:// URIs indicate there is an HTTP authentication URL.
The default for Swift is an HTTPS authentication URL, so swift:// and
swift+https:// are the same...
"""
def process_specs(self):
self.scheme = self.specs.get('scheme', 'swift+https')
self.user = self.specs.get('user')
self.key = self.specs.get('key')
self.auth_or_store_url = self.specs.get('auth_or_store_url')
self.container = self.specs.get('container')
self.obj = self.specs.get('obj')
def _get_credstring(self):
if self.user and self.key:
return '%s:%s' % (urllib.quote(self.user), urllib.quote(self.key))
return ''
def get_uri(self, credentials_included=True):
auth_or_store_url = self.auth_or_store_url
if auth_or_store_url.startswith('http://'):
auth_or_store_url = auth_or_store_url[len('http://'):]
elif auth_or_store_url.startswith('https://'):
auth_or_store_url = auth_or_store_url[len('https://'):]
credstring = self._get_credstring()
auth_or_store_url = auth_or_store_url.strip('/')
container = self.container.strip('/')
obj = self.obj.strip('/')
if not credentials_included:
#Used only in case of an add
#Get the current store from config
store = CONF.glance_store.default_swift_reference
return '%s://%s/%s/%s' % ('swift+config', store, container, obj)
if self.scheme == 'swift+config':
if self.ssl_enabled == True:
self.scheme = 'swift+https'
else:
self.scheme = 'swift+http'
if credstring != '':
credstring = "%s@" % credstring
return '%s://%s%s/%s/%s' % (self.scheme, credstring, auth_or_store_url,
container, obj)
def _get_conf_value_from_account_ref(self, netloc):
try:
self.user = SWIFT_STORE_REF_PARAMS[netloc]['user']
self.key = SWIFT_STORE_REF_PARAMS[netloc]['key']
netloc = SWIFT_STORE_REF_PARAMS[netloc]['auth_address']
self.ssl_enabled = True
if netloc != '':
if netloc.startswith('http://'):
self.ssl_enabled = False
netloc = netloc[len('http://'):]
elif netloc.startswith('https://'):
netloc = netloc[len('https://'):]
except KeyError:
reason = _("Badly formed Swift URI. Credentials not found for "
"account reference")
LOG.info(reason)
raise exceptions.BadStoreUri(message=reason)
return netloc
def _form_uri_parts(self, netloc, path):
if netloc != '':
# > Python 2.6.1
if '@' in netloc:
creds, netloc = netloc.split('@')
else:
creds = None
else:
# Python 2.6.1 compat
# see lp659445 and Python issue7904
if '@' in path:
creds, path = path.split('@')
else:
creds = None
netloc = path[0:path.find('/')].strip('/')
path = path[path.find('/'):].strip('/')
if creds:
cred_parts = creds.split(':')
if len(cred_parts) < 2:
reason = _("Badly formed credentials in Swift URI.")
LOG.info(reason)
raise exceptions.BadStoreUri(message=reason)
key = cred_parts.pop()
user = ':'.join(cred_parts)
creds = urllib.unquote(creds)
try:
self.user, self.key = creds.rsplit(':', 1)
except exceptions.BadStoreConfiguration:
self.user = urllib.unquote(user)
self.key = urllib.unquote(key)
else:
self.user = None
self.key = None
return netloc, path
def _form_auth_or_store_url(self, netloc, path):
path_parts = path.split('/')
try:
self.obj = path_parts.pop()
self.container = path_parts.pop()
if not netloc.startswith('http'):
# push hostname back into the remaining to build full authurl
path_parts.insert(0, netloc)
self.auth_or_store_url = '/'.join(path_parts)
except IndexError:
reason = _("Badly formed Swift URI.")
LOG.info(reason)
raise exceptions.BadStoreUri(message=reason)
def parse_uri(self, uri):
"""
Parse URLs. This method fixes an issue where credentials specified
in the URL are interpreted differently in Python 2.6.1+ than prior
versions of Python. It also deals with the peculiarity that new-style
Swift URIs have where a username can contain a ':', like so:
swift://account:user:pass@authurl.com/container/obj
and for system created locations with account reference
swift+config://account_reference/container/obj
"""
# Make sure that URIs that contain multiple schemes, such as:
# swift://user:pass@http://authurl.com/v1/container/obj
# are immediately rejected.
if uri.count('://') != 1:
reason = _("URI cannot contain more than one occurrence "
"of a scheme. If you have specified a URI like "
"swift://user:pass@http://authurl.com/v1/container/obj"
", you need to change it to use the "
"swift+http:// scheme, like so: "
"swift+http://user:pass@authurl.com/v1/container/obj")
LOG.info(_LI("Invalid store URI: %(reason)s"), {'reason': reason})
raise exceptions.BadStoreUri(message=reason)
pieces = urlparse.urlparse(uri)
assert pieces.scheme in ('swift', 'swift+http', 'swift+https',
'swift+config')
self.scheme = pieces.scheme
netloc = pieces.netloc
path = pieces.path.lstrip('/')
# NOTE(Sridevi): Fix to map the account reference to the
# corresponding CONF value
if self.scheme == 'swift+config':
netloc = self._get_conf_value_from_account_ref(netloc)
else:
netloc, path = self._form_uri_parts(netloc, path)
self._form_auth_or_store_url(netloc, path)
@property
def swift_url(self):
"""
Creates a fully-qualified auth address that the Swift client library
can use. The scheme for the auth_address is determined using the scheme
included in the `location` field.
HTTPS is assumed, unless 'swift+http' is specified.
"""
if self.auth_or_store_url.startswith('http'):
return self.auth_or_store_url
else:
if self.scheme == 'swift+config':
if self.ssl_enabled == True:
self.scheme = 'swift+https'
else:
self.scheme = 'swift+http'
if self.scheme in ('swift+https', 'swift'):
auth_scheme = 'https://'
else:
auth_scheme = 'http://'
return ''.join([auth_scheme, self.auth_or_store_url])
def Store(conf):
try:
conf.register_opts(_SWIFT_OPTS, group='glance_store')
except cfg.DuplicateOptError:
pass
if conf.glance_store.swift_store_multi_tenant:
return MultiTenantStore(conf)
return SingleTenantStore(conf)
class BaseStore(driver.Store):
CHUNKSIZE = 65536
OPTIONS = _SWIFT_OPTS
def get_schemes(self):
return ('swift+https', 'swift', 'swift+http', 'swift+config')
def configure(self):
glance_conf = self.conf.glance_store
_obj_size = self._option_get('swift_store_large_object_size')
self.large_object_size = _obj_size * ONE_MB
_chunk_size = self._option_get('swift_store_large_object_chunk_size')
self.large_object_chunk_size = _chunk_size * ONE_MB
self.admin_tenants = glance_conf.swift_store_admin_tenants
self.region = glance_conf.swift_store_region
self.service_type = glance_conf.swift_store_service_type
self.endpoint_type = glance_conf.swift_store_endpoint_type
self.snet = glance_conf.swift_enable_snet
self.insecure = glance_conf.swift_store_auth_insecure
self.ssl_compression = glance_conf.swift_store_ssl_compression
super(BaseStore, self).configure()
def _get_object(self, location, connection=None, start=None, context=None):
if not connection:
connection = self.get_connection(location, context=context)
headers = {}
if start is not None:
bytes_range = 'bytes=%d-' % start
headers = {'Range': bytes_range}
try:
resp_headers, resp_body = connection.get_object(
container=location.container, obj=location.obj,
resp_chunk_size=self.CHUNKSIZE, headers=headers)
except swiftclient.ClientException as e:
if e.http_status == httplib.NOT_FOUND:
msg = _("Swift could not find object %s.") % location.obj
LOG.warn(msg)
raise exceptions.NotFound(message=msg)
else:
raise
return (resp_headers, resp_body)
def validate_location(self, uri):
pieces = urlparse.urlparse(uri)
if pieces.scheme in ['swift+config']:
reason = (_("Location credentials are invalid"))
raise exceptions.BadStoreUri(message=reason)
def get(self, location, connection=None, context=None):
location = location.store_location
(resp_headers, resp_body) = self._get_object(location, connection,
context=context)
class ResponseIndexable(glance.store.Indexable):
def another(self):
try:
return self.wrapped.next()
except StopIteration:
return ''
length = int(resp_headers.get('content-length', 0))
if self.conf.glance_store.swift_store_retry_get_count > 0:
resp_body = swift_retry_iter(resp_body, length,
self, location, context)
return (ResponseIndexable(resp_body, length), length)
def get_size(self, location, connection=None, context=None):
location = location.store_location
if not connection:
connection = self.get_connection(location, context=context)
try:
resp_headers = connection.head_object(
container=location.container, obj=location.obj)
return int(resp_headers.get('content-length', 0))
except Exception:
return 0
def _option_get(self, param):
result = getattr(self.conf.glance_store, param)
if not result:
reason = (_("Could not find %(param)s in configuration options.")
% param)
LOG.error(reason)
raise exceptions.BadStoreConfiguration(store_name="swift",
reason=reason)
return result
def _delete_stale_chunks(self, connection, container, chunk_list):
for chunk in chunk_list:
LOG.debug("Deleting chunk %s" % chunk)
try:
connection.delete_object(container, chunk)
except Exception:
msg = _("Failed to delete orphaned chunk "
"%(container)s/%(chunk)s")
LOG.exception(msg % {'container': container,
'chunk': chunk})
def add(self, image_id, image_file, image_size,
connection=None, context=None):
location = self.create_location(image_id, context=context)
if not connection:
connection = self.get_connection(location, context=context)
self._create_container_if_missing(location.container, connection)
LOG.debug("Adding image object '%(obj_name)s' "
"to Swift" % dict(obj_name=location.obj))
try:
if image_size > 0 and image_size < self.large_object_size:
# Image size is known, and is less than large_object_size.
# Send to Swift with regular PUT.
obj_etag = connection.put_object(location.container,
location.obj, image_file,
content_length=image_size)
else:
# Write the image into Swift in chunks.
chunk_id = 1
if image_size > 0:
total_chunks = str(int(
math.ceil(float(image_size) /
float(self.large_object_chunk_size))))
else:
# image_size == 0 is when we don't know the size
# of the image. This can occur with older clients
# that don't inspect the payload size.
LOG.debug("Cannot determine image size. Adding as a "
"segmented object to Swift.")
total_chunks = '?'
checksum = hashlib.md5()
written_chunks = []
combined_chunks_size = 0
while True:
chunk_size = self.large_object_chunk_size
if image_size == 0:
content_length = None
else:
left = image_size - combined_chunks_size
if left == 0:
break
if chunk_size > left:
chunk_size = left
content_length = chunk_size
chunk_name = "%s-%05d" % (location.obj, chunk_id)
reader = ChunkReader(image_file, checksum, chunk_size)
try:
chunk_etag = connection.put_object(
location.container, chunk_name, reader,
content_length=content_length)
written_chunks.append(chunk_name)
except Exception:
# Delete orphaned segments from swift backend
with excutils.save_and_reraise_exception():
LOG.exception(_("Error during chunked upload to "
"backend, deleting stale chunks"))
self._delete_stale_chunks(connection,
location.container,
written_chunks)
bytes_read = reader.bytes_read
msg = ("Wrote chunk %(chunk_name)s (%(chunk_id)d/"
"%(total_chunks)s) of length %(bytes_read)d "
"to Swift returning MD5 of content: "
"%(chunk_etag)s" %
{'chunk_name': chunk_name,
'chunk_id': chunk_id,
'total_chunks': total_chunks,
'bytes_read': bytes_read,
'chunk_etag': chunk_etag})
LOG.debug(msg)
if bytes_read == 0:
# Delete the last chunk, because it's of zero size.
# This will happen if size == 0.
LOG.debug("Deleting final zero-length chunk")
connection.delete_object(location.container,
chunk_name)
break
chunk_id += 1
combined_chunks_size += bytes_read
# In the case we have been given an unknown image size,
# set the size to the total size of the combined chunks.
if image_size == 0:
image_size = combined_chunks_size
# Now we write the object manifest and return the
# manifest's etag...
manifest = "%s/%s-" % (location.container, location.obj)
headers = {'ETag': hashlib.md5("").hexdigest(),
'X-Object-Manifest': manifest}
# The ETag returned for the manifest is actually the
# MD5 hash of the concatenated checksums of the strings
# of each chunk...so we ignore this result in favour of
# the MD5 of the entire image file contents, so that
# users can verify the image file contents accordingly
connection.put_object(location.container, location.obj,
None, headers=headers)
obj_etag = checksum.hexdigest()
# NOTE: We return the user and key here! Have to because
# location is used by the API server to return the actual
# image data. We *really* should consider NOT returning
# the location attribute from GET /images/<ID> and
# GET /images/details
if sutils.is_multiple_swift_store_accounts_enabled():
include_creds = False
else:
include_creds = True
return (location.get_uri(credentials_included=include_creds),
image_size, obj_etag, {})
except swiftclient.ClientException as e:
if e.http_status == httplib.CONFLICT:
msg = _("Swift already has an image at this location")
raise exceptions.Duplicate(message=msg)
msg = (_(u"Failed to add object to Swift.\n"
"Got error from Swift: %s") % unicode(e))
LOG.error(msg)
raise glance.store.BackendException(msg)
def delete(self, location, connection=None, context=None):
location = location.store_location
if not connection:
connection = self.get_connection(location, context=context)
try:
# We request the manifest for the object. If one exists,
# that means the object was uploaded in chunks/segments,
# and we need to delete all the chunks as well as the
# manifest.
manifest = None
try:
headers = connection.head_object(
location.container, location.obj)
manifest = headers.get('x-object-manifest')
except swiftclient.ClientException as e:
if e.http_status != httplib.NOT_FOUND:
raise
if manifest:
# Delete all the chunks before the object manifest itself
obj_container, obj_prefix = manifest.split('/', 1)
segments = connection.get_container(
obj_container, prefix=obj_prefix)[1]
for segment in segments:
# TODO(jaypipes): This would be an easy area to parallelize
# since we're simply sending off parallelizable requests
# to Swift to delete stuff. It's not like we're going to
# be hogging up network or file I/O here...
connection.delete_object(obj_container,
segment['name'])
# Delete object (or, in segmented case, the manifest)
connection.delete_object(location.container, location.obj)
except swiftclient.ClientException as e:
if e.http_status == httplib.NOT_FOUND:
msg = _("Swift could not find image at URI.")
raise exceptions.NotFound(message=msg)
else:
raise
def _create_container_if_missing(self, container, connection):
"""
Creates a missing container in Swift if the
``swift_store_create_container_on_put`` option is set.
:param container: Name of container to create
:param connection: Connection to swift service
"""
try:
connection.head_container(container)
except swiftclient.ClientException as e:
if e.http_status == httplib.NOT_FOUND:
if self.conf.glance_store.swift_store_create_container_on_put:
try:
msg = (_LI("Creating swift container %(container)s") %
{'container': container})
LOG.info(msg)
connection.put_container(container)
except swiftclient.ClientException as e:
msg = (_("Failed to add container to Swift.\n"
"Got error from Swift: %(e)s") % {'e': e})
raise glance.store.BackendException(msg)
else:
msg = (_("The container %(container)s does not exist in "
"Swift. Please set the "
"swift_store_create_container_on_put option"
"to add container to Swift automatically.") %
{'container': container})
raise glance.store.BackendException(msg)
else:
raise
def get_connection(self, location, context=None):
raise NotImplementedError()
def create_location(self, image_id, context=None):
raise NotImplementedError()
class SingleTenantStore(BaseStore):
EXAMPLE_URL = "swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<FILE>"
def configure(self):
super(SingleTenantStore, self).configure()
self.auth_version = self._option_get('swift_store_auth_version')
def configure_add(self):
default_swift_reference = \
SWIFT_STORE_REF_PARAMS.get(
self.conf.glance_store.default_swift_reference)
if default_swift_reference:
self.auth_address = default_swift_reference.get('auth_address')
if (not default_swift_reference) or (not self.auth_address):
reason = _("A value for swift_store_auth_address is required.")
LOG.error(reason)
raise exceptions.BadStoreConfiguration(message=reason)
if self.auth_address.startswith('http://'):
self.scheme = 'swift+http'
else:
self.scheme = 'swift+https'
self.container = self.conf.glance_store.swift_store_container
self.user = default_swift_reference.get('user')
self.key = default_swift_reference.get('key')
if not (self.user or self.key):
reason = _("A value for swift_store_ref_params is required.")
LOG.error(reason)
raise exceptions.BadStoreConfiguration(store_name="swift",
reason=reason)
def create_location(self, image_id, context=None):
specs = {'scheme': self.scheme,
'container': self.container,
'obj': str(image_id),
'auth_or_store_url': self.auth_address,
'user': self.user,
'key': self.key}
return StoreLocation(specs)
def get_connection(self, location, context=None):
if not location.user:
reason = _("Location is missing user:password information.")
LOG.info(reason)
raise exceptions.BadStoreUri(message=reason)
auth_url = location.swift_url
if not auth_url.endswith('/'):
auth_url += '/'
if self.auth_version == '2':
try:
tenant_name, user = location.user.split(':')
except ValueError:
reason = (_("Badly formed tenant:user '%(user)s' in "
"Swift URI") % {'user': location.user})
LOG.info(reason)
raise exceptions.BadStoreUri(message=reason)
else:
tenant_name = None
user = location.user
os_options = {}
if self.region:
os_options['region_name'] = self.region
os_options['endpoint_type'] = self.endpoint_type
os_options['service_type'] = self.service_type
return swiftclient.Connection(
auth_url, user, location.key, insecure=self.insecure,
tenant_name=tenant_name, snet=self.snet,
auth_version=self.auth_version, os_options=os_options,
ssl_compression=self.ssl_compression)
class MultiTenantStore(BaseStore):
EXAMPLE_URL = "swift://<SWIFT_URL>/<CONTAINER>/<FILE>"
def _get_endpoint(self, context):
self.container = self.conf.glance_store.swift_store_container
if context is None:
reason = _("Multi-tenant Swift storage requires a context.")
raise exceptions.BadStoreConfiguration(store_name="swift",
reason=reason)
if context.service_catalog is None:
reason = _("Multi-tenant Swift storage requires "
"a service catalog.")
raise exceptions.BadStoreConfiguration(store_name="swift",
reason=reason)
self.storage_url = auth.get_endpoint(
context.service_catalog, service_type=self.service_type,
endpoint_region=self.region, endpoint_type=self.endpoint_type)
if self.storage_url.startswith('http://'):
self.scheme = 'swift+http'
else:
self.scheme = 'swift+https'
return self.storage_url
def delete(self, location, connection=None, context=None):
if not connection:
connection = self.get_connection(location.store_location,
context=context)
super(MultiTenantStore, self).delete(location, connection)
connection.delete_container(location.store_location.container)
def set_acls(self, location, public=False, read_tenants=None,
write_tenants=None, connection=None, context=None):
location = location.store_location
if not connection:
connection = self.get_connection(location, context=context)
if read_tenants is None:
read_tenants = []
if write_tenants is None:
write_tenants = []
headers = {}
if public:
headers['X-Container-Read'] = ".r:*,.rlistings"
elif read_tenants:
headers['X-Container-Read'] = ','.join('%s:*' % i
for i in read_tenants)
else:
headers['X-Container-Read'] = ''
write_tenants.extend(self.admin_tenants)
if write_tenants:
headers['X-Container-Write'] = ','.join('%s:*' % i
for i in write_tenants)
else:
headers['X-Container-Write'] = ''
try:
connection.post_container(location.container, headers=headers)
except swiftclient.ClientException as e:
if e.http_status == httplib.NOT_FOUND:
msg = _("Swift could not find image at URI.")
raise exceptions.NotFound(message=msg)
else:
raise
def create_location(self, image_id, context=None):
ep = self._get_endpoint(context)
specs = {'scheme': self.scheme,
'container': self.container + '_' + str(image_id),
'obj': str(image_id),
'auth_or_store_url': ep}
return StoreLocation(specs)
def get_connection(self, location, context=None):
return swiftclient.Connection(
None, context.user, None,
preauthurl=location.swift_url,
preauthtoken=context.auth_token,
tenant_name=context.tenant,
auth_version='2', snet=self.snet, insecure=self.insecure,
ssl_compression=self.ssl_compression)
class ChunkReader(object):
def __init__(self, fd, checksum, total):
self.fd = fd
self.checksum = checksum
self.total = total
self.bytes_read = 0
def read(self, i):
left = self.total - self.bytes_read
if i > left:
i = left
result = self.fd.read(i)
self.bytes_read += len(result)
self.checksum.update(result)
return result

111
glance/store/_drivers/swift/utils.py Normal file
View File

@ -0,0 +1,111 @@
# Copyright 2014 Rackspace
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import ConfigParser
import logging
try:
from collections import OrderedDict
except ImportError:
from ordereddict import OrderedDict
from oslo.config import cfg
from glance.store import exceptions
from glance.store import i18n
swift_opts = [
cfg.StrOpt('default_swift_reference',
default="ref1",
help=i18n._('The reference to the default swift account/backing'
' store parameters to use for adding new images.')),
cfg.StrOpt('swift_store_auth_address',
help=i18n._('The address where the Swift authentication '
'service is listening.(deprecated)')),
cfg.StrOpt('swift_store_user', secret=True,
help=i18n._('The user to authenticate against the Swift '
'authentication service (deprecated)')),
cfg.StrOpt('swift_store_key', secret=True,
help=i18n._('Auth key for the user authenticating against the '
'Swift authentication service. (deprecated)')),
cfg.StrOpt('swift_store_config_file', secret=True,
help=i18n._('The config file that has the swift account(s)'
'configs.')),
]
# NOTE(bourke): The default dict_type is collections.OrderedDict in py27, but
# we must set manually for compatibility with py26
CONFIG = ConfigParser.SafeConfigParser(dict_type=OrderedDict)
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
CONF.register_opts(swift_opts, group='glance_store')
def is_multiple_swift_store_accounts_enabled():
if CONF.glance_store.swift_store_config_file is None:
return False
return True
class SwiftParams(object):
def __init__(self):
if is_multiple_swift_store_accounts_enabled():
self.params = self._load_config()
else:
self.params = self._form_default_params()
def _form_default_params(self):
default = {}
if (
CONF.glance_store.swift_store_user and
CONF.glance_store.swift_store_key and
CONF.glance_store.swift_store_auth_address
):
glance_store = CONF.glance_store
default['user'] = glance_store.swift_store_user
default['key'] = glance_store.swift_store_key
default['auth_address'] = glance_store.swift_store_auth_address
return {glance_store.default_swift_reference: default}
return {}
def _load_config(self):
try:
scf = CONF.glance_store.swift_store_config_file
conf_file = CONF.find_file(scf)
CONFIG.read(conf_file)
except Exception as e:
msg = (i18n._("swift config file "
"%(conf_file)s:%(exc)s not found") %
{'conf_file': CONF.glance_store.swift_store_config_file,
'exc': e})
LOG.error(msg)
raise exceptions.BadStoreConfiguration(store_name='swift',
reason=msg)
account_params = {}
account_references = CONFIG.sections()
for ref in account_references:
reference = {}
try:
reference['auth_address'] = CONFIG.get(ref, 'auth_address')
reference['user'] = CONFIG.get(ref, 'user')
reference['key'] = CONFIG.get(ref, 'key')
account_params[ref] = reference
except (ValueError, SyntaxError, ConfigParser.NoOptionError) as e:
LOG.exception(i18n._("Invalid format of swift store config"
"cfg"))
return account_params

288
glance/store/common/auth.py Normal file
View File

@ -0,0 +1,288 @@
# Copyright 2011 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
This auth module is intended to allow OpenStack client-tools to select from a
variety of authentication strategies, including NoAuth (the default), and
Keystone (an identity management system).
> auth_plugin = AuthPlugin(creds)
> auth_plugin.authenticate()
> auth_plugin.auth_token
abcdefg
> auth_plugin.management_url
http://service_endpoint/
"""
import httplib2
import logging
import six.moves.urllib.parse as urlparse
from glance.store.openstack.common import jsonutils
LOG = logging.getLogger(__name__)
class BaseStrategy(object):
def __init__(self):
self.auth_token = None
# TODO(sirp): Should expose selecting public/internal/admin URL.
self.management_url = None
def authenticate(self):
raise NotImplementedError
@property
def is_authenticated(self):
raise NotImplementedError
@property
def strategy(self):
raise NotImplementedError
class NoAuthStrategy(BaseStrategy):
def authenticate(self):
pass
@property
def is_authenticated(self):
return True
@property
def strategy(self):
return 'noauth'
class KeystoneStrategy(BaseStrategy):
MAX_REDIRECTS = 10
def __init__(self, creds, insecure=False, configure_via_auth=True):
self.creds = creds
self.insecure = insecure
self.configure_via_auth = configure_via_auth
super(KeystoneStrategy, self).__init__()
def check_auth_params(self):
# Ensure that supplied credential parameters are as required
for required in ('username', 'password', 'auth_url',
'strategy'):
if self.creds.get(required) is None:
raise exception.MissingCredentialError(required=required)
if self.creds['strategy'] != 'keystone':
raise exception.BadAuthStrategy(expected='keystone',
received=self.creds['strategy'])
# For v2.0 also check tenant is present
if self.creds['auth_url'].rstrip('/').endswith('v2.0'):
if self.creds.get("tenant") is None:
raise exception.MissingCredentialError(required='tenant')
def authenticate(self):
"""Authenticate with the Keystone service.
There are a few scenarios to consider here:
1. Which version of Keystone are we using? v1 which uses headers to
pass the credentials, or v2 which uses a JSON encoded request body?
2. Keystone may respond back with a redirection using a 305 status
code.
3. We may attempt a v1 auth when v2 is what's called for. In this
case, we rewrite the url to contain /v2.0/ and retry using the v2
protocol.
"""
def _authenticate(auth_url):
# If OS_AUTH_URL is missing a trailing slash add one
if not auth_url.endswith('/'):
auth_url += '/'
token_url = urlparse.urljoin(auth_url, "tokens")
# 1. Check Keystone version
is_v2 = auth_url.rstrip('/').endswith('v2.0')
if is_v2:
self._v2_auth(token_url)
else:
self._v1_auth(token_url)
self.check_auth_params()
auth_url = self.creds['auth_url']
for _ in range(self.MAX_REDIRECTS):
try:
_authenticate(auth_url)
except exception.AuthorizationRedirect as e:
# 2. Keystone may redirect us
auth_url = e.url
except exception.AuthorizationFailure:
# 3. In some configurations nova makes redirection to
# v2.0 keystone endpoint. Also, new location does not
# contain real endpoint, only hostname and port.
if 'v2.0' not in auth_url:
auth_url = urlparse.urljoin(auth_url, 'v2.0/')
else:
# If we successfully auth'd, then memorize the correct auth_url
# for future use.
self.creds['auth_url'] = auth_url
break
else:
# Guard against a redirection loop
raise exception.MaxRedirectsExceeded(redirects=self.MAX_REDIRECTS)
def _v1_auth(self, token_url):
creds = self.creds
headers = {}
headers['X-Auth-User'] = creds['username']
headers['X-Auth-Key'] = creds['password']
tenant = creds.get('tenant')
if tenant:
headers['X-Auth-Tenant'] = tenant
resp, resp_body = self._do_request(token_url, 'GET', headers=headers)
def _management_url(self, resp):
for url_header in ('x-image-management-url',
'x-server-management-url',
'x-glance'):
try:
return resp[url_header]
except KeyError as e:
not_found = e
raise not_found
if resp.status in (200, 204):
try:
if self.configure_via_auth:
self.management_url = _management_url(self, resp)
self.auth_token = resp['x-auth-token']
except KeyError:
raise exception.AuthorizationFailure()
elif resp.status == 305:
raise exception.AuthorizationRedirect(uri=resp['location'])
elif resp.status == 400:
raise exception.AuthBadRequest(url=token_url)
elif resp.status == 401:
raise exception.NotAuthenticated()
elif resp.status == 404:
raise exception.AuthUrlNotFound(url=token_url)
else:
raise Exception(_('Unexpected response: %s') % resp.status)
def _v2_auth(self, token_url):
creds = self.creds
creds = {
"auth": {
"tenantName": creds['tenant'],
"passwordCredentials": {
"username": creds['username'],
"password": creds['password']
}
}
}
headers = {}
headers['Content-Type'] = 'application/json'
req_body = jsonutils.dumps(creds)
resp, resp_body = self._do_request(
token_url, 'POST', headers=headers, body=req_body)
if resp.status == 200:
resp_auth = jsonutils.loads(resp_body)['access']
creds_region = self.creds.get('region')
if self.configure_via_auth:
endpoint = get_endpoint(resp_auth['serviceCatalog'],
endpoint_region=creds_region)
self.management_url = endpoint
self.auth_token = resp_auth['token']['id']
elif resp.status == 305:
raise exception.RedirectException(resp['location'])
elif resp.status == 400:
raise exception.AuthBadRequest(url=token_url)
elif resp.status == 401:
raise exception.NotAuthenticated()
elif resp.status == 404:
raise exception.AuthUrlNotFound(url=token_url)
else:
raise Exception(_('Unexpected response: %s') % resp.status)
@property
def is_authenticated(self):
return self.auth_token is not None
@property
def strategy(self):
return 'keystone'
def _do_request(self, url, method, headers=None, body=None):
headers = headers or {}
conn = httplib2.Http()
conn.force_exception_to_status_code = True
conn.disable_ssl_certificate_validation = self.insecure
headers['User-Agent'] = 'glance-client'
resp, resp_body = conn.request(url, method, headers=headers, body=body)
return resp, resp_body
def get_plugin_from_strategy(strategy, creds=None, insecure=False,
configure_via_auth=True):
if strategy == 'noauth':
return NoAuthStrategy()
elif strategy == 'keystone':
return KeystoneStrategy(creds, insecure,
configure_via_auth=configure_via_auth)
else:
raise Exception(_("Unknown auth strategy '%s'") % strategy)
def get_endpoint(service_catalog, service_type='image', endpoint_region=None,
endpoint_type='publicURL'):
"""
Select an endpoint from the service catalog
We search the full service catalog for services
matching both type and region. If the client
supplied no region then any 'image' endpoint
is considered a match. There must be one -- and
only one -- successful match in the catalog,
otherwise we will raise an exception.
"""
endpoint = None
for service in service_catalog:
s_type = None
try:
s_type = service['type']
except KeyError:
msg = _('Encountered service with no "type": %s') % s_type
LOG.warn(msg)
continue
if s_type == service_type:
for ep in service['endpoints']:
if endpoint_region is None or endpoint_region == ep['region']:
if endpoint is not None:
# This is a second match, abort
raise exception.RegionAmbiguity(region=endpoint_region)
endpoint = ep
if endpoint and endpoint.get(endpoint_type):
return endpoint[endpoint_type]
else:
raise exception.NoServiceEndpoint()

127
glance/store/openstack/common/context.py Normal file
View File

@ -0,0 +1,127 @@
# Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Simple class that stores security context information in the web request.
Projects should subclass this class if they wish to enhance the request
context or provide additional information in their specific WSGI pipeline.
"""
import itertools
import uuid
def generate_request_id():
return b'req-' + str(uuid.uuid4()).encode('ascii')
class RequestContext(object):
"""Helper class to represent useful information about a request context.
Stores information about the security context under which the user
accesses the system, as well as additional request information.
"""
user_idt_format = '{user} {tenant} {domain} {user_domain} {p_domain}'
def __init__(self, auth_token=None, user=None, tenant=None, domain=None,
user_domain=None, project_domain=None, is_admin=False,
read_only=False, show_deleted=False, request_id=None,
instance_uuid=None, service_catalog=None):
self.auth_token = auth_token
self.user = user
self.tenant = tenant
self.domain = domain
self.user_domain = user_domain
self.project_domain = project_domain
self.is_admin = is_admin
self.read_only = read_only
self.show_deleted = show_deleted
self.instance_uuid = instance_uuid
self.service_catalog = service_catalog
if not request_id:
request_id = generate_request_id()
self.request_id = request_id
def to_dict(self):
user_idt = (
self.user_idt_format.format(user=self.user or '-',
tenant=self.tenant or '-',
domain=self.domain or '-',
user_domain=self.user_domain or '-',
p_domain=self.project_domain or '-'))
return {'user': self.user,
'tenant': self.tenant,
'domain': self.domain,
'user_domain': self.user_domain,
'project_domain': self.project_domain,
'is_admin': self.is_admin,
'read_only': self.read_only,
'show_deleted': self.show_deleted,
'auth_token': self.auth_token,
'request_id': self.request_id,
'instance_uuid': self.instance_uuid,
'user_identity': user_idt}
@classmethod
def from_dict(cls, ctx):
return cls(
auth_token=ctx.get("auth_token"),
user=ctx.get("user"),
tenant=ctx.get("tenant"),
domain=ctx.get("domain"),
user_domain=ctx.get("user_domain"),
project_domain=ctx.get("project_domain"),
is_admin=ctx.get("is_admin", False),
read_only=ctx.get("read_only", False),
show_deleted=ctx.get("show_deleted", False),
request_id=ctx.get("request_id"),
instance_uuid=ctx.get("instance_uuid"))
def get_admin_context(show_deleted=False):
context = RequestContext(None,
tenant=None,
is_admin=True,
show_deleted=show_deleted)
return context
def get_context_from_function_and_args(function, args, kwargs):
"""Find an arg of type RequestContext and return it.
This is useful in a couple of decorators where we don't
know much about the function we're wrapping.
"""
for arg in itertools.chain(kwargs.values(), args):
if isinstance(arg, RequestContext):
return arg
return None
def is_user_context(context):
"""Indicates if the request context is a normal user."""
if not context:
return False
if context.is_admin:
return False
if not context.user_id or not context.project_id:
return False
return True

21
glance/store/tests/base.py
View File

@ -14,19 +14,27 @@
# License for the specific language governing permissions and limitations
# under the License.
import os
import shutil
import fixtures
from oslo.config import cfg
import testtools
from oslotest import base
import glance.store as store
from glance.store import location
class StoreBaseTest(testtools.TestCase):
class StoreBaseTest(base.BaseTestCase):
#NOTE(flaper87): temporary until we
# can move to a fully-local lib.
# (Swift store's fault)
_CONF = cfg.ConfigOpts()
def setUp(self):
super(StoreBaseTest, self).setUp()
self.conf = cfg.ConfigOpts()
self.conf = self._CONF
self.conf(args=[])
store.register_opts(self.conf)
@ -36,6 +44,13 @@ class StoreBaseTest(testtools.TestCase):
store.create_stores(self.conf)
self.addCleanup(setattr, location, 'SCHEME_TO_CLS_MAP', dict())
self.test_dir = self.useFixture(fixtures.TempDir()).path
self.addCleanup(self.conf.reset)
def copy_data_file(self, file_name, dst_dir):
src_file_name = os.path.join('glance/store/tests/etc', file_name)
shutil.copy(src_file_name, dst_dir)
dst_file_name = os.path.join(dst_dir, file_name)
return dst_file_name
def config(self, **kw):
"""Override some configuration values.

34
glance/store/tests/etc/glance-swift.conf Normal file
View File

@ -0,0 +1,34 @@
[ref1]
user = tenant:user1
key = key1
auth_address = example.com
[ref2]
user = user2
key = key2
auth_address = http://example.com
[store_2]
user = tenant:user1
key = key1
auth_address= https://localhost:8080
[store_3]
user= tenant:user2
key= key2
auth_address= https://localhost:8080
[store_4]
user = tenant:user1
key = key1
auth_address = http://localhost:80
[store_5]
user = tenant:user1
key = key1
auth_address = http://localhost
[store_6]
user = tenant:user1
key = key1
auth_address = https://localhost/v1

1
openstack-common.conf
View File

@ -1,6 +1,7 @@
[DEFAULT]
# The list of modules to copy from openstack-common
module=context
module=fileutils
module=gettextutils
module=importutils

3
requirements.txt
View File

@ -2,9 +2,6 @@ oslo.config>=1.2.0
oslo.i18n>=0.1.0
stevedore>=0.12
# For Swift storage backend.
python-swiftclient>=1.5
python-cinderclient>=1.0.6
# Required by openstack.common libraries

1
setup.cfg
View File

@ -29,6 +29,7 @@ namespace_packages =
glance.store.drivers =
file = glance.store._drivers.filesystem:Store
http = glance.store._drivers.http:Store
swift = glance.store._drivers.swift:Store
vmware = glance.store._drivers.vmware_datastore:Store
# TESTS ONLY

4
test-requirements.txt
View File

@ -23,3 +23,7 @@ boto>=2.12.0,!=2.13.0
# For VMware storage backend.
oslo.vmware>=0.4 # Apache-2.0
# Swift Backend
httplib2>=0.7.5
python-swiftclient>=2.0.2

3
tests/unit/test_filesystem_store.py
View File

@ -348,7 +348,8 @@ class TestStore(base.StoreBaseTest):
[store_map[0] + ":100",
store_map[1] + ":200"],
group='glance_store')
self.store.configure_add()
self.store.configure()
"""Test that we can add an image via the filesystem backend"""
ChunkedFile.CHUNKSIZE = 1024

1127
tests/unit/test_swift_store.py Normal file
View File

File diff suppressed because it is too large Load Diff