Add .gitreview to the repo
This patch also removes `swift` related code until we port it. Change-Id: I5c78af7334cb39e6d32afe7ca342f91c5521d30b
This commit is contained in:
parent
80af4f414a
commit
2450528776
4
.gitreview
Normal file
4
.gitreview
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
[gerrit]
|
||||||
|
host=review.openstack.org
|
||||||
|
port=29418
|
||||||
|
project=openstack/glance.store.git
|
65
doc/source/conf.py
Normal file
65
doc/source/conf.py
Normal file
@ -0,0 +1,65 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.abspath('../..'))
|
||||||
|
# -- General configuration ----------------------------------------------------
|
||||||
|
|
||||||
|
# Add any Sphinx extension module names here, as strings. They can be
|
||||||
|
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
|
||||||
|
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'oslosphinx']
|
||||||
|
|
||||||
|
# autodoc generation is a bit aggressive and a nuisance when doing heavy
|
||||||
|
# text edit cycles.
|
||||||
|
# execute "export SPHINX_DEBUG=1" in your terminal to disable
|
||||||
|
|
||||||
|
# Add any paths that contain templates here, relative to this directory.
|
||||||
|
# templates_path = []
|
||||||
|
|
||||||
|
# The suffix of source filenames.
|
||||||
|
source_suffix = '.rst'
|
||||||
|
|
||||||
|
# The master toctree document.
|
||||||
|
master_doc = 'index'
|
||||||
|
|
||||||
|
# General information about the project.
|
||||||
|
project = u'glance.store'
|
||||||
|
copyright = u'2014, OpenStack Foundation'
|
||||||
|
|
||||||
|
# If true, '()' will be appended to :func: etc. cross-reference text.
|
||||||
|
add_function_parentheses = True
|
||||||
|
|
||||||
|
# If true, the current module name will be prepended to all description
|
||||||
|
# unit titles (such as .. function::).
|
||||||
|
add_module_names = True
|
||||||
|
|
||||||
|
# The name of the Pygments (syntax highlighting) style to use.
|
||||||
|
pygments_style = 'sphinx'
|
||||||
|
|
||||||
|
# -- Options for HTML output --------------------------------------------------
|
||||||
|
|
||||||
|
# The theme to use for HTML and HTML Help pages. Major themes that come with
|
||||||
|
# Sphinx are currently 'default' and 'sphinxdoc'.
|
||||||
|
# html_theme_path = ["."]
|
||||||
|
# html_theme = '_theme'
|
||||||
|
html_static_path = ['static']
|
||||||
|
|
||||||
|
# Output file base name for HTML help builder.
|
||||||
|
htmlhelp_basename = '%sdoc' % project
|
||||||
|
|
||||||
|
git_cmd = "git log --pretty=format:'%ad, commit %h' --date=local -n1"
|
||||||
|
html_last_updated_fmt = os.popen(git_cmd).read()
|
||||||
|
|
||||||
|
# Grouping the document tree into LaTeX files. List of tuples
|
||||||
|
# (source start file, target name, title, author, documentclass
|
||||||
|
# [howto/manual]).
|
||||||
|
latex_documents = [
|
||||||
|
('index',
|
||||||
|
'%s.tex' % project,
|
||||||
|
'%s Documentation' % project,
|
||||||
|
'OpenStack Foundation', 'manual'),
|
||||||
|
]
|
||||||
|
|
||||||
|
# Example configuration for intersphinx: refer to the Python standard library.
|
||||||
|
intersphinx_mapping = {'http://docs.python.org/': None}
|
29
doc/source/index.rst
Normal file
29
doc/source/index.rst
Normal file
@ -0,0 +1,29 @@
|
|||||||
|
glance.store
|
||||||
|
============
|
||||||
|
|
||||||
|
The glance.store library supports the creation, deletion and gather of data
|
||||||
|
assets from/to a set of several, different, storage technologies
|
||||||
|
|
||||||
|
Contents
|
||||||
|
========
|
||||||
|
|
||||||
|
.. toctree::
|
||||||
|
:maxdepth: 1
|
||||||
|
|
||||||
|
Release Notes
|
||||||
|
=============
|
||||||
|
|
||||||
|
0.0.1a1
|
||||||
|
-------
|
||||||
|
|
||||||
|
* Initial release of glance.store_.
|
||||||
|
|
||||||
|
.. _glance.store: https://wiki.openstack.org/wiki/Glance/Store
|
||||||
|
|
||||||
|
Indices and tables
|
||||||
|
==================
|
||||||
|
|
||||||
|
* :ref:`genindex`
|
||||||
|
* :ref:`modindex`
|
||||||
|
* :ref:`search`
|
||||||
|
|
@ -1,687 +0,0 @@
|
|||||||
# Copyright 2010-2011 OpenStack Foundation
|
|
||||||
# All Rights Reserved.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
||||||
# not use this file except in compliance with the License. You may obtain
|
|
||||||
# a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
||||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
||||||
# License for the specific language governing permissions and limitations
|
|
||||||
# under the License.
|
|
||||||
|
|
||||||
"""Storage backend for SWIFT"""
|
|
||||||
|
|
||||||
from __future__ import absolute_import
|
|
||||||
|
|
||||||
import hashlib
|
|
||||||
import httplib
|
|
||||||
import logging
|
|
||||||
import math
|
|
||||||
import urllib
|
|
||||||
import urlparse
|
|
||||||
|
|
||||||
from oslo.config import cfg
|
|
||||||
|
|
||||||
from glance.common import auth
|
|
||||||
from glance.common import exception
|
|
||||||
from glance.openstack.common import excutils
|
|
||||||
import glance.store
|
|
||||||
import glance.store.driver
|
|
||||||
import glance.store.location
|
|
||||||
|
|
||||||
try:
|
|
||||||
import swiftclient
|
|
||||||
except ImportError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
LOG = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
DEFAULT_CONTAINER = 'glance'
|
|
||||||
DEFAULT_LARGE_OBJECT_SIZE = 5 * 1024 # 5GB
|
|
||||||
DEFAULT_LARGE_OBJECT_CHUNK_SIZE = 200 # 200M
|
|
||||||
ONE_MB = 1000 * 1024
|
|
||||||
|
|
||||||
swift_opts = [
|
|
||||||
cfg.BoolOpt('swift_enable_snet', default=False,
|
|
||||||
help=_('Whether to use ServiceNET to communicate with the '
|
|
||||||
'Swift storage servers.')),
|
|
||||||
cfg.StrOpt('swift_store_auth_address',
|
|
||||||
help=_('The address where the Swift authentication service '
|
|
||||||
'is listening.')),
|
|
||||||
cfg.StrOpt('swift_store_user', secret=True,
|
|
||||||
help=_('The user to authenticate against the Swift '
|
|
||||||
'authentication service')),
|
|
||||||
cfg.StrOpt('swift_store_key', secret=True,
|
|
||||||
help=_('Auth key for the user authenticating against the '
|
|
||||||
'Swift authentication service.')),
|
|
||||||
cfg.StrOpt('swift_store_auth_version', default='2',
|
|
||||||
help=_('Version of the authentication service to use. '
|
|
||||||
'Valid versions are 2 for keystone and 1 for swauth '
|
|
||||||
'and rackspace')),
|
|
||||||
cfg.BoolOpt('swift_store_auth_insecure', default=False,
|
|
||||||
help=_('If True, swiftclient won\'t check for a valid SSL '
|
|
||||||
'certificate when authenticating.')),
|
|
||||||
cfg.StrOpt('swift_store_region',
|
|
||||||
help=_('The region of the swift endpoint to be used for '
|
|
||||||
'single tenant. This setting is only necessary if the '
|
|
||||||
'tenant has multiple swift endpoints.')),
|
|
||||||
cfg.StrOpt('swift_store_endpoint_type', default='publicURL',
|
|
||||||
help=_('A string giving the endpoint type of the swift '
|
|
||||||
'service to use (publicURL, adminURL or internalURL). '
|
|
||||||
'This setting is only used if swift_store_auth_version '
|
|
||||||
'is 2.')),
|
|
||||||
cfg.StrOpt('swift_store_service_type', default='object-store',
|
|
||||||
help=_('A string giving the service type of the swift service '
|
|
||||||
'to use. This setting is only used if '
|
|
||||||
'swift_store_auth_version is 2.')),
|
|
||||||
cfg.StrOpt('swift_store_container',
|
|
||||||
default=DEFAULT_CONTAINER,
|
|
||||||
help=_('Container within the account that the account should '
|
|
||||||
'use for storing images in Swift.')),
|
|
||||||
cfg.IntOpt('swift_store_large_object_size',
|
|
||||||
default=DEFAULT_LARGE_OBJECT_SIZE,
|
|
||||||
help=_('The size, in MB, that Glance will start chunking image '
|
|
||||||
'files and do a large object manifest in Swift')),
|
|
||||||
cfg.IntOpt('swift_store_large_object_chunk_size',
|
|
||||||
default=DEFAULT_LARGE_OBJECT_CHUNK_SIZE,
|
|
||||||
help=_('The amount of data written to a temporary disk buffer '
|
|
||||||
'during the process of chunking the image file.')),
|
|
||||||
cfg.BoolOpt('swift_store_create_container_on_put', default=False,
|
|
||||||
help=_('A boolean value that determines if we create the '
|
|
||||||
'container if it does not exist.')),
|
|
||||||
cfg.BoolOpt('swift_store_multi_tenant', default=False,
|
|
||||||
help=_('If set to True, enables multi-tenant storage '
|
|
||||||
'mode which causes Glance images to be stored in '
|
|
||||||
'tenant specific Swift accounts.')),
|
|
||||||
cfg.ListOpt('swift_store_admin_tenants', default=[],
|
|
||||||
help=_('A list of tenants that will be granted read/write '
|
|
||||||
'access on all Swift containers created by Glance in '
|
|
||||||
'multi-tenant mode.')),
|
|
||||||
cfg.BoolOpt('swift_store_ssl_compression', default=True,
|
|
||||||
help=_('If set to False, disables SSL layer compression of '
|
|
||||||
'https swift requests. Setting to False may improve '
|
|
||||||
'performance for images which are already in a '
|
|
||||||
'compressed format, eg qcow2.')),
|
|
||||||
]
|
|
||||||
|
|
||||||
CONF = cfg.CONF
|
|
||||||
CONF.register_opts(swift_opts)
|
|
||||||
|
|
||||||
|
|
||||||
class StoreLocation(glance.store.location.StoreLocation):
|
|
||||||
|
|
||||||
"""
|
|
||||||
Class describing a Swift URI. A Swift URI can look like any of
|
|
||||||
the following:
|
|
||||||
|
|
||||||
swift://user:pass@authurl.com/container/obj-id
|
|
||||||
swift://account:user:pass@authurl.com/container/obj-id
|
|
||||||
swift+http://user:pass@authurl.com/container/obj-id
|
|
||||||
swift+https://user:pass@authurl.com/container/obj-id
|
|
||||||
|
|
||||||
When using multi-tenant a URI might look like this (a storage URL):
|
|
||||||
|
|
||||||
swift+https://example.com/container/obj-id
|
|
||||||
|
|
||||||
The swift+http:// URIs indicate there is an HTTP authentication URL.
|
|
||||||
The default for Swift is an HTTPS authentication URL, so swift:// and
|
|
||||||
swift+https:// are the same...
|
|
||||||
"""
|
|
||||||
|
|
||||||
def process_specs(self):
|
|
||||||
self.scheme = self.specs.get('scheme', 'swift+https')
|
|
||||||
self.user = self.specs.get('user')
|
|
||||||
self.key = self.specs.get('key')
|
|
||||||
self.auth_or_store_url = self.specs.get('auth_or_store_url')
|
|
||||||
self.container = self.specs.get('container')
|
|
||||||
self.obj = self.specs.get('obj')
|
|
||||||
|
|
||||||
def _get_credstring(self):
|
|
||||||
if self.user and self.key:
|
|
||||||
return '%s:%s@' % (urllib.quote(self.user), urllib.quote(self.key))
|
|
||||||
return ''
|
|
||||||
|
|
||||||
def get_uri(self):
|
|
||||||
auth_or_store_url = self.auth_or_store_url
|
|
||||||
if auth_or_store_url.startswith('http://'):
|
|
||||||
auth_or_store_url = auth_or_store_url[len('http://'):]
|
|
||||||
elif auth_or_store_url.startswith('https://'):
|
|
||||||
auth_or_store_url = auth_or_store_url[len('https://'):]
|
|
||||||
|
|
||||||
credstring = self._get_credstring()
|
|
||||||
auth_or_store_url = auth_or_store_url.strip('/')
|
|
||||||
container = self.container.strip('/')
|
|
||||||
obj = self.obj.strip('/')
|
|
||||||
|
|
||||||
return '%s://%s%s/%s/%s' % (self.scheme, credstring, auth_or_store_url,
|
|
||||||
container, obj)
|
|
||||||
|
|
||||||
def parse_uri(self, uri):
|
|
||||||
"""
|
|
||||||
Parse URLs. This method fixes an issue where credentials specified
|
|
||||||
in the URL are interpreted differently in Python 2.6.1+ than prior
|
|
||||||
versions of Python. It also deals with the peculiarity that new-style
|
|
||||||
Swift URIs have where a username can contain a ':', like so:
|
|
||||||
|
|
||||||
swift://account:user:pass@authurl.com/container/obj
|
|
||||||
"""
|
|
||||||
# Make sure that URIs that contain multiple schemes, such as:
|
|
||||||
# swift://user:pass@http://authurl.com/v1/container/obj
|
|
||||||
# are immediately rejected.
|
|
||||||
if uri.count('://') != 1:
|
|
||||||
reason = _("URI cannot contain more than one occurrence "
|
|
||||||
"of a scheme. If you have specified a URI like "
|
|
||||||
"swift://user:pass@http://authurl.com/v1/container/obj"
|
|
||||||
", you need to change it to use the "
|
|
||||||
"swift+http:// scheme, like so: "
|
|
||||||
"swift+http://user:pass@authurl.com/v1/container/obj")
|
|
||||||
LOG.debug(_("Invalid store URI: %(reason)s"), {'reason': reason})
|
|
||||||
raise exceptions.BadStoreUri(message=reason)
|
|
||||||
|
|
||||||
pieces = urlparse.urlparse(uri)
|
|
||||||
assert pieces.scheme in ('swift', 'swift+http', 'swift+https')
|
|
||||||
self.scheme = pieces.scheme
|
|
||||||
netloc = pieces.netloc
|
|
||||||
path = pieces.path.lstrip('/')
|
|
||||||
if netloc != '':
|
|
||||||
# > Python 2.6.1
|
|
||||||
if '@' in netloc:
|
|
||||||
creds, netloc = netloc.split('@')
|
|
||||||
else:
|
|
||||||
creds = None
|
|
||||||
else:
|
|
||||||
# Python 2.6.1 compat
|
|
||||||
# see lp659445 and Python issue7904
|
|
||||||
if '@' in path:
|
|
||||||
creds, path = path.split('@')
|
|
||||||
else:
|
|
||||||
creds = None
|
|
||||||
netloc = path[0:path.find('/')].strip('/')
|
|
||||||
path = path[path.find('/'):].strip('/')
|
|
||||||
if creds:
|
|
||||||
cred_parts = creds.split(':')
|
|
||||||
if len(cred_parts) != 2:
|
|
||||||
reason = (_("Badly formed credentials in Swift URI."))
|
|
||||||
LOG.debug(reason)
|
|
||||||
raise exceptions.BadStoreUri()
|
|
||||||
user, key = cred_parts
|
|
||||||
self.user = urllib.unquote(user)
|
|
||||||
self.key = urllib.unquote(key)
|
|
||||||
else:
|
|
||||||
self.user = None
|
|
||||||
self.key = None
|
|
||||||
path_parts = path.split('/')
|
|
||||||
try:
|
|
||||||
self.obj = path_parts.pop()
|
|
||||||
self.container = path_parts.pop()
|
|
||||||
if not netloc.startswith('http'):
|
|
||||||
# push hostname back into the remaining to build full authurl
|
|
||||||
path_parts.insert(0, netloc)
|
|
||||||
self.auth_or_store_url = '/'.join(path_parts)
|
|
||||||
except IndexError:
|
|
||||||
reason = _("Badly formed Swift URI.")
|
|
||||||
LOG.debug(reason)
|
|
||||||
raise exceptions.BadStoreUri()
|
|
||||||
|
|
||||||
@property
|
|
||||||
def swift_url(self):
|
|
||||||
"""
|
|
||||||
Creates a fully-qualified auth url that the Swift client library can
|
|
||||||
use. The scheme for the auth_url is determined using the scheme
|
|
||||||
included in the `location` field.
|
|
||||||
|
|
||||||
HTTPS is assumed, unless 'swift+http' is specified.
|
|
||||||
"""
|
|
||||||
if self.auth_or_store_url.startswith('http'):
|
|
||||||
return self.auth_or_store_url
|
|
||||||
else:
|
|
||||||
if self.scheme in ('swift+https', 'swift'):
|
|
||||||
auth_scheme = 'https://'
|
|
||||||
else:
|
|
||||||
auth_scheme = 'http://'
|
|
||||||
|
|
||||||
return ''.join([auth_scheme, self.auth_or_store_url])
|
|
||||||
|
|
||||||
|
|
||||||
def Store(context=None, loc=None):
|
|
||||||
if (CONF.swift_store_multi_tenant and
|
|
||||||
(loc is None or loc.store_location.user is None)):
|
|
||||||
return MultiTenantStore(context, loc)
|
|
||||||
return SingleTenantStore(context, loc)
|
|
||||||
|
|
||||||
|
|
||||||
class BaseStore(glance.store.driver.Store):
|
|
||||||
CHUNKSIZE = 65536
|
|
||||||
|
|
||||||
def get_schemes(self):
|
|
||||||
return ('swift+https', 'swift', 'swift+http')
|
|
||||||
|
|
||||||
def configure(self):
|
|
||||||
_obj_size = self._option_get('swift_store_large_object_size')
|
|
||||||
self.large_object_size = _obj_size * ONE_MB
|
|
||||||
_chunk_size = self._option_get('swift_store_large_object_chunk_size')
|
|
||||||
self.large_object_chunk_size = _chunk_size * ONE_MB
|
|
||||||
self.admin_tenants = CONF.swift_store_admin_tenants
|
|
||||||
self.region = CONF.swift_store_region
|
|
||||||
self.service_type = CONF.swift_store_service_type
|
|
||||||
self.endpoint_type = CONF.swift_store_endpoint_type
|
|
||||||
self.snet = CONF.swift_enable_snet
|
|
||||||
self.insecure = CONF.swift_store_auth_insecure
|
|
||||||
self.ssl_compression = CONF.swift_store_ssl_compression
|
|
||||||
|
|
||||||
def get(self, location, offset=0, chunk_size=None, context=None):
|
|
||||||
location = location.store_location
|
|
||||||
if not connection:
|
|
||||||
connection = self.get_connection(location)
|
|
||||||
|
|
||||||
try:
|
|
||||||
resp_headers, resp_body = connection.get_object(
|
|
||||||
container=location.container, obj=location.obj,
|
|
||||||
resp_chunk_size=self.CHUNKSIZE)
|
|
||||||
except swiftclient.ClientException as e:
|
|
||||||
if e.http_status == httplib.NOT_FOUND:
|
|
||||||
msg = _("Swift could not find object %s.") % location.obj
|
|
||||||
LOG.warn(msg)
|
|
||||||
raise exceptions.NotFound(msg)
|
|
||||||
else:
|
|
||||||
raise
|
|
||||||
|
|
||||||
class ResponseIndexable(glance.store.Indexable):
|
|
||||||
def another(self):
|
|
||||||
try:
|
|
||||||
return self.wrapped.next()
|
|
||||||
except StopIteration:
|
|
||||||
return ''
|
|
||||||
|
|
||||||
length = int(resp_headers.get('content-length', 0))
|
|
||||||
return (ResponseIndexable(resp_body, length), length)
|
|
||||||
|
|
||||||
def get_size(self, location, connection=None):
|
|
||||||
location = location.store_location
|
|
||||||
if not connection:
|
|
||||||
connection = self.get_connection(location)
|
|
||||||
try:
|
|
||||||
resp_headers = connection.head_object(
|
|
||||||
container=location.container, obj=location.obj)
|
|
||||||
return int(resp_headers.get('content-length', 0))
|
|
||||||
except Exception:
|
|
||||||
return 0
|
|
||||||
|
|
||||||
def _option_get(self, param):
|
|
||||||
result = getattr(CONF, param)
|
|
||||||
if not result:
|
|
||||||
reason = (_("Could not find %(param)s in configuration "
|
|
||||||
"options.") % {'param': param})
|
|
||||||
LOG.error(reason)
|
|
||||||
raise exceptions.BadStoreConfiguration(store_name="swift",
|
|
||||||
reason=reason)
|
|
||||||
return result
|
|
||||||
|
|
||||||
def _delete_stale_chunks(self, connection, container, chunk_list):
|
|
||||||
for chunk in chunk_list:
|
|
||||||
LOG.debug(_("Deleting chunk %s") % chunk)
|
|
||||||
try:
|
|
||||||
connection.delete_object(container, chunk)
|
|
||||||
except Exception:
|
|
||||||
msg = _("Failed to delete orphaned chunk %s/%s")
|
|
||||||
LOG.exception(msg, container, chunk)
|
|
||||||
|
|
||||||
def add(self, image_id, image_file, image_size, connection=None):
|
|
||||||
location = self.create_location(image_id)
|
|
||||||
if not connection:
|
|
||||||
connection = self.get_connection(location)
|
|
||||||
|
|
||||||
self._create_container_if_missing(location.container, connection)
|
|
||||||
|
|
||||||
LOG.debug(_("Adding image object '%(obj_name)s' "
|
|
||||||
"to Swift") % dict(obj_name=location.obj))
|
|
||||||
try:
|
|
||||||
if image_size > 0 and image_size < self.large_object_size:
|
|
||||||
# Image size is known, and is less than large_object_size.
|
|
||||||
# Send to Swift with regular PUT.
|
|
||||||
obj_etag = connection.put_object(location.container,
|
|
||||||
location.obj, image_file,
|
|
||||||
content_length=image_size)
|
|
||||||
else:
|
|
||||||
# Write the image into Swift in chunks.
|
|
||||||
chunk_id = 1
|
|
||||||
if image_size > 0:
|
|
||||||
total_chunks = str(int(
|
|
||||||
math.ceil(float(image_size) /
|
|
||||||
float(self.large_object_chunk_size))))
|
|
||||||
else:
|
|
||||||
# image_size == 0 is when we don't know the size
|
|
||||||
# of the image. This can occur with older clients
|
|
||||||
# that don't inspect the payload size.
|
|
||||||
LOG.debug(_("Cannot determine image size. Adding as a "
|
|
||||||
"segmented object to Swift."))
|
|
||||||
total_chunks = '?'
|
|
||||||
|
|
||||||
checksum = hashlib.md5()
|
|
||||||
written_chunks = []
|
|
||||||
combined_chunks_size = 0
|
|
||||||
while True:
|
|
||||||
chunk_size = self.large_object_chunk_size
|
|
||||||
if image_size == 0:
|
|
||||||
content_length = None
|
|
||||||
else:
|
|
||||||
left = image_size - combined_chunks_size
|
|
||||||
if left == 0:
|
|
||||||
break
|
|
||||||
if chunk_size > left:
|
|
||||||
chunk_size = left
|
|
||||||
content_length = chunk_size
|
|
||||||
|
|
||||||
chunk_name = "%s-%05d" % (location.obj, chunk_id)
|
|
||||||
reader = ChunkReader(image_file, checksum, chunk_size)
|
|
||||||
try:
|
|
||||||
chunk_etag = connection.put_object(
|
|
||||||
location.container, chunk_name, reader,
|
|
||||||
content_length=content_length)
|
|
||||||
written_chunks.append(chunk_name)
|
|
||||||
except Exception:
|
|
||||||
# Delete orphaned segments from swift backend
|
|
||||||
with excutils.save_and_reraise_exception():
|
|
||||||
LOG.exception(_("Error during chunked upload to "
|
|
||||||
"backend, deleting stale chunks"))
|
|
||||||
self._delete_stale_chunks(connection,
|
|
||||||
location.container,
|
|
||||||
written_chunks)
|
|
||||||
|
|
||||||
bytes_read = reader.bytes_read
|
|
||||||
msg = (_("Wrote chunk %(chunk_name)s (%(chunk_id)d/"
|
|
||||||
"%(total_chunks)s) of length %(bytes_read)d "
|
|
||||||
"to Swift returning MD5 of content: "
|
|
||||||
"%(chunk_etag)s") %
|
|
||||||
{'chunk_name': chunk_name,
|
|
||||||
'chunk_id': chunk_id,
|
|
||||||
'total_chunks': total_chunks,
|
|
||||||
'bytes_read': bytes_read,
|
|
||||||
'chunk_etag': chunk_etag})
|
|
||||||
LOG.debug(msg)
|
|
||||||
|
|
||||||
if bytes_read == 0:
|
|
||||||
# Delete the last chunk, because it's of zero size.
|
|
||||||
# This will happen if size == 0.
|
|
||||||
LOG.debug(_("Deleting final zero-length chunk"))
|
|
||||||
connection.delete_object(location.container,
|
|
||||||
chunk_name)
|
|
||||||
break
|
|
||||||
|
|
||||||
chunk_id += 1
|
|
||||||
combined_chunks_size += bytes_read
|
|
||||||
|
|
||||||
# In the case we have been given an unknown image size,
|
|
||||||
# set the size to the total size of the combined chunks.
|
|
||||||
if image_size == 0:
|
|
||||||
image_size = combined_chunks_size
|
|
||||||
|
|
||||||
# Now we write the object manifest and return the
|
|
||||||
# manifest's etag...
|
|
||||||
manifest = "%s/%s-" % (location.container, location.obj)
|
|
||||||
headers = {'ETag': hashlib.md5("").hexdigest(),
|
|
||||||
'X-Object-Manifest': manifest}
|
|
||||||
|
|
||||||
# The ETag returned for the manifest is actually the
|
|
||||||
# MD5 hash of the concatenated checksums of the strings
|
|
||||||
# of each chunk...so we ignore this result in favour of
|
|
||||||
# the MD5 of the entire image file contents, so that
|
|
||||||
# users can verify the image file contents accordingly
|
|
||||||
connection.put_object(location.container, location.obj,
|
|
||||||
None, headers=headers)
|
|
||||||
obj_etag = checksum.hexdigest()
|
|
||||||
|
|
||||||
# NOTE: We return the user and key here! Have to because
|
|
||||||
# location is used by the API server to return the actual
|
|
||||||
# image data. We *really* should consider NOT returning
|
|
||||||
# the location attribute from GET /images/<ID> and
|
|
||||||
# GET /images/details
|
|
||||||
|
|
||||||
return (location.get_uri(), image_size, obj_etag, {})
|
|
||||||
except swiftclient.ClientException as e:
|
|
||||||
if e.http_status == httplib.CONFLICT:
|
|
||||||
raise exceptions.Duplicate(_("Swift already has an image at "
|
|
||||||
"this location"))
|
|
||||||
msg = (_("Failed to add object to Swift.\n"
|
|
||||||
"Got error from Swift: %(e)s") % {'e': e})
|
|
||||||
LOG.error(msg)
|
|
||||||
raise glance.store.BackendException(msg)
|
|
||||||
|
|
||||||
def delete(self, location, connection=None):
|
|
||||||
location = location.store_location
|
|
||||||
if not connection:
|
|
||||||
connection = self.get_connection(location)
|
|
||||||
|
|
||||||
try:
|
|
||||||
# We request the manifest for the object. If one exists,
|
|
||||||
# that means the object was uploaded in chunks/segments,
|
|
||||||
# and we need to delete all the chunks as well as the
|
|
||||||
# manifest.
|
|
||||||
manifest = None
|
|
||||||
try:
|
|
||||||
headers = connection.head_object(
|
|
||||||
location.container, location.obj)
|
|
||||||
manifest = headers.get('x-object-manifest')
|
|
||||||
except swiftclient.ClientException as e:
|
|
||||||
if e.http_status != httplib.NOT_FOUND:
|
|
||||||
raise
|
|
||||||
if manifest:
|
|
||||||
# Delete all the chunks before the object manifest itself
|
|
||||||
obj_container, obj_prefix = manifest.split('/', 1)
|
|
||||||
segments = connection.get_container(
|
|
||||||
obj_container, prefix=obj_prefix)[1]
|
|
||||||
for segment in segments:
|
|
||||||
# TODO(jaypipes): This would be an easy area to parallelize
|
|
||||||
# since we're simply sending off parallelizable requests
|
|
||||||
# to Swift to delete stuff. It's not like we're going to
|
|
||||||
# be hogging up network or file I/O here...
|
|
||||||
connection.delete_object(obj_container,
|
|
||||||
segment['name'])
|
|
||||||
|
|
||||||
# Delete object (or, in segmented case, the manifest)
|
|
||||||
connection.delete_object(location.container, location.obj)
|
|
||||||
|
|
||||||
except swiftclient.ClientException as e:
|
|
||||||
if e.http_status == httplib.NOT_FOUND:
|
|
||||||
msg = _("Swift could not find image at URI.")
|
|
||||||
raise exceptions.NotFound(msg)
|
|
||||||
else:
|
|
||||||
raise
|
|
||||||
|
|
||||||
def _create_container_if_missing(self, container, connection):
|
|
||||||
"""
|
|
||||||
Creates a missing container in Swift if the
|
|
||||||
``swift_store_create_container_on_put`` option is set.
|
|
||||||
|
|
||||||
:param container: Name of container to create
|
|
||||||
:param connection: Connection to swift service
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
connection.head_container(container)
|
|
||||||
except swiftclient.ClientException as e:
|
|
||||||
if e.http_status == httplib.NOT_FOUND:
|
|
||||||
if CONF.swift_store_create_container_on_put:
|
|
||||||
try:
|
|
||||||
connection.put_container(container)
|
|
||||||
except swiftclient.ClientException as e:
|
|
||||||
msg = (_("Failed to add container to Swift.\n"
|
|
||||||
"Got error from Swift: %(e)s") % {'e': e})
|
|
||||||
raise glance.store.BackendException(msg)
|
|
||||||
else:
|
|
||||||
msg = (_("The container %(container)s does not exist in "
|
|
||||||
"Swift. Please set the "
|
|
||||||
"swift_store_create_container_on_put option"
|
|
||||||
"to add container to Swift automatically.") %
|
|
||||||
{'container': container})
|
|
||||||
raise glance.store.BackendException(msg)
|
|
||||||
else:
|
|
||||||
raise
|
|
||||||
|
|
||||||
def get_connection(self):
|
|
||||||
raise NotImplemented()
|
|
||||||
|
|
||||||
def create_location(self):
|
|
||||||
raise NotImplemented()
|
|
||||||
|
|
||||||
|
|
||||||
class SingleTenantStore(BaseStore):
|
|
||||||
EXAMPLE_URL = "swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<FILE>"
|
|
||||||
|
|
||||||
def configure(self):
|
|
||||||
super(SingleTenantStore, self).configure()
|
|
||||||
self.auth_version = self._option_get('swift_store_auth_version')
|
|
||||||
|
|
||||||
def configure_add(self):
|
|
||||||
self.auth_address = self._option_get('swift_store_auth_address')
|
|
||||||
if self.auth_address.startswith('http://'):
|
|
||||||
self.scheme = 'swift+http'
|
|
||||||
else:
|
|
||||||
self.scheme = 'swift+https'
|
|
||||||
self.container = CONF.swift_store_container
|
|
||||||
self.user = self._option_get('swift_store_user')
|
|
||||||
self.key = self._option_get('swift_store_key')
|
|
||||||
|
|
||||||
def create_location(self, image_id):
|
|
||||||
specs = {'scheme': self.scheme,
|
|
||||||
'container': self.container,
|
|
||||||
'obj': str(image_id),
|
|
||||||
'auth_or_store_url': self.auth_address,
|
|
||||||
'user': self.user,
|
|
||||||
'key': self.key}
|
|
||||||
return StoreLocation(specs)
|
|
||||||
|
|
||||||
def get_connection(self, location):
|
|
||||||
if not location.user:
|
|
||||||
reason = (_("Location is missing user:password information."))
|
|
||||||
LOG.debug(reason)
|
|
||||||
raise exceptions.BadStoreUri(message=reason)
|
|
||||||
|
|
||||||
auth_url = location.swift_url
|
|
||||||
if not auth_url.endswith('/'):
|
|
||||||
auth_url += '/'
|
|
||||||
|
|
||||||
if self.auth_version == '2':
|
|
||||||
try:
|
|
||||||
tenant_name, user = location.user.split(':')
|
|
||||||
except ValueError:
|
|
||||||
reason = (_("Badly formed tenant:user '%(user)s' in "
|
|
||||||
"Swift URI") % {'user': location.user})
|
|
||||||
LOG.debug(reason)
|
|
||||||
raise exceptions.BadStoreUri()
|
|
||||||
else:
|
|
||||||
tenant_name = None
|
|
||||||
user = location.user
|
|
||||||
|
|
||||||
os_options = {}
|
|
||||||
if self.region:
|
|
||||||
os_options['region_name'] = self.region
|
|
||||||
os_options['endpoint_type'] = self.endpoint_type
|
|
||||||
os_options['service_type'] = self.service_type
|
|
||||||
|
|
||||||
return swiftclient.Connection(
|
|
||||||
auth_url, user, location.key, insecure=self.insecure,
|
|
||||||
tenant_name=tenant_name, snet=self.snet,
|
|
||||||
auth_version=self.auth_version, os_options=os_options,
|
|
||||||
ssl_compression=self.ssl_compression)
|
|
||||||
|
|
||||||
|
|
||||||
class MultiTenantStore(BaseStore):
|
|
||||||
EXAMPLE_URL = "swift://<SWIFT_URL>/<CONTAINER>/<FILE>"
|
|
||||||
|
|
||||||
def configure_add(self):
|
|
||||||
self.container = CONF.swift_store_container
|
|
||||||
if self.context is None:
|
|
||||||
reason = _("Multi-tenant Swift storage requires a context.")
|
|
||||||
raise exceptions.BadStoreConfiguration(store_name="swift",
|
|
||||||
reason=reason)
|
|
||||||
if self.context.service_catalog is None:
|
|
||||||
reason = _("Multi-tenant Swift storage requires "
|
|
||||||
"a service catalog.")
|
|
||||||
raise exceptions.BadStoreConfiguration(store_name="swift",
|
|
||||||
reason=reason)
|
|
||||||
self.storage_url = auth.get_endpoint(
|
|
||||||
self.context.service_catalog, service_type=self.service_type,
|
|
||||||
endpoint_region=self.region, endpoint_type=self.endpoint_type)
|
|
||||||
if self.storage_url.startswith('http://'):
|
|
||||||
self.scheme = 'swift+http'
|
|
||||||
else:
|
|
||||||
self.scheme = 'swift+https'
|
|
||||||
|
|
||||||
def delete(self, location, connection=None):
|
|
||||||
if not connection:
|
|
||||||
connection = self.get_connection(location.store_location)
|
|
||||||
super(MultiTenantStore, self).delete(location, connection)
|
|
||||||
connection.delete_container(location.store_location.container)
|
|
||||||
|
|
||||||
def set_acls(self, location, public=False, read_tenants=None,
|
|
||||||
write_tenants=None, connection=None):
|
|
||||||
location = location.store_location
|
|
||||||
if not connection:
|
|
||||||
connection = self.get_connection(location)
|
|
||||||
|
|
||||||
if read_tenants is None:
|
|
||||||
read_tenants = []
|
|
||||||
if write_tenants is None:
|
|
||||||
write_tenants = []
|
|
||||||
|
|
||||||
headers = {}
|
|
||||||
if public:
|
|
||||||
headers['X-Container-Read'] = ".r:*,.rlistings"
|
|
||||||
elif read_tenants:
|
|
||||||
headers['X-Container-Read'] = ','.join('%s:*' % i
|
|
||||||
for i in read_tenants)
|
|
||||||
else:
|
|
||||||
headers['X-Container-Read'] = ''
|
|
||||||
|
|
||||||
write_tenants.extend(self.admin_tenants)
|
|
||||||
if write_tenants:
|
|
||||||
headers['X-Container-Write'] = ','.join('%s:*' % i
|
|
||||||
for i in write_tenants)
|
|
||||||
else:
|
|
||||||
headers['X-Container-Write'] = ''
|
|
||||||
|
|
||||||
try:
|
|
||||||
connection.post_container(location.container, headers=headers)
|
|
||||||
except swiftclient.ClientException as e:
|
|
||||||
if e.http_status == httplib.NOT_FOUND:
|
|
||||||
msg = _("Swift could not find image at URI.")
|
|
||||||
raise exceptions.NotFound(msg)
|
|
||||||
else:
|
|
||||||
raise
|
|
||||||
|
|
||||||
def create_location(self, image_id):
|
|
||||||
specs = {'scheme': self.scheme,
|
|
||||||
'container': self.container + '_' + str(image_id),
|
|
||||||
'obj': str(image_id),
|
|
||||||
'auth_or_store_url': self.storage_url}
|
|
||||||
return StoreLocation(specs)
|
|
||||||
|
|
||||||
def get_connection(self, location):
|
|
||||||
return swiftclient.Connection(
|
|
||||||
None, self.context.user, None,
|
|
||||||
preauthurl=location.swift_url,
|
|
||||||
preauthtoken=self.context.auth_tok,
|
|
||||||
tenant_name=self.context.tenant,
|
|
||||||
auth_version='2', snet=self.snet, insecure=self.insecure,
|
|
||||||
ssl_compression=self.ssl_compression)
|
|
||||||
|
|
||||||
|
|
||||||
class ChunkReader(object):
|
|
||||||
def __init__(self, fd, checksum, total):
|
|
||||||
self.fd = fd
|
|
||||||
self.checksum = checksum
|
|
||||||
self.total = total
|
|
||||||
self.bytes_read = 0
|
|
||||||
|
|
||||||
def read(self, i):
|
|
||||||
left = self.total - self.bytes_read
|
|
||||||
if i > left:
|
|
||||||
i = left
|
|
||||||
result = self.fd.read(i)
|
|
||||||
self.bytes_read += len(result)
|
|
||||||
self.checksum.update(result)
|
|
||||||
return result
|
|
@ -176,9 +176,12 @@ class StoreLocation(location.StoreLocation):
|
|||||||
|
|
||||||
return '%s?%s' % (base_url, self.query)
|
return '%s?%s' % (base_url, self.query)
|
||||||
|
|
||||||
def _is_valid_path(self, path):
|
# NOTE(flaper87): Commenting out for now, it's probably better to do
|
||||||
sdir = self.conf.glance_store.vmware_store_image_dir.strip('/')
|
# it during image add/get. This validation relies on a config param
|
||||||
return path.startswith(os.path.join(DS_URL_PREFIX, sdir))
|
# which doesn't make sense to have in the StoreLocation instance.
|
||||||
|
#def _is_valid_path(self, path):
|
||||||
|
# sdir = self.conf.glance_store.vmware_store_image_dir.strip('/')
|
||||||
|
# return path.startswith(os.path.join(DS_URL_PREFIX, sdir))
|
||||||
|
|
||||||
def parse_uri(self, uri):
|
def parse_uri(self, uri):
|
||||||
if not uri.startswith('%s://' % STORE_SCHEME):
|
if not uri.startswith('%s://' % STORE_SCHEME):
|
||||||
@ -189,19 +192,14 @@ class StoreLocation(location.StoreLocation):
|
|||||||
(self.scheme, self.server_host,
|
(self.scheme, self.server_host,
|
||||||
path, params, query, fragment) = urlparse.urlparse(uri)
|
path, params, query, fragment) = urlparse.urlparse(uri)
|
||||||
if not query:
|
if not query:
|
||||||
path = path.split('?')
|
path, query = path.split('?')
|
||||||
if self._is_valid_path(path[0]):
|
|
||||||
self.path = path[0]
|
self.path = path
|
||||||
self.query = path[1]
|
self.query = query
|
||||||
return
|
# NOTE(flaper87): Read comment on `_is_valid_path`
|
||||||
#elif self._is_valid_path(path):
|
#reason = 'Badly formed VMware datastore URI %(uri)s.' % {'uri': uri}
|
||||||
else:
|
#LOG.debug(reason)
|
||||||
self.path = path
|
#raise exceptions.BadStoreUri(reason)
|
||||||
self.query = query
|
|
||||||
return
|
|
||||||
reason = 'Badly formed VMware datastore URI %(uri)s.' % {'uri': uri}
|
|
||||||
LOG.debug(reason)
|
|
||||||
raise exceptions.BadStoreUri(reason)
|
|
||||||
|
|
||||||
|
|
||||||
class Store(glance.store.Store):
|
class Store(glance.store.Store):
|
||||||
|
@ -1,5 +1,3 @@
|
|||||||
pbr>=0.5.21,<1.0
|
|
||||||
|
|
||||||
oslo.config>=1.2.0
|
oslo.config>=1.2.0
|
||||||
oslo.i18n>=0.1.0
|
oslo.i18n>=0.1.0
|
||||||
stevedore>=0.12
|
stevedore>=0.12
|
||||||
|
@ -6,6 +6,20 @@ mock>=1.0
|
|||||||
|
|
||||||
# Unit testing
|
# Unit testing
|
||||||
fixtures>=0.3.14
|
fixtures>=0.3.14
|
||||||
python-subunit
|
python-subunit>=0.0.18
|
||||||
testrepository>=0.0.17
|
testrepository>=0.0.18
|
||||||
testtools>=0.9.32
|
testscenarios>=0.4
|
||||||
|
testtools>=0.9.34
|
||||||
|
oslotest
|
||||||
|
|
||||||
|
# this is required for the docs build jobs
|
||||||
|
sphinx>=1.1.2,!=1.2.0,<1.3
|
||||||
|
oslosphinx
|
||||||
|
|
||||||
|
### Store specific packages
|
||||||
|
|
||||||
|
# For S3 storage backend
|
||||||
|
boto>=2.12.0,!=2.13.0
|
||||||
|
|
||||||
|
# For VMware storage backend.
|
||||||
|
oslo.vmware>=0.4 # Apache-2.0
|
||||||
|
@ -1,965 +0,0 @@
|
|||||||
# Copyright 2011 OpenStack Foundation
|
|
||||||
# All Rights Reserved.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
||||||
# not use this file except in compliance with the License. You may obtain
|
|
||||||
# a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
||||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
||||||
# License for the specific language governing permissions and limitations
|
|
||||||
# under the License.
|
|
||||||
|
|
||||||
"""Tests the Swift backend store"""
|
|
||||||
|
|
||||||
import hashlib
|
|
||||||
import httplib
|
|
||||||
import mock
|
|
||||||
import StringIO
|
|
||||||
import tempfile
|
|
||||||
import urllib
|
|
||||||
import uuid
|
|
||||||
|
|
||||||
from oslo.config import cfg
|
|
||||||
import swiftclient
|
|
||||||
|
|
||||||
from glance.store import exceptions
|
|
||||||
from glance.store.location import get_location_from_uri
|
|
||||||
from glance.store._drivers import swift
|
|
||||||
from glance.tests.unit import base
|
|
||||||
|
|
||||||
CONF = cfg.CONF
|
|
||||||
|
|
||||||
FAKE_UUID = lambda: str(uuid.uuid4())
|
|
||||||
|
|
||||||
Store = glance.store.swift.Store
|
|
||||||
FIVE_KB = 5 * 1024
|
|
||||||
FIVE_GB = 5 * 1024 * 3
|
|
||||||
MAX_SWIFT_OBJECT_SIZE = FIVE_GB
|
|
||||||
SWIFT_PUT_OBJECT_CALLS = 0
|
|
||||||
SWIFT_CONF = {'verbose': True,
|
|
||||||
'debug': True,
|
|
||||||
'known_stores': ['glance.store.swift.Store'],
|
|
||||||
'default_store': 'swift',
|
|
||||||
'swift_store_user': 'user',
|
|
||||||
'swift_store_key': 'key',
|
|
||||||
'swift_store_auth_address': 'localhost:8080',
|
|
||||||
'swift_store_container': 'glance'}
|
|
||||||
|
|
||||||
|
|
||||||
# We stub out as little as possible to ensure that the code paths
|
|
||||||
# between glance.store.swift and swiftclient are tested
|
|
||||||
# thoroughly
|
|
||||||
def stub_out_swiftclient(test, swift_store_auth_version):
|
|
||||||
fixture_containers = ['glance']
|
|
||||||
fixture_container_headers = {}
|
|
||||||
fixture_headers = {
|
|
||||||
'glance/%s' % FAKE_UUID: {
|
|
||||||
'content-length': FIVE_KB,
|
|
||||||
'etag': 'c2e5db72bd7fd153f53ede5da5a06de3'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
fixture_objects = {'glance/%s' % FAKE_UUID:
|
|
||||||
StringIO.StringIO("*" * FIVE_KB)}
|
|
||||||
|
|
||||||
def fake_head_container(url, token, container, **kwargs):
|
|
||||||
if container not in fixture_containers:
|
|
||||||
msg = "No container %s found" % container
|
|
||||||
raise swiftclient.ClientException(msg,
|
|
||||||
http_status=httplib.NOT_FOUND)
|
|
||||||
return fixture_container_headers
|
|
||||||
|
|
||||||
def fake_put_container(url, token, container, **kwargs):
|
|
||||||
fixture_containers.append(container)
|
|
||||||
|
|
||||||
def fake_post_container(url, token, container, headers, http_conn=None):
|
|
||||||
for key, value in headers.iteritems():
|
|
||||||
fixture_container_headers[key] = value
|
|
||||||
|
|
||||||
def fake_put_object(url, token, container, name, contents, **kwargs):
|
|
||||||
# PUT returns the ETag header for the newly-added object
|
|
||||||
# Large object manifest...
|
|
||||||
global SWIFT_PUT_OBJECT_CALLS
|
|
||||||
SWIFT_PUT_OBJECT_CALLS += 1
|
|
||||||
CHUNKSIZE = 64 * units.Ki
|
|
||||||
fixture_key = "%s/%s" % (container, name)
|
|
||||||
if fixture_key not in fixture_headers:
|
|
||||||
if kwargs.get('headers'):
|
|
||||||
etag = kwargs['headers']['ETag']
|
|
||||||
fixture_headers[fixture_key] = {'manifest': True,
|
|
||||||
'etag': etag}
|
|
||||||
return etag
|
|
||||||
if hasattr(contents, 'read'):
|
|
||||||
fixture_object = StringIO.StringIO()
|
|
||||||
chunk = contents.read(CHUNKSIZE)
|
|
||||||
checksum = hashlib.md5()
|
|
||||||
while chunk:
|
|
||||||
fixture_object.write(chunk)
|
|
||||||
checksum.update(chunk)
|
|
||||||
chunk = contents.read(CHUNKSIZE)
|
|
||||||
etag = checksum.hexdigest()
|
|
||||||
else:
|
|
||||||
fixture_object = StringIO.StringIO(contents)
|
|
||||||
etag = hashlib.md5(fixture_object.getvalue()).hexdigest()
|
|
||||||
read_len = fixture_object.len
|
|
||||||
if read_len > MAX_SWIFT_OBJECT_SIZE:
|
|
||||||
msg = ('Image size:%d exceeds Swift max:%d' %
|
|
||||||
(read_len, MAX_SWIFT_OBJECT_SIZE))
|
|
||||||
raise swiftclient.ClientException(
|
|
||||||
msg, http_status=httplib.REQUEST_ENTITY_TOO_LARGE)
|
|
||||||
fixture_objects[fixture_key] = fixture_object
|
|
||||||
fixture_headers[fixture_key] = {
|
|
||||||
'content-length': read_len,
|
|
||||||
'etag': etag}
|
|
||||||
return etag
|
|
||||||
else:
|
|
||||||
msg = ("Object PUT failed - Object with key %s already exists"
|
|
||||||
% fixture_key)
|
|
||||||
raise swiftclient.ClientException(msg,
|
|
||||||
http_status=httplib.CONFLICT)
|
|
||||||
|
|
||||||
def fake_get_object(url, token, container, name, **kwargs):
|
|
||||||
# GET returns the tuple (list of headers, file object)
|
|
||||||
fixture_key = "%s/%s" % (container, name)
|
|
||||||
if fixture_key not in fixture_headers:
|
|
||||||
msg = "Object GET failed"
|
|
||||||
raise swiftclient.ClientException(msg,
|
|
||||||
http_status=httplib.NOT_FOUND)
|
|
||||||
|
|
||||||
fixture = fixture_headers[fixture_key]
|
|
||||||
if 'manifest' in fixture:
|
|
||||||
# Large object manifest... we return a file containing
|
|
||||||
# all objects with prefix of this fixture key
|
|
||||||
chunk_keys = sorted([k for k in fixture_headers.keys()
|
|
||||||
if k.startswith(fixture_key) and
|
|
||||||
k != fixture_key])
|
|
||||||
result = StringIO.StringIO()
|
|
||||||
for key in chunk_keys:
|
|
||||||
result.write(fixture_objects[key].getvalue())
|
|
||||||
return fixture_headers[fixture_key], result
|
|
||||||
|
|
||||||
else:
|
|
||||||
return fixture_headers[fixture_key], fixture_objects[fixture_key]
|
|
||||||
|
|
||||||
def fake_head_object(url, token, container, name, **kwargs):
|
|
||||||
# HEAD returns the list of headers for an object
|
|
||||||
try:
|
|
||||||
fixture_key = "%s/%s" % (container, name)
|
|
||||||
return fixture_headers[fixture_key]
|
|
||||||
except KeyError:
|
|
||||||
msg = "Object HEAD failed - Object does not exist"
|
|
||||||
raise swiftclient.ClientException(msg,
|
|
||||||
http_status=httplib.NOT_FOUND)
|
|
||||||
|
|
||||||
def fake_delete_object(url, token, container, name, **kwargs):
|
|
||||||
# DELETE returns nothing
|
|
||||||
fixture_key = "%s/%s" % (container, name)
|
|
||||||
if fixture_key not in fixture_headers:
|
|
||||||
msg = "Object DELETE failed - Object does not exist"
|
|
||||||
raise swiftclient.ClientException(msg,
|
|
||||||
http_status=httplib.NOT_FOUND)
|
|
||||||
else:
|
|
||||||
del fixture_headers[fixture_key]
|
|
||||||
del fixture_objects[fixture_key]
|
|
||||||
|
|
||||||
def fake_http_connection(*args, **kwargs):
|
|
||||||
return None
|
|
||||||
|
|
||||||
def fake_get_auth(url, user, key, snet, auth_version, **kwargs):
|
|
||||||
if url is None:
|
|
||||||
return None, None
|
|
||||||
if 'http' in url and '://' not in url:
|
|
||||||
raise ValueError('Invalid url %s' % url)
|
|
||||||
# Check the auth version against the configured value
|
|
||||||
if swift_store_auth_version != auth_version:
|
|
||||||
msg = 'AUTHENTICATION failed (version mismatch)'
|
|
||||||
raise swiftclient.ClientException(msg)
|
|
||||||
return None, None
|
|
||||||
|
|
||||||
to_mock = [('head_container', fake_head_container),
|
|
||||||
('put_container', fake_put_container),
|
|
||||||
('post_container', fake_post_container),
|
|
||||||
('put_object', fake_put_object),
|
|
||||||
('delete_object', fake_delete_object),
|
|
||||||
('head_object', fake_head_object),
|
|
||||||
('get_object', fake_get_object),
|
|
||||||
('get_auth', fake_get_auth),
|
|
||||||
('http_connection', fake_http_connection)]
|
|
||||||
|
|
||||||
for (meth, fake_meth) in to_mock:
|
|
||||||
mocked = mock.patch.object(swiftclient.client, meth).start()
|
|
||||||
mocked.side_effect = fake_meth
|
|
||||||
test.add_cleanUp(mocked.stop)
|
|
||||||
|
|
||||||
|
|
||||||
class SwiftTests(object):
|
|
||||||
|
|
||||||
@property
|
|
||||||
def swift_store_user(self):
|
|
||||||
return urllib.quote(CONF.swift_store_user)
|
|
||||||
|
|
||||||
def test_get_size(self):
|
|
||||||
"""
|
|
||||||
Test that we can get the size of an object in the swift store
|
|
||||||
"""
|
|
||||||
uri = "swift://%s:key@auth_address/glance/%s" % (
|
|
||||||
self.swift_store_user, FAKE_UUID)
|
|
||||||
loc = get_location_from_uri(uri)
|
|
||||||
image_size = self.store.get_size(loc)
|
|
||||||
self.assertEqual(image_size, 5120)
|
|
||||||
|
|
||||||
def test_get_size_with_multi_tenant_on(self):
|
|
||||||
"""Test that single tenant uris work with multi tenant on."""
|
|
||||||
uri = ("swift://%s:key@auth_address/glance/%s" %
|
|
||||||
(self.swift_store_user, FAKE_UUID))
|
|
||||||
self.config(swift_store_multi_tenant=True)
|
|
||||||
#NOTE(markwash): ensure the image is found
|
|
||||||
context = glance.context.RequestContext()
|
|
||||||
size = glance.store.get_size_from_backend(context, uri)
|
|
||||||
self.assertEqual(size, 5120)
|
|
||||||
|
|
||||||
def test_get(self):
|
|
||||||
"""Test a "normal" retrieval of an image in chunks"""
|
|
||||||
uri = "swift://%s:key@auth_address/glance/%s" % (
|
|
||||||
self.swift_store_user, FAKE_UUID)
|
|
||||||
loc = get_location_from_uri(uri)
|
|
||||||
(image_swift, image_size) = self.store.get(loc)
|
|
||||||
self.assertEqual(image_size, 5120)
|
|
||||||
|
|
||||||
expected_data = "*" * FIVE_KB
|
|
||||||
data = ""
|
|
||||||
|
|
||||||
for chunk in image_swift:
|
|
||||||
data += chunk
|
|
||||||
self.assertEqual(expected_data, data)
|
|
||||||
|
|
||||||
def test_get_with_http_auth(self):
|
|
||||||
"""
|
|
||||||
Test a retrieval from Swift with an HTTP authurl. This is
|
|
||||||
specified either via a Location header with swift+http:// or using
|
|
||||||
http:// in the swift_store_auth_address config value
|
|
||||||
"""
|
|
||||||
loc = get_location_from_uri("swift+http://%s:key@auth_address/"
|
|
||||||
"glance/%s" %
|
|
||||||
(self.swift_store_user, FAKE_UUID))
|
|
||||||
(image_swift, image_size) = self.store.get(loc)
|
|
||||||
self.assertEqual(image_size, 5120)
|
|
||||||
|
|
||||||
expected_data = "*" * FIVE_KB
|
|
||||||
data = ""
|
|
||||||
|
|
||||||
for chunk in image_swift:
|
|
||||||
data += chunk
|
|
||||||
self.assertEqual(expected_data, data)
|
|
||||||
|
|
||||||
def test_get_non_existing(self):
|
|
||||||
"""
|
|
||||||
Test that trying to retrieve a swift that doesn't exist
|
|
||||||
raises an error
|
|
||||||
"""
|
|
||||||
loc = get_location_from_uri("swift://%s:key@authurl/glance/noexist" % (
|
|
||||||
self.swift_store_user))
|
|
||||||
self.assertRaises(exceptions.NotFound,
|
|
||||||
self.store.get,
|
|
||||||
loc)
|
|
||||||
|
|
||||||
def test_add(self):
|
|
||||||
"""Test that we can add an image via the swift backend"""
|
|
||||||
expected_swift_size = FIVE_KB
|
|
||||||
expected_swift_contents = "*" * expected_swift_size
|
|
||||||
expected_checksum = hashlib.md5(expected_swift_contents).hexdigest()
|
|
||||||
expected_image_id = str(uuid.uuid4())
|
|
||||||
loc = 'swift+https://%s:key@localhost:8080/glance/%s'
|
|
||||||
expected_location = loc % (self.swift_store_user,
|
|
||||||
expected_image_id)
|
|
||||||
image_swift = StringIO.StringIO(expected_swift_contents)
|
|
||||||
|
|
||||||
global SWIFT_PUT_OBJECT_CALLS
|
|
||||||
SWIFT_PUT_OBJECT_CALLS = 0
|
|
||||||
|
|
||||||
location, size, checksum, _ = self.store.add(expected_image_id,
|
|
||||||
image_swift,
|
|
||||||
expected_swift_size)
|
|
||||||
|
|
||||||
self.assertEqual(expected_location, location)
|
|
||||||
self.assertEqual(expected_swift_size, size)
|
|
||||||
self.assertEqual(expected_checksum, checksum)
|
|
||||||
# Expecting a single object to be created on Swift i.e. no chunking.
|
|
||||||
self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 1)
|
|
||||||
|
|
||||||
loc = get_location_from_uri(expected_location)
|
|
||||||
(new_image_swift, new_image_size) = self.store.get(loc)
|
|
||||||
new_image_contents = new_image_swift.getvalue()
|
|
||||||
new_image_swift_size = len(new_image_swift)
|
|
||||||
|
|
||||||
self.assertEqual(expected_swift_contents, new_image_contents)
|
|
||||||
self.assertEqual(expected_swift_size, new_image_swift_size)
|
|
||||||
|
|
||||||
def test_add_auth_url_variations(self):
|
|
||||||
"""
|
|
||||||
Test that we can add an image via the swift backend with
|
|
||||||
a variety of different auth_address values
|
|
||||||
"""
|
|
||||||
variations = {
|
|
||||||
'http://localhost:80': 'swift+http://%s:key@localhost:80'
|
|
||||||
'/glance/%s',
|
|
||||||
'http://localhost': 'swift+http://%s:key@localhost/glance/%s',
|
|
||||||
'http://localhost/v1': 'swift+http://%s:key@localhost'
|
|
||||||
'/v1/glance/%s',
|
|
||||||
'http://localhost/v1/': 'swift+http://%s:key@localhost'
|
|
||||||
'/v1/glance/%s',
|
|
||||||
'https://localhost': 'swift+https://%s:key@localhost/glance/%s',
|
|
||||||
'https://localhost:8080': 'swift+https://%s:key@localhost:8080'
|
|
||||||
'/glance/%s',
|
|
||||||
'https://localhost/v1': 'swift+https://%s:key@localhost'
|
|
||||||
'/v1/glance/%s',
|
|
||||||
'https://localhost/v1/': 'swift+https://%s:key@localhost'
|
|
||||||
'/v1/glance/%s',
|
|
||||||
'localhost': 'swift+https://%s:key@localhost/glance/%s',
|
|
||||||
'localhost:8080/v1': 'swift+https://%s:key@localhost:8080'
|
|
||||||
'/v1/glance/%s',
|
|
||||||
}
|
|
||||||
|
|
||||||
for variation, expected_location in variations.items():
|
|
||||||
image_id = str(uuid.uuid4())
|
|
||||||
expected_location = expected_location % (
|
|
||||||
self.swift_store_user, image_id)
|
|
||||||
expected_swift_size = FIVE_KB
|
|
||||||
expected_swift_contents = "*" * expected_swift_size
|
|
||||||
expected_checksum = \
|
|
||||||
hashlib.md5(expected_swift_contents).hexdigest()
|
|
||||||
|
|
||||||
image_swift = StringIO.StringIO(expected_swift_contents)
|
|
||||||
|
|
||||||
global SWIFT_PUT_OBJECT_CALLS
|
|
||||||
SWIFT_PUT_OBJECT_CALLS = 0
|
|
||||||
|
|
||||||
self.config(swift_store_auth_address=variation)
|
|
||||||
self.store = Store()
|
|
||||||
location, size, checksum, _ = self.store.add(image_id, image_swift,
|
|
||||||
expected_swift_size)
|
|
||||||
|
|
||||||
self.assertEqual(expected_location, location)
|
|
||||||
self.assertEqual(expected_swift_size, size)
|
|
||||||
self.assertEqual(expected_checksum, checksum)
|
|
||||||
self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 1)
|
|
||||||
|
|
||||||
loc = get_location_from_uri(expected_location)
|
|
||||||
(new_image_swift, new_image_size) = self.store.get(loc)
|
|
||||||
new_image_contents = new_image_swift.getvalue()
|
|
||||||
new_image_swift_size = len(new_image_swift)
|
|
||||||
|
|
||||||
self.assertEqual(expected_swift_contents, new_image_contents)
|
|
||||||
self.assertEqual(expected_swift_size, new_image_swift_size)
|
|
||||||
|
|
||||||
def test_add_no_container_no_create(self):
|
|
||||||
"""
|
|
||||||
Tests that adding an image with a non-existing container
|
|
||||||
raises an appropriate exception
|
|
||||||
"""
|
|
||||||
self.config(swift_store_create_container_on_put=False,
|
|
||||||
swift_store_container='noexist')
|
|
||||||
self.store = Store()
|
|
||||||
|
|
||||||
image_swift = StringIO.StringIO("nevergonnamakeit")
|
|
||||||
|
|
||||||
global SWIFT_PUT_OBJECT_CALLS
|
|
||||||
SWIFT_PUT_OBJECT_CALLS = 0
|
|
||||||
|
|
||||||
# We check the exception text to ensure the container
|
|
||||||
# missing text is found in it, otherwise, we would have
|
|
||||||
# simply used self.assertRaises here
|
|
||||||
exception_caught = False
|
|
||||||
try:
|
|
||||||
self.store.add(str(uuid.uuid4()), image_swift, 0)
|
|
||||||
except backend.BackendException as e:
|
|
||||||
exception_caught = True
|
|
||||||
self.assertTrue("container noexist does not exist "
|
|
||||||
"in Swift" in str(e))
|
|
||||||
self.assertTrue(exception_caught)
|
|
||||||
self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 0)
|
|
||||||
|
|
||||||
def test_add_no_container_and_create(self):
|
|
||||||
"""
|
|
||||||
Tests that adding an image with a non-existing container
|
|
||||||
creates the container automatically if flag is set
|
|
||||||
"""
|
|
||||||
expected_swift_size = FIVE_KB
|
|
||||||
expected_swift_contents = "*" * expected_swift_size
|
|
||||||
expected_checksum = hashlib.md5(expected_swift_contents).hexdigest()
|
|
||||||
expected_image_id = str(uuid.uuid4())
|
|
||||||
loc = 'swift+https://%s:key@localhost:8080/noexist/%s'
|
|
||||||
expected_location = loc % (self.swift_store_user,
|
|
||||||
expected_image_id)
|
|
||||||
image_swift = StringIO.StringIO(expected_swift_contents)
|
|
||||||
|
|
||||||
global SWIFT_PUT_OBJECT_CALLS
|
|
||||||
SWIFT_PUT_OBJECT_CALLS = 0
|
|
||||||
|
|
||||||
self.config(swift_store_create_container_on_put=True,
|
|
||||||
swift_store_container='noexist')
|
|
||||||
self.store = Store()
|
|
||||||
location, size, checksum, _ = self.store.add(expected_image_id,
|
|
||||||
image_swift,
|
|
||||||
expected_swift_size)
|
|
||||||
|
|
||||||
self.assertEqual(expected_location, location)
|
|
||||||
self.assertEqual(expected_swift_size, size)
|
|
||||||
self.assertEqual(expected_checksum, checksum)
|
|
||||||
self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 1)
|
|
||||||
|
|
||||||
loc = get_location_from_uri(expected_location)
|
|
||||||
(new_image_swift, new_image_size) = self.store.get(loc)
|
|
||||||
new_image_contents = new_image_swift.getvalue()
|
|
||||||
new_image_swift_size = len(new_image_swift)
|
|
||||||
|
|
||||||
self.assertEqual(expected_swift_contents, new_image_contents)
|
|
||||||
self.assertEqual(expected_swift_size, new_image_swift_size)
|
|
||||||
|
|
||||||
def test_add_large_object(self):
|
|
||||||
"""
|
|
||||||
Tests that adding a very large image. We simulate the large
|
|
||||||
object by setting store.large_object_size to a small number
|
|
||||||
and then verify that there have been a number of calls to
|
|
||||||
put_object()...
|
|
||||||
"""
|
|
||||||
expected_swift_size = FIVE_KB
|
|
||||||
expected_swift_contents = "*" * expected_swift_size
|
|
||||||
expected_checksum = hashlib.md5(expected_swift_contents).hexdigest()
|
|
||||||
expected_image_id = str(uuid.uuid4())
|
|
||||||
loc = 'swift+https://%s:key@localhost:8080/glance/%s'
|
|
||||||
expected_location = loc % (self.swift_store_user,
|
|
||||||
expected_image_id)
|
|
||||||
image_swift = StringIO.StringIO(expected_swift_contents)
|
|
||||||
|
|
||||||
global SWIFT_PUT_OBJECT_CALLS
|
|
||||||
SWIFT_PUT_OBJECT_CALLS = 0
|
|
||||||
|
|
||||||
self.config(swift_store_container='glance')
|
|
||||||
self.store = Store()
|
|
||||||
orig_max_size = self.store.large_object_size
|
|
||||||
orig_temp_size = self.store.large_object_chunk_size
|
|
||||||
try:
|
|
||||||
self.store.large_object_size = 1024
|
|
||||||
self.store.large_object_chunk_size = 1024
|
|
||||||
location, size, checksum, _ = self.store.add(expected_image_id,
|
|
||||||
image_swift,
|
|
||||||
expected_swift_size)
|
|
||||||
finally:
|
|
||||||
self.store.large_object_chunk_size = orig_temp_size
|
|
||||||
self.store.large_object_size = orig_max_size
|
|
||||||
|
|
||||||
self.assertEqual(expected_location, location)
|
|
||||||
self.assertEqual(expected_swift_size, size)
|
|
||||||
self.assertEqual(expected_checksum, checksum)
|
|
||||||
# Expecting 6 objects to be created on Swift -- 5 chunks and 1
|
|
||||||
# manifest.
|
|
||||||
self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 6)
|
|
||||||
|
|
||||||
loc = get_location_from_uri(expected_location)
|
|
||||||
(new_image_swift, new_image_size) = self.store.get(loc)
|
|
||||||
new_image_contents = new_image_swift.getvalue()
|
|
||||||
new_image_swift_size = len(new_image_swift)
|
|
||||||
|
|
||||||
self.assertEqual(expected_swift_contents, new_image_contents)
|
|
||||||
self.assertEqual(expected_swift_size, new_image_swift_size)
|
|
||||||
|
|
||||||
def test_add_large_object_zero_size(self):
|
|
||||||
"""
|
|
||||||
Tests that adding an image to Swift which has both an unknown size and
|
|
||||||
exceeds Swift's maximum limit of 5GB is correctly uploaded.
|
|
||||||
|
|
||||||
We avoid the overhead of creating a 5GB object for this test by
|
|
||||||
temporarily setting MAX_SWIFT_OBJECT_SIZE to 1KB, and then adding
|
|
||||||
an object of 5KB.
|
|
||||||
|
|
||||||
Bug lp:891738
|
|
||||||
"""
|
|
||||||
# Set up a 'large' image of 5KB
|
|
||||||
expected_swift_size = FIVE_KB
|
|
||||||
expected_swift_contents = "*" * expected_swift_size
|
|
||||||
expected_checksum = hashlib.md5(expected_swift_contents).hexdigest()
|
|
||||||
expected_image_id = str(uuid.uuid4())
|
|
||||||
loc = 'swift+https://%s:key@localhost:8080/glance/%s'
|
|
||||||
expected_location = loc % (self.swift_store_user,
|
|
||||||
expected_image_id)
|
|
||||||
image_swift = StringIO.StringIO(expected_swift_contents)
|
|
||||||
|
|
||||||
global SWIFT_PUT_OBJECT_CALLS
|
|
||||||
SWIFT_PUT_OBJECT_CALLS = 0
|
|
||||||
|
|
||||||
# Temporarily set Swift MAX_SWIFT_OBJECT_SIZE to 1KB and add our image,
|
|
||||||
# explicitly setting the image_length to 0
|
|
||||||
self.config(swift_store_container='glance')
|
|
||||||
self.store = Store()
|
|
||||||
orig_max_size = self.store.large_object_size
|
|
||||||
orig_temp_size = self.store.large_object_chunk_size
|
|
||||||
global MAX_SWIFT_OBJECT_SIZE
|
|
||||||
orig_max_swift_object_size = MAX_SWIFT_OBJECT_SIZE
|
|
||||||
try:
|
|
||||||
MAX_SWIFT_OBJECT_SIZE = 1024
|
|
||||||
self.store.large_object_size = 1024
|
|
||||||
self.store.large_object_chunk_size = 1024
|
|
||||||
location, size, checksum, _ = self.store.add(expected_image_id,
|
|
||||||
image_swift, 0)
|
|
||||||
finally:
|
|
||||||
self.store.large_object_chunk_size = orig_temp_size
|
|
||||||
self.store.large_object_size = orig_max_size
|
|
||||||
MAX_SWIFT_OBJECT_SIZE = orig_max_swift_object_size
|
|
||||||
|
|
||||||
self.assertEqual(expected_location, location)
|
|
||||||
self.assertEqual(expected_swift_size, size)
|
|
||||||
self.assertEqual(expected_checksum, checksum)
|
|
||||||
# Expecting 7 calls to put_object -- 5 chunks, a zero chunk which is
|
|
||||||
# then deleted, and the manifest. Note the difference with above
|
|
||||||
# where the image_size is specified in advance (there's no zero chunk
|
|
||||||
# in that case).
|
|
||||||
self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 7)
|
|
||||||
|
|
||||||
loc = get_location_from_uri(expected_location)
|
|
||||||
(new_image_swift, new_image_size) = self.store.get(loc)
|
|
||||||
new_image_contents = new_image_swift.getvalue()
|
|
||||||
new_image_swift_size = len(new_image_swift)
|
|
||||||
|
|
||||||
self.assertEqual(expected_swift_contents, new_image_contents)
|
|
||||||
self.assertEqual(expected_swift_size, new_image_swift_size)
|
|
||||||
|
|
||||||
def test_add_already_existing(self):
|
|
||||||
"""
|
|
||||||
Tests that adding an image with an existing identifier
|
|
||||||
raises an appropriate exception
|
|
||||||
"""
|
|
||||||
image_swift = StringIO.StringIO("nevergonnamakeit")
|
|
||||||
self.assertRaises(exceptions.Duplicate,
|
|
||||||
self.store.add,
|
|
||||||
FAKE_UUID, image_swift, 0)
|
|
||||||
|
|
||||||
def test_add_saves_and_reraises_and_not_uses_wildcard_raise(self):
|
|
||||||
image_id = str(uuid.uuid4())
|
|
||||||
swift_size = self.store.large_object_size = 1024
|
|
||||||
loc = 'swift+https://%s:key@localhost:8080/glance/%s'
|
|
||||||
swift_contents = "*" * swift_size
|
|
||||||
connection = mock.Mock()
|
|
||||||
|
|
||||||
def fake_delete_chunk(connection,
|
|
||||||
container,
|
|
||||||
chunks):
|
|
||||||
try:
|
|
||||||
raise Exception()
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
image_swift = StringIO.StringIO(swift_contents)
|
|
||||||
connection.put_object.side_effect = exceptions.ClientConnectionError
|
|
||||||
self.store._delete_stale_chunks = fake_delete_chunk
|
|
||||||
|
|
||||||
self.assertRaises(exceptions.ClientConnectionError,
|
|
||||||
self.store.add,
|
|
||||||
image_id,
|
|
||||||
image_swift,
|
|
||||||
swift_size,
|
|
||||||
connection)
|
|
||||||
|
|
||||||
def _option_required(self, key):
|
|
||||||
conf = self.getConfig()
|
|
||||||
conf[key] = None
|
|
||||||
|
|
||||||
try:
|
|
||||||
self.config(**conf)
|
|
||||||
self.store = Store()
|
|
||||||
return self.store.add == self.store.add_disabled
|
|
||||||
except Exception:
|
|
||||||
return False
|
|
||||||
return False
|
|
||||||
|
|
||||||
def test_no_user(self):
|
|
||||||
"""
|
|
||||||
Tests that options without user disables the add method
|
|
||||||
"""
|
|
||||||
self.assertTrue(self._option_required('swift_store_user'))
|
|
||||||
|
|
||||||
def test_no_key(self):
|
|
||||||
"""
|
|
||||||
Tests that options without key disables the add method
|
|
||||||
"""
|
|
||||||
self.assertTrue(self._option_required('swift_store_key'))
|
|
||||||
|
|
||||||
def test_no_auth_address(self):
|
|
||||||
"""
|
|
||||||
Tests that options without auth address disables the add method
|
|
||||||
"""
|
|
||||||
self.assertTrue(self._option_required('swift_store_auth_address'))
|
|
||||||
|
|
||||||
def test_delete(self):
|
|
||||||
"""
|
|
||||||
Test we can delete an existing image in the swift store
|
|
||||||
"""
|
|
||||||
uri = "swift://%s:key@authurl/glance/%s" % (
|
|
||||||
self.swift_store_user, FAKE_UUID)
|
|
||||||
loc = get_location_from_uri(uri)
|
|
||||||
self.store.delete(loc)
|
|
||||||
|
|
||||||
self.assertRaises(exceptions.NotFound, self.store.get, loc)
|
|
||||||
|
|
||||||
def test_delete_non_existing(self):
|
|
||||||
"""
|
|
||||||
Test that trying to delete a swift that doesn't exist
|
|
||||||
raises an error
|
|
||||||
"""
|
|
||||||
loc = get_location_from_uri("swift://%s:key@authurl/glance/noexist" % (
|
|
||||||
self.swift_store_user))
|
|
||||||
self.assertRaises(exceptions.NotFound, self.store.delete, loc)
|
|
||||||
|
|
||||||
def test_read_acl_public(self):
|
|
||||||
"""
|
|
||||||
Test that we can set a public read acl.
|
|
||||||
"""
|
|
||||||
self.config(swift_store_multi_tenant=True)
|
|
||||||
context = glance.context.RequestContext()
|
|
||||||
store = Store(context)
|
|
||||||
uri = "swift+http://storeurl/glance/%s" % FAKE_UUID
|
|
||||||
loc = get_location_from_uri(uri)
|
|
||||||
store.set_acls(loc, public=True)
|
|
||||||
container_headers = swiftclient.client.head_container('x', 'y',
|
|
||||||
'glance')
|
|
||||||
self.assertEqual(container_headers['X-Container-Read'],
|
|
||||||
".r:*,.rlistings")
|
|
||||||
|
|
||||||
def test_read_acl_tenants(self):
|
|
||||||
"""
|
|
||||||
Test that we can set read acl for tenants.
|
|
||||||
"""
|
|
||||||
self.config(swift_store_multi_tenant=True)
|
|
||||||
context = glance.context.RequestContext()
|
|
||||||
store = Store(context)
|
|
||||||
uri = "swift+http://storeurl/glance/%s" % FAKE_UUID
|
|
||||||
loc = get_location_from_uri(uri)
|
|
||||||
read_tenants = ['matt', 'mark']
|
|
||||||
store.set_acls(loc, read_tenants=read_tenants)
|
|
||||||
container_headers = swiftclient.client.head_container('x', 'y',
|
|
||||||
'glance')
|
|
||||||
self.assertEqual(container_headers['X-Container-Read'],
|
|
||||||
'matt:*,mark:*')
|
|
||||||
|
|
||||||
def test_write_acls(self):
|
|
||||||
"""
|
|
||||||
Test that we can set write acl for tenants.
|
|
||||||
"""
|
|
||||||
self.config(swift_store_multi_tenant=True)
|
|
||||||
context = glance.context.RequestContext()
|
|
||||||
store = Store(context)
|
|
||||||
uri = "swift+http://storeurl/glance/%s" % FAKE_UUID
|
|
||||||
loc = get_location_from_uri(uri)
|
|
||||||
read_tenants = ['frank', 'jim']
|
|
||||||
store.set_acls(loc, write_tenants=read_tenants)
|
|
||||||
container_headers = swiftclient.client.head_container('x', 'y',
|
|
||||||
'glance')
|
|
||||||
self.assertEqual(container_headers['X-Container-Write'],
|
|
||||||
'frank:*,jim:*')
|
|
||||||
|
|
||||||
|
|
||||||
class TestStoreAuthV1(base.StoreClearingUnitTest, SwiftTests):
|
|
||||||
|
|
||||||
def getConfig(self):
|
|
||||||
conf = SWIFT_CONF.copy()
|
|
||||||
conf['swift_store_auth_version'] = '1'
|
|
||||||
conf['swift_store_user'] = 'user'
|
|
||||||
return conf
|
|
||||||
|
|
||||||
def setUp(self):
|
|
||||||
"""Establish a clean test environment"""
|
|
||||||
conf = self.getConfig()
|
|
||||||
self.config(**conf)
|
|
||||||
super(TestStoreAuthV1, self).setUp()
|
|
||||||
stub_out_swiftclient(self, conf['swift_store_auth_version'])
|
|
||||||
self.store = Store()
|
|
||||||
|
|
||||||
class TestStoreAuthV2(TestStoreAuthV1):
|
|
||||||
|
|
||||||
def getConfig(self):
|
|
||||||
conf = super(TestStoreAuthV2, self).getConfig()
|
|
||||||
conf['swift_store_user'] = 'tenant:user'
|
|
||||||
conf['swift_store_auth_version'] = '2'
|
|
||||||
return conf
|
|
||||||
|
|
||||||
def test_v2_with_no_tenant(self):
|
|
||||||
conf = self.getConfig()
|
|
||||||
conf['swift_store_user'] = 'failme'
|
|
||||||
uri = "swift://%s:key@auth_address/glance/%s" % (
|
|
||||||
conf['swift_store_user'], FAKE_UUID)
|
|
||||||
loc = get_location_from_uri(uri)
|
|
||||||
self.assertRaises(exceptions.BadStoreUri,
|
|
||||||
self.store.get,
|
|
||||||
loc)
|
|
||||||
|
|
||||||
def test_v2_multi_tenant_location(self):
|
|
||||||
conf = self.getConfig()
|
|
||||||
conf['swift_store_multi_tenant'] = True
|
|
||||||
uri = "swift://auth_address/glance/%s" % (FAKE_UUID)
|
|
||||||
loc = get_location_from_uri(uri)
|
|
||||||
self.assertEqual('swift', loc.store_name)
|
|
||||||
|
|
||||||
|
|
||||||
class FakeConnection(object):
|
|
||||||
def __init__(self, authurl, user, key, retries=5, preauthurl=None,
|
|
||||||
preauthtoken=None, snet=False, starting_backoff=1,
|
|
||||||
tenant_name=None, os_options={}, auth_version="1",
|
|
||||||
insecure=False, ssl_compression=True):
|
|
||||||
self.authurl = authurl
|
|
||||||
self.user = user
|
|
||||||
self.key = key
|
|
||||||
self.preauthurl = preauthurl
|
|
||||||
self.preauthtoken = preauthtoken
|
|
||||||
self.snet = snet
|
|
||||||
self.tenant_name = tenant_name
|
|
||||||
self.os_options = os_options
|
|
||||||
self.auth_version = auth_version
|
|
||||||
self.insecure = insecure
|
|
||||||
|
|
||||||
|
|
||||||
class TestSingleTenantStoreConnections(base.IsolatedUnitTest):
|
|
||||||
def setUp(self):
|
|
||||||
super(TestSingleTenantStoreConnections, self).setUp()
|
|
||||||
self.stubs.Set(swiftclient, 'Connection', FakeConnection)
|
|
||||||
self.store = glance.store.swift.SingleTenantStore()
|
|
||||||
specs = {'scheme': 'swift',
|
|
||||||
'auth_or_store_url': 'example.com/v2/',
|
|
||||||
'user': 'tenant:user',
|
|
||||||
'key': 'abcdefg',
|
|
||||||
'container': 'cont',
|
|
||||||
'obj': 'object'}
|
|
||||||
self.location = glance.store.swift.StoreLocation(specs)
|
|
||||||
|
|
||||||
def test_basic_connection(self):
|
|
||||||
connection = self.store.get_connection(self.location)
|
|
||||||
self.assertEqual(connection.authurl, 'https://example.com/v2/')
|
|
||||||
self.assertEqual(connection.auth_version, '2')
|
|
||||||
self.assertEqual(connection.user, 'user')
|
|
||||||
self.assertEqual(connection.tenant_name, 'tenant')
|
|
||||||
self.assertEqual(connection.key, 'abcdefg')
|
|
||||||
self.assertFalse(connection.snet)
|
|
||||||
self.assertEqual(connection.preauthurl, None)
|
|
||||||
self.assertEqual(connection.preauthtoken, None)
|
|
||||||
self.assertFalse(connection.insecure)
|
|
||||||
self.assertEqual(connection.os_options,
|
|
||||||
{'service_type': 'object-store',
|
|
||||||
'endpoint_type': 'publicURL'})
|
|
||||||
|
|
||||||
def test_connection_with_no_trailing_slash(self):
|
|
||||||
self.location.auth_or_store_url = 'example.com/v2'
|
|
||||||
connection = self.store.get_connection(self.location)
|
|
||||||
self.assertEqual(connection.authurl, 'https://example.com/v2/')
|
|
||||||
|
|
||||||
def test_connection_insecure(self):
|
|
||||||
self.config(swift_store_auth_insecure=True)
|
|
||||||
self.store.configure()
|
|
||||||
connection = self.store.get_connection(self.location)
|
|
||||||
self.assertTrue(connection.insecure)
|
|
||||||
|
|
||||||
def test_connection_with_auth_v1(self):
|
|
||||||
self.config(swift_store_auth_version='1')
|
|
||||||
self.store.configure()
|
|
||||||
self.location.user = 'auth_v1_user'
|
|
||||||
connection = self.store.get_connection(self.location)
|
|
||||||
self.assertEqual(connection.auth_version, '1')
|
|
||||||
self.assertEqual(connection.user, 'auth_v1_user')
|
|
||||||
self.assertEqual(connection.tenant_name, None)
|
|
||||||
|
|
||||||
def test_connection_invalid_user(self):
|
|
||||||
self.store.configure()
|
|
||||||
self.location.user = 'invalid:format:user'
|
|
||||||
self.assertRaises(exceptions.BadStoreUri,
|
|
||||||
self.store.get_connection, self.location)
|
|
||||||
|
|
||||||
def test_connection_missing_user(self):
|
|
||||||
self.store.configure()
|
|
||||||
self.location.user = None
|
|
||||||
self.assertRaises(exceptions.BadStoreUri,
|
|
||||||
self.store.get_connection, self.location)
|
|
||||||
|
|
||||||
def test_connection_with_region(self):
|
|
||||||
self.config(swift_store_region='Sahara')
|
|
||||||
self.store.configure()
|
|
||||||
connection = self.store.get_connection(self.location)
|
|
||||||
self.assertEqual(connection.os_options,
|
|
||||||
{'region_name': 'Sahara',
|
|
||||||
'service_type': 'object-store',
|
|
||||||
'endpoint_type': 'publicURL'})
|
|
||||||
|
|
||||||
def test_connection_with_service_type(self):
|
|
||||||
self.config(swift_store_service_type='shoe-store')
|
|
||||||
self.store.configure()
|
|
||||||
connection = self.store.get_connection(self.location)
|
|
||||||
self.assertEqual(connection.os_options,
|
|
||||||
{'service_type': 'shoe-store',
|
|
||||||
'endpoint_type': 'publicURL'})
|
|
||||||
|
|
||||||
def test_connection_with_endpoint_type(self):
|
|
||||||
self.config(swift_store_endpoint_type='internalURL')
|
|
||||||
self.store.configure()
|
|
||||||
connection = self.store.get_connection(self.location)
|
|
||||||
self.assertEqual(connection.os_options,
|
|
||||||
{'service_type': 'object-store',
|
|
||||||
'endpoint_type': 'internalURL'})
|
|
||||||
|
|
||||||
def test_connection_with_snet(self):
|
|
||||||
self.config(swift_enable_snet=True)
|
|
||||||
self.store.configure()
|
|
||||||
connection = self.store.get_connection(self.location)
|
|
||||||
self.assertTrue(connection.snet)
|
|
||||||
|
|
||||||
|
|
||||||
class TestMultiTenantStoreConnections(base.IsolatedUnitTest):
|
|
||||||
def setUp(self):
|
|
||||||
super(TestMultiTenantStoreConnections, self).setUp()
|
|
||||||
self.stubs.Set(swiftclient, 'Connection', FakeConnection)
|
|
||||||
self.context = glance.context.RequestContext(
|
|
||||||
user='user', tenant='tenant', auth_tok='0123')
|
|
||||||
self.store = glance.store.swift.MultiTenantStore(self.context)
|
|
||||||
specs = {'scheme': 'swift',
|
|
||||||
'auth_or_store_url': 'example.com',
|
|
||||||
'container': 'cont',
|
|
||||||
'obj': 'object'}
|
|
||||||
self.location = glance.store.swift.StoreLocation(specs)
|
|
||||||
|
|
||||||
def test_basic_connection(self):
|
|
||||||
self.store.configure()
|
|
||||||
connection = self.store.get_connection(self.location)
|
|
||||||
self.assertEqual(connection.authurl, None)
|
|
||||||
self.assertEqual(connection.auth_version, '2')
|
|
||||||
self.assertEqual(connection.user, 'user')
|
|
||||||
self.assertEqual(connection.tenant_name, 'tenant')
|
|
||||||
self.assertEqual(connection.key, None)
|
|
||||||
self.assertFalse(connection.snet)
|
|
||||||
self.assertEqual(connection.preauthurl, 'https://example.com')
|
|
||||||
self.assertEqual(connection.preauthtoken, '0123')
|
|
||||||
self.assertEqual(connection.os_options, {})
|
|
||||||
|
|
||||||
def test_connection_with_snet(self):
|
|
||||||
self.config(swift_enable_snet=True)
|
|
||||||
self.store.configure()
|
|
||||||
connection = self.store.get_connection(self.location)
|
|
||||||
self.assertTrue(connection.snet)
|
|
||||||
|
|
||||||
|
|
||||||
class FakeGetEndpoint(object):
|
|
||||||
def __init__(self, response):
|
|
||||||
self.response = response
|
|
||||||
|
|
||||||
def __call__(self, service_catalog, service_type=None,
|
|
||||||
endpoint_region=None, endpoint_type=None):
|
|
||||||
self.service_type = service_type
|
|
||||||
self.endpoint_region = endpoint_region
|
|
||||||
self.endpoint_type = endpoint_type
|
|
||||||
return self.response
|
|
||||||
|
|
||||||
|
|
||||||
class TestCreatingLocations(base.IsolatedUnitTest):
|
|
||||||
def test_single_tenant_location(self):
|
|
||||||
self.config(swift_store_auth_address='example.com/v2',
|
|
||||||
swift_store_container='container',
|
|
||||||
swift_store_user='tenant:user',
|
|
||||||
swift_store_key='auth_key')
|
|
||||||
store = glance.store.swift.SingleTenantStore()
|
|
||||||
location = store.create_location('image-id')
|
|
||||||
self.assertEqual(location.scheme, 'swift+https')
|
|
||||||
self.assertEqual(location.swift_url, 'https://example.com/v2')
|
|
||||||
self.assertEqual(location.container, 'container')
|
|
||||||
self.assertEqual(location.obj, 'image-id')
|
|
||||||
self.assertEqual(location.user, 'tenant:user')
|
|
||||||
self.assertEqual(location.key, 'auth_key')
|
|
||||||
|
|
||||||
def test_single_tenant_location_http(self):
|
|
||||||
self.config(swift_store_auth_address='http://example.com/v2',
|
|
||||||
swift_store_container='container',
|
|
||||||
swift_store_user='tenant:user',
|
|
||||||
swift_store_key='auth_key')
|
|
||||||
store = glance.store.swift.SingleTenantStore()
|
|
||||||
location = store.create_location('image-id')
|
|
||||||
self.assertEqual(location.scheme, 'swift+http')
|
|
||||||
self.assertEqual(location.swift_url, 'http://example.com/v2')
|
|
||||||
|
|
||||||
def test_multi_tenant_location(self):
|
|
||||||
self.config(swift_store_container='container')
|
|
||||||
fake_get_endpoint = FakeGetEndpoint('https://some_endpoint')
|
|
||||||
self.stubs.Set(glance.store.common.auth, 'get_endpoint', fake_get_endpoint)
|
|
||||||
context = glance.context.RequestContext(
|
|
||||||
user='user', tenant='tenant', auth_tok='123',
|
|
||||||
service_catalog={})
|
|
||||||
store = glance.store.swift.MultiTenantStore(context)
|
|
||||||
location = store.create_location('image-id')
|
|
||||||
self.assertEqual(location.scheme, 'swift+https')
|
|
||||||
self.assertEqual(location.swift_url, 'https://some_endpoint')
|
|
||||||
self.assertEqual(location.container, 'container_image-id')
|
|
||||||
self.assertEqual(location.obj, 'image-id')
|
|
||||||
self.assertEqual(location.user, None)
|
|
||||||
self.assertEqual(location.key, None)
|
|
||||||
self.assertEqual(fake_get_endpoint.service_type, 'object-store')
|
|
||||||
|
|
||||||
def test_multi_tenant_location_http(self):
|
|
||||||
fake_get_endpoint = FakeGetEndpoint('http://some_endpoint')
|
|
||||||
self.stubs.Set(glance.store.common.auth, 'get_endpoint', fake_get_endpoint)
|
|
||||||
context = glance.context.RequestContext(
|
|
||||||
user='user', tenant='tenant', auth_tok='123',
|
|
||||||
service_catalog={})
|
|
||||||
store = glance.store.swift.MultiTenantStore(context)
|
|
||||||
location = store.create_location('image-id')
|
|
||||||
self.assertEqual(location.scheme, 'swift+http')
|
|
||||||
self.assertEqual(location.swift_url, 'http://some_endpoint')
|
|
||||||
|
|
||||||
def test_multi_tenant_location_with_region(self):
|
|
||||||
self.config(swift_store_region='WestCarolina')
|
|
||||||
fake_get_endpoint = FakeGetEndpoint('https://some_endpoint')
|
|
||||||
self.stubs.Set(glance.store.common.auth, 'get_endpoint', fake_get_endpoint)
|
|
||||||
context = glance.context.RequestContext(
|
|
||||||
user='user', tenant='tenant', auth_tok='123',
|
|
||||||
service_catalog={})
|
|
||||||
store = glance.store.swift.MultiTenantStore(context)
|
|
||||||
self.assertEqual(fake_get_endpoint.endpoint_region, 'WestCarolina')
|
|
||||||
|
|
||||||
def test_multi_tenant_location_custom_service_type(self):
|
|
||||||
self.config(swift_store_service_type='toy-store')
|
|
||||||
fake_get_endpoint = FakeGetEndpoint('https://some_endpoint')
|
|
||||||
self.stubs.Set(glance.store.common.auth, 'get_endpoint', fake_get_endpoint)
|
|
||||||
context = glance.context.RequestContext(
|
|
||||||
user='user', tenant='tenant', auth_tok='123',
|
|
||||||
service_catalog={})
|
|
||||||
store = glance.store.swift.MultiTenantStore(context)
|
|
||||||
self.assertEqual(fake_get_endpoint.service_type, 'toy-store')
|
|
||||||
|
|
||||||
def test_multi_tenant_location_custom_endpoint_type(self):
|
|
||||||
self.config(swift_store_endpoint_type='InternalURL')
|
|
||||||
fake_get_endpoint = FakeGetEndpoint('https://some_endpoint')
|
|
||||||
self.stubs.Set(glance.store.common.auth, 'get_endpoint', fake_get_endpoint)
|
|
||||||
context = glance.context.RequestContext(
|
|
||||||
user='user', tenant='tenant', auth_tok='123',
|
|
||||||
service_catalog={})
|
|
||||||
store = glance.store.swift.MultiTenantStore(context)
|
|
||||||
self.assertEqual(fake_get_endpoint.endpoint_type, 'InternalURL')
|
|
||||||
|
|
||||||
|
|
||||||
class TestChunkReader(base.StoreClearingUnitTest):
|
|
||||||
|
|
||||||
def test_read_all_data(self):
|
|
||||||
"""
|
|
||||||
Replicate what goes on in the Swift driver with the
|
|
||||||
repeated creation of the ChunkReader object
|
|
||||||
"""
|
|
||||||
CHUNKSIZE = 100
|
|
||||||
checksum = hashlib.md5()
|
|
||||||
data_file = tempfile.NamedTemporaryFile()
|
|
||||||
data_file.write('*' * units.Ki)
|
|
||||||
data_file.flush()
|
|
||||||
infile = open(data_file.name, 'rb')
|
|
||||||
bytes_read = 0
|
|
||||||
while True:
|
|
||||||
cr = glance.store.swift.ChunkReader(infile, checksum, CHUNKSIZE)
|
|
||||||
chunk = cr.read(CHUNKSIZE)
|
|
||||||
bytes_read += len(chunk)
|
|
||||||
if not chunk:
|
|
||||||
break
|
|
||||||
self.assertEqual(1024, bytes_read)
|
|
||||||
data_file.close()
|
|
Loading…
Reference in New Issue
Block a user