openstack/glance_store
Code Issues Proposed changes

Enhance configuration handling

Browse Source 
1. Changed drivers to prevent using global CONF but access it from
client by interface.
2. Corrected swift driver to involve options from swift driver utils,
allow glance_store exports and registers them for client.
3. Added missing test cases for the functions of swift driver util.

Change-Id: I8fac6800efde202e29179791ea05c4814ec58435
Signed-off-by: Zhi Yan Liu <zhiyanl@cn.ibm.com>
This commit is contained in:
Zhi Yan Liu 2014-09-17 20:56:35 +08:00 committed by Zhi Yan Liu
parent 56af7fd60c
commit 22dbccb491
19 changed files with 291 additions and 186 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
glance_store/_drivers/gridfs.py
View File

@ -174,7 +174,7 @@ class Store(glance_store.driver.Store):
:raises `glance_store.exceptions.Duplicate` if the image already
existed
"""
loc = StoreLocation({'image_id': image_id})
loc = StoreLocation({'image_id': image_id}, self.conf)
if self.fs.exists(image_id):
raise exceptions.Duplicate(_("GridFS already has an image at "

1
glance_store/_drivers/http.py
View File

@ -192,6 +192,7 @@ class Store(glance_store.driver.Store):
location_class = glance_store.location.Location
new_loc = location_class(location.store_name,
location.store_location.__class__,
self.conf,
uri=location_header,
image_id=location.image_id,
store_specs=location.store_specs)

4
glance_store/_drivers/rbd.py
View File

@ -261,10 +261,10 @@ class Store(driver.Store):
'pool': self.pool,
'image': image_name,
'snapshot': DEFAULT_SNAPNAME,
})
}, self.conf)
else:
librbd.create(ioctx, image_name, size, order, old_format=True)
return StoreLocation({'image': image_name})
return StoreLocation({'image': image_name}, self.conf)
def _delete_image(self, image_name, snapshot_name=None, context=None):
"""

2
glance_store/_drivers/s3.py
View File

@ -352,7 +352,7 @@ class Store(glance_store.driver.Store):
'key': image_id,
's3serviceurl': self.full_s3_host,
'accesskey': self.access_key,
'secretkey': self.secret_key})
'secretkey': self.secret_key}, self.conf)
uformat = self.conf.glance_store.s3_store_bucket_url_format
calling_format = get_calling_format(s3_store_bucket_url_format=uformat)

2
glance_store/_drivers/sheepdog.py
View File

@ -268,7 +268,7 @@ class Store(glance_store.driver.Store):
raise exceptions.Duplicate(_("Sheepdog image %s already exists")
% image_id)
location = StoreLocation({'image': image_id})
location = StoreLocation({'image': image_id}, self.conf)
checksum = hashlib.md5()
image.create(image_size)

43
glance_store/_drivers/swift/store.py
View File

@ -102,10 +102,6 @@ _SWIFT_OPTS = [
'before the request fails.'))
]
CONF = cfg.CONF
SWIFT_STORE_REF_PARAMS = sutils.SwiftParams().params
def swift_retry_iter(resp_iter, length, store, location, context):
length = length if length else (resp_iter.len
@ -139,9 +135,9 @@ def swift_retry_iter(resp_iter, length, store, location, context):
'max_retries': retry_count,
'start': bytes_read,
'end': length})
(resp_headers, resp_iter) = store._get_object(location, None,
bytes_read,
context=context)
(_resp_headers, resp_iter) = store._get_object(location, None,
bytes_read,
context=context)
else:
break
@ -194,7 +190,7 @@ class StoreLocation(location.StoreLocation):
if not credentials_included:
#Used only in case of an add
#Get the current store from config
store = CONF.glance_store.default_swift_reference
store = self.conf.glance_store.default_swift_reference
return '%s://%s/%s/%s' % ('swift+config', store, container, obj)
if self.scheme == 'swift+config':
@ -209,9 +205,10 @@ class StoreLocation(location.StoreLocation):
def _get_conf_value_from_account_ref(self, netloc):
try:
self.user = SWIFT_STORE_REF_PARAMS[netloc]['user']
self.key = SWIFT_STORE_REF_PARAMS[netloc]['key']
netloc = SWIFT_STORE_REF_PARAMS[netloc]['auth_address']
ref_params = sutils.SwiftParams(self.conf).params
self.user = ref_params[netloc]['user']
self.key = ref_params[netloc]['key']
netloc = ref_params[netloc]['auth_address']
self.ssl_enabled = True
if netloc != '':
if netloc.startswith('http://'):
@ -308,7 +305,7 @@ class StoreLocation(location.StoreLocation):
path = pieces.path.lstrip('/')
# NOTE(Sridevi): Fix to map the account reference to the
# corresponding CONF value
# corresponding configuration value
if self.scheme == 'swift+config':
netloc = self._get_conf_value_from_account_ref(netloc)
else:
@ -343,7 +340,8 @@ class StoreLocation(location.StoreLocation):
def Store(conf):
try:
conf.register_opts(_SWIFT_OPTS, group='glance_store')
conf.register_opts(_SWIFT_OPTS + sutils.swift_opts,
group='glance_store')
except cfg.DuplicateOptError:
pass
@ -351,13 +349,13 @@ def Store(conf):
return MultiTenantStore(conf)
return SingleTenantStore(conf)
Store.OPTIONS = _SWIFT_OPTS
Store.OPTIONS = _SWIFT_OPTS + sutils.swift_opts
class BaseStore(driver.Store):
CHUNKSIZE = 65536
OPTIONS = _SWIFT_OPTS
OPTIONS = _SWIFT_OPTS + sutils.swift_opts
def get_schemes(self):
return ('swift+https', 'swift', 'swift+http', 'swift+config')
@ -567,7 +565,7 @@ class BaseStore(driver.Store):
# image data. We *really* should consider NOT returning
# the location attribute from GET /images/<ID> and
# GET /images/details
if sutils.is_multiple_swift_store_accounts_enabled():
if sutils.is_multiple_swift_store_accounts_enabled(self.conf):
include_creds = False
else:
include_creds = True
@ -668,14 +666,17 @@ class BaseStore(driver.Store):
class SingleTenantStore(BaseStore):
EXAMPLE_URL = "swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<FILE>"
def __init__(self, conf):
super(SingleTenantStore, self).__init__(conf)
self.ref_params = sutils.SwiftParams(self.conf).params
def configure(self):
super(SingleTenantStore, self).configure()
self.auth_version = self._option_get('swift_store_auth_version')
def configure_add(self):
default_swift_reference = \
SWIFT_STORE_REF_PARAMS.get(
self.conf.glance_store.default_swift_reference)
default_ref = self.conf.glance_store.default_swift_reference
default_swift_reference = self.ref_params.get(default_ref)
if default_swift_reference:
self.auth_address = default_swift_reference.get('auth_address')
if (not default_swift_reference) or (not self.auth_address):
@ -704,7 +705,7 @@ class SingleTenantStore(BaseStore):
'auth_or_store_url': self.auth_address,
'user': self.user,
'key': self.key}
return StoreLocation(specs)
return StoreLocation(specs, self.conf)
def get_connection(self, location, context=None):
if not location.user:
@ -814,7 +815,7 @@ class MultiTenantStore(BaseStore):
'container': self.container + '_' + str(image_id),
'obj': str(image_id),
'auth_or_store_url': ep}
return StoreLocation(specs)
return StoreLocation(specs, self.conf)
def get_connection(self, location, context=None):
return swiftclient.Connection(

32
glance_store/_drivers/swift/utils.py
View File

@ -50,22 +50,16 @@ CONFIG = ConfigParser.SafeConfigParser(dict_type=OrderedDict)
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
for opt in swift_opts:
opt.deprecated_opts = [cfg.DeprecatedOpt(opt.name,
group='DEFAULT')]
CONF.register_opt(opt, group='glance_store')
def is_multiple_swift_store_accounts_enabled():
if CONF.glance_store.swift_store_config_file is None:
def is_multiple_swift_store_accounts_enabled(conf):
if conf.glance_store.swift_store_config_file is None:
return False
return True
class SwiftParams(object):
def __init__(self):
if is_multiple_swift_store_accounts_enabled():
def __init__(self, conf):
self.conf = conf
if is_multiple_swift_store_accounts_enabled(self.conf):
self.params = self._load_config()
else:
self.params = self._form_default_params()
@ -74,12 +68,12 @@ class SwiftParams(object):
default = {}
if (
CONF.glance_store.swift_store_user and
CONF.glance_store.swift_store_key and
CONF.glance_store.swift_store_auth_address
self.conf.glance_store.swift_store_user and
self.conf.glance_store.swift_store_key and
self.conf.glance_store.swift_store_auth_address
):
glance_store = CONF.glance_store
glance_store = self.conf.glance_store
default['user'] = glance_store.swift_store_user
default['key'] = glance_store.swift_store_key
default['auth_address'] = glance_store.swift_store_auth_address
@ -88,13 +82,13 @@ class SwiftParams(object):
def _load_config(self):
try:
scf = CONF.glance_store.swift_store_config_file
conf_file = CONF.find_file(scf)
scf = self.conf.glance_store.swift_store_config_file
conf_file = self.conf.find_file(scf)
CONFIG.read(conf_file)
except Exception as e:
msg = (i18n._("swift config file "
"%(conf_file)s:%(exc)s not found") %
{'conf_file': CONF.glance_store.swift_store_config_file,
"%(conf)s:%(exc)s not found") %
{'conf': self.conf.glance_store.swift_store_config_file,
'exc': e})
LOG.error(msg)
raise exceptions.BadStoreConfiguration(store_name='swift',

2
glance_store/_drivers/vmware_datastore.py
View File

@ -311,7 +311,7 @@ class Store(glance_store.Store):
'image_dir': self.store_image_dir,
'datacenter_path': self.datacenter_path,
'datastore_name': self.datastore_name,
'image_id': image_id})
'image_id': image_id}, self.conf)
# NOTE(arnaud): use a decorator when the config is not tied to self
for i in range(self.api_retry_count + 1):
cookie = self._build_vim_cookie_header(

15
glance_store/backend.py
View File

@ -25,6 +25,7 @@ from glance_store import i18n
from glance_store import location
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
_ = i18n._
@ -45,7 +46,6 @@ _STORE_OPTS = [
deprecated_opts=[_DEPRECATED_STORE_OPTS[1]])
]
CONF = cfg.CONF
_STORE_CFG_GROUP = 'glance_store'
@ -212,7 +212,7 @@ def create_stores(conf=CONF):
def verify_default_store():
scheme = cfg.CONF.glance_store.default_store
scheme = CONF.glance_store.default_store
try:
get_store_from_scheme(scheme)
except exceptions.UnknownScheme:
@ -250,7 +250,7 @@ def get_store_from_uri(uri):
def get_from_backend(uri, offset=0, chunk_size=None, context=None):
"""Yields chunks of data from backend specified by uri."""
loc = location.get_location_from_uri(uri)
loc = location.get_location_from_uri(uri, conf=CONF)
store = get_store_from_uri(uri)
try:
@ -264,7 +264,7 @@ def get_from_backend(uri, offset=0, chunk_size=None, context=None):
def get_size_from_backend(uri, context=None):
"""Retrieves image size from backend specified by uri."""
loc = location.get_location_from_uri(uri)
loc = location.get_location_from_uri(uri, conf=CONF)
store = get_store_from_uri(uri)
return store.get_size(loc, context=context)
@ -272,7 +272,8 @@ def get_size_from_backend(uri, context=None):
def delete_from_backend(uri, context=None):
"""Removes chunks of data from backend specified by uri."""
loc = location.get_location_from_uri(uri)
loc = location.get_location_from_uri(uri, conf=CONF)
store = get_store_from_uri(uri)
try:
@ -289,7 +290,7 @@ def get_store_from_location(uri):
:param uri: Location to check for the store
"""
loc = location.get_location_from_uri(uri)
loc = location.get_location_from_uri(uri, conf=CONF)
return loc.store_name
@ -361,7 +362,7 @@ def set_acls(location_uri, public=False, read_tenants=[],
if write_tenants is None:
write_tenants = []
loc = location.get_location_from_uri(location_uri)
loc = location.get_location_from_uri(location_uri, conf=CONF)
scheme = get_store_from_location(location_uri)
store = get_store_from_scheme(scheme)
try:

20
glance_store/location.py
View File

@ -40,20 +40,24 @@ credentials and is **not** user-facing.
import logging
import urlparse
from oslo.config import cfg
from glance_store import exceptions
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
SCHEME_TO_CLS_MAP = {}
def get_location_from_uri(uri):
def get_location_from_uri(uri, conf=CONF):
"""
Given a URI, return a Location object that has had an appropriate
store parse the URI.
:param uri: A URI that could come from the end-user in the Location
attribute/header
attribute/header.
:param conf: The global configuration.
Example URIs:
https://user:pass@example.com:80/images/some-id
@ -70,8 +74,8 @@ def get_location_from_uri(uri):
if pieces.scheme not in SCHEME_TO_CLS_MAP.keys():
raise exceptions.UnknownScheme(scheme=pieces.scheme)
scheme_info = SCHEME_TO_CLS_MAP[pieces.scheme]
return Location(pieces.scheme, uri=uri,
store_location_class=scheme_info['location_class'])
return Location(pieces.scheme, scheme_info['location_class'],
conf, uri=uri)
def register_scheme_map(scheme_map):
@ -93,7 +97,7 @@ class Location(object):
Class describing the location of an image that Glance knows about
"""
def __init__(self, store_name, store_location_class,
def __init__(self, store_name, store_location_class, conf,
uri=None, image_id=None, store_specs=None):
"""
Create a new Location object.
@ -111,7 +115,8 @@ class Location(object):
self.store_name = store_name
self.image_id = image_id
self.store_specs = store_specs or {}
self.store_location = store_location_class(self.store_specs)
self.conf = conf
self.store_location = store_location_class(self.store_specs, conf)
if uri:
self.store_location.parse_uri(uri)
@ -132,7 +137,8 @@ class StoreLocation(object):
Base class that must be implemented by each store
"""
def __init__(self, store_specs):
def __init__(self, store_specs, conf):
self.conf = conf
self.specs = store_specs
if self.specs:
self.process_specs()

4
tests/unit/test_cinder_store.py
View File

@ -18,7 +18,7 @@ import mock
from glance_store._drivers import cinder
from glance_store import exceptions
from glance_store.location import get_location_from_uri
from glance_store import location
from glance_store.tests import base
@ -65,7 +65,7 @@ class TestCinderStore(base.StoreBaseTest):
tenant='fake_tenant')
uri = 'cinder://%s' % fake_volumes.keys()[0]
loc = get_location_from_uri(uri)
loc = location.get_location_from_uri(uri, conf=self.conf)
image_size = self.store.get_size(loc, context=fake_context)
self.assertEqual(image_size,
fake_volumes.values()[0].size * (1024 ** 3))

53
tests/unit/test_filesystem_store.py
View File

@ -31,7 +31,7 @@ import six
from glance_store._drivers.filesystem import ChunkedFile
from glance_store._drivers.filesystem import Store
from glance_store import exceptions
from glance_store.location import get_location_from_uri
from glance_store import location
from glance_store.openstack.common import units
from glance_store.tests import base
@ -63,13 +63,13 @@ class TestStore(base.StoreBaseTest):
file_contents = "chunk00000remainder"
image_file = StringIO.StringIO(file_contents)
location, size, checksum, _ = self.store.add(image_id,
image_file,
len(file_contents))
loc, size, checksum, _ = self.store.add(image_id,
image_file,
len(file_contents))
# Now read it back...
uri = "file:///%s/%s" % (self.test_dir, image_id)
loc = get_location_from_uri(uri)
loc = location.get_location_from_uri(uri, conf=self.conf)
(image_file, image_size) = self.store.get(loc)
expected_data = "chunk00000remainder"
@ -90,13 +90,13 @@ class TestStore(base.StoreBaseTest):
file_contents = "chunk00000remainder"
image_file = StringIO.StringIO(file_contents)
location, size, checksum, _ = self.store.add(image_id,
image_file,
len(file_contents))
loc, size, checksum, _ = self.store.add(image_id,
image_file,
len(file_contents))
# Now read it back...
uri = "file:///%s/%s" % (self.test_dir, image_id)
loc = get_location_from_uri(uri)
loc = location.get_location_from_uri(uri, conf=self.conf)
data = ""
for offset in range(len(file_contents)):
@ -124,7 +124,8 @@ class TestStore(base.StoreBaseTest):
Test that trying to retrieve a file that doesn't exist
raises an error
"""
loc = get_location_from_uri("file:///%s/non-existing" % self.test_dir)
loc = location.get_location_from_uri(
"file:///%s/non-existing" % self.test_dir, conf=self.conf)
self.assertRaises(exceptions.NotFound,
self.store.get,
loc)
@ -140,16 +141,16 @@ class TestStore(base.StoreBaseTest):
expected_image_id)
image_file = StringIO.StringIO(expected_file_contents)
location, size, checksum, _ = self.store.add(expected_image_id,
image_file,
expected_file_size)
loc, size, checksum, _ = self.store.add(expected_image_id,
image_file,
expected_file_size)
self.assertEqual(expected_location, location)
self.assertEqual(expected_location, loc)
self.assertEqual(expected_file_size, size)
self.assertEqual(expected_checksum, checksum)
uri = "file:///%s/%s" % (self.test_dir, expected_image_id)
loc = get_location_from_uri(uri)
loc = location.get_location_from_uri(uri, conf=self.conf)
(new_image_file, new_image_size) = self.store.get(loc)
new_image_contents = ""
new_image_file_size = 0
@ -317,13 +318,13 @@ class TestStore(base.StoreBaseTest):
file_contents = "*" * file_size
image_file = StringIO.StringIO(file_contents)
location, size, checksum, _ = self.store.add(image_id,
image_file,
file_size)
loc, size, checksum, _ = self.store.add(image_id,
image_file,
file_size)
# Now check that we can delete it
uri = "file:///%s/%s" % (self.test_dir, image_id)
loc = get_location_from_uri(uri)
loc = location.get_location_from_uri(uri, conf=self.conf)
self.store.delete(loc)
self.assertRaises(exceptions.NotFound, self.store.get, loc)
@ -333,7 +334,8 @@ class TestStore(base.StoreBaseTest):
Test that trying to delete a file that doesn't exist
raises an error
"""
loc = get_location_from_uri("file:///tmp/glance-tests/non-existing")
loc = location.get_location_from_uri(
"file:///tmp/glance-tests/non-existing", conf=self.conf)
self.assertRaises(exceptions.NotFound,
self.store.delete,
loc)
@ -398,15 +400,16 @@ class TestStore(base.StoreBaseTest):
expected_image_id)
image_file = six.StringIO(expected_file_contents)
location, size, checksum, _ = self.store.add(expected_image_id,
image_file,
expected_file_size)
loc, size, checksum, _ = self.store.add(expected_image_id,
image_file,
expected_file_size)
self.assertEqual(expected_location, location)
self.assertEqual(expected_location, loc)
self.assertEqual(expected_file_size, size)
self.assertEqual(expected_checksum, checksum)
loc = get_location_from_uri(expected_location)
loc = location.get_location_from_uri(expected_location,
conf=self.conf)
(new_image_file, new_image_size) = self.store.get(loc)
new_image_contents = ""
new_image_file_size = 0

14
tests/unit/test_http_store.py
View File

@ -18,7 +18,7 @@ import mock
from glance_store._drivers import http
from glance_store import delete_from_backend
from glance_store import exceptions
from glance_store.location import get_location_from_uri
from glance_store import location
from glance_store.tests import base
from glance_store.tests import utils
@ -45,7 +45,7 @@ class TestHttpStore(base.StoreBaseTest):
uri = "http://netloc/path/to/file.tar.gz"
expected_returns = ['I ', 'am', ' a', ' t', 'ea', 'po', 't,', ' s',
'ho', 'rt', ' a', 'nd', ' s', 'to', 'ut', '\n']
loc = get_location_from_uri(uri)
loc = location.get_location_from_uri(uri, conf=self.conf)
(image_file, image_size) = self.store.get(loc)
self.assertEqual(image_size, 31)
chunks = [c for c in image_file]
@ -69,7 +69,7 @@ class TestHttpStore(base.StoreBaseTest):
expected_returns = ['I ', 'am', ' a', ' t', 'ea', 'po', 't,', ' s',
'ho', 'rt', ' a', 'nd', ' s', 'to', 'ut', '\n']
loc = get_location_from_uri(uri)
loc = location.get_location_from_uri(uri, conf=self.conf)
(image_file, image_size) = self.store.get(loc)
self.assertEqual(image_size, 31)
@ -86,7 +86,7 @@ class TestHttpStore(base.StoreBaseTest):
self.response.side_effect = getresponse
uri = "http://netloc/path/to/file.tar.gz"
loc = get_location_from_uri(uri)
loc = location.get_location_from_uri(uri, conf=self.conf)
self.assertRaises(exceptions.MaxRedirectsExceeded, self.store.get, loc)
def test_http_get_redirect_invalid(self):
@ -95,7 +95,7 @@ class TestHttpStore(base.StoreBaseTest):
self.response.return_value = redirect_resp
uri = "http://netloc/path/to/file.tar.gz"
loc = get_location_from_uri(uri)
loc = location.get_location_from_uri(uri, conf=self.conf)
self.assertRaises(exceptions.BadStoreUri, self.store.get, loc)
def test_http_get_not_found(self):
@ -103,12 +103,12 @@ class TestHttpStore(base.StoreBaseTest):
self.response.return_value = fake
uri = "http://netloc/path/to/file.tar.gz"
loc = get_location_from_uri(uri)
loc = location.get_location_from_uri(uri, conf=self.conf)
self.assertRaises(exceptions.NotFound, self.store.get, loc)
def test_http_delete_raise_error(self):
uri = "https://netloc/path/to/file.tar.gz"
loc = get_location_from_uri(uri)
loc = location.get_location_from_uri(uri, conf=self.conf)
self.assertRaises(NotImplementedError, self.store.delete, loc)
self.assertRaises(exceptions.StoreDeleteNotSupported,
delete_from_backend, uri, {})

5
tests/unit/test_opts.py
View File

@ -63,6 +63,7 @@ class OptsTestCase(base.StoreBaseTest):
'cinder_catalog_info',
'cinder_endpoint_template',
'cinder_http_retries',
'default_swift_reference',
'filesystem_store_datadir',
'filesystem_store_datadirs',
'filesystem_store_file_perm',
@ -86,11 +87,14 @@ class OptsTestCase(base.StoreBaseTest):
'sheepdog_store_port',
'swift_enable_snet',
'swift_store_admin_tenants',
'swift_store_auth_address',
'swift_store_auth_insecure',
'swift_store_auth_version',
'swift_store_config_file',
'swift_store_container',
'swift_store_create_container_on_put',
'swift_store_endpoint_type',
'swift_store_key',
'swift_store_large_object_chunk_size',
'swift_store_large_object_size',
'swift_store_multi_tenant',
@ -98,6 +102,7 @@ class OptsTestCase(base.StoreBaseTest):
'swift_store_retry_get_count',
'swift_store_service_type',
'swift_store_ssl_compression',
'swift_store_user',
'vmware_api_insecure',
'vmware_api_retry_count',
'vmware_datacenter_path',

6
tests/unit/test_rbd_store.py
View File

@ -156,7 +156,8 @@ class TestStore(base.StoreBaseTest):
self.called_commands_expected = []
self.store_specs = {'image': 'fake_image',
'snapshot': 'fake_snapshot'}
self.location = rbd_store.StoreLocation(self.store_specs)
self.location = rbd_store.StoreLocation(self.store_specs,
self.conf)
# Provide enough data to get more than one chunk iteration.
self.data_len = 3 * 1024
self.data_iter = StringIO.StringIO('*' * self.data_len)
@ -217,7 +218,8 @@ class TestStore(base.StoreBaseTest):
self.store.delete(Location('test_rbd_store',
rbd_store.StoreLocation,
self.location.get_uri()))
self.conf,
uri=self.location.get_uri()))
self.called_commands_expected = ['remove']
def test_delete_image(self):

55
tests/unit/test_s3_store.py
View File

@ -22,12 +22,10 @@ import uuid
import boto.s3.connection
import mock
from glance_store import exceptions
from glance_store.openstack.common import units
from glance_store._drivers import s3
from glance_store.exceptions import UnsupportedBackend
from glance_store.location import get_location_from_uri
from glance_store import exceptions
from glance_store import location
from glance_store.openstack.common import units
from glance_store.tests import base
@ -118,7 +116,7 @@ def fakers():
def fake_connection_constructor(self, *args, **kwargs):
host = kwargs.get('host')
if host.startswith('http://') or host.startswith('https://'):
raise UnsupportedBackend(host)
raise exceptions.UnsupportedBackend(host)
def fake_get_bucket(bucket_id):
bucket = fixture_buckets.get(bucket_id)
@ -169,8 +167,9 @@ class TestStore(base.StoreBaseTest):
def test_get(self):
"""Test a "normal" retrieval of an image in chunks."""
loc = get_location_from_uri(
"s3://user:key@auth_address/glance/%s" % FAKE_UUID)
loc = location.get_location_from_uri(
"s3://user:key@auth_address/glance/%s" % FAKE_UUID,
conf=self.conf)
(image_s3, image_size) = self.store.get(loc)
self.assertEqual(image_size, FIVE_KB)
@ -194,8 +193,9 @@ class TestStore(base.StoreBaseTest):
with mock.patch.object(s3_connection, '__init__') as m:
m.side_effect = fake_S3Connection_init
loc = get_location_from_uri(
"s3://user:key@auth_address/glance/%s" % FAKE_UUID)
loc = location.get_location_from_uri(
"s3://user:key@auth_address/glance/%s" % FAKE_UUID,
conf=self.conf)
(image_s3, image_size) = self.store.get(loc)
def test_get_calling_format_default(self):
@ -209,8 +209,9 @@ class TestStore(base.StoreBaseTest):
with mock.patch.object(s3_connection, '__init__') as m:
m.side_effect = fake_S3Connection_init
loc = get_location_from_uri(
"s3://user:key@auth_address/glance/%s" % FAKE_UUID)
loc = location.get_location_from_uri(
"s3://user:key@auth_address/glance/%s" % FAKE_UUID,
conf=self.conf)
(image_s3, image_size) = self.store.get(loc)
def test_get_non_existing(self):
@ -219,11 +220,11 @@ class TestStore(base.StoreBaseTest):
raises an error
"""
uri = "s3://user:key@auth_address/badbucket/%s" % FAKE_UUID
loc = get_location_from_uri(uri)
loc = location.get_location_from_uri(uri, conf=self.conf)
self.assertRaises(exceptions.NotFound, self.store.get, loc)
uri = "s3://user:key@auth_address/glance/noexist"
loc = get_location_from_uri(uri)
loc = location.get_location_from_uri(uri, conf=self.conf)
self.assertRaises(exceptions.NotFound, self.store.get, loc)
def test_add(self):
@ -240,15 +241,16 @@ class TestStore(base.StoreBaseTest):
expected_image_id)
image_s3 = StringIO.StringIO(expected_s3_contents)
location, size, checksum, _ = self.store.add(expected_image_id,
image_s3,
expected_s3_size)
loc, size, checksum, _ = self.store.add(expected_image_id,
image_s3,
expected_s3_size)
self.assertEqual(expected_location, location)
self.assertEqual(expected_location, loc)
self.assertEqual(expected_s3_size, size)
self.assertEqual(expected_checksum, checksum)
loc = get_location_from_uri(expected_location)
loc = location.get_location_from_uri(expected_location,
conf=self.conf)
(new_image_s3, new_image_size) = self.store.get(loc)
new_image_contents = StringIO.StringIO()
for chunk in new_image_s3:
@ -291,15 +293,16 @@ class TestStore(base.StoreBaseTest):
self.config(**new_conf)
self.store = s3.Store(self.conf)
self.store.configure()
location, size, checksum, _ = self.store.add(expected_image_id,
image_s3,
expected_s3_size)
loc, size, checksum, _ = self.store.add(expected_image_id,
image_s3,
expected_s3_size)
self.assertEqual(expected_location, location)
self.assertEqual(expected_location, loc)
self.assertEqual(expected_s3_size, size)
self.assertEqual(expected_checksum, checksum)
loc = get_location_from_uri(expected_location)
loc = location.get_location_from_uri(expected_location,
conf=self.conf)
(new_image_s3, new_image_size) = self.store.get(loc)
new_image_contents = new_image_s3.getvalue()
new_image_s3_size = len(new_image_s3)
@ -353,7 +356,7 @@ class TestStore(base.StoreBaseTest):
Test we can delete an existing image in the s3 store
"""
uri = "s3://user:key@auth_address/glance/%s" % FAKE_UUID
loc = get_location_from_uri(uri)
loc = location.get_location_from_uri(uri, conf=self.conf)
self.store.delete(loc)
self.assertRaises(exceptions.NotFound, self.store.get, loc)
@ -364,7 +367,7 @@ class TestStore(base.StoreBaseTest):
raises an error
"""
uri = "s3://user:key@auth_address/glance/noexist"
loc = get_location_from_uri(uri)
loc = location.get_location_from_uri(uri, conf=self.conf)
self.assertRaises(exceptions.NotFound, self.store.delete, loc)
def _do_test_get_s3_location(self, host, loc):

106
tests/unit/test_swift_store.py
View File

@ -29,13 +29,12 @@ import six
import swiftclient
from glance_store._drivers.swift import store as swift
from glance_store._drivers.swift import utils as sutils
from glance_store import backend
from glance_store import BackendException
from glance_store.common import auth
from glance_store.common import utils
from glance_store import exceptions
from glance_store.location import get_location_from_uri
from glance_store import location
from glance_store.openstack.common import context
from glance_store.openstack.common import units
from glance_store.tests import base
@ -234,7 +233,7 @@ class SwiftTests(object):
"""
uri = "swift://%s:key@auth_address/glance/%s" % (
self.swift_store_user, FAKE_UUID)
loc = get_location_from_uri(uri)
loc = location.get_location_from_uri(uri, conf=self.conf)
image_size = self.store.get_size(loc)
self.assertEqual(image_size, 5120)
@ -272,7 +271,7 @@ class SwiftTests(object):
"""Test a "normal" retrieval of an image in chunks."""
uri = "swift://%s:key@auth_address/glance/%s" % (
self.swift_store_user, FAKE_UUID)
loc = get_location_from_uri(uri)
loc = location.get_location_from_uri(uri, conf=self.conf)
(image_swift, image_size) = self.store.get(loc)
self.assertEqual(image_size, 5120)
@ -290,7 +289,7 @@ class SwiftTests(object):
"""
uri = "swift://%s:key@auth_address/glance/%s" % (
self.swift_store_user, FAKE_UUID)
loc = get_location_from_uri(uri)
loc = location.get_location_from_uri(uri, conf=self.conf)
ctxt = context.RequestContext()
(image_swift, image_size) = self.store.get(loc, context=ctxt)
resp_full = ''.join([chunk for chunk in image_swift.wrapped])
@ -314,9 +313,9 @@ class SwiftTests(object):
specified either via a Location header with swift+http:// or using
http:// in the swift_store_auth_address config value
"""
loc = get_location_from_uri("swift+http://%s:key@auth_address/"
"glance/%s" %
(self.swift_store_user, FAKE_UUID))
loc = location.get_location_from_uri(
"swift+http://%s:key@auth_address/glance/%s" %
(self.swift_store_user, FAKE_UUID), conf=self.conf)
ctxt = context.RequestContext()
(image_swift, image_size) = self.store.get(loc, context=ctxt)
@ -334,8 +333,9 @@ class SwiftTests(object):
Test that trying to retrieve a swift that doesn't exist
raises an error
"""
loc = get_location_from_uri("swift://%s:key@authurl/glance/noexist" % (
self.swift_store_user))
loc = location.get_location_from_uri(
"swift://%s:key@authurl/glance/noexist" % (self.swift_store_user),
conf=self.conf)
self.assertRaises(exceptions.NotFound,
self.store.get,
loc)
@ -359,17 +359,17 @@ class SwiftTests(object):
global SWIFT_PUT_OBJECT_CALLS
SWIFT_PUT_OBJECT_CALLS = 0
location, size, checksum, _ = self.store.add(expected_image_id,
image_swift,
expected_swift_size)
loc, size, checksum, _ = self.store.add(expected_image_id,
image_swift,
expected_swift_size)
self.assertEqual(expected_location, location)
self.assertEqual(expected_location, loc)
self.assertEqual(expected_swift_size, size)
self.assertEqual(expected_checksum, checksum)
# Expecting a single object to be created on Swift i.e. no chunking.
self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 1)
loc = get_location_from_uri(expected_location)
loc = location.get_location_from_uri(expected_location, conf=self.conf)
(new_image_swift, new_image_size) = self.store.get(loc)
new_image_contents = ''.join([chunk for chunk in new_image_swift])
new_image_swift_size = len(new_image_swift)
@ -435,15 +435,16 @@ class SwiftTests(object):
reload(swift)
self.store = Store(self.conf)
self.store.configure()
location, size, checksum, _ = self.store.add(image_id, image_swift,
expected_swift_size)
loc, size, checksum, _ = self.store.add(image_id, image_swift,
expected_swift_size)
self.assertEqual(expected_location, location)
self.assertEqual(expected_location, loc)
self.assertEqual(expected_swift_size, size)
self.assertEqual(expected_checksum, checksum)
self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 1)
loc = get_location_from_uri(expected_location)
loc = location.get_location_from_uri(expected_location,
conf=self.conf)
(new_image_swift, new_image_size) = self.store.get(loc)
new_image_contents = ''.join([chunk for chunk in new_image_swift])
new_image_swift_size = len(new_image_swift)
@ -510,16 +511,16 @@ class SwiftTests(object):
reload(swift)
self.store = Store(self.conf)
self.store.configure()
location, size, checksum, _ = self.store.add(expected_image_id,
image_swift,
expected_swift_size)
loc, size, checksum, _ = self.store.add(expected_image_id,
image_swift,
expected_swift_size)
self.assertEqual(expected_location, location)
self.assertEqual(expected_location, loc)
self.assertEqual(expected_swift_size, size)
self.assertEqual(expected_checksum, checksum)
self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 1)
loc = get_location_from_uri(expected_location)
loc = location.get_location_from_uri(expected_location, conf=self.conf)
(new_image_swift, new_image_size) = self.store.get(loc)
new_image_contents = ''.join([chunk for chunk in new_image_swift])
new_image_swift_size = len(new_image_swift)
@ -555,21 +556,21 @@ class SwiftTests(object):
try:
self.store.large_object_size = 1024
self.store.large_object_chunk_size = 1024
location, size, checksum, _ = self.store.add(expected_image_id,
image_swift,
expected_swift_size)
loc, size, checksum, _ = self.store.add(expected_image_id,
image_swift,
expected_swift_size)
finally:
self.store.large_object_chunk_size = orig_temp_size
self.store.large_object_size = orig_max_size
self.assertEqual(expected_location, location)
self.assertEqual(expected_location, loc)
self.assertEqual(expected_swift_size, size)
self.assertEqual(expected_checksum, checksum)
# Expecting 6 objects to be created on Swift -- 5 chunks and 1
# manifest.
self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 6)
loc = get_location_from_uri(expected_location)
loc = location.get_location_from_uri(expected_location, conf=self.conf)
(new_image_swift, new_image_size) = self.store.get(loc)
new_image_contents = ''.join([chunk for chunk in new_image_swift])
new_image_swift_size = len(new_image_contents)
@ -613,14 +614,14 @@ class SwiftTests(object):
MAX_SWIFT_OBJECT_SIZE = 1024
self.store.large_object_size = 1024
self.store.large_object_chunk_size = 1024
location, size, checksum, _ = self.store.add(expected_image_id,
image_swift, 0)
loc, size, checksum, _ = self.store.add(expected_image_id,
image_swift, 0)
finally:
self.store.large_object_chunk_size = orig_temp_size
self.store.large_object_size = orig_max_size
MAX_SWIFT_OBJECT_SIZE = orig_max_swift_object_size
self.assertEqual(expected_location, location)
self.assertEqual(expected_location, loc)
self.assertEqual(expected_swift_size, size)
self.assertEqual(expected_checksum, checksum)
# Expecting 7 calls to put_object -- 5 chunks, a zero chunk which is
@ -629,7 +630,7 @@ class SwiftTests(object):
# in that case).
self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 7)
loc = get_location_from_uri(expected_location)
loc = location.get_location_from_uri(expected_location, conf=self.conf)
(new_image_swift, new_image_size) = self.store.get(loc)
new_image_contents = ''.join([chunk for chunk in new_image_swift])
new_image_swift_size = len(new_image_contents)
@ -642,6 +643,7 @@ class SwiftTests(object):
Tests that adding an image with an existing identifier
raises an appropriate exception
"""
self.store = Store(self.conf)
self.store.configure()
image_swift = six.StringIO("nevergonnamakeit")
self.assertRaises(exceptions.Duplicate,
@ -664,10 +666,10 @@ class SwiftTests(object):
"""
Tests that options without a valid credentials disables the add method
"""
swift.SWIFT_STORE_REF_PARAMS = {'ref1': {'auth_address':
'authurl.com', 'user': '',
'key': ''}}
self.store = Store(self.conf)
self.store.ref_params = {'ref1': {'auth_address':
'authurl.com', 'user': '',
'key': ''}}
self.store.configure()
self.assertEqual(self.store.add, self.store.add_disabled)
@ -675,11 +677,10 @@ class SwiftTests(object):
"""
Tests that options without auth address disables the add method
"""
swift.SWIFT_STORE_REF_PARAMS = {'ref1': {'auth_address':
'', 'user': 'user1',
'key': 'key1'}}
self.store = Store(self.conf)
self.store.ref_params = {'ref1': {'auth_address':
'', 'user': 'user1',
'key': 'key1'}}
self.store.configure()
self.assertEqual(self.store.add, self.store.add_disabled)
@ -689,7 +690,7 @@ class SwiftTests(object):
"""
uri = "swift://%s:key@authurl/glance/%s" % (
self.swift_store_user, FAKE_UUID)
loc = get_location_from_uri(uri)
loc = location.get_location_from_uri(uri, conf=self.conf)
self.store.delete(loc)
self.assertRaises(exceptions.NotFound, self.store.get, loc)
@ -699,7 +700,7 @@ class SwiftTests(object):
Test we can delete an existing image in the swift store
"""
uri = "swift+config://ref1/glance/%s" % (FAKE_UUID)
loc = get_location_from_uri(uri)
loc = location.get_location_from_uri(uri, conf=self.conf)
self.store.delete(loc)
self.assertRaises(exceptions.NotFound, self.store.get, loc)
@ -709,8 +710,9 @@ class SwiftTests(object):
Test that trying to delete a swift that doesn't exist
raises an error
"""
loc = get_location_from_uri("swift://%s:key@authurl/glance/noexist" % (
self.swift_store_user))
loc = location.get_location_from_uri(
"swift://%s:key@authurl/glance/noexist" % (self.swift_store_user),
conf=self.conf)
self.assertRaises(exceptions.NotFound, self.store.delete, loc)
def test_read_acl_public(self):
@ -721,7 +723,7 @@ class SwiftTests(object):
store = Store(self.conf)
store.configure()
uri = "swift+http://storeurl/glance/%s" % FAKE_UUID
loc = get_location_from_uri(uri)
loc = location.get_location_from_uri(uri, conf=self.conf)
ctxt = context.RequestContext()
store.set_acls(loc, public=True, context=ctxt)
container_headers = swiftclient.client.head_container('x', 'y',
@ -737,7 +739,7 @@ class SwiftTests(object):
store = Store(self.conf)
store.configure()
uri = "swift+http://storeurl/glance/%s" % FAKE_UUID
loc = get_location_from_uri(uri)
loc = location.get_location_from_uri(uri, conf=self.conf)
read_tenants = ['matt', 'mark']
ctxt = context.RequestContext()
store.set_acls(loc, read_tenants=read_tenants, context=ctxt)
@ -754,7 +756,7 @@ class SwiftTests(object):
store = Store(self.conf)
store.configure()
uri = "swift+http://storeurl/glance/%s" % FAKE_UUID
loc = get_location_from_uri(uri)
loc = location.get_location_from_uri(uri, conf=self.conf)
read_tenants = ['frank', 'jim']
ctxt = context.RequestContext()
store.set_acls(loc, write_tenants=read_tenants, context=ctxt)
@ -790,7 +792,6 @@ class TestStoreAuthV1(base.StoreBaseTest, SwiftTests):
self.config(**conf)
self.store.configure()
self.register_store_schemes(self.store)
swift.SWIFT_STORE_REF_PARAMS = sutils.SwiftParams().params
self.addCleanup(self.conf.reset)
@ -804,7 +805,7 @@ class TestStoreAuthV2(TestStoreAuthV1):
def test_v2_with_no_tenant(self):
uri = "swift://failme:key@auth_address/glance/%s" % (FAKE_UUID)
loc = get_location_from_uri(uri)
loc = location.get_location_from_uri(uri, conf=self.conf)
self.assertRaises(exceptions.BadStoreUri,
self.store.get,
loc)
@ -813,7 +814,7 @@ class TestStoreAuthV2(TestStoreAuthV1):
conf = self.getConfig()
conf['swift_store_multi_tenant'] = True
uri = "swift://auth_address/glance/%s" % (FAKE_UUID)
loc = get_location_from_uri(uri)
loc = location.get_location_from_uri(uri, conf=self.conf)
self.assertEqual('swift', loc.store_name)
@ -853,7 +854,7 @@ class TestSingleTenantStoreConnections(base.StoreBaseTest):
'key': 'key1',
'container': 'cont',
'obj': 'object'}
self.location = swift.StoreLocation(specs)
self.location = swift.StoreLocation(specs, self.conf)
self.addCleanup(self.conf.reset)
def test_basic_connection(self):
@ -969,7 +970,7 @@ class TestMultiTenantStoreConnections(base.StoreBaseTest):
'auth_or_store_url': 'example.com',
'container': 'cont',
'obj': 'object'}
self.location = swift.StoreLocation(specs)
self.location = swift.StoreLocation(specs, self.conf)
self.addCleanup(self.conf.reset)
def test_basic_connection(self):
@ -1047,7 +1048,6 @@ class TestCreatingLocations(base.StoreBaseTest):
default_swift_reference='ref2',
swift_store_config_file=self.swift_config_file)
swift.SWIFT_STORE_REF_PARAMS = sutils.SwiftParams().params
store = swift.SingleTenantStore(self.conf)
store.configure()
location = store.create_location('image-id')

87
tests/unit/test_swift_store_utils.py Normal file
View File

@ -0,0 +1,87 @@
# Copyright 2014 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#