openstack/glance_store
Code Issues Proposed changes

Remove Python 2 support

Browse Source 
Python 2 has been deprecated for almost two years, and has not been
guaranteed to work with glance_store for a while. This patch removes all
traces of six, unicode strings and Python 2 tweaks.

Co-Authored-By: Cyril Roelandt <cyril@redhat.com>
Change-Id: Ifa78924d7ecf4f2d9a54c677888ab2926530c487
This commit is contained in:
liyou01 2021-01-07 15:57:43 +08:00 committed by Cyril Roelandt
parent aeee48b561
commit 5ff06df97f
32 changed files with 283 additions and 323 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
doc/source/conf.py
View File

@ -52,8 +52,8 @@ source_suffix = '.rst'
master_doc = 'index' master_doc = 'index'
# General information about the project. # General information about the project.
project = u'glance_store' project = 'glance_store'
copyright = u'2014, OpenStack Foundation' copyright = '2014, OpenStack Foundation'
# If true, '()' will be appended to :func: etc. cross-reference text. # If true, '()' will be appended to :func: etc. cross-reference text.
add_function_parentheses = True add_function_parentheses = True

2
glance_store/_drivers/cinder.py
View File

@ -1047,7 +1047,7 @@ class Store(glance_store.driver.Store):
image_metadata = {} image_metadata = {}
location_url = 'cinder://%s' % volume.id location_url = 'cinder://%s' % volume.id
if self.backend_group: if self.backend_group:
image_metadata['store'] = u"%s" % self.backend_group image_metadata['store'] = self.backend_group
location_url = 'cinder://%s/%s' % (self.backend_group, location_url = 'cinder://%s/%s' % (self.backend_group,
volume.id) volume.id)

4
glance_store/_drivers/filesystem.py
View File

@ -22,6 +22,7 @@ import errno
import logging import logging
import os import os
import stat import stat
import urllib
import jsonschema import jsonschema
from oslo_config import cfg from oslo_config import cfg
@ -29,7 +30,6 @@ from oslo_serialization import jsonutils
from oslo_utils import encodeutils from oslo_utils import encodeutils
from oslo_utils import excutils from oslo_utils import excutils
from oslo_utils import units from oslo_utils import units
from six.moves import urllib
import glance_store import glance_store
from glance_store import capabilities from glance_store import capabilities
@ -793,7 +793,7 @@ class Store(glance_store.driver.Store):
# Add store backend information to location metadata # Add store backend information to location metadata
if self.backend_group: if self.backend_group:
metadata['store'] = u"%s" % self.backend_group metadata['store'] = self.backend_group
return ('file://%s' % filepath, return ('file://%s' % filepath,
bytes_written, bytes_written,

2
glance_store/_drivers/http.py
View File

@ -14,11 +14,11 @@
# under the License. # under the License.
import logging import logging
import urllib
from oslo_config import cfg from oslo_config import cfg
from oslo_utils import encodeutils from oslo_utils import encodeutils
from six.moves import urllib
import requests import requests

4
glance_store/_drivers/rbd.py
View File

@ -20,11 +20,11 @@
import contextlib import contextlib
import logging import logging
import math import math
import urllib
from oslo_config import cfg from oslo_config import cfg
from oslo_utils import encodeutils from oslo_utils import encodeutils
from oslo_utils import units from oslo_utils import units
from six.moves import urllib
from glance_store import capabilities from glance_store import capabilities
from glance_store.common import utils from glance_store.common import utils
@ -651,7 +651,7 @@ class Store(driver.Store):
# Add store backend information to location metadata # Add store backend information to location metadata
metadata = {} metadata = {}
if self.backend_group: if self.backend_group:
metadata['store'] = u"%s" % self.backend_group metadata['store'] = self.backend_group
return (loc.get_uri(), return (loc.get_uri(),
image_size, image_size,

10
glance_store/_drivers/s3.py
View File

@ -15,9 +15,11 @@
"""Storage backend for S3 or Storage Servers that follow the S3 Protocol""" """Storage backend for S3 or Storage Servers that follow the S3 Protocol"""
import io
import logging import logging
import math import math
import re import re
import urllib
from boto3 import session as boto_session from boto3 import session as boto_session
from botocore import client as boto_client from botocore import client as boto_client
@ -27,8 +29,6 @@ import eventlet
from oslo_config import cfg from oslo_config import cfg
from oslo_utils import encodeutils from oslo_utils import encodeutils
from oslo_utils import units from oslo_utils import units
import six
from six.moves import urllib
import glance_store import glance_store
from glance_store import capabilities from glance_store import capabilities
@ -706,7 +706,7 @@ class Store(glance_store.driver.Store):
checksum.update(write_chunk) checksum.update(write_chunk)
if verifier: if verifier:
verifier.update(write_chunk) verifier.update(write_chunk)
fp = six.BytesIO(write_chunk) fp = io.BytesIO(write_chunk)
fp.seek(0) fp.seek(0)
part = UploadPart(mpu, fp, cstart + 1, len(write_chunk)) part = UploadPart(mpu, fp, cstart + 1, len(write_chunk))
pool.spawn_n(run_upload, s3_client, bucket, key, part) pool.spawn_n(run_upload, s3_client, bucket, key, part)
@ -721,7 +721,7 @@ class Store(glance_store.driver.Store):
checksum.update(write_chunk) checksum.update(write_chunk)
if verifier: if verifier:
verifier.update(write_chunk) verifier.update(write_chunk)
fp = six.BytesIO(write_chunk) fp = io.BytesIO(write_chunk)
fp.seek(0) fp.seek(0)
part = UploadPart(mpu, fp, cstart + 1, len(write_chunk)) part = UploadPart(mpu, fp, cstart + 1, len(write_chunk))
pool.spawn_n(run_upload, s3_client, bucket, key, part) pool.spawn_n(run_upload, s3_client, bucket, key, part)
@ -888,7 +888,7 @@ class Store(glance_store.driver.Store):
{ {
'PartNumber': pnum, 'PartNumber': pnum,
'ETag': etag 'ETag': etag
} for pnum, etag in six.iteritems(pedict) } for pnum, etag in pedict.items()
] ]
} }

36
glance_store/_drivers/swift/store.py
View File

@ -15,8 +15,11 @@
"""Storage backend for SWIFT""" """Storage backend for SWIFT"""
import http.client
import io
import logging import logging
import math import math
import urllib.parse
from keystoneauth1.access import service_catalog as keystone_sc from keystoneauth1.access import service_catalog as keystone_sc
from keystoneauth1 import identity as ks_identity from keystoneauth1 import identity as ks_identity
@ -26,9 +29,6 @@ from oslo_config import cfg
from oslo_utils import encodeutils from oslo_utils import encodeutils
from oslo_utils import excutils from oslo_utils import excutils
from oslo_utils import units from oslo_utils import units
import six
from six.moves import http_client
from six.moves import urllib
try: try:
import swiftclient import swiftclient
except ImportError: except ImportError:
@ -478,17 +478,13 @@ Related options:
def swift_retry_iter(resp_iter, length, store, location, manager): def swift_retry_iter(resp_iter, length, store, location, manager):
if not length and isinstance(resp_iter, six.BytesIO): if not length and isinstance(resp_iter, io.BytesIO):
if six.PY3: # io.BytesIO does not have a len attribute, instead go the end using
# On Python 3, io.BytesIO does not have a len attribute, instead # seek to get the size of the file
# go the end using seek to get the size of the file
pos = resp_iter.tell() pos = resp_iter.tell()
resp_iter.seek(0, 2) resp_iter.seek(0, 2)
length = resp_iter.tell() length = resp_iter.tell()
resp_iter.seek(pos) resp_iter.seek(pos)
else:
# On Python 2, StringIO has a len attribute
length = resp_iter.len
length = length if length else (resp_iter.len length = length if length else (resp_iter.len
if hasattr(resp_iter, 'len') else 0) if hasattr(resp_iter, 'len') else 0)
@ -773,7 +769,7 @@ Store.OPTIONS = _SWIFT_OPTS + sutils.swift_opts + buffered.BUFFERING_OPTS
def _is_slo(slo_header): def _is_slo(slo_header):
if (slo_header is not None and isinstance(slo_header, six.string_types) if (slo_header is not None and isinstance(slo_header, str)
and slo_header.lower() == 'true'): and slo_header.lower() == 'true'):
return True return True
@ -836,7 +832,7 @@ class BaseStore(driver.Store):
location.container, location.obj, location.container, location.obj,
resp_chunk_size=self.CHUNKSIZE, headers=headers) resp_chunk_size=self.CHUNKSIZE, headers=headers)
except swiftclient.ClientException as e: except swiftclient.ClientException as e:
if e.http_status == http_client.NOT_FOUND: if e.http_status == http.client.NOT_FOUND:
msg = _("Swift could not find object %s.") % location.obj msg = _("Swift could not find object %s.") % location.obj
LOG.warning(msg) LOG.warning(msg)
raise exceptions.NotFound(message=msg) raise exceptions.NotFound(message=msg)
@ -1065,19 +1061,19 @@ class BaseStore(driver.Store):
metadata = {} metadata = {}
if self.backend_group: if self.backend_group:
metadata['store'] = u"%s" % self.backend_group metadata['store'] = self.backend_group
return (location.get_uri(credentials_included=include_creds), return (location.get_uri(credentials_included=include_creds),
image_size, obj_etag, os_hash_value.hexdigest(), image_size, obj_etag, os_hash_value.hexdigest(),
metadata) metadata)
except swiftclient.ClientException as e: except swiftclient.ClientException as e:
if e.http_status == http_client.CONFLICT: if e.http_status == http.client.CONFLICT:
msg = _("Swift already has an image at this location") msg = _("Swift already has an image at this location")
raise exceptions.Duplicate(message=msg) raise exceptions.Duplicate(message=msg)
elif e.http_status == http_client.REQUEST_ENTITY_TOO_LARGE: elif e.http_status == http.client.REQUEST_ENTITY_TOO_LARGE:
raise exceptions.StorageFull(message=e.msg) raise exceptions.StorageFull(message=e.msg)
msg = (_(u"Failed to add object to Swift.\n" msg = (_("Failed to add object to Swift.\n"
"Got error from Swift: %s.") "Got error from Swift: %s.")
% encodeutils.exception_to_unicode(e)) % encodeutils.exception_to_unicode(e))
LOG.error(msg) LOG.error(msg)
@ -1102,7 +1098,7 @@ class BaseStore(driver.Store):
dlo_manifest = headers.get('x-object-manifest') dlo_manifest = headers.get('x-object-manifest')
slo_manifest = headers.get('x-static-large-object') slo_manifest = headers.get('x-static-large-object')
except swiftclient.ClientException as e: except swiftclient.ClientException as e:
if e.http_status != http_client.NOT_FOUND: if e.http_status != http.client.NOT_FOUND:
raise raise
if _is_slo(slo_manifest): if _is_slo(slo_manifest):
@ -1134,7 +1130,7 @@ class BaseStore(driver.Store):
connection.delete_object(location.container, location.obj) connection.delete_object(location.container, location.obj)
except swiftclient.ClientException as e: except swiftclient.ClientException as e:
if e.http_status == http_client.NOT_FOUND: if e.http_status == http.client.NOT_FOUND:
msg = _("Swift could not find image at URI.") msg = _("Swift could not find image at URI.")
raise exceptions.NotFound(message=msg) raise exceptions.NotFound(message=msg)
else: else:
@ -1155,7 +1151,7 @@ class BaseStore(driver.Store):
try: try:
connection.head_container(container) connection.head_container(container)
except swiftclient.ClientException as e: except swiftclient.ClientException as e:
if e.http_status == http_client.NOT_FOUND: if e.http_status == http.client.NOT_FOUND:
if store_conf.swift_store_create_container_on_put: if store_conf.swift_store_create_container_on_put:
try: try:
msg = (_LI("Creating swift container %(container)s") % msg = (_LI("Creating swift container %(container)s") %
@ -1541,7 +1537,7 @@ class MultiTenantStore(BaseStore):
try: try:
connection.post_container(location.container, headers=headers) connection.post_container(location.container, headers=headers)
except swiftclient.ClientException as e: except swiftclient.ClientException as e:
if e.http_status == http_client.NOT_FOUND: if e.http_status == http.client.NOT_FOUND:
msg = _("Swift could not find image at URI.") msg = _("Swift could not find image at URI.")
raise exceptions.NotFound(message=msg) raise exceptions.NotFound(message=msg)
else: else:

17
glance_store/_drivers/swift/utils.py
View File

@ -12,11 +12,10 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
import configparser
import logging import logging
import sys
from oslo_config import cfg from oslo_config import cfg
from six.moves import configparser
from glance_store import exceptions from glance_store import exceptions
from glance_store.i18n import _, _LE from glance_store.i18n import _, _LE
@ -104,16 +103,11 @@ _config_defaults = {'user_domain_id': 'default',
'project_domain_id': 'default', 'project_domain_id': 'default',
'project_domain_name': 'default'} 'project_domain_name': 'default'}
if sys.version_info >= (3, 2):
parser_class = configparser.ConfigParser
else:
parser_class = configparser.SafeConfigParser
class SwiftConfigParser(configparser.ConfigParser):
class SwiftConfigParser(parser_class):
def get(self, *args, **kwargs): def get(self, *args, **kwargs):
value = super(parser_class, self).get(*args, **kwargs) value = super(configparser.ConfigParser, self).get(*args, **kwargs)
return self._process_quotes(value) return self._process_quotes(value)
@staticmethod @staticmethod
@ -127,10 +121,7 @@ class SwiftConfigParser(parser_class):
return value return value
if sys.version_info >= (3,): CONFIG = SwiftConfigParser(defaults=_config_defaults)
CONFIG = SwiftConfigParser(defaults=_config_defaults)
else:
CONFIG = parser_class(defaults=_config_defaults)
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)

19
glance_store/_drivers/vmware_datastore.py
View File

@ -17,6 +17,7 @@
import logging import logging
import os import os
import urllib.parse
from oslo_config import cfg from oslo_config import cfg
from oslo_utils import excutils from oslo_utils import excutils
@ -31,15 +32,9 @@ try:
except ImportError: except ImportError:
api = None api = None
from six.moves import urllib
import six.moves.urllib.parse as urlparse
import requests import requests
from requests import adapters from requests import adapters
from requests.packages.urllib3.util import retry from requests.packages.urllib3.util import retry
import six
# NOTE(jokke): simplified transition to py3, behaves like py2 xrange
from six.moves import range
import glance_store import glance_store
from glance_store import capabilities from glance_store import capabilities
@ -349,9 +344,9 @@ class StoreLocation(location.StoreLocation):
Creates a https url that can be used to upload/download data from a Creates a https url that can be used to upload/download data from a
vmware store. vmware store.
""" """
parsed_url = urlparse.urlparse(self.get_uri()) parsed_url = urllib.parse.urlparse(self.get_uri())
new_url = parsed_url._replace(scheme='https') new_url = parsed_url._replace(scheme='https')
return urlparse.urlunparse(new_url) return urllib.parse.urlunparse(new_url)
class Store(glance_store.Store): class Store(glance_store.Store):
@ -597,7 +592,7 @@ class Store(glance_store.Store):
image_file = _Reader(image_file, hashing_algo, verifier) image_file = _Reader(image_file, hashing_algo, verifier)
headers = {} headers = {}
if image_size > 0: if image_size > 0:
headers.update({'Content-Length': six.text_type(image_size)}) headers.update({'Content-Length': str(image_size)})
data = image_file data = image_file
else: else:
data = utils.chunkiter(image_file, CHUNKSIZE) data = utils.chunkiter(image_file, CHUNKSIZE)
@ -656,7 +651,7 @@ class Store(glance_store.Store):
metadata = {} metadata = {}
if self.backend_group: if self.backend_group:
metadata['store'] = u"%s" % self.backend_group metadata['store'] = self.backend_group
return (loc.get_uri(), return (loc.get_uri(),
image_file.size, image_file.size,
@ -804,9 +799,9 @@ class Store(glance_store.Store):
# Note(sabari): The redirect url will have a scheme 'http(s)', but the # Note(sabari): The redirect url will have a scheme 'http(s)', but the
# store only accepts url with scheme 'vsphere'. Thus, replacing with # store only accepts url with scheme 'vsphere'. Thus, replacing with
# store's scheme. # store's scheme.
parsed_url = urlparse.urlparse(url) parsed_url = urllib.parse.urlparse(url)
new_url = parsed_url._replace(scheme='vsphere') new_url = parsed_url._replace(scheme='vsphere')
vsphere_url = urlparse.urlunparse(new_url) vsphere_url = urllib.parse.urlunparse(new_url)
return glance_store.location.Location(store_name, return glance_store.location.Location(store_name,
store_class, store_class,
self.conf, self.conf,

3
glance_store/backend.py
View File

@ -19,7 +19,6 @@ import logging
from oslo_config import cfg from oslo_config import cfg
from oslo_utils import encodeutils from oslo_utils import encodeutils
import six
from stevedore import driver from stevedore import driver
from stevedore import extension from stevedore import extension
@ -380,7 +379,7 @@ def check_location_metadata(val, key=''):
for v in val: for v in val:
check_location_metadata(v, key='%s[%d]' % (key, ndx)) check_location_metadata(v, key='%s[%d]' % (key, ndx))
ndx = ndx + 1 ndx = ndx + 1
elif not isinstance(val, six.text_type): elif not isinstance(val, str):
raise exceptions.BackendException(_("The image metadata key %(key)s " raise exceptions.BackendException(_("The image metadata key %(key)s "
"has an invalid type of %(type)s. " "has an invalid type of %(type)s. "
"Only dict, list, and unicode are " "Only dict, list, and unicode are "

6
glance_store/driver.py
View File

@ -24,7 +24,6 @@ from oslo_config import cfg
from oslo_utils import encodeutils from oslo_utils import encodeutils
from oslo_utils import importutils from oslo_utils import importutils
from oslo_utils import units from oslo_utils import units
import six
from glance_store import capabilities from glance_store import capabilities
from glance_store import exceptions from glance_store import exceptions
@ -93,7 +92,7 @@ class Store(capabilities.StoreCapability):
self.configure_add() self.configure_add()
except exceptions.BadStoreConfiguration as e: except exceptions.BadStoreConfiguration as e:
self.unset_capabilities(capabilities.BitMasks.WRITE_ACCESS) self.unset_capabilities(capabilities.BitMasks.WRITE_ACCESS)
msg = (_(u"Failed to configure store correctly: %s " msg = (_("Failed to configure store correctly: %s "
"Disabling add method.") "Disabling add method.")
% encodeutils.exception_to_unicode(e)) % encodeutils.exception_to_unicode(e))
LOG.warning(msg) LOG.warning(msg)
@ -257,8 +256,7 @@ def back_compat_add(store_add_fun):
# everything is present, including hashing_algo # everything is present, including hashing_algo
back_compat_required = False back_compat_required = False
elif ('hashing_algo' in kwargs or elif ('hashing_algo' in kwargs or
(num_args >= p_algo + 1 and isinstance(args[p_algo], (num_args >= p_algo + 1 and isinstance(args[p_algo], str))):
six.string_types))):
# there is a hashing_algo argument present # there is a hashing_algo argument present
back_compat_required = False back_compat_required = False
else: else:

11
glance_store/exceptions.py
View File

@ -15,8 +15,7 @@
"""Glance Store exception subclasses""" """Glance Store exception subclasses"""
import six import urllib.parse
import six.moves.urllib.parse as urlparse
from glance_store.i18n import _ from glance_store.i18n import _
@ -31,7 +30,7 @@ class UnsupportedBackend(BackendException):
class RedirectException(Exception): class RedirectException(Exception):
def __init__(self, url): def __init__(self, url):
self.url = urlparse.urlparse(url) self.url = urllib.parse.urlparse(url)
class GlanceStoreException(Exception): class GlanceStoreException(Exception):
@ -55,12 +54,6 @@ class GlanceStoreException(Exception):
self.msg = message self.msg = message
super(GlanceStoreException, self).__init__(message) super(GlanceStoreException, self).__init__(message)
def __unicode__(self):
# NOTE(flwang): By default, self.msg is an instance of Message, which
# can't be converted by str(). Based on the definition of
# __unicode__, it should return unicode always.
return six.text_type(self.msg)
class MissingCredentialError(GlanceStoreException): class MissingCredentialError(GlanceStoreException):
message = _("Missing required credential: %(required)s") message = _("Missing required credential: %(required)s")

2
glance_store/location.py
View File

@ -38,9 +38,9 @@ credentials and is **not** user-facing.
""" """
import logging import logging
import urllib.parse
from oslo_config import cfg from oslo_config import cfg
from six.moves import urllib
from glance_store import exceptions from glance_store import exceptions
from glance_store.i18n import _ from glance_store.i18n import _

3
glance_store/multi_backend.py
View File

@ -20,7 +20,6 @@ import logging
from oslo_config import cfg from oslo_config import cfg
from oslo_utils import encodeutils from oslo_utils import encodeutils
from oslo_utils import units from oslo_utils import units
import six
from stevedore import driver from stevedore import driver
from stevedore import extension from stevedore import extension
@ -495,7 +494,7 @@ def check_location_metadata(val, key=''):
for v in val: for v in val:
check_location_metadata(v, key='%s[%d]' % (key, ndx)) check_location_metadata(v, key='%s[%d]' % (key, ndx))
ndx = ndx + 1 ndx = ndx + 1
elif not isinstance(val, six.text_type): elif not isinstance(val, str):
raise exceptions.BackendException(_("The image metadata key %(key)s " raise exceptions.BackendException(_("The image metadata key %(key)s "
"has an invalid type of %(type)s. " "has an invalid type of %(type)s. "
"Only dict, list, and unicode are " "Only dict, list, and unicode are "

16
glance_store/tests/unit/test_backend.py
View File

@ -122,25 +122,25 @@ class TestStoreAddToBackend(base.StoreBaseTest):
self._good_metadata(metadata) self._good_metadata(metadata)
def test_string(self): def test_string(self):
metadata = {'key': u'somevalue'} metadata = {'key': 'somevalue'}
self._good_metadata(metadata) self._good_metadata(metadata)
def test_list(self): def test_list(self):
m = {'key': [u'somevalue', u'2']} m = {'key': ['somevalue', '2']}
self._good_metadata(m) self._good_metadata(m)
def test_unicode_dict(self): def test_unicode_dict(self):
inner = {'key1': u'somevalue', 'key2': u'somevalue'} inner = {'key1': 'somevalue', 'key2': 'somevalue'}
m = {'topkey': inner} m = {'topkey': inner}
self._good_metadata(m) self._good_metadata(m)
def test_unicode_dict_list(self): def test_unicode_dict_list(self):
inner = {'key1': u'somevalue', 'key2': u'somevalue'} inner = {'key1': 'somevalue', 'key2': 'somevalue'}
m = {'topkey': inner, 'list': [u'somevalue', u'2'], 'u': u'2'} m = {'topkey': inner, 'list': ['somevalue', '2'], 'u': '2'}
self._good_metadata(m) self._good_metadata(m)
def test_nested_dict(self): def test_nested_dict(self):
inner = {'key1': u'somevalue', 'key2': u'somevalue'} inner = {'key1': 'somevalue', 'key2': 'somevalue'}
inner = {'newkey': inner} inner = {'newkey': inner}
inner = {'anotherkey': inner} inner = {'anotherkey': inner}
m = {'topkey': inner} m = {'topkey': inner}
@ -151,9 +151,9 @@ class TestStoreAddToBackend(base.StoreBaseTest):
self._bad_metadata(metadata) self._bad_metadata(metadata)
def test_bad_nonunicode_dict_list(self): def test_bad_nonunicode_dict_list(self):
inner = {'key1': u'somevalue', 'key2': u'somevalue', inner = {'key1': 'somevalue', 'key2': 'somevalue',
'k3': [1, object()]} 'k3': [1, object()]}
m = {'topkey': inner, 'list': [u'somevalue', u'2'], 'u': u'2'} m = {'topkey': inner, 'list': ['somevalue', '2'], 'u': '2'}
self._bad_metadata(m) self._bad_metadata(m)
def test_bad_metadata_not_dict(self): def test_bad_metadata_not_dict(self):

8
glance_store/tests/unit/test_cinder_store.py
View File

@ -43,10 +43,10 @@ class TestCinderStore(base.StoreBaseTest,
self.store.READ_CHUNKSIZE = 4096 self.store.READ_CHUNKSIZE = 4096
self.store.WRITE_CHUNKSIZE = 4096 self.store.WRITE_CHUNKSIZE = 4096
fake_sc = [{u'endpoints': [{u'publicURL': u'http://foo/public_url'}], fake_sc = [{'endpoints': [{'publicURL': 'http://foo/public_url'}],
u'endpoints_links': [], 'endpoints_links': [],
u'name': u'cinder', 'name': 'cinder',
u'type': u'volumev3'}] 'type': 'volumev3'}]
self.context = mock.MagicMock(service_catalog=fake_sc, self.context = mock.MagicMock(service_catalog=fake_sc,
user_id='fake_user', user_id='fake_user',
auth_token='fake_token', auth_token='fake_token',

7
glance_store/tests/unit/test_exceptions.py
View File

@ -14,7 +14,6 @@
# under the License. # under the License.
from oslo_utils import encodeutils from oslo_utils import encodeutils
from oslotest import base from oslotest import base
import six
import glance_store import glance_store
@ -23,11 +22,11 @@ class TestExceptions(base.BaseTestCase):
"""Test routines in glance_store.common.utils.""" """Test routines in glance_store.common.utils."""
def test_backend_exception(self): def test_backend_exception(self):
msg = glance_store.BackendException() msg = glance_store.BackendException()
self.assertIn(u'', encodeutils.exception_to_unicode(msg)) self.assertIn('', encodeutils.exception_to_unicode(msg))
def test_unsupported_backend_exception(self): def test_unsupported_backend_exception(self):
msg = glance_store.UnsupportedBackend() msg = glance_store.UnsupportedBackend()
self.assertIn(u'', encodeutils.exception_to_unicode(msg)) self.assertIn('', encodeutils.exception_to_unicode(msg))
def test_redirect_exception(self): def test_redirect_exception(self):
# Just checks imports work ok # Just checks imports work ok
@ -54,4 +53,4 @@ class TestExceptions(base.BaseTestCase):
def test_non_unicode_error_msg(self): def test_non_unicode_error_msg(self):
exc = glance_store.NotFound(str('test')) exc = glance_store.NotFound(str('test'))
self.assertIsInstance(encodeutils.exception_to_unicode(exc), self.assertIsInstance(encodeutils.exception_to_unicode(exc),
six.text_type) str)

42
glance_store/tests/unit/test_filesystem_store.py
View File

@ -15,8 +15,10 @@
"""Tests the filesystem backend store""" """Tests the filesystem backend store"""
import builtins
import errno import errno
import hashlib import hashlib
import io
import json import json
import os import os
import stat import stat
@ -26,10 +28,6 @@ import uuid
import fixtures import fixtures
from oslo_utils.secretutils import md5 from oslo_utils.secretutils import md5
from oslo_utils import units from oslo_utils import units
import six
from six.moves import builtins
# NOTE(jokke): simplified transition to py3, behaves like py2 xrange
from six.moves import range
from glance_store._drivers import filesystem from glance_store._drivers import filesystem
from glance_store import exceptions from glance_store import exceptions
@ -67,7 +65,7 @@ class TestStore(base.StoreBaseTest,
expected_image_id = str(uuid.uuid4()) expected_image_id = str(uuid.uuid4())
expected_file_size = 10 expected_file_size = 10
expected_file_contents = b"*" * expected_file_size expected_file_contents = b"*" * expected_file_size
image_file = six.BytesIO(expected_file_contents) image_file = io.BytesIO(expected_file_contents)
self.store.FILESYSTEM_STORE_METADATA = in_metadata self.store.FILESYSTEM_STORE_METADATA = in_metadata
return self.store.add(expected_image_id, image_file, return self.store.add(expected_image_id, image_file,
expected_file_size, self.hash_algo) expected_file_size, self.hash_algo)
@ -77,7 +75,7 @@ class TestStore(base.StoreBaseTest,
# First add an image... # First add an image...
image_id = str(uuid.uuid4()) image_id = str(uuid.uuid4())
file_contents = b"chunk00000remainder" file_contents = b"chunk00000remainder"
image_file = six.BytesIO(file_contents) image_file = io.BytesIO(file_contents)
loc, size, checksum, multihash, _ = self.store.add( loc, size, checksum, multihash, _ = self.store.add(
image_id, image_file, len(file_contents), self.hash_algo) image_id, image_file, len(file_contents), self.hash_algo)
@ -103,7 +101,7 @@ class TestStore(base.StoreBaseTest,
# First add an image... # First add an image...
image_id = str(uuid.uuid4()) image_id = str(uuid.uuid4())
file_contents = b"chunk00000remainder" file_contents = b"chunk00000remainder"
image_file = six.BytesIO(file_contents) image_file = io.BytesIO(file_contents)
loc, size, checksum, multihash, _ = self.store.add( loc, size, checksum, multihash, _ = self.store.add(
image_id, image_file, len(file_contents), self.hash_algo) image_id, image_file, len(file_contents), self.hash_algo)
@ -160,7 +158,7 @@ class TestStore(base.StoreBaseTest,
expected_multihash = hashlib.sha256(expected_file_contents).hexdigest() expected_multihash = hashlib.sha256(expected_file_contents).hexdigest()
expected_location = "file://%s/%s" % (self.test_dir, expected_location = "file://%s/%s" % (self.test_dir,
expected_image_id) expected_image_id)
image_file = six.BytesIO(expected_file_contents) image_file = io.BytesIO(expected_file_contents)
loc, size, checksum, multihash, _ = self.store.add( loc, size, checksum, multihash, _ = self.store.add(
expected_image_id, image_file, expected_file_size, self.hash_algo) expected_image_id, image_file, expected_file_size, self.hash_algo)
@ -254,7 +252,7 @@ class TestStore(base.StoreBaseTest,
group='glance_store') group='glance_store')
self.store.configure() self.store.configure()
image_file = six.BytesIO(content) image_file = io.BytesIO(content)
image_id = str(uuid.uuid4()) image_id = str(uuid.uuid4())
with mock.patch.object(builtins, 'open') as popen: with mock.patch.object(builtins, 'open') as popen:
self.store.add(image_id, image_file, size, self.hash_algo) self.store.add(image_id, image_file, size, self.hash_algo)
@ -273,7 +271,7 @@ class TestStore(base.StoreBaseTest,
image_id = str(uuid.uuid4()) image_id = str(uuid.uuid4())
file_size = units.Ki # 1K file_size = units.Ki # 1K
file_contents = b"*" * file_size file_contents = b"*" * file_size
image_file = six.BytesIO(file_contents) image_file = io.BytesIO(file_contents)
self.store.add(image_id, image_file, file_size, self.hash_algo, self.store.add(image_id, image_file, file_size, self.hash_algo,
verifier=verifier) verifier=verifier)
@ -310,7 +308,7 @@ class TestStore(base.StoreBaseTest,
group="glance_store") group="glance_store")
expected_file_size = 10 expected_file_size = 10
expected_file_contents = b"*" * expected_file_size expected_file_contents = b"*" * expected_file_size
image_file = six.BytesIO(expected_file_contents) image_file = io.BytesIO(expected_file_contents)
location, size, checksum, multihash, metadata = self.store.add( location, size, checksum, multihash, metadata = self.store.add(
expected_image_id, image_file, expected_file_size, self.hash_algo) expected_image_id, image_file, expected_file_size, self.hash_algo)
@ -326,11 +324,11 @@ class TestStore(base.StoreBaseTest,
image_id = str(uuid.uuid4()) image_id = str(uuid.uuid4())
file_size = 5 * units.Ki # 5K file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size file_contents = b"*" * file_size
image_file = six.BytesIO(file_contents) image_file = io.BytesIO(file_contents)
location, size, checksum, multihash, _ = self.store.add( location, size, checksum, multihash, _ = self.store.add(
image_id, image_file, file_size, self.hash_algo) image_id, image_file, file_size, self.hash_algo)
image_file = six.BytesIO(b"nevergonnamakeit") image_file = io.BytesIO(b"nevergonnamakeit")
self.assertRaises(exceptions.Duplicate, self.assertRaises(exceptions.Duplicate,
self.store.add, self.store.add,
image_id, image_file, 0, self.hash_algo) image_id, image_file, 0, self.hash_algo)
@ -341,7 +339,7 @@ class TestStore(base.StoreBaseTest,
file_size = 5 * units.Ki # 5K file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size file_contents = b"*" * file_size
path = os.path.join(self.test_dir, image_id) path = os.path.join(self.test_dir, image_id)
image_file = six.BytesIO(file_contents) image_file = io.BytesIO(file_contents)
with mock.patch.object(builtins, 'open') as popen: with mock.patch.object(builtins, 'open') as popen:
e = IOError() e = IOError()
@ -392,7 +390,7 @@ class TestStore(base.StoreBaseTest,
file_size = 5 * units.Ki # 5K file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size file_contents = b"*" * file_size
path = os.path.join(self.test_dir, image_id) path = os.path.join(self.test_dir, image_id)
image_file = six.BytesIO(file_contents) image_file = io.BytesIO(file_contents)
def fake_Error(size): def fake_Error(size):
raise AttributeError() raise AttributeError()
@ -413,7 +411,7 @@ class TestStore(base.StoreBaseTest,
image_id = str(uuid.uuid4()) image_id = str(uuid.uuid4())
file_size = 5 * units.Ki # 5K file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size file_contents = b"*" * file_size
image_file = six.BytesIO(file_contents) image_file = io.BytesIO(file_contents)
loc, size, checksum, multihash, _ = self.store.add( loc, size, checksum, multihash, _ = self.store.add(
image_id, image_file, file_size, self.hash_algo) image_id, image_file, file_size, self.hash_algo)
@ -445,7 +443,7 @@ class TestStore(base.StoreBaseTest,
image_id = str(uuid.uuid4()) image_id = str(uuid.uuid4())
file_size = 5 * units.Ki # 5K file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size file_contents = b"*" * file_size
image_file = six.BytesIO(file_contents) image_file = io.BytesIO(file_contents)
loc, size, checksum, multihash, _ = self.store.add( loc, size, checksum, multihash, _ = self.store.add(
image_id, image_file, file_size, self.hash_algo) image_id, image_file, file_size, self.hash_algo)
@ -611,7 +609,7 @@ class TestStore(base.StoreBaseTest,
expected_multihash = hashlib.sha256(expected_file_contents).hexdigest() expected_multihash = hashlib.sha256(expected_file_contents).hexdigest()
expected_location = "file://%s/%s" % (store_map[1], expected_location = "file://%s/%s" % (store_map[1],
expected_image_id) expected_image_id)
image_file = six.BytesIO(expected_file_contents) image_file = io.BytesIO(expected_file_contents)
loc, size, checksum, multihash, _ = self.store.add( loc, size, checksum, multihash, _ = self.store.add(
expected_image_id, image_file, expected_file_size, self.hash_algo) expected_image_id, image_file, expected_file_size, self.hash_algo)
@ -659,7 +657,7 @@ class TestStore(base.StoreBaseTest,
expected_multihash = hashlib.sha256(expected_file_contents).hexdigest() expected_multihash = hashlib.sha256(expected_file_contents).hexdigest()
expected_location = "file://%s/%s" % (store_map[1], expected_location = "file://%s/%s" % (store_map[1],
expected_image_id) expected_image_id)
image_file = six.BytesIO(expected_file_contents) image_file = io.BytesIO(expected_file_contents)
loc, size, checksum, multihash, _ = self.store.add( loc, size, checksum, multihash, _ = self.store.add(
expected_image_id, image_file, expected_file_size, self.hash_algo) expected_image_id, image_file, expected_file_size, self.hash_algo)
@ -709,7 +707,7 @@ class TestStore(base.StoreBaseTest,
expected_image_id = str(uuid.uuid4()) expected_image_id = str(uuid.uuid4())
expected_file_size = 5 * units.Ki # 5K expected_file_size = 5 * units.Ki # 5K
expected_file_contents = b"*" * expected_file_size expected_file_contents = b"*" * expected_file_size
image_file = six.BytesIO(expected_file_contents) image_file = io.BytesIO(expected_file_contents)
self.assertRaises(exceptions.StorageFull, self.assertRaises(exceptions.StorageFull,
self.store.add, self.store.add,
@ -770,7 +768,7 @@ class TestStore(base.StoreBaseTest,
expected_multihash = hashlib.sha256(expected_file_contents).hexdigest() expected_multihash = hashlib.sha256(expected_file_contents).hexdigest()
expected_location = "file://%s/%s" % (store, expected_location = "file://%s/%s" % (store,
expected_image_id) expected_image_id)
image_file = six.BytesIO(expected_file_contents) image_file = io.BytesIO(expected_file_contents)
location, size, checksum, multihash, _ = self.store.add( location, size, checksum, multihash, _ = self.store.add(
expected_image_id, image_file, expected_file_size, self.hash_algo) expected_image_id, image_file, expected_file_size, self.hash_algo)
@ -813,7 +811,7 @@ class TestStore(base.StoreBaseTest,
expected_multihash = hashlib.sha256(expected_file_contents).hexdigest() expected_multihash = hashlib.sha256(expected_file_contents).hexdigest()
expected_location = "file://%s/%s" % (store, expected_location = "file://%s/%s" % (store,
expected_image_id) expected_image_id)
image_file = six.BytesIO(expected_file_contents) image_file = io.BytesIO(expected_file_contents)
location, size, checksum, multihash, _ = self.store.add( location, size, checksum, multihash, _ = self.store.add(
expected_image_id, image_file, expected_file_size, self.hash_algo) expected_image_id, image_file, expected_file_size, self.hash_algo)

8
glance_store/tests/unit/test_multistore_cinder.py
View File

@ -72,10 +72,10 @@ class TestMultiCinderStore(base.MultiStoreBaseTest,
self.store.READ_CHUNKSIZE = 4096 self.store.READ_CHUNKSIZE = 4096
self.store.WRITE_CHUNKSIZE = 4096 self.store.WRITE_CHUNKSIZE = 4096
fake_sc = [{u'endpoints': [{u'publicURL': u'http://foo/public_url'}], fake_sc = [{'endpoints': [{'publicURL': 'http://foo/public_url'}],
u'endpoints_links': [], 'endpoints_links': [],
u'name': u'cinder', 'name': 'cinder',
u'type': u'volumev3'}] 'type': 'volumev3'}]
self.context = mock.MagicMock(service_catalog=fake_sc, self.context = mock.MagicMock(service_catalog=fake_sc,
user_id='fake_user', user_id='fake_user',
auth_token='fake_token', auth_token='fake_token',

70
glance_store/tests/unit/test_multistore_filesystem.py
View File

@ -15,7 +15,9 @@
"""Tests the filesystem backend store""" """Tests the filesystem backend store"""
import builtins
import errno import errno
import io
import json import json
import os import os
import stat import stat
@ -26,10 +28,6 @@ import fixtures
from oslo_config import cfg from oslo_config import cfg
from oslo_utils.secretutils import md5 from oslo_utils.secretutils import md5
from oslo_utils import units from oslo_utils import units
import six
from six.moves import builtins
# NOTE(jokke): simplified transition to py3, behaves like py2 xrange
from six.moves import range
import glance_store as store import glance_store as store
from glance_store._drivers import filesystem from glance_store._drivers import filesystem
@ -91,7 +89,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
expected_image_id = str(uuid.uuid4()) expected_image_id = str(uuid.uuid4())
expected_file_size = 10 expected_file_size = 10
expected_file_contents = b"*" * expected_file_size expected_file_contents = b"*" * expected_file_size
image_file = six.BytesIO(expected_file_contents) image_file = io.BytesIO(expected_file_contents)
self.store.FILESYSTEM_STORE_METADATA = in_metadata self.store.FILESYSTEM_STORE_METADATA = in_metadata
return self.store.add(expected_image_id, image_file, return self.store.add(expected_image_id, image_file,
expected_file_size) expected_file_size)
@ -105,12 +103,12 @@ class TestMultiStore(base.MultiStoreBaseTest,
# First add an image... # First add an image...
image_id = str(uuid.uuid4()) image_id = str(uuid.uuid4())
file_contents = b"chunk00000remainder" file_contents = b"chunk00000remainder"
image_file = six.BytesIO(file_contents) image_file = io.BytesIO(file_contents)
loc, size, checksum, metadata = self.store.add( loc, size, checksum, metadata = self.store.add(
image_id, image_file, len(file_contents)) image_id, image_file, len(file_contents))
# Check metadata contains 'file1' as a store # Check metadata contains 'file1' as a store
self.assertEqual(u"file1", metadata['store']) self.assertEqual("file1", metadata['store'])
# Now read it back... # Now read it back...
uri = "file:///%s/%s" % (self.test_dir, image_id) uri = "file:///%s/%s" % (self.test_dir, image_id)
@ -134,13 +132,13 @@ class TestMultiStore(base.MultiStoreBaseTest,
# First add an image... # First add an image...
image_id = str(uuid.uuid4()) image_id = str(uuid.uuid4())
file_contents = b"chunk00000remainder" file_contents = b"chunk00000remainder"
image_file = six.BytesIO(file_contents) image_file = io.BytesIO(file_contents)
loc, size, checksum, metadata = self.store.add(image_id, loc, size, checksum, metadata = self.store.add(image_id,
image_file, image_file,
len(file_contents)) len(file_contents))
# Check metadata contains 'file1' as a store # Check metadata contains 'file1' as a store
self.assertEqual(u"file1", metadata['store']) self.assertEqual("file1", metadata['store'])
# Now read it back... # Now read it back...
uri = "file:///%s/%s" % (self.test_dir, image_id) uri = "file:///%s/%s" % (self.test_dir, image_id)
@ -193,7 +191,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
usedforsecurity=False).hexdigest() usedforsecurity=False).hexdigest()
expected_location = "file://%s/%s" % (self.test_dir, expected_location = "file://%s/%s" % (self.test_dir,
expected_image_id) expected_image_id)
image_file = six.BytesIO(expected_file_contents) image_file = io.BytesIO(expected_file_contents)
loc, size, checksum, metadata = self.store.add(expected_image_id, loc, size, checksum, metadata = self.store.add(expected_image_id,
image_file, image_file,
@ -202,7 +200,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
self.assertEqual(expected_location, loc) self.assertEqual(expected_location, loc)
self.assertEqual(expected_file_size, size) self.assertEqual(expected_file_size, size)
self.assertEqual(expected_checksum, checksum) self.assertEqual(expected_checksum, checksum)
self.assertEqual(u"file1", metadata['store']) self.assertEqual("file1", metadata['store'])
uri = "file:///%s/%s" % (self.test_dir, expected_image_id) uri = "file:///%s/%s" % (self.test_dir, expected_image_id)
loc = location.get_location_from_uri_and_backend( loc = location.get_location_from_uri_and_backend(
@ -234,7 +232,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
usedforsecurity=False).hexdigest() usedforsecurity=False).hexdigest()
expected_location = "file://%s/%s" % (self.test_dir, expected_location = "file://%s/%s" % (self.test_dir,
expected_image_id) expected_image_id)
image_file = six.BytesIO(expected_file_contents) image_file = io.BytesIO(expected_file_contents)
loc, size, checksum, metadata = self.store.add(expected_image_id, loc, size, checksum, metadata = self.store.add(expected_image_id,
image_file, image_file,
@ -243,7 +241,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
self.assertEqual(expected_location, loc) self.assertEqual(expected_location, loc)
self.assertEqual(expected_file_size, size) self.assertEqual(expected_file_size, size)
self.assertEqual(expected_checksum, checksum) self.assertEqual(expected_checksum, checksum)
self.assertEqual(u"file2", metadata['store']) self.assertEqual("file2", metadata['store'])
uri = "file:///%s/%s" % (self.test_dir, expected_image_id) uri = "file:///%s/%s" % (self.test_dir, expected_image_id)
loc = location.get_location_from_uri_and_backend( loc = location.get_location_from_uri_and_backend(
@ -263,20 +261,20 @@ class TestMultiStore(base.MultiStoreBaseTest,
in_metadata = [{'id': 'abcdefg', in_metadata = [{'id': 'abcdefg',
'mountpoint': '/xyz/images'}] 'mountpoint': '/xyz/images'}]
location, size, checksum, metadata = self._store_image(in_metadata) location, size, checksum, metadata = self._store_image(in_metadata)
self.assertEqual({'store': u'file1'}, metadata) self.assertEqual({'store': 'file1'}, metadata)
def test_add_check_metadata_list_with_invalid_mountpoint_locations(self): def test_add_check_metadata_list_with_invalid_mountpoint_locations(self):
in_metadata = [{'id': 'abcdefg', 'mountpoint': '/xyz/images'}, in_metadata = [{'id': 'abcdefg', 'mountpoint': '/xyz/images'},
{'id': 'xyz1234', 'mountpoint': '/pqr/images'}] {'id': 'xyz1234', 'mountpoint': '/pqr/images'}]
location, size, checksum, metadata = self._store_image(in_metadata) location, size, checksum, metadata = self._store_image(in_metadata)
self.assertEqual({'store': u'file1'}, metadata) self.assertEqual({'store': 'file1'}, metadata)
def test_add_check_metadata_list_with_valid_mountpoint_locations(self): def test_add_check_metadata_list_with_valid_mountpoint_locations(self):
in_metadata = [{'id': 'abcdefg', 'mountpoint': '/tmp'}, in_metadata = [{'id': 'abcdefg', 'mountpoint': '/tmp'},
{'id': 'xyz1234', 'mountpoint': '/xyz'}] {'id': 'xyz1234', 'mountpoint': '/xyz'}]
location, size, checksum, metadata = self._store_image(in_metadata) location, size, checksum, metadata = self._store_image(in_metadata)
self.assertEqual(in_metadata[0], metadata) self.assertEqual(in_metadata[0], metadata)
self.assertEqual(u"file1", metadata["store"]) self.assertEqual("file1", metadata["store"])
def test_add_check_metadata_bad_nosuch_file(self): def test_add_check_metadata_bad_nosuch_file(self):
expected_image_id = str(uuid.uuid4()) expected_image_id = str(uuid.uuid4())
@ -287,13 +285,13 @@ class TestMultiStore(base.MultiStoreBaseTest,
group="file1") group="file1")
expected_file_size = 10 expected_file_size = 10
expected_file_contents = b"*" * expected_file_size expected_file_contents = b"*" * expected_file_size
image_file = six.BytesIO(expected_file_contents) image_file = io.BytesIO(expected_file_contents)
location, size, checksum, metadata = self.store.add(expected_image_id, location, size, checksum, metadata = self.store.add(expected_image_id,
image_file, image_file,
expected_file_size) expected_file_size)
self.assertEqual({'store': u'file1'}, metadata) self.assertEqual({'store': 'file1'}, metadata)
def test_add_already_existing(self): def test_add_already_existing(self):
""" """
@ -304,14 +302,14 @@ class TestMultiStore(base.MultiStoreBaseTest,
image_id = str(uuid.uuid4()) image_id = str(uuid.uuid4())
file_size = 5 * units.Ki # 5K file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size file_contents = b"*" * file_size
image_file = six.BytesIO(file_contents) image_file = io.BytesIO(file_contents)
location, size, checksum, metadata = self.store.add(image_id, location, size, checksum, metadata = self.store.add(image_id,
image_file, image_file,
file_size) file_size)
self.assertEqual(u"file1", metadata["store"]) self.assertEqual("file1", metadata["store"])
image_file = six.BytesIO(b"nevergonnamakeit") image_file = io.BytesIO(b"nevergonnamakeit")
self.assertRaises(exceptions.Duplicate, self.assertRaises(exceptions.Duplicate,
self.store.add, self.store.add,
image_id, image_file, 0) image_id, image_file, 0)
@ -322,7 +320,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
file_size = 5 * units.Ki # 5K file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size file_contents = b"*" * file_size
path = os.path.join(self.test_dir, image_id) path = os.path.join(self.test_dir, image_id)
image_file = six.BytesIO(file_contents) image_file = io.BytesIO(file_contents)
with mock.patch.object(builtins, 'open') as popen: with mock.patch.object(builtins, 'open') as popen:
e = IOError() e = IOError()
@ -370,7 +368,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
file_size = 5 * units.Ki # 5K file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size file_contents = b"*" * file_size
path = os.path.join(self.test_dir, image_id) path = os.path.join(self.test_dir, image_id)
image_file = six.BytesIO(file_contents) image_file = io.BytesIO(file_contents)
def fake_Error(size): def fake_Error(size):
raise AttributeError() raise AttributeError()
@ -389,12 +387,12 @@ class TestMultiStore(base.MultiStoreBaseTest,
image_id = str(uuid.uuid4()) image_id = str(uuid.uuid4())
file_size = 5 * units.Ki # 5K file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size file_contents = b"*" * file_size
image_file = six.BytesIO(file_contents) image_file = io.BytesIO(file_contents)
loc, size, checksum, metadata = self.store.add(image_id, loc, size, checksum, metadata = self.store.add(image_id,
image_file, image_file,
file_size) file_size)
self.assertEqual(u"file1", metadata["store"]) self.assertEqual("file1", metadata["store"])
# Now check that we can delete it # Now check that we can delete it
uri = "file:///%s/%s" % (self.test_dir, image_id) uri = "file:///%s/%s" % (self.test_dir, image_id)
@ -418,12 +416,12 @@ class TestMultiStore(base.MultiStoreBaseTest,
image_id = str(uuid.uuid4()) image_id = str(uuid.uuid4())
file_size = 5 * units.Ki # 5K file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size file_contents = b"*" * file_size
image_file = six.BytesIO(file_contents) image_file = io.BytesIO(file_contents)
loc, size, checksum, metadata = self.store.add(image_id, loc, size, checksum, metadata = self.store.add(image_id,
image_file, image_file,
file_size) file_size)
self.assertEqual(u"file1", metadata["store"]) self.assertEqual("file1", metadata["store"])
uri = "file:///%s/%s" % (self.test_dir, image_id) uri = "file:///%s/%s" % (self.test_dir, image_id)
loc = location.get_location_from_uri_and_backend(uri, "file1", loc = location.get_location_from_uri_and_backend(uri, "file1",
@ -589,12 +587,12 @@ class TestMultiStore(base.MultiStoreBaseTest,
usedforsecurity=False).hexdigest() usedforsecurity=False).hexdigest()
expected_location = "file://%s/%s" % (store_map[1], expected_location = "file://%s/%s" % (store_map[1],
expected_image_id) expected_image_id)
image_file = six.BytesIO(expected_file_contents) image_file = io.BytesIO(expected_file_contents)
loc, size, checksum, metadata = self.store.add(expected_image_id, loc, size, checksum, metadata = self.store.add(expected_image_id,
image_file, image_file,
expected_file_size) expected_file_size)
self.assertEqual(u"file1", metadata["store"]) self.assertEqual("file1", metadata["store"])
self.assertEqual(expected_location, loc) self.assertEqual(expected_location, loc)
self.assertEqual(expected_file_size, size) self.assertEqual(expected_file_size, size)
@ -637,12 +635,12 @@ class TestMultiStore(base.MultiStoreBaseTest,
usedforsecurity=False).hexdigest() usedforsecurity=False).hexdigest()
expected_location = "file://%s/%s" % (store_map[1], expected_location = "file://%s/%s" % (store_map[1],
expected_image_id) expected_image_id)
image_file = six.BytesIO(expected_file_contents) image_file = io.BytesIO(expected_file_contents)
loc, size, checksum, metadata = self.store.add(expected_image_id, loc, size, checksum, metadata = self.store.add(expected_image_id,
image_file, image_file,
expected_file_size) expected_file_size)
self.assertEqual(u"file1", metadata["store"]) self.assertEqual("file1", metadata["store"])
self.assertEqual(expected_location, loc) self.assertEqual(expected_location, loc)
self.assertEqual(expected_file_size, size) self.assertEqual(expected_file_size, size)
@ -689,7 +687,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
expected_image_id = str(uuid.uuid4()) expected_image_id = str(uuid.uuid4())
expected_file_size = 5 * units.Ki # 5K expected_file_size = 5 * units.Ki # 5K
expected_file_contents = b"*" * expected_file_size expected_file_contents = b"*" * expected_file_size
image_file = six.BytesIO(expected_file_contents) image_file = io.BytesIO(expected_file_contents)
self.assertRaises(exceptions.StorageFull, self.store.add, self.assertRaises(exceptions.StorageFull, self.store.add,
expected_image_id, image_file, expected_image_id, image_file,
@ -749,12 +747,12 @@ class TestMultiStore(base.MultiStoreBaseTest,
usedforsecurity=False).hexdigest() usedforsecurity=False).hexdigest()
expected_location = "file://%s/%s" % (store, expected_location = "file://%s/%s" % (store,
expected_image_id) expected_image_id)
image_file = six.BytesIO(expected_file_contents) image_file = io.BytesIO(expected_file_contents)
location, size, checksum, metadata = self.store.add(expected_image_id, location, size, checksum, metadata = self.store.add(expected_image_id,
image_file, image_file,
expected_file_size) expected_file_size)
self.assertEqual(u"file1", metadata["store"]) self.assertEqual("file1", metadata["store"])
self.assertEqual(expected_location, location) self.assertEqual(expected_location, location)
self.assertEqual(expected_file_size, size) self.assertEqual(expected_file_size, size)
@ -794,12 +792,12 @@ class TestMultiStore(base.MultiStoreBaseTest,
usedforsecurity=False).hexdigest() usedforsecurity=False).hexdigest()
expected_location = "file://%s/%s" % (store, expected_location = "file://%s/%s" % (store,
expected_image_id) expected_image_id)
image_file = six.BytesIO(expected_file_contents) image_file = io.BytesIO(expected_file_contents)
location, size, checksum, metadata = self.store.add(expected_image_id, location, size, checksum, metadata = self.store.add(expected_image_id,
image_file, image_file,
expected_file_size) expected_file_size)
self.assertEqual(u"file1", metadata["store"]) self.assertEqual("file1", metadata["store"])
self.assertEqual(expected_location, location) self.assertEqual(expected_location, location)
self.assertEqual(expected_file_size, size) self.assertEqual(expected_file_size, size)

6
glance_store/tests/unit/test_multistore_rbd.py
View File

@ -13,11 +13,11 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
import io
from unittest import mock from unittest import mock
from oslo_config import cfg from oslo_config import cfg
from oslo_utils import units from oslo_utils import units
import six
import glance_store as store import glance_store as store
from glance_store._drivers import rbd as rbd_store from glance_store._drivers import rbd as rbd_store
@ -222,7 +222,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
self.conf) self.conf)
# Provide enough data to get more than one chunk iteration. # Provide enough data to get more than one chunk iteration.
self.data_len = 3 * units.Ki self.data_len = 3 * units.Ki
self.data_iter = six.BytesIO(b'*' * self.data_len) self.data_iter = io.BytesIO(b'*' * self.data_len)
def test_location_url_prefix_is_set(self): def test_location_url_prefix_is_set(self):
expected_url_prefix = "rbd://" expected_url_prefix = "rbd://"
@ -253,7 +253,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
self.conf) self.conf)
# Provide enough data to get more than one chunk iteration. # Provide enough data to get more than one chunk iteration.
self.data_len = 3 * units.Ki self.data_len = 3 * units.Ki
self.data_iter = six.BytesIO(b'*' * self.data_len) self.data_iter = io.BytesIO(b'*' * self.data_len)
self.store.chunk_size = units.Ki self.store.chunk_size = units.Ki
with mock.patch.object(rbd_store.rbd.Image, 'resize') as resize: with mock.patch.object(rbd_store.rbd.Image, 'resize') as resize:
with mock.patch.object(rbd_store.rbd.Image, 'write') as write: with mock.patch.object(rbd_store.rbd.Image, 'write') as write:

16
glance_store/tests/unit/test_multistore_s3.py
View File

@ -16,6 +16,7 @@
"""Tests the Multiple S3 backend store""" """Tests the Multiple S3 backend store"""
import hashlib import hashlib
import io
from unittest import mock from unittest import mock
import uuid import uuid
@ -26,7 +27,6 @@ from botocore import stub
from oslo_config import cfg from oslo_config import cfg
from oslo_utils.secretutils import md5 from oslo_utils.secretutils import md5
from oslo_utils import units from oslo_utils import units
import six
import glance_store as store import glance_store as store
from glance_store._drivers import s3 from glance_store._drivers import s3
@ -137,7 +137,7 @@ class TestMultiS3Store(base.MultiStoreBaseTest,
"""Test a "normal" retrieval of an image in chunks.""" """Test a "normal" retrieval of an image in chunks."""
bucket, key = 'glance', FAKE_UUID bucket, key = 'glance', FAKE_UUID
fixture_object = { fixture_object = {
'Body': six.BytesIO(b"*" * FIVE_KB), 'Body': io.BytesIO(b"*" * FIVE_KB),
'ContentLength': FIVE_KB 'ContentLength': FIVE_KB
} }
fake_s3_client = botocore.session.get_session().create_client('s3') fake_s3_client = botocore.session.get_session().create_client('s3')
@ -220,7 +220,7 @@ class TestMultiS3Store(base.MultiStoreBaseTest,
S3_CONF['s3_store_host'], S3_CONF['s3_store_host'],
S3_CONF['s3_store_bucket'], S3_CONF['s3_store_bucket'],
expected_image_id) expected_image_id)
image_s3 = six.BytesIO(expected_s3_contents) image_s3 = io.BytesIO(expected_s3_contents)
fake_s3_client = botocore.session.get_session().create_client('s3') fake_s3_client = botocore.session.get_session().create_client('s3')
@ -273,7 +273,7 @@ class TestMultiS3Store(base.MultiStoreBaseTest,
S3_CONF['s3_store_host'], S3_CONF['s3_store_host'],
S3_CONF['s3_store_bucket'], S3_CONF['s3_store_bucket'],
expected_image_id) expected_image_id)
image_s3 = six.BytesIO(expected_s3_contents) image_s3 = io.BytesIO(expected_s3_contents)
fake_s3_client = botocore.session.get_session().create_client('s3') fake_s3_client = botocore.session.get_session().create_client('s3')
@ -327,7 +327,7 @@ class TestMultiS3Store(base.MultiStoreBaseTest,
'http://s3-region2.com', 'http://s3-region2.com',
S3_CONF['s3_store_bucket'], S3_CONF['s3_store_bucket'],
expected_image_id) expected_image_id)
image_s3 = six.BytesIO(expected_s3_contents) image_s3 = io.BytesIO(expected_s3_contents)
fake_s3_client = botocore.session.get_session().create_client('s3') fake_s3_client = botocore.session.get_session().create_client('s3')
@ -369,7 +369,7 @@ class TestMultiS3Store(base.MultiStoreBaseTest,
expected_image_id = str(uuid.uuid4()) expected_image_id = str(uuid.uuid4())
expected_s3_size = FIVE_KB expected_s3_size = FIVE_KB
expected_s3_contents = b"*" * expected_s3_size expected_s3_contents = b"*" * expected_s3_size
image_s3 = six.BytesIO(expected_s3_contents) image_s3 = io.BytesIO(expected_s3_contents)
fake_s3_client = botocore.session.get_session().create_client('s3') fake_s3_client = botocore.session.get_session().create_client('s3')
verifier = mock.MagicMock(name='mock_verifier') verifier = mock.MagicMock(name='mock_verifier')
@ -401,7 +401,7 @@ class TestMultiS3Store(base.MultiStoreBaseTest,
S3_CONF['s3_store_host'], S3_CONF['s3_store_host'],
S3_CONF['s3_store_bucket'], S3_CONF['s3_store_bucket'],
expected_image_id) expected_image_id)
image_s3 = six.BytesIO(expected_s3_contents) image_s3 = io.BytesIO(expected_s3_contents)
fake_s3_client = botocore.session.get_session().create_client('s3') fake_s3_client = botocore.session.get_session().create_client('s3')
@ -483,7 +483,7 @@ class TestMultiS3Store(base.MultiStoreBaseTest,
"""Tests that adding an image with an existing identifier raises an """Tests that adding an image with an existing identifier raises an
appropriate exception appropriate exception
""" """
image_s3 = six.BytesIO(b"never_gonna_make_it") image_s3 = io.BytesIO(b"never_gonna_make_it")
fake_s3_client = botocore.session.get_session().create_client('s3') fake_s3_client = botocore.session.get_session().create_client('s3')

26
glance_store/tests/unit/test_multistore_vmware.py
View File

@ -16,6 +16,7 @@
"""Tests the Multiple VMware Datastore backend store""" """Tests the Multiple VMware Datastore backend store"""
import hashlib import hashlib
import io
from unittest import mock from unittest import mock
import uuid import uuid
@ -26,7 +27,6 @@ from oslo_vmware import api
from oslo_vmware import exceptions as vmware_exceptions from oslo_vmware import exceptions as vmware_exceptions
from oslo_vmware.objects import datacenter as oslo_datacenter from oslo_vmware.objects import datacenter as oslo_datacenter
from oslo_vmware.objects import datastore as oslo_datastore from oslo_vmware.objects import datastore as oslo_datastore
import six
import glance_store as store import glance_store as store
import glance_store._drivers.vmware_datastore as vm_store import glance_store._drivers.vmware_datastore as vm_store
@ -130,7 +130,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
self.store.configure() self.store.configure()
def _mock_http_connection(self): def _mock_http_connection(self):
return mock.patch('six.moves.http_client.HTTPConnection') return mock.patch('http.client.HTTPConnection')
def test_location_url_prefix_is_set(self): def test_location_url_prefix_is_set(self):
expected_url_prefix = "vsphere://127.0.0.1/openstack_glance" expected_url_prefix = "vsphere://127.0.0.1/openstack_glance"
@ -181,7 +181,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
fake_size.__get__ = mock.Mock(return_value=expected_size) fake_size.__get__ = mock.Mock(return_value=expected_size)
expected_cookie = 'vmware_soap_session=fake-uuid' expected_cookie = 'vmware_soap_session=fake-uuid'
fake_cookie.return_value = expected_cookie fake_cookie.return_value = expected_cookie
expected_headers = {'Content-Length': six.text_type(expected_size), expected_headers = {'Content-Length': str(expected_size),
'Cookie': expected_cookie} 'Cookie': expected_cookie}
with mock.patch('hashlib.md5') as md5: with mock.patch('hashlib.md5') as md5:
md5.return_value = hash_code md5.return_value = hash_code
@ -190,7 +190,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
VMWARE_DS['vmware_store_image_dir'], VMWARE_DS['vmware_store_image_dir'],
expected_image_id, expected_image_id,
VMWARE_DS['vmware_datastores']) VMWARE_DS['vmware_datastores'])
image = six.BytesIO(expected_contents) image = io.BytesIO(expected_contents)
with mock.patch('requests.Session.request') as HttpConn: with mock.patch('requests.Session.request') as HttpConn:
HttpConn.return_value = utils.fake_response() HttpConn.return_value = utils.fake_response()
location, size, checksum, metadata = self.store.add( location, size, checksum, metadata = self.store.add(
@ -227,7 +227,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
VMWARE_DS['vmware_store_image_dir'], VMWARE_DS['vmware_store_image_dir'],
expected_image_id, expected_image_id,
VMWARE_DS['vmware_datastores']) VMWARE_DS['vmware_datastores'])
image = six.BytesIO(expected_contents) image = io.BytesIO(expected_contents)
with mock.patch('requests.Session.request') as HttpConn: with mock.patch('requests.Session.request') as HttpConn:
HttpConn.return_value = utils.fake_response() HttpConn.return_value = utils.fake_response()
location, size, checksum, metadata = self.store.add( location, size, checksum, metadata = self.store.add(
@ -247,7 +247,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
image_id = str(uuid.uuid4()) image_id = str(uuid.uuid4())
size = FIVE_KB size = FIVE_KB
contents = b"*" * size contents = b"*" * size
image = six.BytesIO(contents) image = io.BytesIO(contents)
with mock.patch('requests.Session.request') as HttpConn: with mock.patch('requests.Session.request') as HttpConn:
HttpConn.return_value = utils.fake_response() HttpConn.return_value = utils.fake_response()
location, size, checksum, multihash, metadata = self.store.add( location, size, checksum, multihash, metadata = self.store.add(
@ -264,7 +264,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
image_id = str(uuid.uuid4()) image_id = str(uuid.uuid4())
size = FIVE_KB size = FIVE_KB
contents = b"*" * size contents = b"*" * size
image = six.BytesIO(contents) image = io.BytesIO(contents)
with mock.patch('requests.Session.request') as HttpConn: with mock.patch('requests.Session.request') as HttpConn:
HttpConn.return_value = utils.fake_response() HttpConn.return_value = utils.fake_response()
location, size, checksum, multihash, metadata = self.store.add( location, size, checksum, multihash, metadata = self.store.add(
@ -330,7 +330,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
def test_reader_full(self): def test_reader_full(self):
content = b'XXX' content = b'XXX'
image = six.BytesIO(content) image = io.BytesIO(content)
expected_checksum = secretutils.md5(content, expected_checksum = secretutils.md5(content,
usedforsecurity=False).hexdigest() usedforsecurity=False).hexdigest()
expected_multihash = hashlib.sha256(content).hexdigest() expected_multihash = hashlib.sha256(content).hexdigest()
@ -343,7 +343,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
def test_reader_partial(self): def test_reader_partial(self):
content = b'XXX' content = b'XXX'
image = six.BytesIO(content) image = io.BytesIO(content)
expected_checksum = secretutils.md5(b'X', expected_checksum = secretutils.md5(b'X',
usedforsecurity=False).hexdigest() usedforsecurity=False).hexdigest()
expected_multihash = hashlib.sha256(b'X').hexdigest() expected_multihash = hashlib.sha256(b'X').hexdigest()
@ -356,7 +356,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
def test_reader_with_verifier(self): def test_reader_with_verifier(self):
content = b'XXX' content = b'XXX'
image = six.BytesIO(content) image = io.BytesIO(content)
verifier = mock.MagicMock(name='mock_verifier') verifier = mock.MagicMock(name='mock_verifier')
reader = vm_store._Reader(image, self.hash_algo, verifier) reader = vm_store._Reader(image, self.hash_algo, verifier)
reader.read() reader.read()
@ -413,7 +413,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
expected_image_id = str(uuid.uuid4()) expected_image_id = str(uuid.uuid4())
expected_size = FIVE_KB expected_size = FIVE_KB
expected_contents = b"*" * expected_size expected_contents = b"*" * expected_size
image = six.BytesIO(expected_contents) image = io.BytesIO(expected_contents)
self.session = mock.Mock() self.session = mock.Mock()
with mock.patch('requests.Session.request') as HttpConn: with mock.patch('requests.Session.request') as HttpConn:
HttpConn.return_value = utils.fake_response(status_code=401) HttpConn.return_value = utils.fake_response(status_code=401)
@ -428,7 +428,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
expected_image_id = str(uuid.uuid4()) expected_image_id = str(uuid.uuid4())
expected_size = FIVE_KB expected_size = FIVE_KB
expected_contents = b"*" * expected_size expected_contents = b"*" * expected_size
image = six.BytesIO(expected_contents) image = io.BytesIO(expected_contents)
self.session = mock.Mock() self.session = mock.Mock()
with self._mock_http_connection() as HttpConn: with self._mock_http_connection() as HttpConn:
HttpConn.return_value = utils.fake_response(status_code=500, HttpConn.return_value = utils.fake_response(status_code=500,
@ -470,7 +470,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
expected_image_id = str(uuid.uuid4()) expected_image_id = str(uuid.uuid4())
expected_size = FIVE_KB expected_size = FIVE_KB
expected_contents = b"*" * expected_size expected_contents = b"*" * expected_size
image = six.BytesIO(expected_contents) image = io.BytesIO(expected_contents)
self.session = mock.Mock() self.session = mock.Mock()
with mock.patch('requests.Session.request') as HttpConn: with mock.patch('requests.Session.request') as HttpConn:
HttpConn.request.side_effect = IOError HttpConn.request.side_effect = IOError

10
glance_store/tests/unit/test_rbd_store.py
View File

@ -14,11 +14,11 @@
# under the License. # under the License.
import hashlib import hashlib
import io
from unittest import mock from unittest import mock
from oslo_utils.secretutils import md5 from oslo_utils.secretutils import md5
from oslo_utils import units from oslo_utils import units
import six
from glance_store._drivers import rbd as rbd_store from glance_store._drivers import rbd as rbd_store
from glance_store import exceptions from glance_store import exceptions
@ -315,7 +315,7 @@ class TestStore(base.StoreBaseTest,
self.conf) self.conf)
# Provide enough data to get more than one chunk iteration. # Provide enough data to get more than one chunk iteration.
self.data_len = 3 * units.Ki self.data_len = 3 * units.Ki
self.data_iter = six.BytesIO(b'*' * self.data_len) self.data_iter = io.BytesIO(b'*' * self.data_len)
self.hash_algo = 'sha256' self.hash_algo = 'sha256'
def test_thin_provisioning_is_disabled_by_default(self): def test_thin_provisioning_is_disabled_by_default(self):
@ -409,7 +409,7 @@ class TestStore(base.StoreBaseTest,
image_id = 'fake_image_id' image_id = 'fake_image_id'
file_size = 5 * units.Ki # 5K file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size file_contents = b"*" * file_size
image_file = six.BytesIO(file_contents) image_file = io.BytesIO(file_contents)
with mock.patch.object(rbd_store.rbd.Image, 'write'): with mock.patch.object(rbd_store.rbd.Image, 'write'):
self.store.add(image_id, image_file, file_size, self.hash_algo, self.store.add(image_id, image_file, file_size, self.hash_algo,
@ -422,7 +422,7 @@ class TestStore(base.StoreBaseTest,
image_id = 'fake_image_id' image_id = 'fake_image_id'
file_size = 5 * units.Ki # 5K file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size file_contents = b"*" * file_size
image_file = six.BytesIO(file_contents) image_file = io.BytesIO(file_contents)
expected_checksum = md5(file_contents, expected_checksum = md5(file_contents,
usedforsecurity=False).hexdigest() usedforsecurity=False).hexdigest()
expected_multihash = hashlib.sha256(file_contents).hexdigest() expected_multihash = hashlib.sha256(file_contents).hexdigest()
@ -496,7 +496,7 @@ class TestStore(base.StoreBaseTest,
self.store.configure() self.store.configure()
image_id = 'fake_image_id' image_id = 'fake_image_id'
image_file = six.BytesIO(content) image_file = io.BytesIO(content)
expected_checksum = md5(content, expected_checksum = md5(content,
usedforsecurity=False).hexdigest() usedforsecurity=False).hexdigest()
expected_multihash = hashlib.sha256(content).hexdigest() expected_multihash = hashlib.sha256(content).hexdigest()

14
glance_store/tests/unit/test_s3_store.py
View File

@ -16,6 +16,7 @@
"""Tests the S3 backend store""" """Tests the S3 backend store"""
import hashlib import hashlib
import io
from unittest import mock from unittest import mock
import uuid import uuid
@ -25,7 +26,6 @@ from botocore import exceptions as boto_exceptions
from botocore import stub from botocore import stub
from oslo_utils.secretutils import md5 from oslo_utils.secretutils import md5
from oslo_utils import units from oslo_utils import units
import six
from glance_store._drivers import s3 from glance_store._drivers import s3
from glance_store import capabilities from glance_store import capabilities
@ -89,7 +89,7 @@ class TestStore(base.StoreBaseTest,
"""Test a "normal" retrieval of an image in chunks.""" """Test a "normal" retrieval of an image in chunks."""
bucket, key = 'glance', FAKE_UUID bucket, key = 'glance', FAKE_UUID
fixture_object = { fixture_object = {
'Body': six.BytesIO(b"*" * FIVE_KB), 'Body': io.BytesIO(b"*" * FIVE_KB),
'ContentLength': FIVE_KB 'ContentLength': FIVE_KB
} }
fake_s3_client = botocore.session.get_session().create_client('s3') fake_s3_client = botocore.session.get_session().create_client('s3')
@ -169,7 +169,7 @@ class TestStore(base.StoreBaseTest,
S3_CONF['s3_store_host'], S3_CONF['s3_store_host'],
S3_CONF['s3_store_bucket'], S3_CONF['s3_store_bucket'],
expected_image_id) expected_image_id)
image_s3 = six.BytesIO(expected_s3_contents) image_s3 = io.BytesIO(expected_s3_contents)
fake_s3_client = botocore.session.get_session().create_client('s3') fake_s3_client = botocore.session.get_session().create_client('s3')
@ -221,7 +221,7 @@ class TestStore(base.StoreBaseTest,
S3_CONF['s3_store_host'], S3_CONF['s3_store_host'],
S3_CONF['s3_store_bucket'], S3_CONF['s3_store_bucket'],
expected_image_id) expected_image_id)
image_s3 = six.BytesIO(expected_s3_contents) image_s3 = io.BytesIO(expected_s3_contents)
fake_s3_client = botocore.session.get_session().create_client('s3') fake_s3_client = botocore.session.get_session().create_client('s3')
@ -262,7 +262,7 @@ class TestStore(base.StoreBaseTest,
expected_image_id = str(uuid.uuid4()) expected_image_id = str(uuid.uuid4())
expected_s3_size = FIVE_KB expected_s3_size = FIVE_KB
expected_s3_contents = b"*" * expected_s3_size expected_s3_contents = b"*" * expected_s3_size
image_s3 = six.BytesIO(expected_s3_contents) image_s3 = io.BytesIO(expected_s3_contents)
fake_s3_client = botocore.session.get_session().create_client('s3') fake_s3_client = botocore.session.get_session().create_client('s3')
verifier = mock.MagicMock(name='mock_verifier') verifier = mock.MagicMock(name='mock_verifier')
@ -294,7 +294,7 @@ class TestStore(base.StoreBaseTest,
S3_CONF['s3_store_host'], S3_CONF['s3_store_host'],
S3_CONF['s3_store_bucket'], S3_CONF['s3_store_bucket'],
expected_image_id) expected_image_id)
image_s3 = six.BytesIO(expected_s3_contents) image_s3 = io.BytesIO(expected_s3_contents)
fake_s3_client = botocore.session.get_session().create_client('s3') fake_s3_client = botocore.session.get_session().create_client('s3')
@ -375,7 +375,7 @@ class TestStore(base.StoreBaseTest,
"""Tests that adding an image with an existing identifier """Tests that adding an image with an existing identifier
raises an appropriate exception raises an appropriate exception
""" """
image_s3 = six.BytesIO(b"never_gonna_make_it") image_s3 = io.BytesIO(b"never_gonna_make_it")
fake_s3_client = botocore.session.get_session().create_client('s3') fake_s3_client = botocore.session.get_session().create_client('s3')

106
glance_store/tests/unit/test_swift_store.py
View File

@ -20,6 +20,9 @@ from unittest import mock
import fixtures import fixtures
import hashlib import hashlib
import http.client
import importlib
import io
import tempfile import tempfile
import uuid import uuid
@ -28,11 +31,6 @@ from oslo_utils import encodeutils
from oslo_utils.secretutils import md5 from oslo_utils.secretutils import md5
from oslo_utils import units from oslo_utils import units
import requests_mock import requests_mock
import six
from six import moves
from six.moves import http_client
# NOTE(jokke): simplified transition to py3, behaves like py2 xrange
from six.moves import range
import swiftclient import swiftclient
from glance_store._drivers.swift import buffered from glance_store._drivers.swift import buffered
@ -92,15 +90,15 @@ class SwiftTests(object):
}, },
} }
fixture_objects = { fixture_objects = {
'glance/%s' % FAKE_UUID: six.BytesIO(b"*" * FIVE_KB), 'glance/%s' % FAKE_UUID: io.BytesIO(b"*" * FIVE_KB),
'glance/%s' % FAKE_UUID2: six.BytesIO(b"*" * FIVE_KB), 'glance/%s' % FAKE_UUID2: io.BytesIO(b"*" * FIVE_KB),
'glance/%s' % FAKE_UUID3: six.BytesIO(), 'glance/%s' % FAKE_UUID3: io.BytesIO(),
} }
def fake_head_container(url, token, container, **kwargs): def fake_head_container(url, token, container, **kwargs):
if container not in fixture_containers: if container not in fixture_containers:
msg = "No container %s found" % container msg = "No container %s found" % container
status = http_client.NOT_FOUND status = http.client.NOT_FOUND
raise swiftclient.ClientException(msg, http_status=status) raise swiftclient.ClientException(msg, http_status=status)
return fixture_container_headers return fixture_container_headers
@ -132,7 +130,7 @@ class SwiftTests(object):
fixture_objects[fixture_key] = None fixture_objects[fixture_key] = None
return etag return etag
if hasattr(contents, 'read'): if hasattr(contents, 'read'):
fixture_object = six.BytesIO() fixture_object = io.BytesIO()
read_len = 0 read_len = 0
chunk = contents.read(CHUNKSIZE) chunk = contents.read(CHUNKSIZE)
checksum = md5(usedforsecurity=False) checksum = md5(usedforsecurity=False)
@ -143,7 +141,7 @@ class SwiftTests(object):
chunk = contents.read(CHUNKSIZE) chunk = contents.read(CHUNKSIZE)
etag = checksum.hexdigest() etag = checksum.hexdigest()
else: else:
fixture_object = six.BytesIO(contents) fixture_object = io.BytesIO(contents)
read_len = len(contents) read_len = len(contents)
etag = md5(fixture_object.getvalue(), etag = md5(fixture_object.getvalue(),
usedforsecurity=False).hexdigest() usedforsecurity=False).hexdigest()
@ -151,7 +149,7 @@ class SwiftTests(object):
msg = ('Image size:%d exceeds Swift max:%d' % msg = ('Image size:%d exceeds Swift max:%d' %
(read_len, MAX_SWIFT_OBJECT_SIZE)) (read_len, MAX_SWIFT_OBJECT_SIZE))
raise swiftclient.ClientException( raise swiftclient.ClientException(
msg, http_status=http_client.REQUEST_ENTITY_TOO_LARGE) msg, http_status=http.client.REQUEST_ENTITY_TOO_LARGE)
fixture_objects[fixture_key] = fixture_object fixture_objects[fixture_key] = fixture_object
fixture_headers[fixture_key] = { fixture_headers[fixture_key] = {
'content-length': read_len, 'content-length': read_len,
@ -161,14 +159,14 @@ class SwiftTests(object):
msg = ("Object PUT failed - Object with key %s already exists" msg = ("Object PUT failed - Object with key %s already exists"
% fixture_key) % fixture_key)
raise swiftclient.ClientException( raise swiftclient.ClientException(
msg, http_status=http_client.CONFLICT) msg, http_status=http.client.CONFLICT)
def fake_get_object(conn, container, name, **kwargs): def fake_get_object(conn, container, name, **kwargs):
# GET returns the tuple (list of headers, file object) # GET returns the tuple (list of headers, file object)
fixture_key = "%s/%s" % (container, name) fixture_key = "%s/%s" % (container, name)
if fixture_key not in fixture_headers: if fixture_key not in fixture_headers:
msg = "Object GET failed" msg = "Object GET failed"
status = http_client.NOT_FOUND status = http.client.NOT_FOUND
raise swiftclient.ClientException(msg, http_status=status) raise swiftclient.ClientException(msg, http_status=status)
byte_range = None byte_range = None
@ -185,7 +183,7 @@ class SwiftTests(object):
chunk_keys = sorted([k for k in fixture_headers.keys() chunk_keys = sorted([k for k in fixture_headers.keys()
if k.startswith(fixture_key) and if k.startswith(fixture_key) and
k != fixture_key]) k != fixture_key])
result = six.BytesIO() result = io.BytesIO()
for key in chunk_keys: for key in chunk_keys:
result.write(fixture_objects[key].getvalue()) result.write(fixture_objects[key].getvalue())
else: else:
@ -193,7 +191,7 @@ class SwiftTests(object):
if byte_range is not None: if byte_range is not None:
start = int(byte_range.split('=')[1].strip('-')) start = int(byte_range.split('=')[1].strip('-'))
result = six.BytesIO(result.getvalue()[start:]) result = io.BytesIO(result.getvalue()[start:])
fixture_headers[fixture_key]['content-length'] = len( fixture_headers[fixture_key]['content-length'] = len(
result.getvalue()) result.getvalue())
@ -206,7 +204,7 @@ class SwiftTests(object):
return fixture_headers[fixture_key] return fixture_headers[fixture_key]
except KeyError: except KeyError:
msg = "Object HEAD failed - Object does not exist" msg = "Object HEAD failed - Object does not exist"
status = http_client.NOT_FOUND status = http.client.NOT_FOUND
raise swiftclient.ClientException(msg, http_status=status) raise swiftclient.ClientException(msg, http_status=status)
def fake_delete_object(url, token, container, name, **kwargs): def fake_delete_object(url, token, container, name, **kwargs):
@ -214,7 +212,7 @@ class SwiftTests(object):
fixture_key = "%s/%s" % (container, name) fixture_key = "%s/%s" % (container, name)
if fixture_key not in fixture_headers: if fixture_key not in fixture_headers:
msg = "Object DELETE failed - Object does not exist" msg = "Object DELETE failed - Object does not exist"
status = http_client.NOT_FOUND status = http.client.NOT_FOUND
raise swiftclient.ClientException(msg, http_status=status) raise swiftclient.ClientException(msg, http_status=status)
else: else:
del fixture_headers[fixture_key] del fixture_headers[fixture_key]
@ -342,7 +340,7 @@ class SwiftTests(object):
(image_swift, image_size) = self.store.get(loc, context=ctxt) (image_swift, image_size) = self.store.get(loc, context=ctxt)
resp_full = b''.join([chunk for chunk in image_swift.wrapped]) resp_full = b''.join([chunk for chunk in image_swift.wrapped])
resp_half = resp_full[:len(resp_full) // 2] resp_half = resp_full[:len(resp_full) // 2]
resp_half = six.BytesIO(resp_half) resp_half = io.BytesIO(resp_half)
manager = self.store.get_manager(loc.store_location, ctxt) manager = self.store.get_manager(loc.store_location, ctxt)
image_swift.wrapped = swift.swift_retry_iter(resp_half, image_size, image_swift.wrapped = swift.swift_retry_iter(resp_half, image_size,
@ -418,7 +416,7 @@ class SwiftTests(object):
mock.Mock(return_value=False)) mock.Mock(return_value=False))
def test_add(self): def test_add(self):
"""Test that we can add an image via the swift backend.""" """Test that we can add an image via the swift backend."""
moves.reload_module(swift) importlib.reload(swift)
self.mock_keystone_client() self.mock_keystone_client()
self.store = Store(self.conf) self.store = Store(self.conf)
self.store.configure() self.store.configure()
@ -431,7 +429,7 @@ class SwiftTests(object):
expected_image_id = str(uuid.uuid4()) expected_image_id = str(uuid.uuid4())
loc = "swift+https://tenant%%3Auser1:key@localhost:8080/glance/%s" loc = "swift+https://tenant%%3Auser1:key@localhost:8080/glance/%s"
expected_location = loc % (expected_image_id) expected_location = loc % (expected_image_id)
image_swift = six.BytesIO(expected_swift_contents) image_swift = io.BytesIO(expected_swift_contents)
global SWIFT_PUT_OBJECT_CALLS global SWIFT_PUT_OBJECT_CALLS
SWIFT_PUT_OBJECT_CALLS = 0 SWIFT_PUT_OBJECT_CALLS = 0
@ -460,7 +458,7 @@ class SwiftTests(object):
conf = copy.deepcopy(SWIFT_CONF) conf = copy.deepcopy(SWIFT_CONF)
conf['default_swift_reference'] = 'store_2' conf['default_swift_reference'] = 'store_2'
self.config(**conf) self.config(**conf)
moves.reload_module(swift) importlib.reload(swift)
self.mock_keystone_client() self.mock_keystone_client()
self.store = Store(self.conf) self.store = Store(self.conf)
self.store.configure() self.store.configure()
@ -468,7 +466,7 @@ class SwiftTests(object):
expected_swift_size = FIVE_KB expected_swift_size = FIVE_KB
expected_swift_contents = b"*" * expected_swift_size expected_swift_contents = b"*" * expected_swift_size
expected_image_id = str(uuid.uuid4()) expected_image_id = str(uuid.uuid4())
image_swift = six.BytesIO(expected_swift_contents) image_swift = io.BytesIO(expected_swift_contents)
global SWIFT_PUT_OBJECT_CALLS global SWIFT_PUT_OBJECT_CALLS
SWIFT_PUT_OBJECT_CALLS = 0 SWIFT_PUT_OBJECT_CALLS = 0
loc = 'swift+config://store_2/glance/%s' loc = 'swift+config://store_2/glance/%s'
@ -484,7 +482,7 @@ class SwiftTests(object):
conf = copy.deepcopy(SWIFT_CONF) conf = copy.deepcopy(SWIFT_CONF)
conf['default_swift_reference'] = 'store_2' conf['default_swift_reference'] = 'store_2'
self.config(**conf) self.config(**conf)
moves.reload_module(swift) importlib.reload(swift)
self.mock_keystone_client() self.mock_keystone_client()
self.store = Store(self.conf) self.store = Store(self.conf)
self.store.configure() self.store.configure()
@ -492,7 +490,7 @@ class SwiftTests(object):
def fake_put_object_entity_too_large(*args, **kwargs): def fake_put_object_entity_too_large(*args, **kwargs):
msg = "Test Out of Quota" msg = "Test Out of Quota"
raise swiftclient.ClientException( raise swiftclient.ClientException(
msg, http_status=http_client.REQUEST_ENTITY_TOO_LARGE) msg, http_status=http.client.REQUEST_ENTITY_TOO_LARGE)
self.useFixture(fixtures.MockPatch( self.useFixture(fixtures.MockPatch(
'swiftclient.client.put_object', fake_put_object_entity_too_large)) 'swiftclient.client.put_object', fake_put_object_entity_too_large))
@ -500,7 +498,7 @@ class SwiftTests(object):
expected_swift_size = FIVE_KB expected_swift_size = FIVE_KB
expected_swift_contents = b"*" * expected_swift_size expected_swift_contents = b"*" * expected_swift_size
expected_image_id = str(uuid.uuid4()) expected_image_id = str(uuid.uuid4())
image_swift = six.BytesIO(expected_swift_contents) image_swift = io.BytesIO(expected_swift_contents)
self.assertRaises(exceptions.StorageFull, self.store.add, self.assertRaises(exceptions.StorageFull, self.store.add,
expected_image_id, image_swift, expected_image_id, image_swift,
@ -516,7 +514,7 @@ class SwiftTests(object):
expected_container = 'container_' + expected_image_id expected_container = 'container_' + expected_image_id
loc = 'swift+https://some_endpoint/%s/%s' loc = 'swift+https://some_endpoint/%s/%s'
expected_location = loc % (expected_container, expected_image_id) expected_location = loc % (expected_container, expected_image_id)
image_swift = six.BytesIO(expected_swift_contents) image_swift = io.BytesIO(expected_swift_contents)
global SWIFT_PUT_OBJECT_CALLS global SWIFT_PUT_OBJECT_CALLS
SWIFT_PUT_OBJECT_CALLS = 0 SWIFT_PUT_OBJECT_CALLS = 0
@ -577,13 +575,13 @@ class SwiftTests(object):
expected_multihash = \ expected_multihash = \
hashlib.sha256(expected_swift_contents).hexdigest() hashlib.sha256(expected_swift_contents).hexdigest()
image_swift = six.BytesIO(expected_swift_contents) image_swift = io.BytesIO(expected_swift_contents)
global SWIFT_PUT_OBJECT_CALLS global SWIFT_PUT_OBJECT_CALLS
SWIFT_PUT_OBJECT_CALLS = 0 SWIFT_PUT_OBJECT_CALLS = 0
conf['default_swift_reference'] = variation conf['default_swift_reference'] = variation
self.config(**conf) self.config(**conf)
moves.reload_module(swift) importlib.reload(swift)
self.mock_keystone_client() self.mock_keystone_client()
self.store = Store(self.conf) self.store = Store(self.conf)
self.store.configure() self.store.configure()
@ -615,13 +613,13 @@ class SwiftTests(object):
conf['swift_store_create_container_on_put'] = False conf['swift_store_create_container_on_put'] = False
conf['swift_store_container'] = 'noexist' conf['swift_store_container'] = 'noexist'
self.config(**conf) self.config(**conf)
moves.reload_module(swift) importlib.reload(swift)
self.mock_keystone_client() self.mock_keystone_client()
self.store = Store(self.conf) self.store = Store(self.conf)
self.store.configure() self.store.configure()
image_swift = six.BytesIO(b"nevergonnamakeit") image_swift = io.BytesIO(b"nevergonnamakeit")
global SWIFT_PUT_OBJECT_CALLS global SWIFT_PUT_OBJECT_CALLS
SWIFT_PUT_OBJECT_CALLS = 0 SWIFT_PUT_OBJECT_CALLS = 0
@ -656,7 +654,7 @@ class SwiftTests(object):
expected_image_id = str(uuid.uuid4()) expected_image_id = str(uuid.uuid4())
loc = 'swift+config://ref1/noexist/%s' loc = 'swift+config://ref1/noexist/%s'
expected_location = loc % (expected_image_id) expected_location = loc % (expected_image_id)
image_swift = six.BytesIO(expected_swift_contents) image_swift = io.BytesIO(expected_swift_contents)
global SWIFT_PUT_OBJECT_CALLS global SWIFT_PUT_OBJECT_CALLS
SWIFT_PUT_OBJECT_CALLS = 0 SWIFT_PUT_OBJECT_CALLS = 0
@ -665,7 +663,7 @@ class SwiftTests(object):
conf['swift_store_create_container_on_put'] = True conf['swift_store_create_container_on_put'] = True
conf['swift_store_container'] = 'noexist' conf['swift_store_container'] = 'noexist'
self.config(**conf) self.config(**conf)
moves.reload_module(swift) importlib.reload(swift)
self.mock_keystone_client() self.mock_keystone_client()
self.store = Store(self.conf) self.store = Store(self.conf)
self.store.configure() self.store.configure()
@ -704,7 +702,7 @@ class SwiftTests(object):
container = 'randomname_' + expected_image_id[:2] container = 'randomname_' + expected_image_id[:2]
loc = 'swift+config://ref1/%s/%s' loc = 'swift+config://ref1/%s/%s'
expected_location = loc % (container, expected_image_id) expected_location = loc % (container, expected_image_id)
image_swift = six.BytesIO(expected_swift_contents) image_swift = io.BytesIO(expected_swift_contents)
global SWIFT_PUT_OBJECT_CALLS global SWIFT_PUT_OBJECT_CALLS
SWIFT_PUT_OBJECT_CALLS = 0 SWIFT_PUT_OBJECT_CALLS = 0
@ -714,7 +712,7 @@ class SwiftTests(object):
conf['swift_store_container'] = 'randomname' conf['swift_store_container'] = 'randomname'
conf['swift_store_multiple_containers_seed'] = 2 conf['swift_store_multiple_containers_seed'] = 2
self.config(**conf) self.config(**conf)
moves.reload_module(swift) importlib.reload(swift)
self.mock_keystone_client() self.mock_keystone_client()
self.store = Store(self.conf) self.store = Store(self.conf)
@ -750,7 +748,7 @@ class SwiftTests(object):
conf['swift_store_container'] = 'randomname' conf['swift_store_container'] = 'randomname'
conf['swift_store_multiple_containers_seed'] = 2 conf['swift_store_multiple_containers_seed'] = 2
self.config(**conf) self.config(**conf)
moves.reload_module(swift) importlib.reload(swift)
self.mock_keystone_client() self.mock_keystone_client()
expected_image_id = str(uuid.uuid4()) expected_image_id = str(uuid.uuid4())
@ -759,7 +757,7 @@ class SwiftTests(object):
self.store = Store(self.conf) self.store = Store(self.conf)
self.store.configure() self.store.configure()
image_swift = six.BytesIO(b"nevergonnamakeit") image_swift = io.BytesIO(b"nevergonnamakeit")
global SWIFT_PUT_OBJECT_CALLS global SWIFT_PUT_OBJECT_CALLS
SWIFT_PUT_OBJECT_CALLS = 0 SWIFT_PUT_OBJECT_CALLS = 0
@ -787,7 +785,7 @@ class SwiftTests(object):
base_byte = b"12345678" base_byte = b"12345678"
swift_contents = base_byte * (swift_size // 8) swift_contents = base_byte * (swift_size // 8)
image_id = str(uuid.uuid4()) image_id = str(uuid.uuid4())
image_swift = six.BytesIO(swift_contents) image_swift = io.BytesIO(swift_contents)
self.store = Store(self.conf) self.store = Store(self.conf)
self.store.configure() self.store.configure()
@ -834,7 +832,7 @@ class SwiftTests(object):
base_byte = b"12345678" base_byte = b"12345678"
swift_contents = base_byte * (swift_size // 8) swift_contents = base_byte * (swift_size // 8)
image_id = str(uuid.uuid4()) image_id = str(uuid.uuid4())
image_swift = six.BytesIO(swift_contents) image_swift = io.BytesIO(swift_contents)
self.store = Store(self.conf) self.store = Store(self.conf)
self.store.configure() self.store.configure()
@ -873,7 +871,7 @@ class SwiftTests(object):
expected_container = 'container_' + expected_image_id expected_container = 'container_' + expected_image_id
loc = 'swift+https://some_endpoint/%s/%s' loc = 'swift+https://some_endpoint/%s/%s'
expected_location = loc % (expected_container, expected_image_id) expected_location = loc % (expected_container, expected_image_id)
image_swift = six.BytesIO(expected_swift_contents) image_swift = io.BytesIO(expected_swift_contents)
global SWIFT_PUT_OBJECT_CALLS global SWIFT_PUT_OBJECT_CALLS
SWIFT_PUT_OBJECT_CALLS = 0 SWIFT_PUT_OBJECT_CALLS = 0
@ -925,7 +923,7 @@ class SwiftTests(object):
expected_image_id = str(uuid.uuid4()) expected_image_id = str(uuid.uuid4())
loc = 'swift+config://ref1/glance/%s' loc = 'swift+config://ref1/glance/%s'
expected_location = loc % (expected_image_id) expected_location = loc % (expected_image_id)
image_swift = six.BytesIO(expected_swift_contents) image_swift = io.BytesIO(expected_swift_contents)
global SWIFT_PUT_OBJECT_CALLS global SWIFT_PUT_OBJECT_CALLS
SWIFT_PUT_OBJECT_CALLS = 0 SWIFT_PUT_OBJECT_CALLS = 0
@ -980,7 +978,7 @@ class SwiftTests(object):
expected_image_id = str(uuid.uuid4()) expected_image_id = str(uuid.uuid4())
loc = 'swift+config://ref1/glance/%s' loc = 'swift+config://ref1/glance/%s'
expected_location = loc % (expected_image_id) expected_location = loc % (expected_image_id)
image_swift = six.BytesIO(expected_swift_contents) image_swift = io.BytesIO(expected_swift_contents)
global SWIFT_PUT_OBJECT_CALLS global SWIFT_PUT_OBJECT_CALLS
SWIFT_PUT_OBJECT_CALLS = 0 SWIFT_PUT_OBJECT_CALLS = 0
@ -1027,7 +1025,7 @@ class SwiftTests(object):
""" """
self.store = Store(self.conf) self.store = Store(self.conf)
self.store.configure() self.store.configure()
image_swift = six.BytesIO(b"nevergonnamakeit") image_swift = io.BytesIO(b"nevergonnamakeit")
self.assertRaises(exceptions.Duplicate, self.assertRaises(exceptions.Duplicate,
self.store.add, self.store.add,
FAKE_UUID, image_swift, 0, HASH_ALGO) FAKE_UUID, image_swift, 0, HASH_ALGO)
@ -1074,7 +1072,7 @@ class SwiftTests(object):
""" """
conf = copy.deepcopy(SWIFT_CONF) conf = copy.deepcopy(SWIFT_CONF)
self.config(**conf) self.config(**conf)
moves.reload_module(swift) importlib.reload(swift)
self.mock_keystone_client() self.mock_keystone_client()
self.store = Store(self.conf) self.store = Store(self.conf)
self.store.configure() self.store.configure()
@ -1093,7 +1091,7 @@ class SwiftTests(object):
""" """
conf = copy.deepcopy(SWIFT_CONF) conf = copy.deepcopy(SWIFT_CONF)
self.config(**conf) self.config(**conf)
moves.reload_module(swift) importlib.reload(swift)
self.store = Store(self.conf) self.store = Store(self.conf)
self.store.configure() self.store.configure()
@ -1114,7 +1112,7 @@ class SwiftTests(object):
""" """
conf = copy.deepcopy(SWIFT_CONF) conf = copy.deepcopy(SWIFT_CONF)
self.config(**conf) self.config(**conf)
moves.reload_module(swift) importlib.reload(swift)
self.mock_keystone_client() self.mock_keystone_client()
self.store = Store(self.conf) self.store = Store(self.conf)
self.store.configure() self.store.configure()
@ -1134,7 +1132,7 @@ class SwiftTests(object):
""" """
conf = copy.deepcopy(SWIFT_CONF) conf = copy.deepcopy(SWIFT_CONF)
self.config(**conf) self.config(**conf)
moves.reload_module(swift) importlib.reload(swift)
# mock client because v3 uses it to receive auth_info # mock client because v3 uses it to receive auth_info
self.mock_keystone_client() self.mock_keystone_client()
self.store = Store(self.conf) self.store = Store(self.conf)
@ -1153,7 +1151,7 @@ class SwiftTests(object):
""" """
conf = copy.deepcopy(SWIFT_CONF) conf = copy.deepcopy(SWIFT_CONF)
self.config(**conf) self.config(**conf)
moves.reload_module(swift) importlib.reload(swift)
self.store = Store(self.conf) self.store = Store(self.conf)
self.store.configure() self.store.configure()
@ -1192,7 +1190,7 @@ class SwiftTests(object):
conf = copy.deepcopy(SWIFT_CONF) conf = copy.deepcopy(SWIFT_CONF)
self.config(**conf) self.config(**conf)
moves.reload_module(swift) importlib.reload(swift)
self.store = Store(self.conf) self.store = Store(self.conf)
self.store.configure() self.store.configure()
@ -1847,7 +1845,7 @@ class TestMultiTenantStoreContext(base.StoreBaseTest):
store = Store(self.conf) store = Store(self.conf)
store.configure() store.configure()
content = b'Some data' content = b'Some data'
pseudo_file = six.BytesIO(content) pseudo_file = io.BytesIO(content)
store.add('123', pseudo_file, len(content), HASH_ALGO, store.add('123', pseudo_file, len(content), HASH_ALGO,
context=self.ctx) context=self.ctx)
self.assertEqual(b'0123', self.assertEqual(b'0123',
@ -1864,7 +1862,7 @@ class TestCreatingLocations(base.StoreBaseTest):
conf = copy.deepcopy(SWIFT_CONF) conf = copy.deepcopy(SWIFT_CONF)
self.store = Store(self.conf) self.store = Store(self.conf)
self.config(**conf) self.config(**conf)
moves.reload_module(swift) importlib.reload(swift)
self.addCleanup(self.conf.reset) self.addCleanup(self.conf.reset)
service_catalog = [ service_catalog = [
@ -1894,7 +1892,7 @@ class TestCreatingLocations(base.StoreBaseTest):
conf.update({'swift_store_config_file': self.swift_config_file}) conf.update({'swift_store_config_file': self.swift_config_file})
conf['default_swift_reference'] = 'ref1' conf['default_swift_reference'] = 'ref1'
self.config(**conf) self.config(**conf)
moves.reload_module(swift) importlib.reload(swift)
store = swift.SingleTenantStore(self.conf) store = swift.SingleTenantStore(self.conf)
store.configure() store.configure()
@ -2107,7 +2105,7 @@ class TestBufferedReader(base.StoreBaseTest):
super(TestBufferedReader, self).setUp() super(TestBufferedReader, self).setUp()
self.config(swift_upload_buffer_dir=self.test_dir) self.config(swift_upload_buffer_dir=self.test_dir)
s = b'1234567890' s = b'1234567890'
self.infile = six.BytesIO(s) self.infile = io.BytesIO(s)
self.infile.seek(0) self.infile.seek(0)
self.checksum = md5(usedforsecurity=False) self.checksum = md5(usedforsecurity=False)
@ -2286,7 +2284,7 @@ class TestBufferedReader(base.StoreBaseTest):
# simulate testing where there is less in the buffer than a # simulate testing where there is less in the buffer than a
# full segment # full segment
s = b'12' s = b'12'
infile = six.BytesIO(s) infile = io.BytesIO(s)
infile.seek(0) infile.seek(0)
total = 7 total = 7
checksum = md5(usedforsecurity=False) checksum = md5(usedforsecurity=False)

94
glance_store/tests/unit/test_swift_store_multibackend.py
View File

@ -20,6 +20,9 @@ from unittest import mock
import fixtures import fixtures
import hashlib import hashlib
import http.client
import importlib
import io
import tempfile import tempfile
import uuid import uuid
@ -28,11 +31,6 @@ from oslo_utils import encodeutils
from oslo_utils.secretutils import md5 from oslo_utils.secretutils import md5
from oslo_utils import units from oslo_utils import units
import requests_mock import requests_mock
import six
from six import moves
from six.moves import http_client
# NOTE(jokke): simplified transition to py3, behaves like py2 xrange
from six.moves import range
import swiftclient import swiftclient
from glance_store._drivers.swift import connection_manager as manager from glance_store._drivers.swift import connection_manager as manager
@ -84,14 +82,14 @@ class SwiftTests(object):
'glance/%s' % FAKE_UUID2: {'x-static-large-object': 'true', }, 'glance/%s' % FAKE_UUID2: {'x-static-large-object': 'true', },
} }
fixture_objects = { fixture_objects = {
'glance/%s' % FAKE_UUID: six.BytesIO(b"*" * FIVE_KB), 'glance/%s' % FAKE_UUID: io.BytesIO(b"*" * FIVE_KB),
'glance/%s' % FAKE_UUID2: six.BytesIO(b"*" * FIVE_KB), 'glance/%s' % FAKE_UUID2: io.BytesIO(b"*" * FIVE_KB),
} }
def fake_head_container(url, token, container, **kwargs): def fake_head_container(url, token, container, **kwargs):
if container not in fixture_containers: if container not in fixture_containers:
msg = "No container %s found" % container msg = "No container %s found" % container
status = http_client.NOT_FOUND status = http.client.NOT_FOUND
raise swiftclient.ClientException(msg, http_status=status) raise swiftclient.ClientException(msg, http_status=status)
return fixture_container_headers return fixture_container_headers
@ -123,7 +121,7 @@ class SwiftTests(object):
fixture_objects[fixture_key] = None fixture_objects[fixture_key] = None
return etag return etag
if hasattr(contents, 'read'): if hasattr(contents, 'read'):
fixture_object = six.BytesIO() fixture_object = io.BytesIO()
read_len = 0 read_len = 0
chunk = contents.read(CHUNKSIZE) chunk = contents.read(CHUNKSIZE)
checksum = md5(usedforsecurity=False) checksum = md5(usedforsecurity=False)
@ -134,7 +132,7 @@ class SwiftTests(object):
chunk = contents.read(CHUNKSIZE) chunk = contents.read(CHUNKSIZE)
etag = checksum.hexdigest() etag = checksum.hexdigest()
else: else:
fixture_object = six.BytesIO(contents) fixture_object = io.BytesIO(contents)
read_len = len(contents) read_len = len(contents)
etag = md5(fixture_object.getvalue(), etag = md5(fixture_object.getvalue(),
usedforsecurity=False).hexdigest() usedforsecurity=False).hexdigest()
@ -142,7 +140,7 @@ class SwiftTests(object):
msg = ('Image size:%d exceeds Swift max:%d' % msg = ('Image size:%d exceeds Swift max:%d' %
(read_len, MAX_SWIFT_OBJECT_SIZE)) (read_len, MAX_SWIFT_OBJECT_SIZE))
raise swiftclient.ClientException( raise swiftclient.ClientException(
msg, http_status=http_client.REQUEST_ENTITY_TOO_LARGE) msg, http_status=http.client.REQUEST_ENTITY_TOO_LARGE)
fixture_objects[fixture_key] = fixture_object fixture_objects[fixture_key] = fixture_object
fixture_headers[fixture_key] = { fixture_headers[fixture_key] = {
'content-length': read_len, 'content-length': read_len,
@ -152,14 +150,14 @@ class SwiftTests(object):
msg = ("Object PUT failed - Object with key %s already exists" msg = ("Object PUT failed - Object with key %s already exists"
% fixture_key) % fixture_key)
raise swiftclient.ClientException( raise swiftclient.ClientException(
msg, http_status=http_client.CONFLICT) msg, http_status=http.client.CONFLICT)
def fake_get_object(conn, container, name, **kwargs): def fake_get_object(conn, container, name, **kwargs):
# GET returns the tuple (list of headers, file object) # GET returns the tuple (list of headers, file object)
fixture_key = "%s/%s" % (container, name) fixture_key = "%s/%s" % (container, name)
if fixture_key not in fixture_headers: if fixture_key not in fixture_headers:
msg = "Object GET failed" msg = "Object GET failed"
status = http_client.NOT_FOUND status = http.client.NOT_FOUND
raise swiftclient.ClientException(msg, http_status=status) raise swiftclient.ClientException(msg, http_status=status)
byte_range = None byte_range = None
@ -176,7 +174,7 @@ class SwiftTests(object):
chunk_keys = sorted([k for k in fixture_headers.keys() chunk_keys = sorted([k for k in fixture_headers.keys()
if k.startswith(fixture_key) and if k.startswith(fixture_key) and
k != fixture_key]) k != fixture_key])
result = six.BytesIO() result = io.BytesIO()
for key in chunk_keys: for key in chunk_keys:
result.write(fixture_objects[key].getvalue()) result.write(fixture_objects[key].getvalue())
else: else:
@ -184,7 +182,7 @@ class SwiftTests(object):
if byte_range is not None: if byte_range is not None:
start = int(byte_range.split('=')[1].strip('-')) start = int(byte_range.split('=')[1].strip('-'))
result = six.BytesIO(result.getvalue()[start:]) result = io.BytesIO(result.getvalue()[start:])
fixture_headers[fixture_key]['content-length'] = len( fixture_headers[fixture_key]['content-length'] = len(
result.getvalue()) result.getvalue())
@ -197,7 +195,7 @@ class SwiftTests(object):
return fixture_headers[fixture_key] return fixture_headers[fixture_key]
except KeyError: except KeyError:
msg = "Object HEAD failed - Object does not exist" msg = "Object HEAD failed - Object does not exist"
status = http_client.NOT_FOUND status = http.client.NOT_FOUND
raise swiftclient.ClientException(msg, http_status=status) raise swiftclient.ClientException(msg, http_status=status)
def fake_delete_object(url, token, container, name, **kwargs): def fake_delete_object(url, token, container, name, **kwargs):
@ -205,7 +203,7 @@ class SwiftTests(object):
fixture_key = "%s/%s" % (container, name) fixture_key = "%s/%s" % (container, name)
if fixture_key not in fixture_headers: if fixture_key not in fixture_headers:
msg = "Object DELETE failed - Object does not exist" msg = "Object DELETE failed - Object does not exist"
status = http_client.NOT_FOUND status = http.client.NOT_FOUND
raise swiftclient.ClientException(msg, http_status=status) raise swiftclient.ClientException(msg, http_status=status)
else: else:
del fixture_headers[fixture_key] del fixture_headers[fixture_key]
@ -317,7 +315,7 @@ class SwiftTests(object):
(image_swift, image_size) = self.store.get(loc, context=ctxt) (image_swift, image_size) = self.store.get(loc, context=ctxt)
resp_full = b''.join([chunk for chunk in image_swift.wrapped]) resp_full = b''.join([chunk for chunk in image_swift.wrapped])
resp_half = resp_full[:len(resp_full) // 2] resp_half = resp_full[:len(resp_full) // 2]
resp_half = six.BytesIO(resp_half) resp_half = io.BytesIO(resp_half)
manager = self.store.get_manager(loc.store_location, ctxt) manager = self.store.get_manager(loc.store_location, ctxt)
image_swift.wrapped = swift.swift_retry_iter(resp_half, image_size, image_swift.wrapped = swift.swift_retry_iter(resp_half, image_size,
@ -393,7 +391,7 @@ class SwiftTests(object):
mock.Mock(return_value=False)) mock.Mock(return_value=False))
def test_add(self): def test_add(self):
"""Test that we can add an image via the swift backend.""" """Test that we can add an image via the swift backend."""
moves.reload_module(swift) importlib.reload(swift)
self.mock_keystone_client() self.mock_keystone_client()
self.store = Store(self.conf, backend="swift1") self.store = Store(self.conf, backend="swift1")
self.store.configure() self.store.configure()
@ -404,7 +402,7 @@ class SwiftTests(object):
expected_image_id = str(uuid.uuid4()) expected_image_id = str(uuid.uuid4())
loc = "swift+https://tenant%%3Auser1:key@localhost:8080/glance/%s" loc = "swift+https://tenant%%3Auser1:key@localhost:8080/glance/%s"
expected_location = loc % (expected_image_id) expected_location = loc % (expected_image_id)
image_swift = six.BytesIO(expected_swift_contents) image_swift = io.BytesIO(expected_swift_contents)
global SWIFT_PUT_OBJECT_CALLS global SWIFT_PUT_OBJECT_CALLS
SWIFT_PUT_OBJECT_CALLS = 0 SWIFT_PUT_OBJECT_CALLS = 0
@ -433,7 +431,7 @@ class SwiftTests(object):
conf = copy.deepcopy(SWIFT_CONF) conf = copy.deepcopy(SWIFT_CONF)
conf['default_swift_reference'] = 'store_2' conf['default_swift_reference'] = 'store_2'
self.config(group="swift1", **conf) self.config(group="swift1", **conf)
moves.reload_module(swift) importlib.reload(swift)
self.mock_keystone_client() self.mock_keystone_client()
self.store = Store(self.conf, backend="swift1") self.store = Store(self.conf, backend="swift1")
self.store.configure() self.store.configure()
@ -441,7 +439,7 @@ class SwiftTests(object):
expected_swift_size = FIVE_KB expected_swift_size = FIVE_KB
expected_swift_contents = b"*" * expected_swift_size expected_swift_contents = b"*" * expected_swift_size
expected_image_id = str(uuid.uuid4()) expected_image_id = str(uuid.uuid4())
image_swift = six.BytesIO(expected_swift_contents) image_swift = io.BytesIO(expected_swift_contents)
global SWIFT_PUT_OBJECT_CALLS global SWIFT_PUT_OBJECT_CALLS
SWIFT_PUT_OBJECT_CALLS = 0 SWIFT_PUT_OBJECT_CALLS = 0
loc = 'swift+config://store_2/glance/%s' loc = 'swift+config://store_2/glance/%s'
@ -464,7 +462,7 @@ class SwiftTests(object):
expected_container = 'container_' + expected_image_id expected_container = 'container_' + expected_image_id
loc = 'swift+https://some_endpoint/%s/%s' loc = 'swift+https://some_endpoint/%s/%s'
expected_location = loc % (expected_container, expected_image_id) expected_location = loc % (expected_container, expected_image_id)
image_swift = six.BytesIO(expected_swift_contents) image_swift = io.BytesIO(expected_swift_contents)
global SWIFT_PUT_OBJECT_CALLS global SWIFT_PUT_OBJECT_CALLS
SWIFT_PUT_OBJECT_CALLS = 0 SWIFT_PUT_OBJECT_CALLS = 0
@ -526,13 +524,13 @@ class SwiftTests(object):
expected_checksum = \ expected_checksum = \
md5(expected_swift_contents, usedforsecurity=False).hexdigest() md5(expected_swift_contents, usedforsecurity=False).hexdigest()
image_swift = six.BytesIO(expected_swift_contents) image_swift = io.BytesIO(expected_swift_contents)
global SWIFT_PUT_OBJECT_CALLS global SWIFT_PUT_OBJECT_CALLS
SWIFT_PUT_OBJECT_CALLS = 0 SWIFT_PUT_OBJECT_CALLS = 0
conf['default_swift_reference'] = variation conf['default_swift_reference'] = variation
self.config(group="swift1", **conf) self.config(group="swift1", **conf)
moves.reload_module(swift) importlib.reload(swift)
self.mock_keystone_client() self.mock_keystone_client()
self.store = Store(self.conf, backend="swift1") self.store = Store(self.conf, backend="swift1")
self.store.configure() self.store.configure()
@ -565,13 +563,13 @@ class SwiftTests(object):
conf['swift_store_create_container_on_put'] = False conf['swift_store_create_container_on_put'] = False
conf['swift_store_container'] = 'noexist' conf['swift_store_container'] = 'noexist'
self.config(group="swift1", **conf) self.config(group="swift1", **conf)
moves.reload_module(swift) importlib.reload(swift)
self.mock_keystone_client() self.mock_keystone_client()
self.store = Store(self.conf, backend='swift1') self.store = Store(self.conf, backend='swift1')
self.store.configure() self.store.configure()
image_swift = six.BytesIO(b"nevergonnamakeit") image_swift = io.BytesIO(b"nevergonnamakeit")
global SWIFT_PUT_OBJECT_CALLS global SWIFT_PUT_OBJECT_CALLS
SWIFT_PUT_OBJECT_CALLS = 0 SWIFT_PUT_OBJECT_CALLS = 0
@ -604,7 +602,7 @@ class SwiftTests(object):
expected_image_id = str(uuid.uuid4()) expected_image_id = str(uuid.uuid4())
loc = 'swift+config://ref1/noexist/%s' loc = 'swift+config://ref1/noexist/%s'
expected_location = loc % (expected_image_id) expected_location = loc % (expected_image_id)
image_swift = six.BytesIO(expected_swift_contents) image_swift = io.BytesIO(expected_swift_contents)
global SWIFT_PUT_OBJECT_CALLS global SWIFT_PUT_OBJECT_CALLS
SWIFT_PUT_OBJECT_CALLS = 0 SWIFT_PUT_OBJECT_CALLS = 0
@ -613,7 +611,7 @@ class SwiftTests(object):
conf['swift_store_create_container_on_put'] = True conf['swift_store_create_container_on_put'] = True
conf['swift_store_container'] = 'noexist' conf['swift_store_container'] = 'noexist'
self.config(group="swift1", **conf) self.config(group="swift1", **conf)
moves.reload_module(swift) importlib.reload(swift)
self.mock_keystone_client() self.mock_keystone_client()
self.store = Store(self.conf, backend="swift1") self.store = Store(self.conf, backend="swift1")
self.store.configure() self.store.configure()
@ -652,7 +650,7 @@ class SwiftTests(object):
container = 'randomname_' + expected_image_id[:2] container = 'randomname_' + expected_image_id[:2]
loc = 'swift+config://ref1/%s/%s' loc = 'swift+config://ref1/%s/%s'
expected_location = loc % (container, expected_image_id) expected_location = loc % (container, expected_image_id)
image_swift = six.BytesIO(expected_swift_contents) image_swift = io.BytesIO(expected_swift_contents)
global SWIFT_PUT_OBJECT_CALLS global SWIFT_PUT_OBJECT_CALLS
SWIFT_PUT_OBJECT_CALLS = 0 SWIFT_PUT_OBJECT_CALLS = 0
@ -662,7 +660,7 @@ class SwiftTests(object):
conf['swift_store_container'] = 'randomname' conf['swift_store_container'] = 'randomname'
conf['swift_store_multiple_containers_seed'] = 2 conf['swift_store_multiple_containers_seed'] = 2
self.config(group="swift1", **conf) self.config(group="swift1", **conf)
moves.reload_module(swift) importlib.reload(swift)
self.mock_keystone_client() self.mock_keystone_client()
self.store = Store(self.conf, backend="swift1") self.store = Store(self.conf, backend="swift1")
@ -700,7 +698,7 @@ class SwiftTests(object):
conf['swift_store_container'] = 'randomname' conf['swift_store_container'] = 'randomname'
conf['swift_store_multiple_containers_seed'] = 2 conf['swift_store_multiple_containers_seed'] = 2
self.config(group="swift1", **conf) self.config(group="swift1", **conf)
moves.reload_module(swift) importlib.reload(swift)
self.mock_keystone_client() self.mock_keystone_client()
expected_image_id = str(uuid.uuid4()) expected_image_id = str(uuid.uuid4())
@ -709,7 +707,7 @@ class SwiftTests(object):
self.store = Store(self.conf, backend="swift1") self.store = Store(self.conf, backend="swift1")
self.store.configure() self.store.configure()
image_swift = six.BytesIO(b"nevergonnamakeit") image_swift = io.BytesIO(b"nevergonnamakeit")
global SWIFT_PUT_OBJECT_CALLS global SWIFT_PUT_OBJECT_CALLS
SWIFT_PUT_OBJECT_CALLS = 0 SWIFT_PUT_OBJECT_CALLS = 0
@ -737,7 +735,7 @@ class SwiftTests(object):
base_byte = b"12345678" base_byte = b"12345678"
swift_contents = base_byte * (swift_size // 8) swift_contents = base_byte * (swift_size // 8)
image_id = str(uuid.uuid4()) image_id = str(uuid.uuid4())
image_swift = six.BytesIO(swift_contents) image_swift = io.BytesIO(swift_contents)
self.store = Store(self.conf, backend="swift1") self.store = Store(self.conf, backend="swift1")
self.store.configure() self.store.configure()
@ -784,7 +782,7 @@ class SwiftTests(object):
base_byte = b"12345678" base_byte = b"12345678"
swift_contents = base_byte * (swift_size // 8) swift_contents = base_byte * (swift_size // 8)
image_id = str(uuid.uuid4()) image_id = str(uuid.uuid4())
image_swift = six.BytesIO(swift_contents) image_swift = io.BytesIO(swift_contents)
self.store = Store(self.conf, backend="swift1") self.store = Store(self.conf, backend="swift1")
self.store.configure() self.store.configure()
@ -823,7 +821,7 @@ class SwiftTests(object):
expected_container = 'container_' + expected_image_id expected_container = 'container_' + expected_image_id
loc = 'swift+https://some_endpoint/%s/%s' loc = 'swift+https://some_endpoint/%s/%s'
expected_location = loc % (expected_container, expected_image_id) expected_location = loc % (expected_container, expected_image_id)
image_swift = six.BytesIO(expected_swift_contents) image_swift = io.BytesIO(expected_swift_contents)
global SWIFT_PUT_OBJECT_CALLS global SWIFT_PUT_OBJECT_CALLS
SWIFT_PUT_OBJECT_CALLS = 0 SWIFT_PUT_OBJECT_CALLS = 0
@ -876,7 +874,7 @@ class SwiftTests(object):
expected_image_id = str(uuid.uuid4()) expected_image_id = str(uuid.uuid4())
loc = 'swift+config://ref1/glance/%s' loc = 'swift+config://ref1/glance/%s'
expected_location = loc % (expected_image_id) expected_location = loc % (expected_image_id)
image_swift = six.BytesIO(expected_swift_contents) image_swift = io.BytesIO(expected_swift_contents)
global SWIFT_PUT_OBJECT_CALLS global SWIFT_PUT_OBJECT_CALLS
SWIFT_PUT_OBJECT_CALLS = 0 SWIFT_PUT_OBJECT_CALLS = 0
@ -931,7 +929,7 @@ class SwiftTests(object):
expected_image_id = str(uuid.uuid4()) expected_image_id = str(uuid.uuid4())
loc = 'swift+config://ref1/glance/%s' loc = 'swift+config://ref1/glance/%s'
expected_location = loc % (expected_image_id) expected_location = loc % (expected_image_id)
image_swift = six.BytesIO(expected_swift_contents) image_swift = io.BytesIO(expected_swift_contents)
global SWIFT_PUT_OBJECT_CALLS global SWIFT_PUT_OBJECT_CALLS
SWIFT_PUT_OBJECT_CALLS = 0 SWIFT_PUT_OBJECT_CALLS = 0
@ -986,7 +984,7 @@ class SwiftTests(object):
""" """
self.store = Store(self.conf, backend="swift1") self.store = Store(self.conf, backend="swift1")
self.store.configure() self.store.configure()
image_swift = six.BytesIO(b"nevergonnamakeit") image_swift = io.BytesIO(b"nevergonnamakeit")
self.assertRaises(exceptions.Duplicate, self.assertRaises(exceptions.Duplicate,
self.store.add, self.store.add,
FAKE_UUID, image_swift, 0) FAKE_UUID, image_swift, 0)
@ -1033,7 +1031,7 @@ class SwiftTests(object):
""" """
conf = copy.deepcopy(SWIFT_CONF) conf = copy.deepcopy(SWIFT_CONF)
self.config(group="swift1", **conf) self.config(group="swift1", **conf)
moves.reload_module(swift) importlib.reload(swift)
self.mock_keystone_client() self.mock_keystone_client()
self.store = Store(self.conf, backend="swift1") self.store = Store(self.conf, backend="swift1")
self.store.configure() self.store.configure()
@ -1053,7 +1051,7 @@ class SwiftTests(object):
""" """
conf = copy.deepcopy(SWIFT_CONF) conf = copy.deepcopy(SWIFT_CONF)
self.config(group="swift1", **conf) self.config(group="swift1", **conf)
moves.reload_module(swift) importlib.reload(swift)
self.store = Store(self.conf, backend="swift1") self.store = Store(self.conf, backend="swift1")
self.store.configure() self.store.configure()
@ -1075,7 +1073,7 @@ class SwiftTests(object):
""" """
conf = copy.deepcopy(SWIFT_CONF) conf = copy.deepcopy(SWIFT_CONF)
self.config(group="swift1", **conf) self.config(group="swift1", **conf)
moves.reload_module(swift) importlib.reload(swift)
self.mock_keystone_client() self.mock_keystone_client()
self.store = Store(self.conf, backend="swift1") self.store = Store(self.conf, backend="swift1")
self.store.configure() self.store.configure()
@ -1096,7 +1094,7 @@ class SwiftTests(object):
""" """
conf = copy.deepcopy(SWIFT_CONF) conf = copy.deepcopy(SWIFT_CONF)
self.config(group="swift1", **conf) self.config(group="swift1", **conf)
moves.reload_module(swift) importlib.reload(swift)
# mock client because v3 uses it to receive auth_info # mock client because v3 uses it to receive auth_info
self.mock_keystone_client() self.mock_keystone_client()
self.store = Store(self.conf, backend="swift1") self.store = Store(self.conf, backend="swift1")
@ -1116,7 +1114,7 @@ class SwiftTests(object):
""" """
conf = copy.deepcopy(SWIFT_CONF) conf = copy.deepcopy(SWIFT_CONF)
self.config(group="swift1", **conf) self.config(group="swift1", **conf)
moves.reload_module(swift) importlib.reload(swift)
self.store = Store(self.conf, backend="swift1") self.store = Store(self.conf, backend="swift1")
self.store.configure() self.store.configure()
@ -1155,7 +1153,7 @@ class SwiftTests(object):
conf = copy.deepcopy(SWIFT_CONF) conf = copy.deepcopy(SWIFT_CONF)
self.config(group="swift1", **conf) self.config(group="swift1", **conf)
moves.reload_module(swift) importlib.reload(swift)
self.store = Store(self.conf, backend="swift1") self.store = Store(self.conf, backend="swift1")
self.store.configure() self.store.configure()
@ -1896,7 +1894,7 @@ class TestMultiTenantStoreContext(base.MultiStoreBaseTest):
store = Store(self.conf, backend="swift1") store = Store(self.conf, backend="swift1")
store.configure() store.configure()
content = b'Some data' content = b'Some data'
pseudo_file = six.BytesIO(content) pseudo_file = io.BytesIO(content)
store.add('123', pseudo_file, len(content), store.add('123', pseudo_file, len(content),
context=self.ctx) context=self.ctx)
self.assertEqual(b'0123', self.assertEqual(b'0123',
@ -1939,7 +1937,7 @@ class TestCreatingLocations(base.MultiStoreBaseTest):
self.store.configure() self.store.configure()
self.register_store_backend_schemes(self.store, 'swift', 'swift1') self.register_store_backend_schemes(self.store, 'swift', 'swift1')
moves.reload_module(swift) importlib.reload(swift)
self.addCleanup(self.conf.reset) self.addCleanup(self.conf.reset)
service_catalog = [ service_catalog = [
@ -1969,7 +1967,7 @@ class TestCreatingLocations(base.MultiStoreBaseTest):
conf.update({'swift_store_config_file': self.swift_config_file}) conf.update({'swift_store_config_file': self.swift_config_file})
conf['default_swift_reference'] = 'ref1' conf['default_swift_reference'] = 'ref1'
self.config(group="swift1", **conf) self.config(group="swift1", **conf)
moves.reload_module(swift) importlib.reload(swift)
store = swift.SingleTenantStore(self.conf, backend="swift1") store = swift.SingleTenantStore(self.conf, backend="swift1")
store.configure() store.configure()

26
glance_store/tests/unit/test_vmware_store.py
View File

@ -16,6 +16,7 @@
"""Tests the VMware Datastore backend store""" """Tests the VMware Datastore backend store"""
import hashlib import hashlib
import io
from unittest import mock from unittest import mock
import uuid import uuid
@ -25,7 +26,6 @@ from oslo_vmware import api
from oslo_vmware import exceptions as vmware_exceptions from oslo_vmware import exceptions as vmware_exceptions
from oslo_vmware.objects import datacenter as oslo_datacenter from oslo_vmware.objects import datacenter as oslo_datacenter
from oslo_vmware.objects import datastore as oslo_datastore from oslo_vmware.objects import datastore as oslo_datastore
import six
import glance_store._drivers.vmware_datastore as vm_store import glance_store._drivers.vmware_datastore as vm_store
from glance_store import backend from glance_store import backend
@ -105,7 +105,7 @@ class TestStore(base.StoreBaseTest,
self.hash_algo = 'sha256' self.hash_algo = 'sha256'
def _mock_http_connection(self): def _mock_http_connection(self):
return mock.patch('six.moves.http_client.HTTPConnection') return mock.patch('http.client.HTTPConnection')
@mock.patch('oslo_vmware.api.VMwareAPISession') @mock.patch('oslo_vmware.api.VMwareAPISession')
def test_get(self, mock_api_session): def test_get(self, mock_api_session):
@ -153,7 +153,7 @@ class TestStore(base.StoreBaseTest,
fake_size.__get__ = mock.Mock(return_value=expected_size) fake_size.__get__ = mock.Mock(return_value=expected_size)
expected_cookie = 'vmware_soap_session=fake-uuid' expected_cookie = 'vmware_soap_session=fake-uuid'
fake_cookie.return_value = expected_cookie fake_cookie.return_value = expected_cookie
expected_headers = {'Content-Length': six.text_type(expected_size), expected_headers = {'Content-Length': str(expected_size),
'Cookie': expected_cookie} 'Cookie': expected_cookie}
with mock.patch('hashlib.md5') as md5: with mock.patch('hashlib.md5') as md5:
with mock.patch('hashlib.new') as fake_new: with mock.patch('hashlib.new') as fake_new:
@ -164,7 +164,7 @@ class TestStore(base.StoreBaseTest,
VMWARE_DS['vmware_store_image_dir'], VMWARE_DS['vmware_store_image_dir'],
expected_image_id, expected_image_id,
VMWARE_DS['vmware_datastores']) VMWARE_DS['vmware_datastores'])
image = six.BytesIO(expected_contents) image = io.BytesIO(expected_contents)
with mock.patch('requests.Session.request') as HttpConn: with mock.patch('requests.Session.request') as HttpConn:
HttpConn.return_value = utils.fake_response() HttpConn.return_value = utils.fake_response()
location, size, checksum, multihash, _ = self.store.add( location, size, checksum, multihash, _ = self.store.add(
@ -205,7 +205,7 @@ class TestStore(base.StoreBaseTest,
VMWARE_DS['vmware_store_image_dir'], VMWARE_DS['vmware_store_image_dir'],
expected_image_id, expected_image_id,
VMWARE_DS['vmware_datastores']) VMWARE_DS['vmware_datastores'])
image = six.BytesIO(expected_contents) image = io.BytesIO(expected_contents)
with mock.patch('requests.Session.request') as HttpConn: with mock.patch('requests.Session.request') as HttpConn:
HttpConn.return_value = utils.fake_response() HttpConn.return_value = utils.fake_response()
location, size, checksum, multihash, _ = self.store.add( location, size, checksum, multihash, _ = self.store.add(
@ -224,7 +224,7 @@ class TestStore(base.StoreBaseTest,
image_id = str(uuid.uuid4()) image_id = str(uuid.uuid4())
size = FIVE_KB size = FIVE_KB
contents = b"*" * size contents = b"*" * size
image = six.BytesIO(contents) image = io.BytesIO(contents)
with mock.patch('requests.Session.request') as HttpConn: with mock.patch('requests.Session.request') as HttpConn:
HttpConn.return_value = utils.fake_response() HttpConn.return_value = utils.fake_response()
self.store.add(image_id, image, size, self.hash_algo, self.store.add(image_id, image, size, self.hash_algo,
@ -240,7 +240,7 @@ class TestStore(base.StoreBaseTest,
image_id = str(uuid.uuid4()) image_id = str(uuid.uuid4())
size = FIVE_KB size = FIVE_KB
contents = b"*" * size contents = b"*" * size
image = six.BytesIO(contents) image = io.BytesIO(contents)
with mock.patch('requests.Session.request') as HttpConn: with mock.patch('requests.Session.request') as HttpConn:
HttpConn.return_value = utils.fake_response() HttpConn.return_value = utils.fake_response()
self.store.add(image_id, image, 0, self.hash_algo, self.store.add(image_id, image, 0, self.hash_algo,
@ -303,7 +303,7 @@ class TestStore(base.StoreBaseTest,
def test_reader_full(self): def test_reader_full(self):
content = b'XXX' content = b'XXX'
image = six.BytesIO(content) image = io.BytesIO(content)
expected_checksum = secretutils.md5(content, expected_checksum = secretutils.md5(content,
usedforsecurity=False).hexdigest() usedforsecurity=False).hexdigest()
expected_multihash = hashlib.sha256(content).hexdigest() expected_multihash = hashlib.sha256(content).hexdigest()
@ -316,7 +316,7 @@ class TestStore(base.StoreBaseTest,
def test_reader_partial(self): def test_reader_partial(self):
content = b'XXX' content = b'XXX'
image = six.BytesIO(content) image = io.BytesIO(content)
expected_checksum = secretutils.md5(b'X', expected_checksum = secretutils.md5(b'X',
usedforsecurity=False).hexdigest() usedforsecurity=False).hexdigest()
expected_multihash = hashlib.sha256(b'X').hexdigest() expected_multihash = hashlib.sha256(b'X').hexdigest()
@ -329,7 +329,7 @@ class TestStore(base.StoreBaseTest,
def test_reader_with_verifier(self): def test_reader_with_verifier(self):
content = b'XXX' content = b'XXX'
image = six.BytesIO(content) image = io.BytesIO(content)
verifier = mock.MagicMock(name='mock_verifier') verifier = mock.MagicMock(name='mock_verifier')
reader = vm_store._Reader(image, self.hash_algo, verifier) reader = vm_store._Reader(image, self.hash_algo, verifier)
reader.read() reader.read()
@ -414,7 +414,7 @@ class TestStore(base.StoreBaseTest,
expected_image_id = str(uuid.uuid4()) expected_image_id = str(uuid.uuid4())
expected_size = FIVE_KB expected_size = FIVE_KB
expected_contents = b"*" * expected_size expected_contents = b"*" * expected_size
image = six.BytesIO(expected_contents) image = io.BytesIO(expected_contents)
self.session = mock.Mock() self.session = mock.Mock()
with mock.patch('requests.Session.request') as HttpConn: with mock.patch('requests.Session.request') as HttpConn:
HttpConn.return_value = utils.fake_response(status_code=401) HttpConn.return_value = utils.fake_response(status_code=401)
@ -430,7 +430,7 @@ class TestStore(base.StoreBaseTest,
expected_image_id = str(uuid.uuid4()) expected_image_id = str(uuid.uuid4())
expected_size = FIVE_KB expected_size = FIVE_KB
expected_contents = b"*" * expected_size expected_contents = b"*" * expected_size
image = six.BytesIO(expected_contents) image = io.BytesIO(expected_contents)
self.session = mock.Mock() self.session = mock.Mock()
with self._mock_http_connection() as HttpConn: with self._mock_http_connection() as HttpConn:
HttpConn.return_value = utils.fake_response(status_code=500, HttpConn.return_value = utils.fake_response(status_code=500,
@ -473,7 +473,7 @@ class TestStore(base.StoreBaseTest,
expected_image_id = str(uuid.uuid4()) expected_image_id = str(uuid.uuid4())
expected_size = FIVE_KB expected_size = FIVE_KB
expected_contents = b"*" * expected_size expected_contents = b"*" * expected_size
image = six.BytesIO(expected_contents) image = io.BytesIO(expected_contents)
self.session = mock.Mock() self.session = mock.Mock()
with mock.patch('requests.Session.request') as HttpConn: with mock.patch('requests.Session.request') as HttpConn:
HttpConn.request.side_effect = IOError HttpConn.request.side_effect = IOError

6
glance_store/tests/utils.py
View File

@ -13,8 +13,8 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
import six import io
from six.moves import urllib import urllib.parse
from oslo_utils import units from oslo_utils import units
import requests import requests
@ -45,7 +45,7 @@ def sort_url_by_qs_keys(url):
class FakeHTTPResponse(object): class FakeHTTPResponse(object):
def __init__(self, status=200, headers=None, data=None, *args, **kwargs): def __init__(self, status=200, headers=None, data=None, *args, **kwargs):
data = data or 'I am a teapot, short and stout\n' data = data or 'I am a teapot, short and stout\n'
self.data = six.StringIO(data) self.data = io.StringIO(data)
self.read = self.data.read self.read = self.data.read
self.status = status self.status = status
self.headers = headers or {'content-length': len(data)} self.headers = headers or {'content-length': len(data)}

16
releasenotes/source/conf.py
View File

@ -56,8 +56,8 @@ source_suffix = '.rst'
master_doc = 'index' master_doc = 'index'
# General information about the project. # General information about the project.
project = u'Glance_store Release Notes' project = 'Glance_store Release Notes'
copyright = u'2015, Openstack Foundation' copyright = '2015, Openstack Foundation'
# Release notes are unversioned, so we don't need to set version or release # Release notes are unversioned, so we don't need to set version or release
version = '' version = ''
@ -206,8 +206,8 @@ latex_elements = {
latex_documents = [ latex_documents = [
('index', ('index',
'GlanceStoreReleaseNotes.tex', 'GlanceStoreReleaseNotes.tex',
u'Glance_store Release Notes Documentation', 'Glance_store Release Notes Documentation',
u'Glance_store Developers', 'Glance_store Developers',
'manual'), 'manual'),
] ]
@ -239,8 +239,8 @@ latex_documents = [
man_pages = [ man_pages = [
('index', ('index',
'glancestorereleasenotes', 'glancestorereleasenotes',
u'Glance_store Release Notes Documentation', 'Glance_store Release Notes Documentation',
[u'Glance_store Developers'], ['Glance_store Developers'],
1) 1)
] ]
@ -256,8 +256,8 @@ man_pages = [
texinfo_documents = [ texinfo_documents = [
('index', ('index',
'GlanceStoreReleaseNotes', 'GlanceStoreReleaseNotes',
u'Glance_store Release Notes Documentation', 'Glance_store Release Notes Documentation',
u'Glance_store Developers', 'Glance_store Developers',
'GlanceStoreReleaseNotes', 'GlanceStoreReleaseNotes',
'One line description of project.', 'One line description of project.',
'Miscellaneous'), 'Miscellaneous'),

1
requirements.txt
View File

@ -8,7 +8,6 @@ oslo.utils>=4.7.0 # Apache-2.0
oslo.concurrency>=3.26.0 # Apache-2.0 oslo.concurrency>=3.26.0 # Apache-2.0
stevedore>=1.20.0 # Apache-2.0 stevedore>=1.20.0 # Apache-2.0
eventlet!=0.18.3,!=0.20.1,>=0.18.2 # MIT eventlet!=0.18.3,!=0.20.1,>=0.18.2 # MIT
six>=1.11.0 # MIT
jsonschema>=3.2.0 # MIT jsonschema>=3.2.0 # MIT
keystoneauth1>=3.4.0 # Apache-2.0 keystoneauth1>=3.4.0 # Apache-2.0

3
tools/colorizer.py
View File

@ -42,7 +42,6 @@
"""Display a subunit stream through a colorized unittest test runner.""" """Display a subunit stream through a colorized unittest test runner."""
import heapq import heapq
import six
import subunit import subunit
import sys import sys
import unittest import unittest
@ -277,7 +276,7 @@ class SubunitTestResult(testtools.TestResult):
self.stopTestRun() self.stopTestRun()
def stopTestRun(self): def stopTestRun(self):
for cls in list(six.iterkeys(self.results)): for cls in self.results:
self.writeTestCase(cls) self.writeTestCase(cls)
self.stream.writeln() self.stream.writeln()
self.writeSlowTests() self.writeSlowTests()