openstack/glance_store
Code Issues Proposed changes

Remove Python 2 support

Browse Source 
Python 2 has been deprecated for almost two years, and has not been
guaranteed to work with glance_store for a while. This patch removes all
traces of six, unicode strings and Python 2 tweaks.

Co-Authored-By: Cyril Roelandt <cyril@redhat.com>
Change-Id: Ifa78924d7ecf4f2d9a54c677888ab2926530c487
This commit is contained in:
liyou01 2021-01-07 15:57:43 +08:00 committed by Cyril Roelandt
parent aeee48b561
commit 5ff06df97f
32 changed files with 283 additions and 323 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
doc/source/conf.py
View File

@ -52,8 +52,8 @@ source_suffix = '.rst'
master_doc = 'index'
# General information about the project.
project = u'glance_store'
copyright = u'2014, OpenStack Foundation'
project = 'glance_store'
copyright = '2014, OpenStack Foundation'
# If true, '()' will be appended to :func: etc. cross-reference text.
add_function_parentheses = True

2
glance_store/_drivers/cinder.py
View File

@ -1047,7 +1047,7 @@ class Store(glance_store.driver.Store):
image_metadata = {}
location_url = 'cinder://%s' % volume.id
if self.backend_group:
image_metadata['store'] = u"%s" % self.backend_group
image_metadata['store'] = self.backend_group
location_url = 'cinder://%s/%s' % (self.backend_group,
volume.id)

4
glance_store/_drivers/filesystem.py
View File

@ -22,6 +22,7 @@ import errno
import logging
import os
import stat
import urllib
import jsonschema
from oslo_config import cfg
@ -29,7 +30,6 @@ from oslo_serialization import jsonutils
from oslo_utils import encodeutils
from oslo_utils import excutils
from oslo_utils import units
from six.moves import urllib
import glance_store
from glance_store import capabilities
@ -793,7 +793,7 @@ class Store(glance_store.driver.Store):
# Add store backend information to location metadata
if self.backend_group:
metadata['store'] = u"%s" % self.backend_group
metadata['store'] = self.backend_group
return ('file://%s' % filepath,
bytes_written,

2
glance_store/_drivers/http.py
View File

@ -14,11 +14,11 @@
# under the License.
import logging
import urllib
from oslo_config import cfg
from oslo_utils import encodeutils
from six.moves import urllib
import requests

4
glance_store/_drivers/rbd.py
View File

@ -20,11 +20,11 @@
import contextlib
import logging
import math
import urllib
from oslo_config import cfg
from oslo_utils import encodeutils
from oslo_utils import units
from six.moves import urllib
from glance_store import capabilities
from glance_store.common import utils
@ -651,7 +651,7 @@ class Store(driver.Store):
# Add store backend information to location metadata
metadata = {}
if self.backend_group:
metadata['store'] = u"%s" % self.backend_group
metadata['store'] = self.backend_group
return (loc.get_uri(),
image_size,

10
glance_store/_drivers/s3.py
View File

@ -15,9 +15,11 @@
"""Storage backend for S3 or Storage Servers that follow the S3 Protocol"""
import io
import logging
import math
import re
import urllib
from boto3 import session as boto_session
from botocore import client as boto_client
@ -27,8 +29,6 @@ import eventlet
from oslo_config import cfg
from oslo_utils import encodeutils
from oslo_utils import units
import six
from six.moves import urllib
import glance_store
from glance_store import capabilities
@ -706,7 +706,7 @@ class Store(glance_store.driver.Store):
checksum.update(write_chunk)
if verifier:
verifier.update(write_chunk)
fp = six.BytesIO(write_chunk)
fp = io.BytesIO(write_chunk)
fp.seek(0)
part = UploadPart(mpu, fp, cstart + 1, len(write_chunk))
pool.spawn_n(run_upload, s3_client, bucket, key, part)
@ -721,7 +721,7 @@ class Store(glance_store.driver.Store):
checksum.update(write_chunk)
if verifier:
verifier.update(write_chunk)
fp = six.BytesIO(write_chunk)
fp = io.BytesIO(write_chunk)
fp.seek(0)
part = UploadPart(mpu, fp, cstart + 1, len(write_chunk))
pool.spawn_n(run_upload, s3_client, bucket, key, part)
@ -888,7 +888,7 @@ class Store(glance_store.driver.Store):
{
'PartNumber': pnum,
'ETag': etag
} for pnum, etag in six.iteritems(pedict)
} for pnum, etag in pedict.items()
]
}

44
glance_store/_drivers/swift/store.py
View File

@ -15,8 +15,11 @@
"""Storage backend for SWIFT"""
import http.client
import io
import logging
import math
import urllib.parse
from keystoneauth1.access import service_catalog as keystone_sc
from keystoneauth1 import identity as ks_identity
@ -26,9 +29,6 @@ from oslo_config import cfg
from oslo_utils import encodeutils
from oslo_utils import excutils
from oslo_utils import units
import six
from six.moves import http_client
from six.moves import urllib
try:
import swiftclient
except ImportError:
@ -478,17 +478,13 @@ Related options:
def swift_retry_iter(resp_iter, length, store, location, manager):
if not length and isinstance(resp_iter, six.BytesIO):
if six.PY3:
# On Python 3, io.BytesIO does not have a len attribute, instead
# go the end using seek to get the size of the file
pos = resp_iter.tell()
resp_iter.seek(0, 2)
length = resp_iter.tell()
resp_iter.seek(pos)
else:
# On Python 2, StringIO has a len attribute
length = resp_iter.len
if not length and isinstance(resp_iter, io.BytesIO):
# io.BytesIO does not have a len attribute, instead go the end using
# seek to get the size of the file
pos = resp_iter.tell()
resp_iter.seek(0, 2)
length = resp_iter.tell()
resp_iter.seek(pos)
length = length if length else (resp_iter.len
if hasattr(resp_iter, 'len') else 0)
@ -773,7 +769,7 @@ Store.OPTIONS = _SWIFT_OPTS + sutils.swift_opts + buffered.BUFFERING_OPTS
def _is_slo(slo_header):
if (slo_header is not None and isinstance(slo_header, six.string_types)
if (slo_header is not None and isinstance(slo_header, str)
and slo_header.lower() == 'true'):
return True
@ -836,7 +832,7 @@ class BaseStore(driver.Store):
location.container, location.obj,
resp_chunk_size=self.CHUNKSIZE, headers=headers)
except swiftclient.ClientException as e:
if e.http_status == http_client.NOT_FOUND:
if e.http_status == http.client.NOT_FOUND:
msg = _("Swift could not find object %s.") % location.obj
LOG.warning(msg)
raise exceptions.NotFound(message=msg)
@ -1065,19 +1061,19 @@ class BaseStore(driver.Store):
metadata = {}
if self.backend_group:
metadata['store'] = u"%s" % self.backend_group
metadata['store'] = self.backend_group
return (location.get_uri(credentials_included=include_creds),
image_size, obj_etag, os_hash_value.hexdigest(),
metadata)
except swiftclient.ClientException as e:
if e.http_status == http_client.CONFLICT:
if e.http_status == http.client.CONFLICT:
msg = _("Swift already has an image at this location")
raise exceptions.Duplicate(message=msg)
elif e.http_status == http_client.REQUEST_ENTITY_TOO_LARGE:
elif e.http_status == http.client.REQUEST_ENTITY_TOO_LARGE:
raise exceptions.StorageFull(message=e.msg)
msg = (_(u"Failed to add object to Swift.\n"
msg = (_("Failed to add object to Swift.\n"
"Got error from Swift: %s.")
% encodeutils.exception_to_unicode(e))
LOG.error(msg)
@ -1102,7 +1098,7 @@ class BaseStore(driver.Store):
dlo_manifest = headers.get('x-object-manifest')
slo_manifest = headers.get('x-static-large-object')
except swiftclient.ClientException as e:
if e.http_status != http_client.NOT_FOUND:
if e.http_status != http.client.NOT_FOUND:
raise
if _is_slo(slo_manifest):
@ -1134,7 +1130,7 @@ class BaseStore(driver.Store):
connection.delete_object(location.container, location.obj)
except swiftclient.ClientException as e:
if e.http_status == http_client.NOT_FOUND:
if e.http_status == http.client.NOT_FOUND:
msg = _("Swift could not find image at URI.")
raise exceptions.NotFound(message=msg)
else:
@ -1155,7 +1151,7 @@ class BaseStore(driver.Store):
try:
connection.head_container(container)
except swiftclient.ClientException as e:
if e.http_status == http_client.NOT_FOUND:
if e.http_status == http.client.NOT_FOUND:
if store_conf.swift_store_create_container_on_put:
try:
msg = (_LI("Creating swift container %(container)s") %
@ -1541,7 +1537,7 @@ class MultiTenantStore(BaseStore):
try:
connection.post_container(location.container, headers=headers)
except swiftclient.ClientException as e:
if e.http_status == http_client.NOT_FOUND:
if e.http_status == http.client.NOT_FOUND:
msg = _("Swift could not find image at URI.")
raise exceptions.NotFound(message=msg)
else:

17
glance_store/_drivers/swift/utils.py
View File

@ -12,11 +12,10 @@
# License for the specific language governing permissions and limitations
# under the License.
import configparser
import logging
import sys
from oslo_config import cfg
from six.moves import configparser
from glance_store import exceptions
from glance_store.i18n import _, _LE
@ -104,16 +103,11 @@ _config_defaults = {'user_domain_id': 'default',
'project_domain_id': 'default',
'project_domain_name': 'default'}
if sys.version_info >= (3, 2):
parser_class = configparser.ConfigParser
else:
parser_class = configparser.SafeConfigParser
class SwiftConfigParser(parser_class):
class SwiftConfigParser(configparser.ConfigParser):
def get(self, *args, **kwargs):
value = super(parser_class, self).get(*args, **kwargs)
value = super(configparser.ConfigParser, self).get(*args, **kwargs)
return self._process_quotes(value)
@staticmethod
@ -127,10 +121,7 @@ class SwiftConfigParser(parser_class):
return value
if sys.version_info >= (3,):
CONFIG = SwiftConfigParser(defaults=_config_defaults)
else:
CONFIG = parser_class(defaults=_config_defaults)
CONFIG = SwiftConfigParser(defaults=_config_defaults)
LOG = logging.getLogger(__name__)

19
glance_store/_drivers/vmware_datastore.py
View File

@ -17,6 +17,7 @@
import logging
import os
import urllib.parse
from oslo_config import cfg
from oslo_utils import excutils
@ -31,15 +32,9 @@ try:
except ImportError:
api = None
from six.moves import urllib
import six.moves.urllib.parse as urlparse
import requests
from requests import adapters
from requests.packages.urllib3.util import retry
import six
# NOTE(jokke): simplified transition to py3, behaves like py2 xrange
from six.moves import range
import glance_store
from glance_store import capabilities
@ -349,9 +344,9 @@ class StoreLocation(location.StoreLocation):
Creates a https url that can be used to upload/download data from a
vmware store.
"""
parsed_url = urlparse.urlparse(self.get_uri())
parsed_url = urllib.parse.urlparse(self.get_uri())
new_url = parsed_url._replace(scheme='https')
return urlparse.urlunparse(new_url)
return urllib.parse.urlunparse(new_url)
class Store(glance_store.Store):
@ -597,7 +592,7 @@ class Store(glance_store.Store):
image_file = _Reader(image_file, hashing_algo, verifier)
headers = {}
if image_size > 0:
headers.update({'Content-Length': six.text_type(image_size)})
headers.update({'Content-Length': str(image_size)})
data = image_file
else:
data = utils.chunkiter(image_file, CHUNKSIZE)
@ -656,7 +651,7 @@ class Store(glance_store.Store):
metadata = {}
if self.backend_group:
metadata['store'] = u"%s" % self.backend_group
metadata['store'] = self.backend_group
return (loc.get_uri(),
image_file.size,
@ -804,9 +799,9 @@ class Store(glance_store.Store):
# Note(sabari): The redirect url will have a scheme 'http(s)', but the
# store only accepts url with scheme 'vsphere'. Thus, replacing with
# store's scheme.
parsed_url = urlparse.urlparse(url)
parsed_url = urllib.parse.urlparse(url)
new_url = parsed_url._replace(scheme='vsphere')
vsphere_url = urlparse.urlunparse(new_url)
vsphere_url = urllib.parse.urlunparse(new_url)
return glance_store.location.Location(store_name,
store_class,
self.conf,

3
glance_store/backend.py
View File

@ -19,7 +19,6 @@ import logging
from oslo_config import cfg
from oslo_utils import encodeutils
import six
from stevedore import driver
from stevedore import extension
@ -380,7 +379,7 @@ def check_location_metadata(val, key=''):
for v in val:
check_location_metadata(v, key='%s[%d]' % (key, ndx))
ndx = ndx + 1
elif not isinstance(val, six.text_type):
elif not isinstance(val, str):
raise exceptions.BackendException(_("The image metadata key %(key)s "
"has an invalid type of %(type)s. "
"Only dict, list, and unicode are "

6
glance_store/driver.py
View File

@ -24,7 +24,6 @@ from oslo_config import cfg
from oslo_utils import encodeutils
from oslo_utils import importutils
from oslo_utils import units
import six
from glance_store import capabilities
from glance_store import exceptions
@ -93,7 +92,7 @@ class Store(capabilities.StoreCapability):
self.configure_add()
except exceptions.BadStoreConfiguration as e:
self.unset_capabilities(capabilities.BitMasks.WRITE_ACCESS)
msg = (_(u"Failed to configure store correctly: %s "
msg = (_("Failed to configure store correctly: %s "
"Disabling add method.")
% encodeutils.exception_to_unicode(e))
LOG.warning(msg)
@ -257,8 +256,7 @@ def back_compat_add(store_add_fun):
# everything is present, including hashing_algo
back_compat_required = False
elif ('hashing_algo' in kwargs or
(num_args >= p_algo + 1 and isinstance(args[p_algo],
six.string_types))):
(num_args >= p_algo + 1 and isinstance(args[p_algo], str))):
# there is a hashing_algo argument present
back_compat_required = False
else:

11
glance_store/exceptions.py
View File

@ -15,8 +15,7 @@
"""Glance Store exception subclasses"""
import six
import six.moves.urllib.parse as urlparse
import urllib.parse
from glance_store.i18n import _
@ -31,7 +30,7 @@ class UnsupportedBackend(BackendException):
class RedirectException(Exception):
def __init__(self, url):
self.url = urlparse.urlparse(url)
self.url = urllib.parse.urlparse(url)
class GlanceStoreException(Exception):
@ -55,12 +54,6 @@ class GlanceStoreException(Exception):
self.msg = message
super(GlanceStoreException, self).__init__(message)
def __unicode__(self):
# NOTE(flwang): By default, self.msg is an instance of Message, which
# can't be converted by str(). Based on the definition of
# __unicode__, it should return unicode always.
return six.text_type(self.msg)
class MissingCredentialError(GlanceStoreException):
message = _("Missing required credential: %(required)s")

2
glance_store/location.py
View File

@ -38,9 +38,9 @@ credentials and is **not** user-facing.
"""
import logging
import urllib.parse
from oslo_config import cfg
from six.moves import urllib
from glance_store import exceptions
from glance_store.i18n import _

3
glance_store/multi_backend.py
View File

@ -20,7 +20,6 @@ import logging
from oslo_config import cfg
from oslo_utils import encodeutils
from oslo_utils import units
import six
from stevedore import driver
from stevedore import extension
@ -495,7 +494,7 @@ def check_location_metadata(val, key=''):
for v in val:
check_location_metadata(v, key='%s[%d]' % (key, ndx))
ndx = ndx + 1
elif not isinstance(val, six.text_type):
elif not isinstance(val, str):
raise exceptions.BackendException(_("The image metadata key %(key)s "
"has an invalid type of %(type)s. "
"Only dict, list, and unicode are "

16
glance_store/tests/unit/test_backend.py
View File

@ -122,25 +122,25 @@ class TestStoreAddToBackend(base.StoreBaseTest):
self._good_metadata(metadata)
def test_string(self):
metadata = {'key': u'somevalue'}
metadata = {'key': 'somevalue'}
self._good_metadata(metadata)
def test_list(self):
m = {'key': [u'somevalue', u'2']}
m = {'key': ['somevalue', '2']}
self._good_metadata(m)
def test_unicode_dict(self):
inner = {'key1': u'somevalue', 'key2': u'somevalue'}
inner = {'key1': 'somevalue', 'key2': 'somevalue'}
m = {'topkey': inner}
self._good_metadata(m)
def test_unicode_dict_list(self):
inner = {'key1': u'somevalue', 'key2': u'somevalue'}
m = {'topkey': inner, 'list': [u'somevalue', u'2'], 'u': u'2'}
inner = {'key1': 'somevalue', 'key2': 'somevalue'}
m = {'topkey': inner, 'list': ['somevalue', '2'], 'u': '2'}
self._good_metadata(m)
def test_nested_dict(self):
inner = {'key1': u'somevalue', 'key2': u'somevalue'}
inner = {'key1': 'somevalue', 'key2': 'somevalue'}
inner = {'newkey': inner}
inner = {'anotherkey': inner}
m = {'topkey': inner}
@ -151,9 +151,9 @@ class TestStoreAddToBackend(base.StoreBaseTest):
self._bad_metadata(metadata)
def test_bad_nonunicode_dict_list(self):
inner = {'key1': u'somevalue', 'key2': u'somevalue',
inner = {'key1': 'somevalue', 'key2': 'somevalue',
'k3': [1, object()]}
m = {'topkey': inner, 'list': [u'somevalue', u'2'], 'u': u'2'}
m = {'topkey': inner, 'list': ['somevalue', '2'], 'u': '2'}
self._bad_metadata(m)
def test_bad_metadata_not_dict(self):

8
glance_store/tests/unit/test_cinder_store.py
View File

@ -43,10 +43,10 @@ class TestCinderStore(base.StoreBaseTest,
self.store.READ_CHUNKSIZE = 4096
self.store.WRITE_CHUNKSIZE = 4096
fake_sc = [{u'endpoints': [{u'publicURL': u'http://foo/public_url'}],
u'endpoints_links': [],
u'name': u'cinder',
u'type': u'volumev3'}]
fake_sc = [{'endpoints': [{'publicURL': 'http://foo/public_url'}],
'endpoints_links': [],
'name': 'cinder',
'type': 'volumev3'}]
self.context = mock.MagicMock(service_catalog=fake_sc,
user_id='fake_user',
auth_token='fake_token',

7
glance_store/tests/unit/test_exceptions.py
View File

@ -14,7 +14,6 @@
# under the License.
from oslo_utils import encodeutils
from oslotest import base
import six
import glance_store
@ -23,11 +22,11 @@ class TestExceptions(base.BaseTestCase):
"""Test routines in glance_store.common.utils."""
def test_backend_exception(self):
msg = glance_store.BackendException()
self.assertIn(u'', encodeutils.exception_to_unicode(msg))
self.assertIn('', encodeutils.exception_to_unicode(msg))
def test_unsupported_backend_exception(self):
msg = glance_store.UnsupportedBackend()
self.assertIn(u'', encodeutils.exception_to_unicode(msg))
self.assertIn('', encodeutils.exception_to_unicode(msg))
def test_redirect_exception(self):
# Just checks imports work ok
@ -54,4 +53,4 @@ class TestExceptions(base.BaseTestCase):
def test_non_unicode_error_msg(self):
exc = glance_store.NotFound(str('test'))
self.assertIsInstance(encodeutils.exception_to_unicode(exc),
six.text_type)
str)

42
glance_store/tests/unit/test_filesystem_store.py
View File

@ -15,8 +15,10 @@
"""Tests the filesystem backend store"""
import builtins
import errno
import hashlib
import io
import json
import os
import stat
@ -26,10 +28,6 @@ import uuid
import fixtures
from oslo_utils.secretutils import md5
from oslo_utils import units
import six
from six.moves import builtins
# NOTE(jokke): simplified transition to py3, behaves like py2 xrange
from six.moves import range
from glance_store._drivers import filesystem
from glance_store import exceptions
@ -67,7 +65,7 @@ class TestStore(base.StoreBaseTest,
expected_image_id = str(uuid.uuid4())
expected_file_size = 10
expected_file_contents = b"*" * expected_file_size
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
self.store.FILESYSTEM_STORE_METADATA = in_metadata
return self.store.add(expected_image_id, image_file,
expected_file_size, self.hash_algo)
@ -77,7 +75,7 @@ class TestStore(base.StoreBaseTest,
# First add an image...
image_id = str(uuid.uuid4())
file_contents = b"chunk00000remainder"
image_file = six.BytesIO(file_contents)
image_file = io.BytesIO(file_contents)
loc, size, checksum, multihash, _ = self.store.add(
image_id, image_file, len(file_contents), self.hash_algo)
@ -103,7 +101,7 @@ class TestStore(base.StoreBaseTest,
# First add an image...
image_id = str(uuid.uuid4())
file_contents = b"chunk00000remainder"
image_file = six.BytesIO(file_contents)
image_file = io.BytesIO(file_contents)
loc, size, checksum, multihash, _ = self.store.add(
image_id, image_file, len(file_contents), self.hash_algo)
@ -160,7 +158,7 @@ class TestStore(base.StoreBaseTest,
expected_multihash = hashlib.sha256(expected_file_contents).hexdigest()
expected_location = "file://%s/%s" % (self.test_dir,
expected_image_id)
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
loc, size, checksum, multihash, _ = self.store.add(
expected_image_id, image_file, expected_file_size, self.hash_algo)
@ -254,7 +252,7 @@ class TestStore(base.StoreBaseTest,
group='glance_store')
self.store.configure()
image_file = six.BytesIO(content)
image_file = io.BytesIO(content)
image_id = str(uuid.uuid4())
with mock.patch.object(builtins, 'open') as popen:
self.store.add(image_id, image_file, size, self.hash_algo)
@ -273,7 +271,7 @@ class TestStore(base.StoreBaseTest,
image_id = str(uuid.uuid4())
file_size = units.Ki # 1K
file_contents = b"*" * file_size
image_file = six.BytesIO(file_contents)
image_file = io.BytesIO(file_contents)
self.store.add(image_id, image_file, file_size, self.hash_algo,
verifier=verifier)
@ -310,7 +308,7 @@ class TestStore(base.StoreBaseTest,
group="glance_store")
expected_file_size = 10
expected_file_contents = b"*" * expected_file_size
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
location, size, checksum, multihash, metadata = self.store.add(
expected_image_id, image_file, expected_file_size, self.hash_algo)
@ -326,11 +324,11 @@ class TestStore(base.StoreBaseTest,
image_id = str(uuid.uuid4())
file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size
image_file = six.BytesIO(file_contents)
image_file = io.BytesIO(file_contents)
location, size, checksum, multihash, _ = self.store.add(
image_id, image_file, file_size, self.hash_algo)
image_file = six.BytesIO(b"nevergonnamakeit")
image_file = io.BytesIO(b"nevergonnamakeit")
self.assertRaises(exceptions.Duplicate,
self.store.add,
image_id, image_file, 0, self.hash_algo)
@ -341,7 +339,7 @@ class TestStore(base.StoreBaseTest,
file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size
path = os.path.join(self.test_dir, image_id)
image_file = six.BytesIO(file_contents)
image_file = io.BytesIO(file_contents)
with mock.patch.object(builtins, 'open') as popen:
e = IOError()
@ -392,7 +390,7 @@ class TestStore(base.StoreBaseTest,
file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size
path = os.path.join(self.test_dir, image_id)
image_file = six.BytesIO(file_contents)
image_file = io.BytesIO(file_contents)
def fake_Error(size):
raise AttributeError()
@ -413,7 +411,7 @@ class TestStore(base.StoreBaseTest,
image_id = str(uuid.uuid4())
file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size
image_file = six.BytesIO(file_contents)
image_file = io.BytesIO(file_contents)
loc, size, checksum, multihash, _ = self.store.add(
image_id, image_file, file_size, self.hash_algo)
@ -445,7 +443,7 @@ class TestStore(base.StoreBaseTest,
image_id = str(uuid.uuid4())
file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size
image_file = six.BytesIO(file_contents)
image_file = io.BytesIO(file_contents)
loc, size, checksum, multihash, _ = self.store.add(
image_id, image_file, file_size, self.hash_algo)
@ -611,7 +609,7 @@ class TestStore(base.StoreBaseTest,
expected_multihash = hashlib.sha256(expected_file_contents).hexdigest()
expected_location = "file://%s/%s" % (store_map[1],
expected_image_id)
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
loc, size, checksum, multihash, _ = self.store.add(
expected_image_id, image_file, expected_file_size, self.hash_algo)
@ -659,7 +657,7 @@ class TestStore(base.StoreBaseTest,
expected_multihash = hashlib.sha256(expected_file_contents).hexdigest()
expected_location = "file://%s/%s" % (store_map[1],
expected_image_id)
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
loc, size, checksum, multihash, _ = self.store.add(
expected_image_id, image_file, expected_file_size, self.hash_algo)
@ -709,7 +707,7 @@ class TestStore(base.StoreBaseTest,
expected_image_id = str(uuid.uuid4())
expected_file_size = 5 * units.Ki # 5K
expected_file_contents = b"*" * expected_file_size
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
self.assertRaises(exceptions.StorageFull,
self.store.add,
@ -770,7 +768,7 @@ class TestStore(base.StoreBaseTest,
expected_multihash = hashlib.sha256(expected_file_contents).hexdigest()
expected_location = "file://%s/%s" % (store,
expected_image_id)
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
location, size, checksum, multihash, _ = self.store.add(
expected_image_id, image_file, expected_file_size, self.hash_algo)
@ -813,7 +811,7 @@ class TestStore(base.StoreBaseTest,
expected_multihash = hashlib.sha256(expected_file_contents).hexdigest()
expected_location = "file://%s/%s" % (store,
expected_image_id)
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
location, size, checksum, multihash, _ = self.store.add(
expected_image_id, image_file, expected_file_size, self.hash_algo)

8
glance_store/tests/unit/test_multistore_cinder.py
View File

@ -72,10 +72,10 @@ class TestMultiCinderStore(base.MultiStoreBaseTest,
self.store.READ_CHUNKSIZE = 4096
self.store.WRITE_CHUNKSIZE = 4096
fake_sc = [{u'endpoints': [{u'publicURL': u'http://foo/public_url'}],
u'endpoints_links': [],
u'name': u'cinder',
u'type': u'volumev3'}]
fake_sc = [{'endpoints': [{'publicURL': 'http://foo/public_url'}],
'endpoints_links': [],
'name': 'cinder',
'type': 'volumev3'}]
self.context = mock.MagicMock(service_catalog=fake_sc,
user_id='fake_user',
auth_token='fake_token',

70
glance_store/tests/unit/test_multistore_filesystem.py
View File

@ -15,7 +15,9 @@
"""Tests the filesystem backend store"""
import builtins
import errno
import io
import json
import os
import stat
@ -26,10 +28,6 @@ import fixtures
from oslo_config import cfg
from oslo_utils.secretutils import md5
from oslo_utils import units
import six
from six.moves import builtins
# NOTE(jokke): simplified transition to py3, behaves like py2 xrange
from six.moves import range
import glance_store as store
from glance_store._drivers import filesystem
@ -91,7 +89,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
expected_image_id = str(uuid.uuid4())
expected_file_size = 10
expected_file_contents = b"*" * expected_file_size
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
self.store.FILESYSTEM_STORE_METADATA = in_metadata
return self.store.add(expected_image_id, image_file,
expected_file_size)
@ -105,12 +103,12 @@ class TestMultiStore(base.MultiStoreBaseTest,
# First add an image...
image_id = str(uuid.uuid4())
file_contents = b"chunk00000remainder"
image_file = six.BytesIO(file_contents)
image_file = io.BytesIO(file_contents)
loc, size, checksum, metadata = self.store.add(
image_id, image_file, len(file_contents))
# Check metadata contains 'file1' as a store
self.assertEqual(u"file1", metadata['store'])
self.assertEqual("file1", metadata['store'])
# Now read it back...
uri = "file:///%s/%s" % (self.test_dir, image_id)
@ -134,13 +132,13 @@ class TestMultiStore(base.MultiStoreBaseTest,
# First add an image...
image_id = str(uuid.uuid4())
file_contents = b"chunk00000remainder"
image_file = six.BytesIO(file_contents)
image_file = io.BytesIO(file_contents)
loc, size, checksum, metadata = self.store.add(image_id,
image_file,
len(file_contents))
# Check metadata contains 'file1' as a store
self.assertEqual(u"file1", metadata['store'])
self.assertEqual("file1", metadata['store'])
# Now read it back...
uri = "file:///%s/%s" % (self.test_dir, image_id)
@ -193,7 +191,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
usedforsecurity=False).hexdigest()
expected_location = "file://%s/%s" % (self.test_dir,
expected_image_id)
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
loc, size, checksum, metadata = self.store.add(expected_image_id,
image_file,
@ -202,7 +200,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
self.assertEqual(expected_location, loc)
self.assertEqual(expected_file_size, size)
self.assertEqual(expected_checksum, checksum)
self.assertEqual(u"file1", metadata['store'])
self.assertEqual("file1", metadata['store'])
uri = "file:///%s/%s" % (self.test_dir, expected_image_id)
loc = location.get_location_from_uri_and_backend(
@ -234,7 +232,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
usedforsecurity=False).hexdigest()
expected_location = "file://%s/%s" % (self.test_dir,
expected_image_id)
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
loc, size, checksum, metadata = self.store.add(expected_image_id,
image_file,
@ -243,7 +241,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
self.assertEqual(expected_location, loc)
self.assertEqual(expected_file_size, size)
self.assertEqual(expected_checksum, checksum)
self.assertEqual(u"file2", metadata['store'])
self.assertEqual("file2", metadata['store'])
uri = "file:///%s/%s" % (self.test_dir, expected_image_id)
loc = location.get_location_from_uri_and_backend(
@ -263,20 +261,20 @@ class TestMultiStore(base.MultiStoreBaseTest,
in_metadata = [{'id': 'abcdefg',
'mountpoint': '/xyz/images'}]
location, size, checksum, metadata = self._store_image(in_metadata)
self.assertEqual({'store': u'file1'}, metadata)
self.assertEqual({'store': 'file1'}, metadata)
def test_add_check_metadata_list_with_invalid_mountpoint_locations(self):
in_metadata = [{'id': 'abcdefg', 'mountpoint': '/xyz/images'},
{'id': 'xyz1234', 'mountpoint': '/pqr/images'}]
location, size, checksum, metadata = self._store_image(in_metadata)
self.assertEqual({'store': u'file1'}, metadata)
self.assertEqual({'store': 'file1'}, metadata)
def test_add_check_metadata_list_with_valid_mountpoint_locations(self):
in_metadata = [{'id': 'abcdefg', 'mountpoint': '/tmp'},
{'id': 'xyz1234', 'mountpoint': '/xyz'}]
location, size, checksum, metadata = self._store_image(in_metadata)
self.assertEqual(in_metadata[0], metadata)
self.assertEqual(u"file1", metadata["store"])
self.assertEqual("file1", metadata["store"])
def test_add_check_metadata_bad_nosuch_file(self):
expected_image_id = str(uuid.uuid4())
@ -287,13 +285,13 @@ class TestMultiStore(base.MultiStoreBaseTest,
group="file1")
expected_file_size = 10
expected_file_contents = b"*" * expected_file_size
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
location, size, checksum, metadata = self.store.add(expected_image_id,
image_file,
expected_file_size)
self.assertEqual({'store': u'file1'}, metadata)
self.assertEqual({'store': 'file1'}, metadata)
def test_add_already_existing(self):
"""
@ -304,14 +302,14 @@ class TestMultiStore(base.MultiStoreBaseTest,
image_id = str(uuid.uuid4())
file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size
image_file = six.BytesIO(file_contents)
image_file = io.BytesIO(file_contents)
location, size, checksum, metadata = self.store.add(image_id,
image_file,
file_size)
self.assertEqual(u"file1", metadata["store"])
self.assertEqual("file1", metadata["store"])
image_file = six.BytesIO(b"nevergonnamakeit")
image_file = io.BytesIO(b"nevergonnamakeit")
self.assertRaises(exceptions.Duplicate,
self.store.add,
image_id, image_file, 0)
@ -322,7 +320,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size
path = os.path.join(self.test_dir, image_id)
image_file = six.BytesIO(file_contents)
image_file = io.BytesIO(file_contents)
with mock.patch.object(builtins, 'open') as popen:
e = IOError()
@ -370,7 +368,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size
path = os.path.join(self.test_dir, image_id)
image_file = six.BytesIO(file_contents)
image_file = io.BytesIO(file_contents)
def fake_Error(size):
raise AttributeError()
@ -389,12 +387,12 @@ class TestMultiStore(base.MultiStoreBaseTest,
image_id = str(uuid.uuid4())
file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size
image_file = six.BytesIO(file_contents)
image_file = io.BytesIO(file_contents)
loc, size, checksum, metadata = self.store.add(image_id,
image_file,
file_size)
self.assertEqual(u"file1", metadata["store"])
self.assertEqual("file1", metadata["store"])
# Now check that we can delete it
uri = "file:///%s/%s" % (self.test_dir, image_id)
@ -418,12 +416,12 @@ class TestMultiStore(base.MultiStoreBaseTest,
image_id = str(uuid.uuid4())
file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size
image_file = six.BytesIO(file_contents)
image_file = io.BytesIO(file_contents)
loc, size, checksum, metadata = self.store.add(image_id,
image_file,
file_size)
self.assertEqual(u"file1", metadata["store"])
self.assertEqual("file1", metadata["store"])
uri = "file:///%s/%s" % (self.test_dir, image_id)
loc = location.get_location_from_uri_and_backend(uri, "file1",
@ -589,12 +587,12 @@ class TestMultiStore(base.MultiStoreBaseTest,
usedforsecurity=False).hexdigest()
expected_location = "file://%s/%s" % (store_map[1],
expected_image_id)
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
loc, size, checksum, metadata = self.store.add(expected_image_id,
image_file,
expected_file_size)
self.assertEqual(u"file1", metadata["store"])
self.assertEqual("file1", metadata["store"])
self.assertEqual(expected_location, loc)
self.assertEqual(expected_file_size, size)
@ -637,12 +635,12 @@ class TestMultiStore(base.MultiStoreBaseTest,
usedforsecurity=False).hexdigest()
expected_location = "file://%s/%s" % (store_map[1],
expected_image_id)
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
loc, size, checksum, metadata = self.store.add(expected_image_id,
image_file,
expected_file_size)
self.assertEqual(u"file1", metadata["store"])
self.assertEqual("file1", metadata["store"])
self.assertEqual(expected_location, loc)
self.assertEqual(expected_file_size, size)
@ -689,7 +687,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
expected_image_id = str(uuid.uuid4())
expected_file_size = 5 * units.Ki # 5K
expected_file_contents = b"*" * expected_file_size
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
self.assertRaises(exceptions.StorageFull, self.store.add,
expected_image_id, image_file,
@ -749,12 +747,12 @@ class TestMultiStore(base.MultiStoreBaseTest,
usedforsecurity=False).hexdigest()
expected_location = "file://%s/%s" % (store,
expected_image_id)
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
location, size, checksum, metadata = self.store.add(expected_image_id,
image_file,
expected_file_size)
self.assertEqual(u"file1", metadata["store"])
self.assertEqual("file1", metadata["store"])
self.assertEqual(expected_location, location)
self.assertEqual(expected_file_size, size)
@ -794,12 +792,12 @@ class TestMultiStore(base.MultiStoreBaseTest,
usedforsecurity=False).hexdigest()
expected_location = "file://%s/%s" % (store,
expected_image_id)
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
location, size, checksum, metadata = self.store.add(expected_image_id,
image_file,
expected_file_size)
self.assertEqual(u"file1", metadata["store"])
self.assertEqual("file1", metadata["store"])
self.assertEqual(expected_location, location)
self.assertEqual(expected_file_size, size)

6
glance_store/tests/unit/test_multistore_rbd.py
View File

@ -13,11 +13,11 @@
# License for the specific language governing permissions and limitations
# under the License.
import io
from unittest import mock
from oslo_config import cfg
from oslo_utils import units
import six
import glance_store as store
from glance_store._drivers import rbd as rbd_store
@ -222,7 +222,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
self.conf)
# Provide enough data to get more than one chunk iteration.
self.data_len = 3 * units.Ki
self.data_iter = six.BytesIO(b'*' * self.data_len)
self.data_iter = io.BytesIO(b'*' * self.data_len)
def test_location_url_prefix_is_set(self):
expected_url_prefix = "rbd://"
@ -253,7 +253,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
self.conf)
# Provide enough data to get more than one chunk iteration.
self.data_len = 3 * units.Ki
self.data_iter = six.BytesIO(b'*' * self.data_len)
self.data_iter = io.BytesIO(b'*' * self.data_len)
self.store.chunk_size = units.Ki
with mock.patch.object(rbd_store.rbd.Image, 'resize') as resize:
with mock.patch.object(rbd_store.rbd.Image, 'write') as write:

16
glance_store/tests/unit/test_multistore_s3.py
View File

@ -16,6 +16,7 @@
"""Tests the Multiple S3 backend store"""
import hashlib
import io
from unittest import mock
import uuid
@ -26,7 +27,6 @@ from botocore import stub
from oslo_config import cfg
from oslo_utils.secretutils import md5
from oslo_utils import units
import six
import glance_store as store
from glance_store._drivers import s3
@ -137,7 +137,7 @@ class TestMultiS3Store(base.MultiStoreBaseTest,