openstack
/
glance_store
Code Issues Proposed changes

Remove Python 2 support

Browse Source 
Python 2 has been deprecated for almost two years, and has not been
guaranteed to work with glance_store for a while. This patch removes all
traces of six, unicode strings and Python 2 tweaks.

Co-Authored-By: Cyril Roelandt <cyril@redhat.com>
Change-Id: Ifa78924d7ecf4f2d9a54c677888ab2926530c487
changes/82/769682/4
liyou01 2 years ago committed by Cyril Roelandt
parent aeee48b561
commit 5ff06df97f
32 changed files with 283 additions and 323 deletions
Show Stats Download Patch File Download Diff File

4
doc/source/conf.py
Unescape View File

@ -52,8 +52,8 @@ source_suffix = '.rst'
master_doc = 'index'
# General information about the project.
project = u'glance_store'
copyright = u'2014, OpenStack Foundation'
project = 'glance_store'
copyright = '2014, OpenStack Foundation'
# If true, '()' will be appended to :func: etc. cross-reference text.
add_function_parentheses = True

2
glance_store/_drivers/cinder.py
Unescape View File

@ -1047,7 +1047,7 @@ class Store(glance_store.driver.Store):
image_metadata = {}
location_url = 'cinder://%s' % volume.id
if self.backend_group:
image_metadata['store'] = u"%s" % self.backend_group
image_metadata['store'] = self.backend_group
location_url = 'cinder://%s/%s' % (self.backend_group,
volume.id)

4
glance_store/_drivers/filesystem.py
Unescape View File

@ -22,6 +22,7 @@ import errno
import logging
import os
import stat
import urllib
import jsonschema
from oslo_config import cfg
@ -29,7 +30,6 @@ from oslo_serialization import jsonutils
from oslo_utils import encodeutils
from oslo_utils import excutils
from oslo_utils import units
from six.moves import urllib
import glance_store
from glance_store import capabilities
@ -793,7 +793,7 @@ class Store(glance_store.driver.Store):
# Add store backend information to location metadata
if self.backend_group:
metadata['store'] = u"%s" % self.backend_group
metadata['store'] = self.backend_group
return ('file://%s' % filepath,
bytes_written,

2
glance_store/_drivers/http.py
Unescape View File

@ -14,11 +14,11 @@
# under the License.
import logging
import urllib
from oslo_config import cfg
from oslo_utils import encodeutils
from six.moves import urllib
import requests

4
glance_store/_drivers/rbd.py
Unescape View File

@ -20,11 +20,11 @@
import contextlib
import logging
import math
import urllib
from oslo_config import cfg
from oslo_utils import encodeutils
from oslo_utils import units
from six.moves import urllib
from glance_store import capabilities
from glance_store.common import utils
@ -651,7 +651,7 @@ class Store(driver.Store):
# Add store backend information to location metadata
metadata = {}
if self.backend_group:
metadata['store'] = u"%s" % self.backend_group
metadata['store'] = self.backend_group
return (loc.get_uri(),
image_size,

10
glance_store/_drivers/s3.py
Unescape View File

@ -15,9 +15,11 @@
"""Storage backend for S3 or Storage Servers that follow the S3 Protocol"""
import io
import logging
import math
import re
import urllib
from boto3 import session as boto_session
from botocore import client as boto_client
@ -27,8 +29,6 @@ import eventlet
from oslo_config import cfg
from oslo_utils import encodeutils
from oslo_utils import units
import six
from six.moves import urllib
import glance_store
from glance_store import capabilities
@ -706,7 +706,7 @@ class Store(glance_store.driver.Store):
checksum.update(write_chunk)
if verifier:
verifier.update(write_chunk)
fp = six.BytesIO(write_chunk)
fp = io.BytesIO(write_chunk)
fp.seek(0)
part = UploadPart(mpu, fp, cstart + 1, len(write_chunk))
pool.spawn_n(run_upload, s3_client, bucket, key, part)
@ -721,7 +721,7 @@ class Store(glance_store.driver.Store):
checksum.update(write_chunk)
if verifier:
verifier.update(write_chunk)
fp = six.BytesIO(write_chunk)
fp = io.BytesIO(write_chunk)
fp.seek(0)
part = UploadPart(mpu, fp, cstart + 1, len(write_chunk))
pool.spawn_n(run_upload, s3_client, bucket, key, part)
@ -888,7 +888,7 @@ class Store(glance_store.driver.Store):
{
'PartNumber': pnum,
'ETag': etag
} for pnum, etag in six.iteritems(pedict)
} for pnum, etag in pedict.items()
]
}

44
glance_store/_drivers/swift/store.py
Unescape View File

@ -15,8 +15,11 @@
"""Storage backend for SWIFT"""
import http.client
import io
import logging
import math
import urllib.parse
from keystoneauth1.access import service_catalog as keystone_sc
from keystoneauth1 import identity as ks_identity
@ -26,9 +29,6 @@ from oslo_config import cfg
from oslo_utils import encodeutils
from oslo_utils import excutils
from oslo_utils import units
import six
from six.moves import http_client
from six.moves import urllib
try:
import swiftclient
except ImportError:
@ -478,17 +478,13 @@ Related options:
def swift_retry_iter(resp_iter, length, store, location, manager):
if not length and isinstance(resp_iter, six.BytesIO):
if six.PY3:
# On Python 3, io.BytesIO does not have a len attribute, instead
# go the end using seek to get the size of the file
pos = resp_iter.tell()
resp_iter.seek(0, 2)
length = resp_iter.tell()
resp_iter.seek(pos)
else:
# On Python 2, StringIO has a len attribute
length = resp_iter.len
if not length and isinstance(resp_iter, io.BytesIO):
# io.BytesIO does not have a len attribute, instead go the end using
# seek to get the size of the file
pos = resp_iter.tell()
resp_iter.seek(0, 2)
length = resp_iter.tell()
resp_iter.seek(pos)
length = length if length else (resp_iter.len
if hasattr(resp_iter, 'len') else 0)
@ -773,7 +769,7 @@ Store.OPTIONS = _SWIFT_OPTS + sutils.swift_opts + buffered.BUFFERING_OPTS
def _is_slo(slo_header):
if (slo_header is not None and isinstance(slo_header, six.string_types)
if (slo_header is not None and isinstance(slo_header, str)
and slo_header.lower() == 'true'):
return True
@ -836,7 +832,7 @@ class BaseStore(driver.Store):
location.container, location.obj,
resp_chunk_size=self.CHUNKSIZE, headers=headers)
except swiftclient.ClientException as e:
if e.http_status == http_client.NOT_FOUND:
if e.http_status == http.client.NOT_FOUND:
msg = _("Swift could not find object %s.") % location.obj
LOG.warning(msg)
raise exceptions.NotFound(message=msg)
@ -1065,19 +1061,19 @@ class BaseStore(driver.Store):
metadata = {}
if self.backend_group:
metadata['store'] = u"%s" % self.backend_group
metadata['store'] = self.backend_group
return (location.get_uri(credentials_included=include_creds),
image_size, obj_etag, os_hash_value.hexdigest(),
metadata)
except swiftclient.ClientException as e:
if e.http_status == http_client.CONFLICT:
if e.http_status == http.client.CONFLICT:
msg = _("Swift already has an image at this location")
raise exceptions.Duplicate(message=msg)
elif e.http_status == http_client.REQUEST_ENTITY_TOO_LARGE:
elif e.http_status == http.client.REQUEST_ENTITY_TOO_LARGE:
raise exceptions.StorageFull(message=e.msg)
msg = (_(u"Failed to add object to Swift.\n"
msg = (_("Failed to add object to Swift.\n"
"Got error from Swift: %s.")
% encodeutils.exception_to_unicode(e))
LOG.error(msg)
@ -1102,7 +1098,7 @@ class BaseStore(driver.Store):
dlo_manifest = headers.get('x-object-manifest')
slo_manifest = headers.get('x-static-large-object')
except swiftclient.ClientException as e:
if e.http_status != http_client.NOT_FOUND:
if e.http_status != http.client.NOT_FOUND:
raise
if _is_slo(slo_manifest):
@ -1134,7 +1130,7 @@ class BaseStore(driver.Store):
connection.delete_object(location.container, location.obj)
except swiftclient.ClientException as e:
if e.http_status == http_client.NOT_FOUND:
if e.http_status == http.client.NOT_FOUND:
msg = _("Swift could not find image at URI.")
raise exceptions.NotFound(message=msg)
else:
@ -1155,7 +1151,7 @@ class BaseStore(driver.Store):
try:
connection.head_container(container)
except swiftclient.ClientException as e:
if e.http_status == http_client.NOT_FOUND:
if e.http_status == http.client.NOT_FOUND:
if store_conf.swift_store_create_container_on_put:
try:
msg = (_LI("Creating swift container %(container)s") %
@ -1541,7 +1537,7 @@ class MultiTenantStore(BaseStore):
try:
connection.post_container(location.container, headers=headers)
except swiftclient.ClientException as e:
if e.http_status == http_client.NOT_FOUND:
if e.http_status == http.client.NOT_FOUND:
msg = _("Swift could not find image at URI.")
raise exceptions.NotFound(message=msg)
else:

17
glance_store/_drivers/swift/utils.py
Unescape View File

@ -12,11 +12,10 @@
# License for the specific language governing permissions and limitations
# under the License.
import configparser
import logging
import sys
from oslo_config import cfg
from six.moves import configparser
from glance_store import exceptions
from glance_store.i18n import _, _LE
@ -104,16 +103,11 @@ _config_defaults = {'user_domain_id': 'default',
'project_domain_id': 'default',
'project_domain_name': 'default'}
if sys.version_info >= (3, 2):
parser_class = configparser.ConfigParser
else:
parser_class = configparser.SafeConfigParser
class SwiftConfigParser(parser_class):
class SwiftConfigParser(configparser.ConfigParser):
def get(self, *args, **kwargs):
value = super(parser_class, self).get(*args, **kwargs)
value = super(configparser.ConfigParser, self).get(*args, **kwargs)
return self._process_quotes(value)
@staticmethod
@ -127,10 +121,7 @@ class SwiftConfigParser(parser_class):
return value
if sys.version_info >= (3,):
CONFIG = SwiftConfigParser(defaults=_config_defaults)
else:
CONFIG = parser_class(defaults=_config_defaults)
CONFIG = SwiftConfigParser(defaults=_config_defaults)
LOG = logging.getLogger(__name__)

19
glance_store/_drivers/vmware_datastore.py
Unescape View File

@ -17,6 +17,7 @@
import logging
import os
import urllib.parse
from oslo_config import cfg
from oslo_utils import excutils
@ -31,15 +32,9 @@ try:
except ImportError:
api = None
from six.moves import urllib
import six.moves.urllib.parse as urlparse
import requests
from requests import adapters
from requests.packages.urllib3.util import retry
import six
# NOTE(jokke): simplified transition to py3, behaves like py2 xrange
from six.moves import range
import glance_store
from glance_store import capabilities
@ -349,9 +344,9 @@ class StoreLocation(location.StoreLocation):
Creates a https url that can be used to upload/download data from a
vmware store.
"""
parsed_url = urlparse.urlparse(self.get_uri())
parsed_url = urllib.parse.urlparse(self.get_uri())
new_url = parsed_url._replace(scheme='https')
return urlparse.urlunparse(new_url)
return urllib.parse.urlunparse(new_url)
class Store(glance_store.Store):
@ -597,7 +592,7 @@ class Store(glance_store.Store):
image_file = _Reader(image_file, hashing_algo, verifier)
headers = {}
if image_size > 0:
headers.update({'Content-Length': six.text_type(image_size)})
headers.update({'Content-Length': str(image_size)})
data = image_file
else:
data = utils.chunkiter(image_file, CHUNKSIZE)
@ -656,7 +651,7 @@ class Store(glance_store.Store):
metadata = {}
if self.backend_group:
metadata['store'] = u"%s" % self.backend_group
metadata['store'] = self.backend_group
return (loc.get_uri(),
image_file.size,
@ -804,9 +799,9 @@ class Store(glance_store.Store):
# Note(sabari): The redirect url will have a scheme 'http(s)', but the
# store only accepts url with scheme 'vsphere'. Thus, replacing with
# store's scheme.
parsed_url = urlparse.urlparse(url)
parsed_url = urllib.parse.urlparse(url)
new_url = parsed_url._replace(scheme='vsphere')
vsphere_url = urlparse.urlunparse(new_url)
vsphere_url = urllib.parse.urlunparse(new_url)
return glance_store.location.Location(store_name,
store_class,
self.conf,

3
glance_store/backend.py
Unescape View File

@ -19,7 +19,6 @@ import logging
from oslo_config import cfg
from oslo_utils import encodeutils
import six
from stevedore import driver
from stevedore import extension
@ -380,7 +379,7 @@ def check_location_metadata(val, key=''):
for v in val:
check_location_metadata(v, key='%s[%d]' % (key, ndx))
ndx = ndx + 1
elif not isinstance(val, six.text_type):
elif not isinstance(val, str):
raise exceptions.BackendException(_("The image metadata key %(key)s "
"has an invalid type of %(type)s. "
"Only dict, list, and unicode are "

6
glance_store/driver.py
Unescape View File

@ -24,7 +24,6 @@ from oslo_config import cfg
from oslo_utils import encodeutils
from oslo_utils import importutils
from oslo_utils import units
import six
from glance_store import capabilities
from glance_store import exceptions
@ -93,7 +92,7 @@ class Store(capabilities.StoreCapability):
self.configure_add()
except exceptions.BadStoreConfiguration as e:
self.unset_capabilities(capabilities.BitMasks.WRITE_ACCESS)
msg = (_(u"Failed to configure store correctly: %s "
msg = (_("Failed to configure store correctly: %s "
"Disabling add method.")
% encodeutils.exception_to_unicode(e))
LOG.warning(msg)
@ -257,8 +256,7 @@ def back_compat_add(store_add_fun):
# everything is present, including hashing_algo
back_compat_required = False
elif ('hashing_algo' in kwargs or
(num_args >= p_algo + 1 and isinstance(args[p_algo],
six.string_types))):
(num_args >= p_algo + 1 and isinstance(args[p_algo], str))):
# there is a hashing_algo argument present
back_compat_required = False
else:

11
glance_store/exceptions.py
Unescape View File

@ -15,8 +15,7 @@
"""Glance Store exception subclasses"""
import six
import six.moves.urllib.parse as urlparse
import urllib.parse
from glance_store.i18n import _
@ -31,7 +30,7 @@ class UnsupportedBackend(BackendException):
class RedirectException(Exception):
def __init__(self, url):
self.url = urlparse.urlparse(url)
self.url = urllib.parse.urlparse(url)
class GlanceStoreException(Exception):
@ -55,12 +54,6 @@ class GlanceStoreException(Exception):
self.msg = message
super(GlanceStoreException, self).__init__(message)
def __unicode__(self):
# NOTE(flwang): By default, self.msg is an instance of Message, which
# can't be converted by str(). Based on the definition of
# __unicode__, it should return unicode always.
return six.text_type(self.msg)
class MissingCredentialError(GlanceStoreException):
message = _("Missing required credential: %(required)s")

2
glance_store/location.py
Unescape View File

@ -38,9 +38,9 @@ credentials and is **not** user-facing.
"""
import logging
import urllib.parse
from oslo_config import cfg
from six.moves import urllib
from glance_store import exceptions
from glance_store.i18n import _

3
glance_store/multi_backend.py
Unescape View File

@ -20,7 +20,6 @@ import logging
from oslo_config import cfg
from oslo_utils import encodeutils
from oslo_utils import units
import six
from stevedore import driver
from stevedore import extension
@ -495,7 +494,7 @@ def check_location_metadata(val, key=''):
for v in val:
check_location_metadata(v, key='%s[%d]' % (key, ndx))
ndx = ndx + 1
elif not isinstance(val, six.text_type):
elif not isinstance(val, str):
raise exceptions.BackendException(_("The image metadata key %(key)s "
"has an invalid type of %(type)s. "
"Only dict, list, and unicode are "

16
glance_store/tests/unit/test_backend.py
Unescape View File

@ -122,25 +122,25 @@ class TestStoreAddToBackend(base.StoreBaseTest):
self._good_metadata(metadata)
def test_string(self):
metadata = {'key': u'somevalue'}
metadata = {'key': 'somevalue'}
self._good_metadata(metadata)
def test_list(self):
m = {'key': [u'somevalue', u'2']}
m = {'key': ['somevalue', '2']}
self._good_metadata(m)
def test_unicode_dict(self):
inner = {'key1': u'somevalue', 'key2': u'somevalue'}
inner = {'key1': 'somevalue', 'key2': 'somevalue'}
m = {'topkey': inner}
self._good_metadata(m)
def test_unicode_dict_list(self):
inner = {'key1': u'somevalue', 'key2': u'somevalue'}
m = {'topkey': inner, 'list': [u'somevalue', u'2'], 'u': u'2'}
inner = {'key1': 'somevalue', 'key2': 'somevalue'}
m = {'topkey': inner, 'list': ['somevalue', '2'], 'u': '2'}
self._good_metadata(m)
def test_nested_dict(self):
inner = {'key1': u'somevalue', 'key2': u'somevalue'}
inner = {'key1': 'somevalue', 'key2': 'somevalue'}
inner = {'newkey': inner}
inner = {'anotherkey': inner}
m = {'topkey': inner}
@ -151,9 +151,9 @@ class TestStoreAddToBackend(base.StoreBaseTest):
self._bad_metadata(metadata)
def test_bad_nonunicode_dict_list(self):
inner = {'key1': u'somevalue', 'key2': u'somevalue',
inner = {'key1': 'somevalue', 'key2': 'somevalue',
'k3': [1, object()]}
m = {'topkey': inner, 'list': [u'somevalue', u'2'], 'u': u'2'}
m = {'topkey': inner, 'list': ['somevalue', '2'], 'u': '2'}
self._bad_metadata(m)
def test_bad_metadata_not_dict(self):

8
glance_store/tests/unit/test_cinder_store.py
Unescape View File

@ -43,10 +43,10 @@ class TestCinderStore(base.StoreBaseTest,
self.store.READ_CHUNKSIZE = 4096
self.store.WRITE_CHUNKSIZE = 4096
fake_sc = [{u'endpoints': [{u'publicURL': u'http://foo/public_url'}],
u'endpoints_links': [],
u'name': u'cinder',
u'type': u'volumev3'}]
fake_sc = [{'endpoints': [{'publicURL': 'http://foo/public_url'}],
'endpoints_links': [],
'name': 'cinder',
'type': 'volumev3'}]
self.context = mock.MagicMock(service_catalog=fake_sc,
user_id='fake_user',
auth_token='fake_token',

7
glance_store/tests/unit/test_exceptions.py
Unescape View File

@ -14,7 +14,6 @@
# under the License.
from oslo_utils import encodeutils
from oslotest import base
import six
import glance_store
@ -23,11 +22,11 @@ class TestExceptions(base.BaseTestCase):
"""Test routines in glance_store.common.utils."""
def test_backend_exception(self):
msg = glance_store.BackendException()
self.assertIn(u'', encodeutils.exception_to_unicode(msg))
self.assertIn('', encodeutils.exception_to_unicode(msg))
def test_unsupported_backend_exception(self):
msg = glance_store.UnsupportedBackend()
self.assertIn(u'', encodeutils.exception_to_unicode(msg))
self.assertIn('', encodeutils.exception_to_unicode(msg))
def test_redirect_exception(self):
# Just checks imports work ok
@ -54,4 +53,4 @@ class TestExceptions(base.BaseTestCase):
def test_non_unicode_error_msg(self):
exc = glance_store.NotFound(str('test'))
self.assertIsInstance(encodeutils.exception_to_unicode(exc),
six.text_type)
str)

42
glance_store/tests/unit/test_filesystem_store.py
Unescape View File

@ -15,8 +15,10 @@
"""Tests the filesystem backend store"""
import builtins
import errno
import hashlib
import io
import json
import os
import stat
@ -26,10 +28,6 @@ import uuid
import fixtures
from oslo_utils.secretutils import md5
from oslo_utils import units
import six
from six.moves import builtins
# NOTE(jokke): simplified transition to py3, behaves like py2 xrange
from six.moves import range
from glance_store._drivers import filesystem
from glance_store import exceptions
@ -67,7 +65,7 @@ class TestStore(base.StoreBaseTest,
expected_image_id = str(uuid.uuid4())
expected_file_size = 10
expected_file_contents = b"*" * expected_file_size
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
self.store.FILESYSTEM_STORE_METADATA = in_metadata
return self.store.add(expected_image_id, image_file,
expected_file_size, self.hash_algo)
@ -77,7 +75,7 @@ class TestStore(base.StoreBaseTest,
# First add an image...
image_id = str(uuid.uuid4())
file_contents = b"chunk00000remainder"
image_file = six.BytesIO(file_contents)
image_file = io.BytesIO(file_contents)
loc, size, checksum, multihash, _ = self.store.add(
image_id, image_file, len(file_contents), self.hash_algo)
@ -103,7 +101,7 @@ class TestStore(base.StoreBaseTest,
# First add an image...
image_id = str(uuid.uuid4())
file_contents = b"chunk00000remainder"
image_file = six.BytesIO(file_contents)
image_file = io.BytesIO(file_contents)
loc, size, checksum, multihash, _ = self.store.add(
image_id, image_file, len(file_contents), self.hash_algo)
@ -160,7 +158,7 @@ class TestStore(base.StoreBaseTest,
expected_multihash = hashlib.sha256(expected_file_contents).hexdigest()
expected_location = "file://%s/%s" % (self.test_dir,
expected_image_id)
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
loc, size, checksum, multihash, _ = self.store.add(
expected_image_id, image_file, expected_file_size, self.hash_algo)
@ -254,7 +252,7 @@ class TestStore(base.StoreBaseTest,
group='glance_store')
self.store.configure()
image_file = six.BytesIO(content)
image_file = io.BytesIO(content)
image_id = str(uuid.uuid4())
with mock.patch.object(builtins, 'open') as popen:
self.store.add(image_id, image_file, size, self.hash_algo)
@ -273,7 +271,7 @@ class TestStore(base.StoreBaseTest,
image_id = str(uuid.uuid4())
file_size = units.Ki # 1K
file_contents = b"*" * file_size
image_file = six.BytesIO(file_contents)
image_file = io.BytesIO(file_contents)
self.store.add(image_id, image_file, file_size, self.hash_algo,
verifier=verifier)
@ -310,7 +308,7 @@ class TestStore(base.StoreBaseTest,
group="glance_store")
expected_file_size = 10
expected_file_contents = b"*" * expected_file_size
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
location, size, checksum, multihash, metadata = self.store.add(
expected_image_id, image_file, expected_file_size, self.hash_algo)
@ -326,11 +324,11 @@ class TestStore(base.StoreBaseTest,
image_id = str(uuid.uuid4())
file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size
image_file = six.BytesIO(file_contents)
image_file = io.BytesIO(file_contents)
location, size, checksum, multihash, _ = self.store.add(
image_id, image_file, file_size, self.hash_algo)
image_file = six.BytesIO(b"nevergonnamakeit")
image_file = io.BytesIO(b"nevergonnamakeit")
self.assertRaises(exceptions.Duplicate,
self.store.add,
image_id, image_file, 0, self.hash_algo)
@ -341,7 +339,7 @@ class TestStore(base.StoreBaseTest,
file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size
path = os.path.join(self.test_dir, image_id)
image_file = six.BytesIO(file_contents)
image_file = io.BytesIO(file_contents)
with mock.patch.object(builtins, 'open') as popen:
e = IOError()
@ -392,7 +390,7 @@ class TestStore(base.StoreBaseTest,
file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size
path = os.path.join(self.test_dir, image_id)
image_file = six.BytesIO(file_contents)
image_file = io.BytesIO(file_contents)
def fake_Error(size):
raise AttributeError()
@ -413,7 +411,7 @@ class TestStore(base.StoreBaseTest,
image_id = str(uuid.uuid4())
file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size
image_file = six.BytesIO(file_contents)
image_file = io.BytesIO(file_contents)
loc, size, checksum, multihash, _ = self.store.add(
image_id, image_file, file_size, self.hash_algo)
@ -445,7 +443,7 @@ class TestStore(base.StoreBaseTest,
image_id = str(uuid.uuid4())
file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size
image_file = six.BytesIO(file_contents)
image_file = io.BytesIO(file_contents)
loc, size, checksum, multihash, _ = self.store.add(
image_id, image_file, file_size, self.hash_algo)
@ -611,7 +609,7 @@ class TestStore(base.StoreBaseTest,
expected_multihash = hashlib.sha256(expected_file_contents).hexdigest()
expected_location = "file://%s/%s" % (store_map[1],
expected_image_id)
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
loc, size, checksum, multihash, _ = self.store.add(
expected_image_id, image_file, expected_file_size, self.hash_algo)
@ -659,7 +657,7 @@ class TestStore(base.StoreBaseTest,
expected_multihash = hashlib.sha256(expected_file_contents).hexdigest()
expected_location = "file://%s/%s" % (store_map[1],
expected_image_id)
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
loc, size, checksum, multihash, _ = self.store.add(
expected_image_id, image_file, expected_file_size, self.hash_algo)
@ -709,7 +707,7 @@ class TestStore(base.StoreBaseTest,
expected_image_id = str(uuid.uuid4())
expected_file_size = 5 * units.Ki # 5K
expected_file_contents = b"*" * expected_file_size
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
self.assertRaises(exceptions.StorageFull,
self.store.add,
@ -770,7 +768,7 @@ class TestStore(base.StoreBaseTest,
expected_multihash = hashlib.sha256(expected_file_contents).hexdigest()
expected_location = "file://%s/%s" % (store,
expected_image_id)
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
location, size, checksum, multihash, _ = self.store.add(
expected_image_id, image_file, expected_file_size, self.hash_algo)
@ -813,7 +811,7 @@ class TestStore(base.StoreBaseTest,
expected_multihash = hashlib.sha256(expected_file_contents).hexdigest()
expected_location = "file://%s/%s" % (store,
expected_image_id)
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
location, size, checksum, multihash, _ = self.store.add(
expected_image_id, image_file, expected_file_size, self.hash_algo)

8
glance_store/tests/unit/test_multistore_cinder.py
Unescape View File

@ -72,10 +72,10 @@ class TestMultiCinderStore(base.MultiStoreBaseTest,
self.store.READ_CHUNKSIZE = 4096
self.store.WRITE_CHUNKSIZE = 4096
fake_sc = [{u'endpoints': [{u'publicURL': u'http://foo/public_url'}],
u'endpoints_links': [],
u'name': u'cinder',
u'type': u'volumev3'}]
fake_sc = [{'endpoints': [{'publicURL': 'http://foo/public_url'}],
'endpoints_links': [],
'name': 'cinder',
'type': 'volumev3'}]
self.context = mock.MagicMock(service_catalog=fake_sc,
user_id='fake_user',
auth_token='fake_token',

70
glance_store/tests/unit/test_multistore_filesystem.py
Unescape View File

@ -15,7 +15,9 @@
"""Tests the filesystem backend store"""
import builtins
import errno
import io
import json
import os
import stat
@ -26,10 +28,6 @@ import fixtures
from oslo_config import cfg
from oslo_utils.secretutils import md5
from oslo_utils import units
import six
from six.moves import builtins
# NOTE(jokke): simplified transition to py3, behaves like py2 xrange
from six.moves import range
import glance_store as store
from glance_store._drivers import filesystem
@ -91,7 +89,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
expected_image_id = str(uuid.uuid4())
expected_file_size = 10
expected_file_contents = b"*" * expected_file_size
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
self.store.FILESYSTEM_STORE_METADATA = in_metadata
return self.store.add(expected_image_id, image_file,
expected_file_size)
@ -105,12 +103,12 @@ class TestMultiStore(base.MultiStoreBaseTest,
# First add an image...
image_id = str(uuid.uuid4())
file_contents = b"chunk00000remainder"
image_file = six.BytesIO(file_contents)
image_file = io.BytesIO(file_contents)
loc, size, checksum, metadata = self.store.add(
image_id, image_file, len(file_contents))
# Check metadata contains 'file1' as a store
self.assertEqual(u"file1", metadata['store'])
self.assertEqual("file1", metadata['store'])
# Now read it back...
uri = "file:///%s/%s" % (self.test_dir, image_id)
@ -134,13 +132,13 @@ class TestMultiStore(base.MultiStoreBaseTest,
# First add an image...
image_id = str(uuid.uuid4())
file_contents = b"chunk00000remainder"
image_file = six.BytesIO(file_contents)
image_file = io.BytesIO(file_contents)
loc, size, checksum, metadata = self.store.add(image_id,
image_file,
len(file_contents))
# Check metadata contains 'file1' as a store
self.assertEqual(u"file1", metadata['store'])
self.assertEqual("file1", metadata['store'])
# Now read it back...
uri = "file:///%s/%s" % (self.test_dir, image_id)
@ -193,7 +191,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
usedforsecurity=False).hexdigest()
expected_location = "file://%s/%s" % (self.test_dir,
expected_image_id)
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
loc, size, checksum, metadata = self.store.add(expected_image_id,
image_file,
@ -202,7 +200,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
self.assertEqual(expected_location, loc)
self.assertEqual(expected_file_size, size)
self.assertEqual(expected_checksum, checksum)
self.assertEqual(u"file1", metadata['store'])
self.assertEqual("file1", metadata['store'])
uri = "file:///%s/%s" % (self.test_dir, expected_image_id)
loc = location.get_location_from_uri_and_backend(
@ -234,7 +232,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
usedforsecurity=False).hexdigest()
expected_location = "file://%s/%s" % (self.test_dir,
expected_image_id)
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
loc, size, checksum, metadata = self.store.add(expected_image_id,
image_file,
@ -243,7 +241,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
self.assertEqual(expected_location, loc)
self.assertEqual(expected_file_size, size)
self.assertEqual(expected_checksum, checksum)
self.assertEqual(u"file2", metadata['store'])
self.assertEqual("file2", metadata['store'])
uri = "file:///%s/%s" % (self.test_dir, expected_image_id)
loc = location.get_location_from_uri_and_backend(
@ -263,20 +261,20 @@ class TestMultiStore(base.MultiStoreBaseTest,
in_metadata = [{'id': 'abcdefg',
'mountpoint': '/xyz/images'}]
location, size, checksum, metadata = self._store_image(in_metadata)
self.assertEqual({'store': u'file1'}, metadata)
self.assertEqual({'store': 'file1'}, metadata)
def test_add_check_metadata_list_with_invalid_mountpoint_locations(self):
in_metadata = [{'id': 'abcdefg', 'mountpoint': '/xyz/images'},
{'id': 'xyz1234', 'mountpoint': '/pqr/images'}]
location, size, checksum, metadata = self._store_image(in_metadata)
self.assertEqual({'store': u'file1'}, metadata)
self.assertEqual({'store': 'file1'}, metadata)
def test_add_check_metadata_list_with_valid_mountpoint_locations(self):
in_metadata = [{'id': 'abcdefg', 'mountpoint': '/tmp'},
{'id': 'xyz1234', 'mountpoint': '/xyz'}]
location, size, checksum, metadata = self._store_image(in_metadata)
self.assertEqual(in_metadata[0], metadata)
self.assertEqual(u"file1", metadata["store"])
self.assertEqual("file1", metadata["store"])
def test_add_check_metadata_bad_nosuch_file(self):
expected_image_id = str(uuid.uuid4())
@ -287,13 +285,13 @@ class TestMultiStore(base.MultiStoreBaseTest,
group="file1")
expected_file_size = 10
expected_file_contents = b"*" * expected_file_size
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
location, size, checksum, metadata = self.store.add(expected_image_id,
image_file,
expected_file_size)
self.assertEqual({'store': u'file1'}, metadata)
self.assertEqual({'store': 'file1'}, metadata)
def test_add_already_existing(self):
"""
@ -304,14 +302,14 @@ class TestMultiStore(base.MultiStoreBaseTest,
image_id = str(uuid.uuid4())
file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size
image_file = six.BytesIO(file_contents)
image_file = io.BytesIO(file_contents)
location, size, checksum, metadata = self.store.add(image_id,
image_file,
file_size)
self.assertEqual(u"file1", metadata["store"])
self.assertEqual("file1", metadata["store"])
image_file = six.BytesIO(b"nevergonnamakeit")
image_file = io.BytesIO(b"nevergonnamakeit")
self.assertRaises(exceptions.Duplicate,
self.store.add,
image_id, image_file, 0)
@ -322,7 +320,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size
path = os.path.join(self.test_dir, image_id)
image_file = six.BytesIO(file_contents)
image_file = io.BytesIO(file_contents)
with mock.patch.object(builtins, 'open') as popen:
e = IOError()
@ -370,7 +368,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size
path = os.path.join(self.test_dir, image_id)
image_file = six.BytesIO(file_contents)
image_file = io.BytesIO(file_contents)
def fake_Error(size):
raise AttributeError()
@ -389,12 +387,12 @@ class TestMultiStore(base.MultiStoreBaseTest,
image_id = str(uuid.uuid4())
file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size
image_file = six.BytesIO(file_contents)
image_file = io.BytesIO(file_contents)
loc, size, checksum, metadata = self.store.add(image_id,
image_file,
file_size)
self.assertEqual(u"file1", metadata["store"])
self.assertEqual("file1", metadata["store"])
# Now check that we can delete it
uri = "file:///%s/%s" % (self.test_dir, image_id)
@ -418,12 +416,12 @@ class TestMultiStore(base.MultiStoreBaseTest,
image_id = str(uuid.uuid4())
file_size = 5 * units.Ki # 5K
file_contents = b"*" * file_size
image_file = six.BytesIO(file_contents)
image_file = io.BytesIO(file_contents)
loc, size, checksum, metadata = self.store.add(image_id,
image_file,
file_size)
self.assertEqual(u"file1", metadata["store"])
self.assertEqual("file1", metadata["store"])
uri = "file:///%s/%s" % (self.test_dir, image_id)
loc = location.get_location_from_uri_and_backend(uri, "file1",
@ -589,12 +587,12 @@ class TestMultiStore(base.MultiStoreBaseTest,
usedforsecurity=False).hexdigest()
expected_location = "file://%s/%s" % (store_map[1],
expected_image_id)
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
loc, size, checksum, metadata = self.store.add(expected_image_id,
image_file,
expected_file_size)
self.assertEqual(u"file1", metadata["store"])
self.assertEqual("file1", metadata["store"])
self.assertEqual(expected_location, loc)
self.assertEqual(expected_file_size, size)
@ -637,12 +635,12 @@ class TestMultiStore(base.MultiStoreBaseTest,
usedforsecurity=False).hexdigest()
expected_location = "file://%s/%s" % (store_map[1],
expected_image_id)
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
loc, size, checksum, metadata = self.store.add(expected_image_id,
image_file,
expected_file_size)
self.assertEqual(u"file1", metadata["store"])
self.assertEqual("file1", metadata["store"])
self.assertEqual(expected_location, loc)
self.assertEqual(expected_file_size, size)
@ -689,7 +687,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
expected_image_id = str(uuid.uuid4())
expected_file_size = 5 * units.Ki # 5K
expected_file_contents = b"*" * expected_file_size
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
self.assertRaises(exceptions.StorageFull, self.store.add,
expected_image_id, image_file,
@ -749,12 +747,12 @@ class TestMultiStore(base.MultiStoreBaseTest,
usedforsecurity=False).hexdigest()
expected_location = "file://%s/%s" % (store,
expected_image_id)
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
location, size, checksum, metadata = self.store.add(expected_image_id,
image_file,
expected_file_size)
self.assertEqual(u"file1", metadata["store"])
self.assertEqual("file1", metadata["store"])
self.assertEqual(expected_location, location)
self.assertEqual(expected_file_size, size)
@ -794,12 +792,12 @@ class TestMultiStore(base.MultiStoreBaseTest,
usedforsecurity=False).hexdigest()
expected_location = "file://%s/%s" % (store,
expected_image_id)
image_file = six.BytesIO(expected_file_contents)
image_file = io.BytesIO(expected_file_contents)
location, size, checksum, metadata = self.store.add(expected_image_id,
image_file,
expected_file_size)
self.assertEqual(u"file1", metadata["store"])
self.assertEqual("file1", metadata["store"])
self.assertEqual(expected_location, location)
self.assertEqual(expected_file_size, size)

6
glance_store/tests/unit/test_multistore_rbd.py
Unescape View File

@ -13,11 +13,11 @@
# License for the specific language governing permissions and limitations
# under the License.
import io
from unittest import mock
from oslo_config import cfg
from oslo_utils import units
import six
import glance_store as store
from glance_store._drivers import rbd as rbd_store
@ -222,7 +222,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
self.conf)
# Provide enough data to get more than one chunk iteration.
self.data_len = 3 * units.Ki
self.data_iter = six.BytesIO(b'*' * self.data_len)
self.data_iter = io.BytesIO(b'*' * self.data_len)
def test_location_url_prefix_is_set(self):
expected_url_prefix = "rbd://"
@ -253,7 +253,7 @@ class TestMultiStore(base.MultiStoreBaseTest,
self.conf)
# Provide enough data to get more than one chunk iteration.
self.data_len = 3 * units.Ki
self.data_iter = six.BytesIO(b'*' * self.data_len)
self.data_iter = io.BytesIO(b'*' * self.data_len)
self.store.chunk_size = units.Ki
with mock.patch.object(rbd_store.rbd.Image, 'resize') as resize:
with mock.patch.object(rbd_store.rbd.Image, 'write') as write:

16
glance_store/tests/unit/test_multistore_s3.py
Unescape View File

@ -16,6 +16,7 @@
"""Tests the Multiple S3 backend store"""
import hashlib
import io
from unittest import mock
import uuid
@ -26,7 +27,6 @@ from botocore import stub
from oslo_config import cfg
from oslo_utils.secretutils import md5
from oslo_utils import units
import six
import glance_store as store
from glance_store._drivers import s3
@ -137,7 +137,7 @@ class TestMultiS3Store(base.MultiStoreBaseTest,
"""Test a "normal" retrieval of an image in chunks."""
bucket, key = 'glance', FAKE_UUID
fixture_object = {
'Body': six.BytesIO(b"*" * FIVE_KB),
'Body': io.BytesIO(b"*" * FIVE_KB),
'ContentLength': FIVE_KB
}
fake_s3_client = botocore.session.get_session().create_client('s3')
@ -220,7 +220,7 @@ class TestMultiS3Store(base.MultiStoreBaseTest,
S3_CONF['s3_store_host'],
S3_CONF['s3_store_bucket'],
expected_image_id)
image_s3 = six.BytesIO(expected_s3_contents)
image_s3 = io.BytesIO(expected_s3_contents)
fake_s3_client = botocore.session.get_session().create_client('s3')
@ -273,7 +273,7 @@ class TestMultiS3Store(base.MultiStoreBaseTest,
S3_CONF['s3_store_host'],
S3_CONF['s3_store_bucket'],
expected_image_id)
image_s3 = six.BytesIO(expected_s3_contents)
image_s3 = io.BytesIO(expected_s3_contents)
fake_s3_client = botocore.session.get_session().create_client('s3')
@ -327,7 +327,7 @@ class TestMultiS3Store(base.MultiStoreBaseTest,