openstack/swift
Code Issues Proposed changes

Merge "py3: object server"

Browse Source
This commit is contained in:
Zuul 2019-01-15 05:49:42 +00:00 committed by Gerrit Code Review
commit 8c030b75ce
6 changed files with 487 additions and 413 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
swift/common/swob.py
View File

@ -282,12 +282,16 @@ class HeaderEnvironProxy(MutableMapping):
def wsgi_to_bytes(wsgi_str):
if wsgi_str is None:
return None
if six.PY2:
return wsgi_str
return wsgi_str.encode('latin1')
def wsgi_to_str(wsgi_str):
if wsgi_str is None:
return None
if six.PY2:
return wsgi_str
return wsgi_to_bytes(wsgi_str).decode('utf8', errors='surrogateescape')

22
swift/obj/diskfile.py
View File

@ -31,6 +31,7 @@ are also not considered part of the backend API.
"""
import six.moves.cPickle as pickle
import binascii
import copy
import errno
import fcntl
@ -1073,7 +1074,22 @@ class BaseDiskFileManager(object):
:param path: full path to directory
"""
hashes = defaultdict(md5)
if six.PY2:
hashes = defaultdict(md5)
else:
class shim(object):
def __init__(self):
self.md5 = md5()
def update(self, s):
if isinstance(s, str):
self.md5.update(s.encode('utf-8'))
else:
self.md5.update(s)
def hexdigest(self):
return self.md5.hexdigest()
hashes = defaultdict(shim)
try:
path_contents = sorted(os.listdir(path))
except OSError as err:
@ -1213,7 +1229,7 @@ class BaseDiskFileManager(object):
modified = True
self.logger.debug('Run listdir on %s', partition_path)
hashes.update((suffix, None) for suffix in recalculate)
for suffix, hash_ in hashes.items():
for suffix, hash_ in list(hashes.items()):
if not hash_:
suffix_dir = join(partition_path, suffix)
try:
@ -2073,7 +2089,7 @@ class BaseDiskFileReader(object):
# returning the correct value.
if self._bytes_read > 0:
bin_checksum = os.read(md5_sockfd, 16)
hex_checksum = ''.join("%02x" % ord(c) for c in bin_checksum)
hex_checksum = binascii.hexlify(bin_checksum).decode('ascii')
else:
hex_checksum = MD5_OF_EMPTY_STRING
self._md5_of_sent_bytes = hex_checksum

34
swift/obj/server.py
View File

@ -95,17 +95,24 @@ def _make_backend_fragments_header(fragments):
return None
class EventletPlungerString(str):
"""
Eventlet won't send headers until it's accumulated at least
eventlet.wsgi.MINIMUM_CHUNK_SIZE bytes or the app iter is exhausted. If we
want to send the response body behind Eventlet's back, perhaps with some
zero-copy wizardry, then we have to unclog the plumbing in eventlet.wsgi
to force the headers out, so we use an EventletPlungerString to empty out
all of Eventlet's buffers.
"""
def __len__(self):
return wsgi.MINIMUM_CHUNK_SIZE + 1
if six.PY2:
class EventletPlungerString(str):
"""
Eventlet won't send headers until it's accumulated at least
eventlet.wsgi.MINIMUM_CHUNK_SIZE bytes or the app iter is exhausted.
If we want to send the response body behind Eventlet's back, perhaps
with some zero-copy wizardry, then we have to unclog the plumbing in
eventlet.wsgi to force the headers out, so we use an
EventletPlungerString to empty out all of Eventlet's buffers.
"""
def __len__(self):
return wsgi.MINIMUM_CHUNK_SIZE + 1
else:
# Eventlet of 0.23.0 does encode('ascii') and strips our __len__.
# Avoid it by inheriting from bytes.
class EventletPlungerString(bytes):
def __len__(self):
return wsgi.MINIMUM_CHUNK_SIZE + 1
class ObjectController(BaseStorageServer):
@ -377,7 +384,10 @@ class ObjectController(BaseStorageServer):
contpath = None
if contpartition:
updates = zip(conthosts, contdevices)
# In py3, zip() continues to work for our purposes... But when
# we want to log an error, consumed items are not longer present
# in the zip, making the logs useless for operators. So, list().
updates = list(zip(conthosts, contdevices))
else:
updates = []

21
test/unit/__init__.py
View File

@ -19,8 +19,6 @@ from __future__ import print_function
import os
import copy
import logging
from six.moves import range
from six import BytesIO
import sys
from contextlib import contextmanager, closing
from collections import defaultdict, Iterable
@ -39,21 +37,22 @@ import random
import errno
import xattr
from swift.common import storage_policy, swob, utils
from swift.common.storage_policy import (StoragePolicy, ECStoragePolicy,
VALID_EC_TYPES)
from swift.common.utils import Timestamp, NOTICE
from test import get_config
from swift.common import utils
from swift.common.header_key_dict import HeaderKeyDict
from swift.common.ring import Ring, RingData, RingBuilder
from swift.obj import server
from hashlib import md5
import logging.handlers
import six
from six.moves import range
from six import BytesIO
from six.moves.http_client import HTTPException
from swift.common import storage_policy
from swift.common.storage_policy import (StoragePolicy, ECStoragePolicy,
VALID_EC_TYPES)
from swift.common import swob
import functools
import six.moves.cPickle as pickle
from gzip import GzipFile
@ -870,7 +869,7 @@ def fake_http_connect(*code_iter, **kwargs):
class FakeConn(object):
def __init__(self, status, etag=None, body='', timestamp='1',
def __init__(self, status, etag=None, body=b'', timestamp='1',
headers=None, expect_headers=None, connection_id=None,
give_send=None, give_expect=None):
if not isinstance(status, FakeStatus):
@ -925,7 +924,7 @@ def fake_http_connect(*code_iter, **kwargs):
def getheaders(self):
etag = self.etag
if not etag:
if isinstance(self.body, str):
if isinstance(self.body, six.binary_type):
etag = '"' + md5(self.body).hexdigest() + '"'
else:
etag = '"68b329da9893e34099c7d8ad5cb9c940"'
@ -1190,7 +1189,7 @@ def encode_frag_archive_bodies(policy, body):
fragment_payloads.append(fragments)
# join up the fragment payloads per node
ec_archive_bodies = [''.join(frags)
ec_archive_bodies = [b''.join(frags)
for frags in zip(*fragment_payloads)]
return ec_archive_bodies

818
test/unit/obj/test_server.py
View File

File diff suppressed because it is too large Load Diff

1
tox.ini
View File

@ -84,6 +84,7 @@ commands =
test/unit/container/test_auditor.py \
test/unit/container/test_replicator.py \
test/unit/container/test_sync_store.py \
test/unit/obj/test_server.py \
test/unit/proxy/controllers/test_info.py}
[testenv:py36]