py3: object server

This does not do anything about replicator or other daemons,
only ports the server.

- wsgi_to_bytes(something.get('X-Foo')) assumes that None is possible
- Dunno if del-in-for-in-dict calls for clone or list(), using list()
- Fixed the zero-copy with EventletPlungerString(bytes)
- Yet another reminder that Request.blank() takes a WSGI string path

Curiously enough, the sleep(0) before checking for logging was
already present in the tests. The py3 scheduling merely forces us
to do it consistently throughout.

Change-Id: I203a54fddddbd4352be0e6ea476a628e3f747dc1

swift/common/swob.py

@@ -282,12 +282,16 @@ class HeaderEnvironProxy(MutableMapping):
 
 
 def wsgi_to_bytes(wsgi_str):
+    if wsgi_str is None:
+        return None
     if six.PY2:
         return wsgi_str
     return wsgi_str.encode('latin1')
 
 
 def wsgi_to_str(wsgi_str):
+    if wsgi_str is None:
+        return None
     if six.PY2:
         return wsgi_str
     return wsgi_to_bytes(wsgi_str).decode('utf8', errors='surrogateescape')

swift/obj/diskfile.py

@@ -31,6 +31,7 @@ are also not considered part of the backend API.
 """
 
 import six.moves.cPickle as pickle
+import binascii
 import copy
 import errno
 import fcntl
@@ -1073,7 +1074,22 @@ class BaseDiskFileManager(object):
 
         :param path: full path to directory
         """
-        hashes = defaultdict(md5)
+        if six.PY2:
+            hashes = defaultdict(md5)
+        else:
+            class shim(object):
+                def __init__(self):
+                    self.md5 = md5()
+
+                def update(self, s):
+                    if isinstance(s, str):
+                        self.md5.update(s.encode('utf-8'))
+                    else:
+                        self.md5.update(s)
+
+                def hexdigest(self):
+                    return self.md5.hexdigest()
+            hashes = defaultdict(shim)
         try:
             path_contents = sorted(os.listdir(path))
         except OSError as err:
@@ -1213,7 +1229,7 @@ class BaseDiskFileManager(object):
             modified = True
             self.logger.debug('Run listdir on %s', partition_path)
         hashes.update((suffix, None) for suffix in recalculate)
-        for suffix, hash_ in hashes.items():
+        for suffix, hash_ in list(hashes.items()):
             if not hash_:
                 suffix_dir = join(partition_path, suffix)
                 try:
@@ -2073,7 +2089,7 @@ class BaseDiskFileReader(object):
             # returning the correct value.
             if self._bytes_read > 0:
                 bin_checksum = os.read(md5_sockfd, 16)
-                hex_checksum = ''.join("%02x" % ord(c) for c in bin_checksum)
+                hex_checksum = binascii.hexlify(bin_checksum).decode('ascii')
             else:
                 hex_checksum = MD5_OF_EMPTY_STRING
             self._md5_of_sent_bytes = hex_checksum

swift/obj/server.py

@@ -95,17 +95,24 @@ def _make_backend_fragments_header(fragments):
     return None
 
 
-class EventletPlungerString(str):
-    """
-    Eventlet won't send headers until it's accumulated at least
-    eventlet.wsgi.MINIMUM_CHUNK_SIZE bytes or the app iter is exhausted. If we
-    want to send the response body behind Eventlet's back, perhaps with some
-    zero-copy wizardry, then we have to unclog the plumbing in eventlet.wsgi
-    to force the headers out, so we use an EventletPlungerString to empty out
-    all of Eventlet's buffers.
-    """
-    def __len__(self):
-        return wsgi.MINIMUM_CHUNK_SIZE + 1
+if six.PY2:
+    class EventletPlungerString(str):
+        """
+        Eventlet won't send headers until it's accumulated at least
+        eventlet.wsgi.MINIMUM_CHUNK_SIZE bytes or the app iter is exhausted.
+        If we want to send the response body behind Eventlet's back, perhaps
+        with some zero-copy wizardry, then we have to unclog the plumbing in
+        eventlet.wsgi to force the headers out, so we use an
+        EventletPlungerString to empty out all of Eventlet's buffers.
+        """
+        def __len__(self):
+            return wsgi.MINIMUM_CHUNK_SIZE + 1
+else:
+    # Eventlet of 0.23.0 does encode('ascii') and strips our __len__.
+    # Avoid it by inheriting from bytes.
+    class EventletPlungerString(bytes):
+        def __len__(self):
+            return wsgi.MINIMUM_CHUNK_SIZE + 1
 
 
 class ObjectController(BaseStorageServer):
@@ -377,7 +384,10 @@ class ObjectController(BaseStorageServer):
                 contpath = None
 
         if contpartition:
-            updates = zip(conthosts, contdevices)
+            # In py3, zip() continues to work for our purposes... But when
+            # we want to log an error, consumed items are not longer present
+            # in the zip, making the logs useless for operators. So, list().
+            updates = list(zip(conthosts, contdevices))
         else:
             updates = []
 

test/unit/__init__.py

@@ -19,8 +19,6 @@ from __future__ import print_function
 import os
 import copy
 import logging
-from six.moves import range
-from six import BytesIO
 import sys
 from contextlib import contextmanager, closing
 from collections import defaultdict, Iterable
@@ -39,21 +37,22 @@ import random
 import errno
 import xattr
 
-
+from swift.common import storage_policy, swob, utils
+from swift.common.storage_policy import (StoragePolicy, ECStoragePolicy,
+                                         VALID_EC_TYPES)
 from swift.common.utils import Timestamp, NOTICE
 from test import get_config
-from swift.common import utils
 from swift.common.header_key_dict import HeaderKeyDict
 from swift.common.ring import Ring, RingData, RingBuilder
 from swift.obj import server
 from hashlib import md5
 import logging.handlers
 
+import six
+from six.moves import range
+from six import BytesIO
 from six.moves.http_client import HTTPException
-from swift.common import storage_policy
-from swift.common.storage_policy import (StoragePolicy, ECStoragePolicy,
-                                         VALID_EC_TYPES)
-from swift.common import swob
+
 import functools
 import six.moves.cPickle as pickle
 from gzip import GzipFile
@@ -870,7 +869,7 @@ def fake_http_connect(*code_iter, **kwargs):
 
     class FakeConn(object):
 
-        def __init__(self, status, etag=None, body='', timestamp='1',
+        def __init__(self, status, etag=None, body=b'', timestamp='1',
                      headers=None, expect_headers=None, connection_id=None,
                      give_send=None, give_expect=None):
             if not isinstance(status, FakeStatus):
@@ -925,7 +924,7 @@ def fake_http_connect(*code_iter, **kwargs):
         def getheaders(self):
             etag = self.etag
             if not etag:
-                if isinstance(self.body, str):
+                if isinstance(self.body, six.binary_type):
                     etag = '"' + md5(self.body).hexdigest() + '"'
                 else:
                     etag = '"68b329da9893e34099c7d8ad5cb9c940"'
@@ -1190,7 +1189,7 @@ def encode_frag_archive_bodies(policy, body):
         fragment_payloads.append(fragments)
 
     # join up the fragment payloads per node
-    ec_archive_bodies = [''.join(frags)
+    ec_archive_bodies = [b''.join(frags)
                          for frags in zip(*fragment_payloads)]
     return ec_archive_bodies
 

tox.ini

@@ -84,6 +84,7 @@ commands =
       test/unit/container/test_auditor.py \
       test/unit/container/test_replicator.py \
       test/unit/container/test_sync_store.py \
+      test/unit/obj/test_server.py \
       test/unit/proxy/controllers/test_info.py}
 
 [testenv:py36]