py3: add swift-dsvm-functional-py3 job

Note that keystone wants to stick some UTF-8 encoded bytes into
memcached, but we want to store it as JSON... or something?

Also, make sure we can hit memcache for containers with invalid UTF-8.
Although maybe it'd be better to catch that before we ever try memcache?

Change-Id: I1fbe133c8ec73ef6644ecfcbb1931ddef94e0400

.zuul.yaml

@@ -193,6 +193,20 @@
     run: tools/playbooks/dsvm/run.yaml
     post-run: tools/playbooks/dsvm/post.yaml
 
+- job:
+    name: swift-dsvm-functional-py3
+    parent: swift-dsvm-functional
+    description: |
+      Setup a Swift/Keystone environment under py3 and run Swift's func tests
+      (also under py3).
+    vars:
+      # only run the subset expected to work on py3
+      tox_envlist: func-py3
+      devstack_localrc:
+        USE_PYTHON3: true
+        # explicitly clear swift's default-disabled status
+        DISABLED_PYTHON3_PACKAGES: ''
+
 - job:
     name: swift-dsvm-functional-ipv6
     parent: swift-dsvm-functional
@@ -432,6 +446,11 @@
               - ^(api-ref|doc|releasenotes)/.*$
               - ^test/probe/.*$
               - ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$
+        - swift-dsvm-functional-py3:
+            irrelevant-files:
+              - ^(api-ref|doc|releasenotes)/.*$
+              - ^test/probe/.*$
+              - ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$
         - swift-dsvm-functional-ipv6:
             irrelevant-files:
               - ^(api-ref|doc|releasenotes)/.*$
@@ -479,6 +498,11 @@
               - ^(api-ref|doc|releasenotes)/.*$
               - ^test/probe/.*$
               - ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$
+        - swift-dsvm-functional-py3:
+            irrelevant-files:
+              - ^(api-ref|doc|releasenotes)/.*$
+              - ^test/probe/.*$
+              - ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$
         - swift-dsvm-functional-ipv6:
             irrelevant-files:
               - ^(api-ref|doc|releasenotes)/.*$

etc/object-server.conf-sample

@@ -473,7 +473,7 @@ use = egg:swift#recon
 # to 86400 (1 day).
 # rsync_tempfile_timeout = auto
 
-# [object-expirer]
+[object-expirer]
 # If this true, this expirer will execute tasks from legacy expirer task queue,
 # at least one object server should run with dequeue_from_legacy = true
 # dequeue_from_legacy = false

swift/common/memcached.py

@@ -44,6 +44,7 @@ version is at:
 http://github.com/memcached/memcached/blob/1.4.2/doc/protocol.txt
 """
 
+import six
 import six.moves.cPickle as pickle
 import json
 import logging
@@ -77,7 +78,10 @@ ERROR_LIMIT_DURATION = 60
 
 def md5hash(key):
     if not isinstance(key, bytes):
-        key = key.encode('utf-8')
+        if six.PY2:
+            key = key.encode('utf-8')
+        else:
+            key = key.encode('utf-8', errors='surrogateescape')
     return md5(key).hexdigest().encode('ascii')
 
 
@@ -269,6 +273,8 @@ class MemcacheRing(object):
             value = pickle.dumps(value, PICKLE_PROTOCOL)
             flags |= PICKLE_FLAG
         elif serialize:
+            if isinstance(value, bytes):
+                value = value.decode('utf8')
             value = json.dumps(value).encode('ascii')
             flags |= JSON_FLAG
         elif not isinstance(value, bytes):
@@ -438,6 +444,8 @@ class MemcacheRing(object):
                 value = pickle.dumps(value, PICKLE_PROTOCOL)
                 flags |= PICKLE_FLAG
             elif serialize:
+                if isinstance(value, bytes):
+                    value = value.decode('utf8')
                 value = json.dumps(value).encode('ascii')
                 flags |= JSON_FLAG
             msg.append(set_msg(key, flags, timeout, value))

test/functional/tests.py

@@ -407,7 +407,17 @@ class TestAccount(Base):
         quoted_hax = urllib.parse.quote(hax)
         conn.connection.request('GET', '/v1/' + quoted_hax, None, {})
         resp = conn.connection.getresponse()
-        resp_headers = dict((h.lower(), v) for h, v in resp.getheaders())
+
+        resp_headers = {}
+        for h, v in resp.getheaders():
+            h = h.lower()
+            if h in resp_headers:
+                # py2 would do this for us, but py3 apparently keeps them
+                # separate? Not sure which I like more...
+                resp_headers[h] += ',' + v
+            else:
+                resp_headers[h] = v
+
         self.assertIn('www-authenticate', resp_headers)
         actual = resp_headers['www-authenticate']
         expected = 'Swift realm="%s"' % quoted_hax