py3: port staticweb and domain_remap func tests

Drive-by: Tighten domain_remap assertions on listings, which required
that we fix proxy pipeline placement. Add a note about it to the sample
config.

Change-Id: I41835148051294088a2c0fb4ed4e7a7b61273e5f

.zuul.yaml

@@ -96,6 +96,18 @@
       bindep_profile: test py37
       python_version: 3.7
 
+- job:
+    name: swift-tox-func-domain-remap-staticweb-py37
+    parent: swift-tox-func-py37
+    description: |
+      Run functional tests for swift under cPython version 3.7.
+
+      Uses tox with the ``func-domain-remap-staticweb-py3`` environment.
+      It sets TMPDIR to an XFS mount point created via
+      tools/test-setup.sh.
+    vars:
+      tox_envlist: func-domain-remap-staticweb-py3
+
 - job:
     name: swift-tox-func-centos-7
     parent: swift-tox-func
@@ -412,6 +424,11 @@
               - ^(api-ref|doc|releasenotes)/.*$
               - ^test/probe/.*$
               - ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$
+        - swift-tox-func-domain-remap-staticweb-py37:
+            irrelevant-files:
+              - ^(api-ref|doc|releasenotes)/.*$
+              - ^test/probe/.*$
+              - ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$
         - swift-tox-func-s3api-ceph-s3tests-tempauth:
             irrelevant-files:
               - ^(api-ref|releasenotes)/.*$
@@ -467,6 +484,7 @@
         - swift-tox-func-domain-remap-staticweb
         - swift-tox-func-ec
         - swift-tox-func-s3api
+        - swift-tox-func-domain-remap-staticweb-py37
         - swift-probetests-centos-7:
             irrelevant-files:
               - ^(api-ref|releasenotes)/.*$

etc/proxy-server.conf-sample

@@ -715,6 +715,8 @@ use = egg:swift#read_only
 # allow_deletes = false
 # Note: Put after ratelimit in the pipeline.
 
+# Note: needs to be placed before listing_formats;
+# otherwise remapped listings will always be JSON
 [filter:domain_remap]
 use = egg:swift#domain_remap
 # You can override the default log routing for this filter here:

test/functional/__init__.py

@@ -398,7 +398,10 @@ def _load_domain_remap_staticweb(proxy_conf_file, swift_conf_file, **kwargs):
         old_pipeline = conf.get(section, 'pipeline')
         pipeline = old_pipeline.replace(
             " tempauth ",
-            " domain_remap tempauth staticweb ")
+            " tempauth staticweb ")
+        pipeline = pipeline.replace(
+            " listing_formats ",
+            " domain_remap listing_formats ")
         if pipeline == old_pipeline:
             raise InProcessException(
                 "Failed to insert domain_remap and staticweb into pipeline: %s"

test/functional/test_domain_remap.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 
 from unittest2 import SkipTest
+import six
 
 import test.functional as tf
 from test.functional import cluster_info
@@ -53,10 +54,10 @@ class TestDomainRemapEnv(BaseEnv):
             raise ResponseError(cls.conn.response)
 
         cls.obj = cls.container.file(Utils.create_name())
-        cls.obj.write('obj contents')
+        cls.obj.write(b'obj contents')
 
         cls.obj_slash = cls.container.file('/v1')
-        cls.obj_slash.write('obj contents')
+        cls.obj_slash.write(b'obj contents')
 
 
 class TestDomainRemap(Base):
@@ -103,7 +104,9 @@ class TestDomainRemap(Base):
                                                cfg={'absolute_path': True})
             self.assert_status(200)
             body = self.env.account.conn.response.read()
-            self.assertIn(self.env.container.name, body)
+            if not six.PY2:
+                body = body.decode('utf8')
+            self.assertIn(self.env.container.name, body.split('\n'))
 
             path = '/'.join(['', self.env.container.name])
             self.env.account.conn.make_request('GET', path,
@@ -111,8 +114,10 @@ class TestDomainRemap(Base):
                                                cfg={'absolute_path': True})
             self.assert_status(200)
             body = self.env.account.conn.response.read()
-            self.assertIn(self.env.obj.name, body)
+            if not six.PY2:
-            self.assertIn(self.env.obj_slash.name, body)
+                body = body.decode('utf8')
+            self.assertIn(self.env.obj.name, body.split('\n'))
+            self.assertIn(self.env.obj_slash.name, body.split('\n'))
 
             for obj in (self.env.obj, self.env.obj_slash):
                 path = '/'.join(['', self.env.container.name, obj.name])
@@ -143,7 +148,7 @@ class TestDomainRemap(Base):
                                                cfg={'absolute_path': True})
             self.assert_status(201)
             new_obj = self.env.container.file(new_obj_name)
-            self.assertEqual(new_obj.read(), 'new obj contents')
+            self.assertEqual(new_obj.read(), b'new obj contents')
 
     def test_GET_remapped_container(self):
         for domain in (self.cont_domain_dash, self.cont_domain_underscore):
@@ -152,8 +157,10 @@ class TestDomainRemap(Base):
                                                cfg={'absolute_path': True})
             self.assert_status(200)
             body = self.env.account.conn.response.read()
-            self.assertIn(self.env.obj.name, body)
+            if not six.PY2:
-            self.assertIn(self.env.obj_slash.name, body)
+                body = body.decode('utf8')
+            self.assertIn(self.env.obj.name, body.split('\n'))
+            self.assertIn(self.env.obj_slash.name, body.split('\n'))
 
             for obj in (self.env.obj, self.env.obj_slash):
                 path = '/'.join(['', obj.name])
@@ -174,4 +181,4 @@ class TestDomainRemap(Base):
             self.assert_status(201)
 
             new_obj = self.env.container.file(new_obj_name)
-            self.assertEqual(new_obj.read(), 'new obj contents')
+            self.assertEqual(new_obj.read(), b'new obj contents')

test/functional/test_staticweb.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 
 import functools
+import six
 from unittest2 import SkipTest
 from six.moves.urllib.parse import unquote
 from swift.common.utils import quote
@@ -94,7 +95,7 @@ class TestStaticWebEnv(BaseEnv):
                     'Content-Type': 'application/directory'})
             else:
                 cls.objects[item] = cls.container.file(path)
-                cls.objects[item].write('%s contents' % item)
+                cls.objects[item].write(('%s contents' % item).encode('utf8'))
 
 
 class TestStaticWeb(Base):
@@ -155,11 +156,10 @@ class TestStaticWeb(Base):
 
     def _test_redirect_with_slash(self, host, path, anonymous=False):
         self._set_staticweb_headers(listings=True)
-        self.env.account.conn.make_request('GET', path,
+        self.env.account.conn.make_request(
-                                           hdrs={'X-Web-Mode': not anonymous,
+            'GET', path,
-                                                 'Host': host},
+            hdrs={'X-Web-Mode': str(not anonymous), 'Host': host},
-                                           cfg={'no_auth_token': anonymous,
+            cfg={'no_auth_token': anonymous, 'absolute_path': True})
-                                                'absolute_path': True})
 
         self.assert_status(301)
         expected = '%s://%s%s/' % (
@@ -214,13 +214,14 @@ class TestStaticWeb(Base):
 
     def _test_get_path(self, host, path, anonymous=False, expected_status=200,
                        expected_in=[], expected_not_in=[]):
-        self.env.account.conn.make_request('GET', path,
+        self.env.account.conn.make_request(
-                                           hdrs={'X-Web-Mode': not anonymous,
+            'GET', path,
-                                                 'Host': host},
+            hdrs={'X-Web-Mode': str(not anonymous), 'Host': host},
-                                           cfg={'no_auth_token': anonymous,
+            cfg={'no_auth_token': anonymous, 'absolute_path': True})
-                                                'absolute_path': True})
         self.assert_status(expected_status)
         body = self.env.account.conn.response.read()
+        if not six.PY2:
+            body = body.decode('utf8')
         for string in expected_in:
             self.assertIn(string, body)
         for string in expected_not_in:

tox.ini

@@ -48,10 +48,18 @@ commands = ./.functests {posargs}
 basepython = python3
 commands =
   nosetests {posargs: \
+    test/functional/test_domain_remap.py \
+    test/functional/test_staticweb.py \
     test/functional/test_symlink.py \
     test/functional/test_tempurl.py \
     test/functional/tests.py}
 
+
+[testenv:func-domain-remap-staticweb-py3]
+basepython = python3
+commands = {[testenv:func-py3]commands}
+setenv = SWIFT_TEST_IN_PROCESS=1
+         SWIFT_TEST_IN_PROCESS_CONF_LOADER=domain_remap_staticweb
 [testenv:func-encryption]
 commands = ./.functests {posargs}
 setenv = SWIFT_TEST_IN_PROCESS=1