Improve python3 compatibility

The review:
 * adds a python3 job in order to test the refstack-client with
   python3 as well.
 * adds a job which runs unit tests by python3.7
 * changes default python3 (when -p 3 is specified) in setup_env script
   to version 3.6.0
 * converts downloaded list of tests from bytes to str in order to unify
   the types used for python2 and python3 compatibility.
 * edits -p argument in the setup_env script so that it also accepts a full
   version of Python a user wants to run with, before there were 2 options:
    * python2.7.8
    * python3.6.0
 * sets object-storage.operator_role to Member in tempest.conf in order to run
   tempest.api.object_storage.test_container_services.ContainerTest tests

Change-Id: I961f0f093bd7d40fde7e448ea12ef9907c61d126

.zuul.yaml

@@ -3,17 +3,18 @@
       - openstack-python-jobs
       - openstack-python35-jobs
       - openstack-python36-jobs
+      - openstack-python37-jobs
     check:
       jobs:
         - refstack-client-devstack-tempestconf
-        - openstack-tox-py35
+        - refstack-client-devstack-tempestconf-py3
     gate:
       jobs:
         - refstack-client-devstack-tempestconf
-        - openstack-tox-py35
+        - refstack-client-devstack-tempestconf-py3
 
 - job:
-    name: refstack-client-devstack-tempestconf
+    name: refstack-client-devstack-tempestconf-base
     parent: devstack
     description: |
       Refstack client job for testing python-tempestconf and RefStack Integration
@@ -38,3 +39,21 @@
       - ^doc/.*$
       - ^releasenotes/.*$
       - ^.*\.rst$
+
+- job:
+    name: refstack-client-devstack-tempestconf
+    parent: refstack-client-devstack-tempestconf-base
+    description: |
+      Refstack client job for testing python-tempestconf and RefStack Integration
+      using python2.
+
+- job:
+    name: refstack-client-devstack-tempestconf-py3
+    parent: refstack-client-devstack-tempestconf-base
+    description: |
+      Refstack client job for testing python-tempestconf and RefStack Integration
+      using python3.
+    vars:
+      python_3_args: -p 3
+      devstack_localrc:
+        USE_PYTHON3: true

playbooks/tempestconf-refstack-devstack.yaml

@@ -6,7 +6,7 @@
   vars:
     set_auth_url: "OS_AUTH_URL=$SERVICE_PROTOCOL://$SERVICE_HOST/identity/v3"
     devstack_base_dir: "/opt/stack"
-    aditional_tempestconf_params: "auth.tempest_roles Member"
+    aditional_tempestconf_params: "auth.tempest_roles Member object-storage.operator_role Member"
   tasks:
     - name: Setup Tempest Run Directory
       include_role:
@@ -28,7 +28,6 @@
       include_role:
         name: generate-accounts-file
       vars:
-        aditional_tempestconf_params: "auth.tempest_roles Member"
         source_credentials_commands: "export HOST_IP={{ ansible_default_ipv4.address }}; source {{ devstack_base_dir }}/devstack/openrc admin admin; {{ set_auth_url }}"
         accounts_file_destination: "/etc/openstack"
         tempest_config_file: "/etc/openstack/tempest_admin.conf"

refstack_client/list_parser.py

@@ -65,9 +65,10 @@ class TestListParser(object):
             self.logger.error(stderr)
             raise subprocess.CalledProcessError(process.returncode,
                                                 ' '.join(cmd))
-
+        try:
-        testcase_list = stdout.split('\n')
+            return self._form_test_id_mappings(stdout.split('\n'))
-        return self._form_test_id_mappings(testcase_list)
+        except TypeError:
+            return self._form_test_id_mappings(stdout.decode().split('\n'))
 
     def _form_test_id_mappings(self, test_list):
         """This takes in a list of full test IDs and forms a dict containing

refstack_client/tests/unit/test_client.py

@@ -861,7 +861,10 @@ class TestRefstackClient(unittest.TestCase):
         with httmock.HTTMock(refstack_api_mock):
             results = client.yield_results("http://127.0.0.1")
             self.assertEqual(expected_response['results'], next(results))
-            self.assertRaises(StopIteration, next, results)
+            # Since Python3.7 StopIteration exceptions are transformed into
+            # RuntimeError (PEP 479):
+            # https://docs.python.org/3/whatsnew/3.7.html
+            self.assertRaises((StopIteration, RuntimeError), next, results)
 
     @mock.patch('six.moves.input', side_effect=KeyboardInterrupt)
     @mock.patch('sys.stdout', new_callable=MagicMock)

roles/generate-tempestconf-refstack/tasks/main.yaml

@@ -6,7 +6,7 @@
     shell: |
         set -ex
         export PATH=$PATH:/usr/local/sbin:/usr/sbin
-        ./setup_env -c master -s ../python-tempestconf
+        ./setup_env -c master -s ../python-tempestconf {{ python_3_args | default('') }}
     args:
         chdir: "{{ refstack_client_src_relative_path }}"
         executable: /bin/bash

setup.cfg

@@ -18,6 +18,8 @@ classifier =
     Programming Language :: Python :: 2.7
     Programming Language :: Python :: 3
     Programming Language :: Python :: 3.5
+    Programming Language :: Python :: 3.6
+    Programming Language :: Python :: 3.7
 
 [files]
 packages =

setup_env

@@ -14,7 +14,8 @@ function usage {
     echo "  -h  Print this usage message"
     echo "  -c  Tempest test runner commit. You can specify SHA or branch here"
     echo "      If no commit or tag is specified, tempest will be install from commit"
-    echo "  -p  [ 2 | 3 ] - Uses either python 2.7 or 3.5. Default to python 2.7"
+    echo "  -p  [ 2 | 3 | X.X.X ] - Uses either python 2.7 (if -p 2), 3.6 (if -p 3)"
+    echo "      or given specific version (if -p X.X.X). Default to python 2.7"
     echo "  -q  Run quietly. If .tempest folder exists, refstack-client is considered as installed"
     echo "  -s  Use python-tempestconf from the given source (path), used when running f.e. in Zuul"
     echo "  -t  Tempest test runner tag. You can specify tag here"
@@ -40,7 +41,9 @@ while getopts c:p:t:qs:h FLAG; do
             ;;
         p)
             if [ ${OPTARG} == '3' ]; then
-                PY_VERSION="3.5.2"
+                PY_VERSION="3.6.0"
+            else
+                PY_VERSION=${OPTARG}
             fi
             ;;
         t)

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = pep8,py35,py27
+envlist = pep8,py35,py36,py37,py27
 minversion = 1.6
 skipsdist = True