More compatibility with Python 3

- use the six version of configparser and urllib, and depends on six;
- remove relative imports;
- adapt few tests to the changes.

The changes above should be noop from the point of view of
functionalities, at least on python 2.

And also:
- replace the py34 tox virtualenv with py35;
- add a non-voting py35 job (locally for now, it will be enabled
  to project-config also for gating when stable).

Story: 2002574
Task: 22142
Change-Id: I0a35abaae6f5b7095ebae765fbe2163046e0a4da

.zuul.yaml

@@ -1,6 +1,8 @@
 - project:
     check:
       jobs:
+        - openstack-tox-py35:
+            voting: false
         - python-tempestconf-tox-cover
         - python-tempestconf-tempest-devstack-admin
         - python-tempestconf-tempest-devstack-demo

config_tempest/flavors.py

@@ -13,7 +13,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-from constants import LOG
+from config_tempest.constants import LOG
 
 
 class Flavors(object):

config_tempest/main.py

@@ -37,22 +37,23 @@ obtained by querying the cloud.
 """
 
 import argparse
-import ConfigParser
 import logging
 import os
 import sys
 
-import accounts
-from clients import ClientManager
-import constants as C
-from constants import LOG
-from credentials import Credentials
-from flavors import Flavors
 import os_client_config
 from oslo_config import cfg
-from services.services import Services
+from six.moves import configparser
-import tempest_conf
+
-from users import Users
+from config_tempest import accounts
+from config_tempest.clients import ClientManager
+from config_tempest import constants as C
+from config_tempest.constants import LOG
+from config_tempest.credentials import Credentials
+from config_tempest.flavors import Flavors
+from config_tempest.services.services import Services
+from config_tempest.tempest_conf import TempestConf
+from config_tempest.users import Users
 
 
 def set_logging(debug, verbose):
@@ -111,7 +112,7 @@ def read_deployer_input(deployer_input_file, conf):
     """
     LOG.info("Adding options from deployer-input file '%s'",
              deployer_input_file)
-    deployer_input = ConfigParser.SafeConfigParser()
+    deployer_input = configparser.SafeConfigParser()
     deployer_input.read(deployer_input_file)
     for section in deployer_input.sections():
         # There are no deployer input options in DEFAULT
@@ -368,7 +369,7 @@ def config_tempest(**kwargs):
     set_logging(kwargs.get('debug', False), kwargs.get('verbose', False))
 
     write_credentials = kwargs.get('test_accounts') is None
-    conf = tempest_conf.TempestConf(write_credentials=write_credentials)
+    conf = TempestConf(write_credentials=write_credentials)
     set_options(conf, kwargs.get('deployer_input'),
                 kwargs.get('non_admin', False),
                 kwargs.get('overrides', []), kwargs.get('test_accounts'),

config_tempest/services/base.py

@@ -16,7 +16,8 @@
 import json
 import re
 import urllib3
-import urlparse
+
+from six.moves import urllib
 
 from config_tempest.constants import LOG
 MULTIPLE_SLASH = re.compile(r'/+')
@@ -39,13 +40,13 @@ class Service(object):
         self.versions = []
 
     def do_get(self, url, top_level=False, top_level_path=""):
-        parts = list(urlparse.urlparse(url))
+        parts = list(urllib.parse.urlparse(url))
         # 2 is the path offset
         if top_level:
             parts[2] = '/' + top_level_path
 
         parts[2] = MULTIPLE_SLASH.sub('/', parts[2])
-        url = urlparse.urlunparse(parts)
+        url = urllib.parse.urlunparse(parts)
 
         try:
             if self.disable_ssl_validation:

config_tempest/services/ceilometer.py

@@ -13,7 +13,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import config_tempest.constants as C
+from config_tempest import constants as C
 from tempest.lib import exceptions
 
 

config_tempest/services/compute.py

@@ -13,11 +13,13 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-from base import VersionedService
-import config_tempest.constants as C
 import json
+
 from tempest.lib import exceptions
 
+from config_tempest import constants as C
+from config_tempest.services.base import VersionedService
+
 
 class ComputeService(VersionedService):
     def set_extensions(self):

config_tempest/services/horizon.py

@@ -13,20 +13,20 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import urllib2
+from six.moves import urllib
 
 
 def configure_horizon(conf):
     """Derive the horizon URIs from the identity's URI."""
     uri = conf.get('identity', 'uri')
-    u = urllib2.urlparse.urlparse(uri)
+    u = urllib.parse.urlparse(uri)
     base = '%s://%s%s' % (u.scheme, u.netloc.replace(
         ':' + str(u.port), ''), '/dashboard')
     assert base.startswith('http:') or base.startswith('https:')
     has_horizon = True
     try:
-        urllib2.urlopen(base)
+        urllib.request.urlopen(base)
-    except urllib2.URLError:
+    except urllib.error.URLError:
         has_horizon = False
     conf.set('service_available', 'horizon', str(has_horizon))
     conf.set('dashboard', 'dashboard_url', base + '/')

config_tempest/services/identity.py

@@ -15,10 +15,11 @@
 
 import json
 import requests
-import urlparse
 
-from base import VersionedService
+from six.moves import urllib
+
 from config_tempest.constants import LOG
+from config_tempest.services.base import VersionedService
 
 
 class IdentityService(VersionedService):
@@ -30,7 +31,7 @@ class IdentityService(VersionedService):
         version = ''
         if 'v2' in self.service_url:
             version = '/v2.0'
-            url_parse = urlparse.urlparse(self.service_url)
+            url_parse = urllib.parse.urlparse(self.service_url)
             self.service_url = '{}://{}{}'.format(url_parse.scheme,
                                                   url_parse.netloc, version)
 

config_tempest/services/image.py

@@ -15,12 +15,13 @@
 
 import os
 import shutil
-import urllib2
 
-from base import VersionedService
+from six.moves import urllib
-from config_tempest.constants import LOG
 from tempest.lib import exceptions
 
+from config_tempest.constants import LOG
+from config_tempest.services.base import VersionedService
+
 
 class ImageService(VersionedService):
 
@@ -173,7 +174,7 @@ class ImageService(VersionedService):
             LOG.info("Image '%s' already fetched to '%s'.", url, destination)
             return
         LOG.info("Downloading '%s' and saving as '%s'", url, destination)
-        f = urllib2.urlopen(url)
+        f = urllib.request.urlopen(url)
         data = f.read()
         with open(destination, "wb") as dest:
             dest.write(data)

config_tempest/services/network.py

@@ -15,8 +15,8 @@
 
 import json
 
-from base import VersionedService
 from config_tempest.constants import LOG
+from config_tempest.services.base import VersionedService
 
 
 class NetworkService(VersionedService):

config_tempest/services/object_storage.py

@@ -13,13 +13,14 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import ConfigParser
 import json
 
-from base import Service
+from six.moves import configparser
-from config_tempest.constants import LOG
 from tempest.lib import exceptions
 
+from config_tempest.constants import LOG
+from config_tempest.services.base import Service
+
 
 class ObjectStorageService(Service):
     def set_extensions(self):
@@ -71,7 +72,7 @@ class ObjectStorageService(Service):
                     'object-storage-feature-enabled',
                     'discoverability')):
                 return False
-        except ConfigParser.NoSectionError:
+        except configparser.NoSectionError:
             # Turning http://.../v1/foobar into http://.../
             self.client.accounts.skip_path()
             resp, _ = self.client.accounts.get("healthcheck", {})

config_tempest/services/services.py

@@ -13,20 +13,22 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import urlparse
 
-from base import Service
+from six.moves import urllib
-import boto
+
-import ceilometer
+from config_tempest import constants as C
-from compute import ComputeService
+from config_tempest.services.base import Service
-import config_tempest.constants as C
+from config_tempest.services import boto
-import horizon
+from config_tempest.services import ceilometer
-from identity import IdentityService
+from config_tempest.services.compute import ComputeService
-from image import ImageService
+from config_tempest.services import horizon
-from network import NetworkService
+from config_tempest.services.identity import IdentityService
-from object_storage import ObjectStorageService
+from config_tempest.services.image import ImageService
-from octavia import LoadBalancerService
+from config_tempest.services.network import NetworkService
-import volume
+from config_tempest.services.object_storage import ObjectStorageService
+from config_tempest.services.octavia import LoadBalancerService
+from config_tempest.services import volume
+
 
 service_dict = {'compute': ComputeService,
                 'image': ImageService,
@@ -148,7 +150,7 @@ class Services(object):
 
         # self._clients.auth_provider.auth_url stores identity.uri(_v3) value
         # from TempestConf
-        port = urlparse.urlparse(self._clients.auth_provider.auth_url).port
+        port = urllib.parse.urlparse(self._clients.auth_provider.auth_url).port
         if port is None:
             port = ""
         else:

config_tempest/services/volume.py

@@ -15,10 +15,11 @@
 
 import json
 
-from base import VersionedService
-import config_tempest.constants as C
 from tempest.lib import exceptions
 
+from config_tempest import constants as C
+from config_tempest.services.base import VersionedService
+
 
 class VolumeService(VersionedService):
     def set_extensions(self):

config_tempest/tempest_conf.py

@@ -13,16 +13,16 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import ConfigParser
 import os
 import sys
 
-import constants as C
+from config_tempest import constants as C
 from oslo_config import cfg
+from six.moves import configparser
 import tempest.config
 
 
-class TempestConf(ConfigParser.SafeConfigParser):
+class TempestConf(configparser.SafeConfigParser):
     # causes the config parser to preserve case of the options
     optionxform = str
 
@@ -34,7 +34,7 @@ class TempestConf(ConfigParser.SafeConfigParser):
 
     def __init__(self, write_credentials=True, **kwargs):
         self.write_credentials = write_credentials
-        ConfigParser.SafeConfigParser.__init__(self, **kwargs)
+        configparser.SafeConfigParser.__init__(self, **kwargs)
 
     def get_bool_value(self, value):
         """Returns boolean value of the string value given.
@@ -101,7 +101,7 @@ class TempestConf(ConfigParser.SafeConfigParser):
         if priority:
             self.priority_sectionkeys.add((section, key))
         C.LOG.debug("Setting [%s] %s = %s", section, key, value)
-        ConfigParser.SafeConfigParser.set(self, section, key, value)
+        configparser.SafeConfigParser.set(self, section, key, value)
         return True
 
     def write(self, out_path):
@@ -111,7 +111,7 @@ class TempestConf(ConfigParser.SafeConfigParser):
                        "writing credentials is disabled.")
             self.remove_values(C.ALL_CREDENTIALS_KEYS)
         with open(out_path, 'w') as f:
-            ConfigParser.SafeConfigParser.write(self, f)
+            configparser.SafeConfigParser.write(self, f)
 
     def remove_values(self, to_remove):
         """Remove values from configuration file specified in arguments.
@@ -138,9 +138,9 @@ class TempestConf(ConfigParser.SafeConfigParser):
                     # and preserve the original order of items
                     conf_values = [v for v in conf_values if v not in remove]
                     self.set(section, key, ",".join(conf_values))
-            except ConfigParser.NoOptionError:
+            except configparser.NoOptionError:
                 # only inform a user, option specified by him doesn't exist
                 C.LOG.error(sys.exc_info()[1])
-            except ConfigParser.NoSectionError:
+            except configparser.NoSectionError:
                 # only inform a user, section specified by him doesn't exist
                 C.LOG.error(sys.exc_info()[1])

config_tempest/tests/services/test_ceilometer.py

@@ -13,7 +13,8 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import ConfigParser
+
+from six.moves import configparser
 
 from config_tempest.services import ceilometer
 from config_tempest.tempest_conf import TempestConf
@@ -29,7 +30,7 @@ class TestCeilometerService(BaseServiceTest):
         client_service_mock = self.FakeServiceClient(services={})
         ceilometer.check_ceilometer_service(self.conf, client_service_mock)
 
-        self._assert_conf_get_not_raises(ConfigParser.NoSectionError,
+        self._assert_conf_get_not_raises(configparser.NoSectionError,
                                          "service_available",
                                          "ceilometer")
 

config_tempest/tests/services/test_horizon.py

@@ -28,7 +28,8 @@ class TestConfigTempest(BaseConfigTempestTest):
 
     def test_configure_horizon_ipv4(self):
         mock_function = mock.Mock(return_value=True)
-        self.useFixture(MonkeyPatch('urllib2.urlopen', mock_function))
+        self.useFixture(MonkeyPatch('six.moves.urllib.request.urlopen',
+                                    mock_function))
         horizon.configure_horizon(self.conf)
         self.assertEqual(self.conf.get('service_available', 'horizon'), "True")
         self.assertEqual(self.conf.get('dashboard', 'dashboard_url'),
@@ -38,7 +39,8 @@ class TestConfigTempest(BaseConfigTempestTest):
 
     def test_configure_horizon_ipv6(self):
         mock_function = mock.Mock(return_value=True)
-        self.useFixture(MonkeyPatch('urllib2.urlopen', mock_function))
+        self.useFixture(MonkeyPatch('six.moves.urllib.request.urlopen',
+                                    mock_function))
         self.conf.set('identity', 'uri', 'http://[::1]:5000/v3', priority=True)
         horizon.configure_horizon(self.conf)
         self.assertEqual(self.conf.get('service_available', 'horizon'), "True")

config_tempest/users.py

@@ -13,7 +13,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-from constants import LOG
+from config_tempest.constants import LOG
 from tempest.lib import exceptions
 
 

requirements.txt

@@ -3,6 +3,7 @@
 # process, which may cause wedges in the gate later.
 
 pbr>=1.8 # Apache-2.0
+six>=1.10.0 # MIT
 tempest>=14.0.0 # Apache-2.0
 requests>=2.10.0,!=2.12.2 # Apache-2.0
 os-client-config>=1.26.0 # Apache-2.0

tox.ini

@@ -1,6 +1,6 @@
 [tox]
 minversion = 2.0
-envlist = py34,py27,pypy,pep8
+envlist = py35,py27,pypy,pep8
 skipsdist = True
 
 [testenv]