From cdbc98572d2b15ef89a7d5d71e788f4276c3e26c Mon Sep 17 00:00:00 2001 From: Luigi Toscano Date: Tue, 26 Jun 2018 23:20:52 +0200 Subject: [PATCH] More compatibility with Python 3 - use the six version of configparser and urllib, and depends on six; - remove relative imports; - adapt few tests to the changes. The changes above should be noop from the point of view of functionalities, at least on python 2. And also: - replace the py34 tox virtualenv with py35; - add a non-voting py35 job (locally for now, it will be enabled to project-config also for gating when stable). Story: 2002574 Task: 22142 Change-Id: I0a35abaae6f5b7095ebae765fbe2163046e0a4da --- .zuul.yaml | 2 ++ config_tempest/flavors.py | 2 +- config_tempest/main.py | 25 ++++++++-------- config_tempest/services/base.py | 7 +++-- config_tempest/services/ceilometer.py | 2 +- config_tempest/services/compute.py | 6 ++-- config_tempest/services/horizon.py | 8 ++--- config_tempest/services/identity.py | 7 +++-- config_tempest/services/image.py | 9 +++--- config_tempest/services/network.py | 2 +- config_tempest/services/object_storage.py | 9 +++--- config_tempest/services/services.py | 30 ++++++++++--------- config_tempest/services/volume.py | 5 ++-- config_tempest/tempest_conf.py | 16 +++++----- .../tests/services/test_ceilometer.py | 5 ++-- config_tempest/tests/services/test_horizon.py | 6 ++-- config_tempest/users.py | 2 +- requirements.txt | 1 + tox.ini | 2 +- 19 files changed, 81 insertions(+), 65 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index c65cc7e6..cdb7cf83 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,6 +1,8 @@ - project: check: jobs: + - openstack-tox-py35: + voting: false - python-tempestconf-tox-cover - python-tempestconf-tempest-devstack-admin - python-tempestconf-tempest-devstack-demo diff --git a/config_tempest/flavors.py b/config_tempest/flavors.py index 19f9d22f..d5f7b81e 100644 --- a/config_tempest/flavors.py +++ b/config_tempest/flavors.py @@ -13,7 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. -from constants import LOG +from config_tempest.constants import LOG class Flavors(object): diff --git a/config_tempest/main.py b/config_tempest/main.py index a6117831..345c48f0 100755 --- a/config_tempest/main.py +++ b/config_tempest/main.py @@ -37,22 +37,23 @@ obtained by querying the cloud. """ import argparse -import ConfigParser import logging import os import sys -import accounts -from clients import ClientManager -import constants as C -from constants import LOG -from credentials import Credentials -from flavors import Flavors import os_client_config from oslo_config import cfg -from services.services import Services -import tempest_conf -from users import Users +from six.moves import configparser + +from config_tempest import accounts +from config_tempest.clients import ClientManager +from config_tempest import constants as C +from config_tempest.constants import LOG +from config_tempest.credentials import Credentials +from config_tempest.flavors import Flavors +from config_tempest.services.services import Services +from config_tempest.tempest_conf import TempestConf +from config_tempest.users import Users def set_logging(debug, verbose): @@ -111,7 +112,7 @@ def read_deployer_input(deployer_input_file, conf): """ LOG.info("Adding options from deployer-input file '%s'", deployer_input_file) - deployer_input = ConfigParser.SafeConfigParser() + deployer_input = configparser.SafeConfigParser() deployer_input.read(deployer_input_file) for section in deployer_input.sections(): # There are no deployer input options in DEFAULT @@ -368,7 +369,7 @@ def config_tempest(**kwargs): set_logging(kwargs.get('debug', False), kwargs.get('verbose', False)) write_credentials = kwargs.get('test_accounts') is None - conf = tempest_conf.TempestConf(write_credentials=write_credentials) + conf = TempestConf(write_credentials=write_credentials) set_options(conf, kwargs.get('deployer_input'), kwargs.get('non_admin', False), kwargs.get('overrides', []), kwargs.get('test_accounts'), diff --git a/config_tempest/services/base.py b/config_tempest/services/base.py index 37319198..f50efd9c 100644 --- a/config_tempest/services/base.py +++ b/config_tempest/services/base.py @@ -16,7 +16,8 @@ import json import re import urllib3 -import urlparse + +from six.moves import urllib from config_tempest.constants import LOG MULTIPLE_SLASH = re.compile(r'/+') @@ -39,13 +40,13 @@ class Service(object): self.versions = [] def do_get(self, url, top_level=False, top_level_path=""): - parts = list(urlparse.urlparse(url)) + parts = list(urllib.parse.urlparse(url)) # 2 is the path offset if top_level: parts[2] = '/' + top_level_path parts[2] = MULTIPLE_SLASH.sub('/', parts[2]) - url = urlparse.urlunparse(parts) + url = urllib.parse.urlunparse(parts) try: if self.disable_ssl_validation: diff --git a/config_tempest/services/ceilometer.py b/config_tempest/services/ceilometer.py index a47d3798..8a7cc4b8 100644 --- a/config_tempest/services/ceilometer.py +++ b/config_tempest/services/ceilometer.py @@ -13,7 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. -import config_tempest.constants as C +from config_tempest import constants as C from tempest.lib import exceptions diff --git a/config_tempest/services/compute.py b/config_tempest/services/compute.py index 152ae8a4..ced89009 100644 --- a/config_tempest/services/compute.py +++ b/config_tempest/services/compute.py @@ -13,11 +13,13 @@ # License for the specific language governing permissions and limitations # under the License. -from base import VersionedService -import config_tempest.constants as C import json + from tempest.lib import exceptions +from config_tempest import constants as C +from config_tempest.services.base import VersionedService + class ComputeService(VersionedService): def set_extensions(self): diff --git a/config_tempest/services/horizon.py b/config_tempest/services/horizon.py index 330ad762..93f66ff8 100644 --- a/config_tempest/services/horizon.py +++ b/config_tempest/services/horizon.py @@ -13,20 +13,20 @@ # License for the specific language governing permissions and limitations # under the License. -import urllib2 +from six.moves import urllib def configure_horizon(conf): """Derive the horizon URIs from the identity's URI.""" uri = conf.get('identity', 'uri') - u = urllib2.urlparse.urlparse(uri) + u = urllib.parse.urlparse(uri) base = '%s://%s%s' % (u.scheme, u.netloc.replace( ':' + str(u.port), ''), '/dashboard') assert base.startswith('http:') or base.startswith('https:') has_horizon = True try: - urllib2.urlopen(base) - except urllib2.URLError: + urllib.request.urlopen(base) + except urllib.error.URLError: has_horizon = False conf.set('service_available', 'horizon', str(has_horizon)) conf.set('dashboard', 'dashboard_url', base + '/') diff --git a/config_tempest/services/identity.py b/config_tempest/services/identity.py index 59bd9f7f..ba5e39d8 100644 --- a/config_tempest/services/identity.py +++ b/config_tempest/services/identity.py @@ -15,10 +15,11 @@ import json import requests -import urlparse -from base import VersionedService +from six.moves import urllib + from config_tempest.constants import LOG +from config_tempest.services.base import VersionedService class IdentityService(VersionedService): @@ -30,7 +31,7 @@ class IdentityService(VersionedService): version = '' if 'v2' in self.service_url: version = '/v2.0' - url_parse = urlparse.urlparse(self.service_url) + url_parse = urllib.parse.urlparse(self.service_url) self.service_url = '{}://{}{}'.format(url_parse.scheme, url_parse.netloc, version) diff --git a/config_tempest/services/image.py b/config_tempest/services/image.py index e1007514..fe35e10a 100644 --- a/config_tempest/services/image.py +++ b/config_tempest/services/image.py @@ -15,12 +15,13 @@ import os import shutil -import urllib2 -from base import VersionedService -from config_tempest.constants import LOG +from six.moves import urllib from tempest.lib import exceptions +from config_tempest.constants import LOG +from config_tempest.services.base import VersionedService + class ImageService(VersionedService): @@ -173,7 +174,7 @@ class ImageService(VersionedService): LOG.info("Image '%s' already fetched to '%s'.", url, destination) return LOG.info("Downloading '%s' and saving as '%s'", url, destination) - f = urllib2.urlopen(url) + f = urllib.request.urlopen(url) data = f.read() with open(destination, "wb") as dest: dest.write(data) diff --git a/config_tempest/services/network.py b/config_tempest/services/network.py index 967ecbd7..5f7ba4ea 100644 --- a/config_tempest/services/network.py +++ b/config_tempest/services/network.py @@ -15,8 +15,8 @@ import json -from base import VersionedService from config_tempest.constants import LOG +from config_tempest.services.base import VersionedService class NetworkService(VersionedService): diff --git a/config_tempest/services/object_storage.py b/config_tempest/services/object_storage.py index 8a5575f2..bdeee2fd 100644 --- a/config_tempest/services/object_storage.py +++ b/config_tempest/services/object_storage.py @@ -13,13 +13,14 @@ # License for the specific language governing permissions and limitations # under the License. -import ConfigParser import json -from base import Service -from config_tempest.constants import LOG +from six.moves import configparser from tempest.lib import exceptions +from config_tempest.constants import LOG +from config_tempest.services.base import Service + class ObjectStorageService(Service): def set_extensions(self): @@ -71,7 +72,7 @@ class ObjectStorageService(Service): 'object-storage-feature-enabled', 'discoverability')): return False - except ConfigParser.NoSectionError: + except configparser.NoSectionError: # Turning http://.../v1/foobar into http://.../ self.client.accounts.skip_path() resp, _ = self.client.accounts.get("healthcheck", {}) diff --git a/config_tempest/services/services.py b/config_tempest/services/services.py index 835eb1a2..0ad51b7c 100644 --- a/config_tempest/services/services.py +++ b/config_tempest/services/services.py @@ -13,20 +13,22 @@ # License for the specific language governing permissions and limitations # under the License. -import urlparse -from base import Service -import boto -import ceilometer -from compute import ComputeService -import config_tempest.constants as C -import horizon -from identity import IdentityService -from image import ImageService -from network import NetworkService -from object_storage import ObjectStorageService -from octavia import LoadBalancerService -import volume +from six.moves import urllib + +from config_tempest import constants as C +from config_tempest.services.base import Service +from config_tempest.services import boto +from config_tempest.services import ceilometer +from config_tempest.services.compute import ComputeService +from config_tempest.services import horizon +from config_tempest.services.identity import IdentityService +from config_tempest.services.image import ImageService +from config_tempest.services.network import NetworkService +from config_tempest.services.object_storage import ObjectStorageService +from config_tempest.services.octavia import LoadBalancerService +from config_tempest.services import volume + service_dict = {'compute': ComputeService, 'image': ImageService, @@ -148,7 +150,7 @@ class Services(object): # self._clients.auth_provider.auth_url stores identity.uri(_v3) value # from TempestConf - port = urlparse.urlparse(self._clients.auth_provider.auth_url).port + port = urllib.parse.urlparse(self._clients.auth_provider.auth_url).port if port is None: port = "" else: diff --git a/config_tempest/services/volume.py b/config_tempest/services/volume.py index e1e5d498..088b6f6b 100644 --- a/config_tempest/services/volume.py +++ b/config_tempest/services/volume.py @@ -15,10 +15,11 @@ import json -from base import VersionedService -import config_tempest.constants as C from tempest.lib import exceptions +from config_tempest import constants as C +from config_tempest.services.base import VersionedService + class VolumeService(VersionedService): def set_extensions(self): diff --git a/config_tempest/tempest_conf.py b/config_tempest/tempest_conf.py index d62b3408..e64818ae 100644 --- a/config_tempest/tempest_conf.py +++ b/config_tempest/tempest_conf.py @@ -13,16 +13,16 @@ # License for the specific language governing permissions and limitations # under the License. -import ConfigParser import os import sys -import constants as C +from config_tempest import constants as C from oslo_config import cfg +from six.moves import configparser import tempest.config -class TempestConf(ConfigParser.SafeConfigParser): +class TempestConf(configparser.SafeConfigParser): # causes the config parser to preserve case of the options optionxform = str @@ -34,7 +34,7 @@ class TempestConf(ConfigParser.SafeConfigParser): def __init__(self, write_credentials=True, **kwargs): self.write_credentials = write_credentials - ConfigParser.SafeConfigParser.__init__(self, **kwargs) + configparser.SafeConfigParser.__init__(self, **kwargs) def get_bool_value(self, value): """Returns boolean value of the string value given. @@ -101,7 +101,7 @@ class TempestConf(ConfigParser.SafeConfigParser): if priority: self.priority_sectionkeys.add((section, key)) C.LOG.debug("Setting [%s] %s = %s", section, key, value) - ConfigParser.SafeConfigParser.set(self, section, key, value) + configparser.SafeConfigParser.set(self, section, key, value) return True def write(self, out_path): @@ -111,7 +111,7 @@ class TempestConf(ConfigParser.SafeConfigParser): "writing credentials is disabled.") self.remove_values(C.ALL_CREDENTIALS_KEYS) with open(out_path, 'w') as f: - ConfigParser.SafeConfigParser.write(self, f) + configparser.SafeConfigParser.write(self, f) def remove_values(self, to_remove): """Remove values from configuration file specified in arguments. @@ -138,9 +138,9 @@ class TempestConf(ConfigParser.SafeConfigParser): # and preserve the original order of items conf_values = [v for v in conf_values if v not in remove] self.set(section, key, ",".join(conf_values)) - except ConfigParser.NoOptionError: + except configparser.NoOptionError: # only inform a user, option specified by him doesn't exist C.LOG.error(sys.exc_info()[1]) - except ConfigParser.NoSectionError: + except configparser.NoSectionError: # only inform a user, section specified by him doesn't exist C.LOG.error(sys.exc_info()[1]) diff --git a/config_tempest/tests/services/test_ceilometer.py b/config_tempest/tests/services/test_ceilometer.py index 01480604..40217be3 100644 --- a/config_tempest/tests/services/test_ceilometer.py +++ b/config_tempest/tests/services/test_ceilometer.py @@ -13,7 +13,8 @@ # License for the specific language governing permissions and limitations # under the License. -import ConfigParser + +from six.moves import configparser from config_tempest.services import ceilometer from config_tempest.tempest_conf import TempestConf @@ -29,7 +30,7 @@ class TestCeilometerService(BaseServiceTest): client_service_mock = self.FakeServiceClient(services={}) ceilometer.check_ceilometer_service(self.conf, client_service_mock) - self._assert_conf_get_not_raises(ConfigParser.NoSectionError, + self._assert_conf_get_not_raises(configparser.NoSectionError, "service_available", "ceilometer") diff --git a/config_tempest/tests/services/test_horizon.py b/config_tempest/tests/services/test_horizon.py index 8b19a898..33e847ac 100644 --- a/config_tempest/tests/services/test_horizon.py +++ b/config_tempest/tests/services/test_horizon.py @@ -28,7 +28,8 @@ class TestConfigTempest(BaseConfigTempestTest): def test_configure_horizon_ipv4(self): mock_function = mock.Mock(return_value=True) - self.useFixture(MonkeyPatch('urllib2.urlopen', mock_function)) + self.useFixture(MonkeyPatch('six.moves.urllib.request.urlopen', + mock_function)) horizon.configure_horizon(self.conf) self.assertEqual(self.conf.get('service_available', 'horizon'), "True") self.assertEqual(self.conf.get('dashboard', 'dashboard_url'), @@ -38,7 +39,8 @@ class TestConfigTempest(BaseConfigTempestTest): def test_configure_horizon_ipv6(self): mock_function = mock.Mock(return_value=True) - self.useFixture(MonkeyPatch('urllib2.urlopen', mock_function)) + self.useFixture(MonkeyPatch('six.moves.urllib.request.urlopen', + mock_function)) self.conf.set('identity', 'uri', 'http://[::1]:5000/v3', priority=True) horizon.configure_horizon(self.conf) self.assertEqual(self.conf.get('service_available', 'horizon'), "True") diff --git a/config_tempest/users.py b/config_tempest/users.py index 74e78ca4..769f46cb 100644 --- a/config_tempest/users.py +++ b/config_tempest/users.py @@ -13,7 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. -from constants import LOG +from config_tempest.constants import LOG from tempest.lib import exceptions diff --git a/requirements.txt b/requirements.txt index 0d74e6b7..a46c41ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ # process, which may cause wedges in the gate later. pbr>=1.8 # Apache-2.0 +six>=1.10.0 # MIT tempest>=14.0.0 # Apache-2.0 requests>=2.10.0,!=2.12.2 # Apache-2.0 os-client-config>=1.26.0 # Apache-2.0 diff --git a/tox.ini b/tox.ini index 9e9f963f..13634418 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] minversion = 2.0 -envlist = py34,py27,pypy,pep8 +envlist = py35,py27,pypy,pep8 skipsdist = True [testenv]