diff --git a/sahara/api/middleware/log_exchange.py b/sahara/api/middleware/log_exchange.py index a8785ed7..9ef57f94 100644 --- a/sahara/api/middleware/log_exchange.py +++ b/sahara/api/middleware/log_exchange.py @@ -21,6 +21,7 @@ from __future__ import print_function import sys from oslo_middleware import base +import six import webob.dec @@ -47,7 +48,7 @@ class LogExchange(base.Middleware): resp = req.get_response(self.application) print(("*" * 40) + " RESPONSE HEADERS") - for (key, value) in resp.headers.iteritems(): + for (key, value) in six.iteritems(resp.headers): print(key, "=", value) print() diff --git a/sahara/conductor/resource.py b/sahara/conductor/resource.py index 152ad6ae..1930e5a5 100644 --- a/sahara/conductor/resource.py +++ b/sahara/conductor/resource.py @@ -87,7 +87,7 @@ class Resource(types.FrozenDict): def __init__(self, dct): super(Resource, self).__setattr__('_initial_dict', dct) newdct = dict() - for refname, entity in dct.iteritems(): + for refname, entity in six.iteritems(dct): newdct[refname] = self._wrap_entity(refname, entity) super(Resource, self).__init__(newdct) @@ -143,7 +143,7 @@ class Resource(types.FrozenDict): def _to_dict(self, backref): dct = dict() - for refname, entity in self.iteritems(): + for refname, entity in six.iteritems(self): if refname != backref and refname not in self._filter_fields: childs_backref = None if refname in self._children: diff --git a/sahara/plugins/cdh/client/http_client.py b/sahara/plugins/cdh/client/http_client.py index 925f20cb..b591bc8e 100644 --- a/sahara/plugins/cdh/client/http_client.py +++ b/sahara/plugins/cdh/client/http_client.py @@ -22,15 +22,13 @@ # To satisfy the pep8 and python3 tests, we did some changes to the codes. # We also change some importings to use Sahara inherited classes. -import cookielib import posixpath import types -import urllib -import urllib2 from oslo_log import log as logging from oslo_serialization import jsonutils as json import six +from six.moves import urllib from sahara.i18n import _LW from sahara.plugins.cdh import exceptions as ex @@ -53,15 +51,15 @@ class HttpClient(object): self._headers = {} # Make a basic auth handler that does nothing. Set credentials later. - self._passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm() - authhandler = urllib2.HTTPBasicAuthHandler(self._passmgr) + self._passmgr = urllib.request.HTTPPasswordMgrWithDefaultRealm() + authhandler = urllib.request.HTTPBasicAuthHandler(self._passmgr) # Make a cookie processor - cookiejar = cookielib.CookieJar() + cookiejar = six.moves.http_cookiejar.CookieJar() - self._opener = urllib2.build_opener( - urllib2.HTTPErrorProcessor(), - urllib2.HTTPCookieProcessor(cookiejar), + self._opener = urllib.request.build_opener( + urllib.request.HTTPErrorProcessor(), + urllib.request.HTTPCookieProcessor(cookiejar), authhandler) def set_basic_auth(self, username, password, realm): @@ -103,7 +101,7 @@ class HttpClient(object): :param data: The data to attach to the body of the request. :param headers: The headers to set for this request. - :return: The result of urllib2.urlopen() + :return: The result of urllib.request.urlopen() """ # Prepare URL and params url = self._make_url(path, params) @@ -115,7 +113,7 @@ class HttpClient(object): data = None # Setup the request - request = urllib2.Request(url, data) + request = urllib.request.Request(url, data) # Hack/workaround because urllib2 only does GET and POST request.get_method = lambda: http_method @@ -128,7 +126,7 @@ class HttpClient(object): url=url)) try: return self._opener.open(request) - except urllib2.HTTPError as ex: + except urllib.error.HTTPError as ex: message = six.text_type(ex) try: json_body = json.loads(message) @@ -142,7 +140,7 @@ class HttpClient(object): if path: res += posixpath.normpath('/' + path.lstrip('/')) if params: - param_str = urllib.urlencode(params, True) + param_str = urllib.parse.urlencode(params, True) res += '?' + param_str return iri_to_uri(res) @@ -169,14 +167,14 @@ def iri_to_uri(iri): # sub-delims = "!" / "$" / "&" / "'" / "(" / ")" # / "*" / "+" / "," / ";" / "=" # unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~" - # Of the unreserved characters, urllib.quote already considers all but - # the ~ safe. + # Of the unreserved characters, urllib.parse.quote already considers all + # but the ~ safe. # The % character is also added to the list of safe characters here, as the # end of section 3.1 of RFC 3987 specifically mentions that % must not be # converted. if iri is None: return iri - return urllib.quote(smart_str(iri), safe="/#%[]=:;$&()+,!?*@'~") + return urllib.parse.quote(smart_str(iri), safe="/#%[]=:;$&()+,!?*@'~") # diff --git a/sahara/plugins/cdh/client/resource.py b/sahara/plugins/cdh/client/resource.py index 939d543b..92e81ec2 100644 --- a/sahara/plugins/cdh/client/resource.py +++ b/sahara/plugins/cdh/client/resource.py @@ -24,11 +24,11 @@ import posixpath import socket -import urllib2 from oslo_log import log as logging from oslo_serialization import jsonutils as json import six +from six.moves import urllib from sahara import context from sahara.i18n import _ @@ -111,7 +111,7 @@ class Resource(object): context.sleep(self.retry_sleep) try: return self.invoke("GET", relpath, params) - except (socket.error, urllib2.URLError) as e: + except (socket.error, urllib.error.URLError) as e: if "timed out" in six.text_type(e).lower(): if retry < self.retries: LOG.warning(_LW("Timeout issuing GET request for " diff --git a/sahara/plugins/cdh/v5/resources/cdh_config.py b/sahara/plugins/cdh/v5/resources/cdh_config.py index 86839724..9a9cd676 100644 --- a/sahara/plugins/cdh/v5/resources/cdh_config.py +++ b/sahara/plugins/cdh/v5/resources/cdh_config.py @@ -14,6 +14,7 @@ # limitations under the License. from oslo_serialization import jsonutils as json +import six from sahara.plugins.cdh.client import api_client @@ -58,7 +59,7 @@ def process_service(service, service_name): def parse_config(config): cfg = [] - for name, value in config.iteritems(): + for name, value in six.iteritems(config): p = { 'name': value.name, 'value': value.default, diff --git a/sahara/plugins/cdh/v5_3_0/resources/cdh_config.py b/sahara/plugins/cdh/v5_3_0/resources/cdh_config.py index b9fd0d88..65dc3185 100644 --- a/sahara/plugins/cdh/v5_3_0/resources/cdh_config.py +++ b/sahara/plugins/cdh/v5_3_0/resources/cdh_config.py @@ -14,6 +14,7 @@ # limitations under the License. from oslo_serialization import jsonutils as json +import six from sahara.plugins.cdh.client import api_client @@ -63,7 +64,7 @@ def process_service(service, service_name): def parse_config(config): cfg = [] - for name, value in config.iteritems(): + for name, value in six.iteritems(config): p = { 'name': value.name, 'value': value.default, diff --git a/sahara/plugins/hdp/versions/versionhandlerfactory.py b/sahara/plugins/hdp/versions/versionhandlerfactory.py index 42572f01..6b319a8b 100644 --- a/sahara/plugins/hdp/versions/versionhandlerfactory.py +++ b/sahara/plugins/hdp/versions/versionhandlerfactory.py @@ -29,7 +29,8 @@ class VersionHandlerFactory(object): src_dir = os.path.join(os.path.dirname(__file__), '') versions = [name[8:].replace('_', '.') for name in os.listdir(src_dir) - if os.path.isdir(os.path.join(src_dir, name))] + if os.path.isdir(os.path.join(src_dir, name)) + and name.startswith('version_')] versions.sort(key=general.natural_sort_key) VersionHandlerFactory.versions = versions diff --git a/sahara/plugins/spark/config_helper.py b/sahara/plugins/spark/config_helper.py index 2f62036d..6e3abb7a 100644 --- a/sahara/plugins/spark/config_helper.py +++ b/sahara/plugins/spark/config_helper.py @@ -186,7 +186,7 @@ PRIORITY_1_CONFS += CLUSTER_WIDE_CONFS def _initialise_configs(): configs = [] - for service, config_lists in XML_CONFS.iteritems(): + for service, config_lists in six.iteritems(XML_CONFS): for config_list in config_lists: for config in config_list: if config['name'] not in HIDDEN_CONFS: @@ -206,13 +206,13 @@ def _initialise_configs(): cfg.priority = 1 configs.append(cfg) - for service, config_items in ENV_CONFS.iteritems(): - for name, param_format_str in config_items.iteritems(): + for service, config_items in six.iteritems(ENV_CONFS): + for name, param_format_str in six.iteritems(config_items): configs.append(p.Config(name, service, "node", default_value=1024, priority=1, config_type="int")) - for service, config_items in SPARK_CONFS.iteritems(): + for service, config_items in six.iteritems(SPARK_CONFS): for item in config_items['OPTIONS']: cfg = p.Config(name=item["name"], description=item["description"], diff --git a/sahara/plugins/vanilla/hadoop2/config_helper.py b/sahara/plugins/vanilla/hadoop2/config_helper.py index 7d294a6f..686283fe 100644 --- a/sahara/plugins/vanilla/hadoop2/config_helper.py +++ b/sahara/plugins/vanilla/hadoop2/config_helper.py @@ -14,6 +14,7 @@ # limitations under the License. from oslo_config import cfg +import six from sahara import exceptions as ex from sahara.i18n import _ @@ -90,7 +91,7 @@ PRIORITY_1_CONFS += CLUSTER_WIDE_CONFS def init_xml_configs(xml_confs): configs = [] - for service, config_lists in xml_confs.iteritems(): + for service, config_lists in six.iteritems(xml_confs): for config_list in config_lists: for config in config_list: if config['name'] not in HIDDEN_CONFS: @@ -150,8 +151,8 @@ DATANODES_STARTUP_TIMEOUT = p.Config( def init_env_configs(env_confs): configs = [] - for service, config_items in env_confs.iteritems(): - for name, value in config_items.iteritems(): + for service, config_items in six.iteritems(env_confs): + for name, value in six.iteritems(config_items): configs.append(p.Config(name, service, "node", default_value=value, priority=1, config_type="int")) diff --git a/sahara/plugins/vanilla/v1_2_1/config_helper.py b/sahara/plugins/vanilla/v1_2_1/config_helper.py index 8e685d26..d777d729 100644 --- a/sahara/plugins/vanilla/v1_2_1/config_helper.py +++ b/sahara/plugins/vanilla/v1_2_1/config_helper.py @@ -15,6 +15,7 @@ from oslo_config import cfg from oslo_log import log as logging +import six from sahara import conductor as c from sahara import context @@ -135,7 +136,7 @@ PRIORITY_1_CONFS += CLUSTER_WIDE_CONFS def _initialise_configs(): configs = [] - for service, config_lists in XML_CONFS.iteritems(): + for service, config_lists in six.iteritems(XML_CONFS): for config_list in config_lists: for config in config_list: if config['name'] not in HIDDEN_CONFS: @@ -155,8 +156,8 @@ def _initialise_configs(): cfg.priority = 1 configs.append(cfg) - for service, config_items in ENV_CONFS.iteritems(): - for name, param_format_str in config_items.iteritems(): + for service, config_items in six.iteritems(ENV_CONFS): + for name, param_format_str in six.iteritems(config_items): configs.append(p.Config(name, service, "node", default_value=1024, priority=1, config_type="int")) diff --git a/sahara/service/validations/base.py b/sahara/service/validations/base.py index d3743c67..23f7bfc3 100644 --- a/sahara/service/validations/base.py +++ b/sahara/service/validations/base.py @@ -52,7 +52,7 @@ def _get_plugin_configs(plugin_name, hadoop_version, scope=None): def _check_duplicates(lst, message): invalid = [] lst = collections.Counter(lst) - for (key, value) in lst.iteritems(): + for key, value in six.iteritems(lst): if value > 1: invalid.append(key) diff --git a/sahara/utils/resources.py b/sahara/utils/resources.py index 3b075d66..0234be46 100644 --- a/sahara/utils/resources.py +++ b/sahara/utils/resources.py @@ -15,6 +15,8 @@ import inspect +import six + class BaseResource(object): __resource_name__ = 'base' @@ -43,7 +45,7 @@ class BaseResource(object): def to_dict(self): dictionary = self.__dict__.copy() - return {k: v for k, v in dictionary.iteritems() + return {k: v for k, v in six.iteritems(dictionary) if not self._filter_field(k)} def as_resource(self):