Improve compatible with python3

Changes:
* use six.iteritems instead dict.iteritems
* use urllib in CDH API client
* filtering directories in HDP versionhandler because py3 creates
  __pycache__ directory

Unit tests result:

Ran: 750 tests in 13.4569 sec.
 - Passed: 669
 - Skipped: 5
 - Expected Fail: 0
 - Unexpected Success: 0
 - Failed: 76

Change-Id: Ib36b24f04a933c6e134bdace4994a5020ff4166b

sahara/api/middleware/log_exchange.py

@@ -21,6 +21,7 @@ from __future__ import print_function
 import sys
 
 from oslo_middleware import base
+import six
 import webob.dec
 
 
@@ -47,7 +48,7 @@ class LogExchange(base.Middleware):
         resp = req.get_response(self.application)
 
         print(("*" * 40) + " RESPONSE HEADERS")
-        for (key, value) in resp.headers.iteritems():
+        for (key, value) in six.iteritems(resp.headers):
             print(key, "=", value)
         print()
 

sahara/conductor/resource.py

@@ -87,7 +87,7 @@ class Resource(types.FrozenDict):
     def __init__(self, dct):
         super(Resource, self).__setattr__('_initial_dict', dct)
         newdct = dict()
-        for refname, entity in dct.iteritems():
+        for refname, entity in six.iteritems(dct):
             newdct[refname] = self._wrap_entity(refname, entity)
 
         super(Resource, self).__init__(newdct)
@@ -143,7 +143,7 @@ class Resource(types.FrozenDict):
 
     def _to_dict(self, backref):
         dct = dict()
-        for refname, entity in self.iteritems():
+        for refname, entity in six.iteritems(self):
             if refname != backref and refname not in self._filter_fields:
                 childs_backref = None
                 if refname in self._children:

sahara/plugins/cdh/client/http_client.py

@@ -22,15 +22,13 @@
 # To satisfy the pep8 and python3 tests, we did some changes to the codes.
 # We also change some importings to use Sahara inherited classes.
 
-import cookielib
 import posixpath
 import types
-import urllib
-import urllib2
 
 from oslo_log import log as logging
 from oslo_serialization import jsonutils as json
 import six
+from six.moves import urllib
 
 from sahara.i18n import _LW
 from sahara.plugins.cdh import exceptions as ex
@@ -53,15 +51,15 @@ class HttpClient(object):
         self._headers = {}
 
         # Make a basic auth handler that does nothing. Set credentials later.
-        self._passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
+        self._passmgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
-        authhandler = urllib2.HTTPBasicAuthHandler(self._passmgr)
+        authhandler = urllib.request.HTTPBasicAuthHandler(self._passmgr)
 
         # Make a cookie processor
-        cookiejar = cookielib.CookieJar()
+        cookiejar = six.moves.http_cookiejar.CookieJar()
 
-        self._opener = urllib2.build_opener(
+        self._opener = urllib.request.build_opener(
-            urllib2.HTTPErrorProcessor(),
+            urllib.request.HTTPErrorProcessor(),
-            urllib2.HTTPCookieProcessor(cookiejar),
+            urllib.request.HTTPCookieProcessor(cookiejar),
             authhandler)
 
     def set_basic_auth(self, username, password, realm):
@@ -103,7 +101,7 @@ class HttpClient(object):
         :param data: The data to attach to the body of the request.
         :param headers: The headers to set for this request.
 
-        :return: The result of urllib2.urlopen()
+        :return: The result of urllib.request.urlopen()
         """
         # Prepare URL and params
         url = self._make_url(path, params)
@@ -115,7 +113,7 @@ class HttpClient(object):
                 data = None
 
         # Setup the request
-        request = urllib2.Request(url, data)
+        request = urllib.request.Request(url, data)
         # Hack/workaround because urllib2 only does GET and POST
         request.get_method = lambda: http_method
 
@@ -128,7 +126,7 @@ class HttpClient(object):
                                                         url=url))
         try:
             return self._opener.open(request)
-        except urllib2.HTTPError as ex:
+        except urllib.error.HTTPError as ex:
             message = six.text_type(ex)
             try:
                 json_body = json.loads(message)
@@ -142,7 +140,7 @@ class HttpClient(object):
         if path:
             res += posixpath.normpath('/' + path.lstrip('/'))
         if params:
-            param_str = urllib.urlencode(params, True)
+            param_str = urllib.parse.urlencode(params, True)
             res += '?' + param_str
         return iri_to_uri(res)
 
@@ -169,14 +167,14 @@ def iri_to_uri(iri):
     #     sub-delims  = "!" / "$" / "&" / "'" / "(" / ")"
     #                   / "*" / "+" / "," / ";" / "="
     #     unreserved  = ALPHA / DIGIT / "-" / "." / "_" / "~"
-    # Of the unreserved characters, urllib.quote already considers all but
+    # Of the unreserved characters, urllib.parse.quote already considers all
-    # the ~ safe.
+    # but the ~ safe.
     # The % character is also added to the list of safe characters here, as the
     # end of section 3.1 of RFC 3987 specifically mentions that % must not be
     # converted.
     if iri is None:
         return iri
-    return urllib.quote(smart_str(iri), safe="/#%[]=:;$&()+,!?*@'~")
+    return urllib.parse.quote(smart_str(iri), safe="/#%[]=:;$&()+,!?*@'~")
 
 
 #

sahara/plugins/cdh/client/resource.py

@@ -24,11 +24,11 @@
 
 import posixpath
 import socket
-import urllib2
 
 from oslo_log import log as logging
 from oslo_serialization import jsonutils as json
 import six
+from six.moves import urllib
 
 from sahara import context
 from sahara.i18n import _
@@ -111,7 +111,7 @@ class Resource(object):
                 context.sleep(self.retry_sleep)
             try:
                 return self.invoke("GET", relpath, params)
-            except (socket.error, urllib2.URLError) as e:
+            except (socket.error, urllib.error.URLError) as e:
                 if "timed out" in six.text_type(e).lower():
                     if retry < self.retries:
                         LOG.warning(_LW("Timeout issuing GET request for "

sahara/plugins/cdh/v5/resources/cdh_config.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 from oslo_serialization import jsonutils as json
+import six
 
 from sahara.plugins.cdh.client import api_client
 
@@ -58,7 +59,7 @@ def process_service(service, service_name):
 
 def parse_config(config):
     cfg = []
-    for name, value in config.iteritems():
+    for name, value in six.iteritems(config):
         p = {
             'name': value.name,
             'value': value.default,

sahara/plugins/cdh/v5_3_0/resources/cdh_config.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 from oslo_serialization import jsonutils as json
+import six
 
 from sahara.plugins.cdh.client import api_client
 
@@ -63,7 +64,7 @@ def process_service(service, service_name):
 
 def parse_config(config):
     cfg = []
-    for name, value in config.iteritems():
+    for name, value in six.iteritems(config):
         p = {
             'name': value.name,
             'value': value.default,

sahara/plugins/hdp/versions/versionhandlerfactory.py

@@ -29,7 +29,8 @@ class VersionHandlerFactory(object):
             src_dir = os.path.join(os.path.dirname(__file__), '')
             versions = [name[8:].replace('_', '.')
                         for name in os.listdir(src_dir)
-                        if os.path.isdir(os.path.join(src_dir, name))]
+                        if os.path.isdir(os.path.join(src_dir, name))
+                        and name.startswith('version_')]
             versions.sort(key=general.natural_sort_key)
             VersionHandlerFactory.versions = versions
 

sahara/plugins/spark/config_helper.py

@@ -186,7 +186,7 @@ PRIORITY_1_CONFS += CLUSTER_WIDE_CONFS
 
 def _initialise_configs():
     configs = []
-    for service, config_lists in XML_CONFS.iteritems():
+    for service, config_lists in six.iteritems(XML_CONFS):
         for config_list in config_lists:
             for config in config_list:
                 if config['name'] not in HIDDEN_CONFS:
@@ -206,13 +206,13 @@ def _initialise_configs():
                         cfg.priority = 1
                     configs.append(cfg)
 
-    for service, config_items in ENV_CONFS.iteritems():
+    for service, config_items in six.iteritems(ENV_CONFS):
-        for name, param_format_str in config_items.iteritems():
+        for name, param_format_str in six.iteritems(config_items):
             configs.append(p.Config(name, service, "node",
                                     default_value=1024, priority=1,
                                     config_type="int"))
 
-    for service, config_items in SPARK_CONFS.iteritems():
+    for service, config_items in six.iteritems(SPARK_CONFS):
         for item in config_items['OPTIONS']:
             cfg = p.Config(name=item["name"],
                            description=item["description"],

sahara/plugins/vanilla/hadoop2/config_helper.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 from oslo_config import cfg
+import six
 
 from sahara import exceptions as ex
 from sahara.i18n import _
@@ -90,7 +91,7 @@ PRIORITY_1_CONFS += CLUSTER_WIDE_CONFS
 
 def init_xml_configs(xml_confs):
     configs = []
-    for service, config_lists in xml_confs.iteritems():
+    for service, config_lists in six.iteritems(xml_confs):
         for config_list in config_lists:
             for config in config_list:
                 if config['name'] not in HIDDEN_CONFS:
@@ -150,8 +151,8 @@ DATANODES_STARTUP_TIMEOUT = p.Config(
 
 def init_env_configs(env_confs):
     configs = []
-    for service, config_items in env_confs.iteritems():
+    for service, config_items in six.iteritems(env_confs):
-        for name, value in config_items.iteritems():
+        for name, value in six.iteritems(config_items):
             configs.append(p.Config(name, service, "node",
                                     default_value=value, priority=1,
                                     config_type="int"))

sahara/plugins/vanilla/v1_2_1/config_helper.py

@@ -15,6 +15,7 @@
 
 from oslo_config import cfg
 from oslo_log import log as logging
+import six
 
 from sahara import conductor as c
 from sahara import context
@@ -135,7 +136,7 @@ PRIORITY_1_CONFS += CLUSTER_WIDE_CONFS
 
 def _initialise_configs():
     configs = []
-    for service, config_lists in XML_CONFS.iteritems():
+    for service, config_lists in six.iteritems(XML_CONFS):
         for config_list in config_lists:
             for config in config_list:
                 if config['name'] not in HIDDEN_CONFS:
@@ -155,8 +156,8 @@ def _initialise_configs():
                         cfg.priority = 1
                     configs.append(cfg)
 
-    for service, config_items in ENV_CONFS.iteritems():
+    for service, config_items in six.iteritems(ENV_CONFS):
-        for name, param_format_str in config_items.iteritems():
+        for name, param_format_str in six.iteritems(config_items):
             configs.append(p.Config(name, service, "node",
                                     default_value=1024, priority=1,
                                     config_type="int"))

sahara/service/validations/base.py

@@ -52,7 +52,7 @@ def _get_plugin_configs(plugin_name, hadoop_version, scope=None):
 def _check_duplicates(lst, message):
     invalid = []
     lst = collections.Counter(lst)
-    for (key, value) in lst.iteritems():
+    for key, value in six.iteritems(lst):
         if value > 1:
             invalid.append(key)
 

sahara/utils/resources.py

@@ -15,6 +15,8 @@
 
 import inspect
 
+import six
+
 
 class BaseResource(object):
     __resource_name__ = 'base'
@@ -43,7 +45,7 @@ class BaseResource(object):
 
     def to_dict(self):
         dictionary = self.__dict__.copy()
-        return {k: v for k, v in dictionary.iteritems()
+        return {k: v for k, v in six.iteritems(dictionary)
                 if not self._filter_field(k)}
 
     def as_resource(self):