openstack/deb-sahara
Code Issues Proposed changes

Improve compatible with python3

Browse Source 
Changes:
* use six.iteritems instead dict.iteritems
* use urllib in CDH API client
* filtering directories in HDP versionhandler because py3 creates
  __pycache__ directory

Unit tests result:

Ran: 750 tests in 13.4569 sec.
 - Passed: 669
 - Skipped: 5
 - Expected Fail: 0
 - Unexpected Success: 0
 - Failed: 76

Change-Id: Ib36b24f04a933c6e134bdace4994a5020ff4166b
This commit is contained in:
Sergey Reshetnyak 2015-05-10 22:48:41 +03:00
parent 1af382f7a4
commit 1f9d4089f9
12 changed files with 42 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
sahara/api/middleware/log_exchange.py
View File

@ -21,6 +21,7 @@ from __future__ import print_function
import sys
from oslo_middleware import base
import six
import webob.dec
@ -47,7 +48,7 @@ class LogExchange(base.Middleware):
resp = req.get_response(self.application)
print(("*" * 40) + " RESPONSE HEADERS")
for (key, value) in resp.headers.iteritems():
for (key, value) in six.iteritems(resp.headers):
print(key, "=", value)
print()

4
sahara/conductor/resource.py
View File

@ -87,7 +87,7 @@ class Resource(types.FrozenDict):
def __init__(self, dct):
super(Resource, self).__setattr__('_initial_dict', dct)
newdct = dict()
for refname, entity in dct.iteritems():
for refname, entity in six.iteritems(dct):
newdct[refname] = self._wrap_entity(refname, entity)
super(Resource, self).__init__(newdct)
@ -143,7 +143,7 @@ class Resource(types.FrozenDict):
def _to_dict(self, backref):
dct = dict()
for refname, entity in self.iteritems():
for refname, entity in six.iteritems(self):
if refname != backref and refname not in self._filter_fields:
childs_backref = None
if refname in self._children:

30
sahara/plugins/cdh/client/http_client.py
View File

@ -22,15 +22,13 @@
# To satisfy the pep8 and python3 tests, we did some changes to the codes.
# We also change some importings to use Sahara inherited classes.
import cookielib
import posixpath
import types
import urllib
import urllib2
from oslo_log import log as logging
from oslo_serialization import jsonutils as json
import six
from six.moves import urllib
from sahara.i18n import _LW
from sahara.plugins.cdh import exceptions as ex
@ -53,15 +51,15 @@ class HttpClient(object):
self._headers = {}
# Make a basic auth handler that does nothing. Set credentials later.
self._passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
authhandler = urllib2.HTTPBasicAuthHandler(self._passmgr)
self._passmgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
authhandler = urllib.request.HTTPBasicAuthHandler(self._passmgr)
# Make a cookie processor
cookiejar = cookielib.CookieJar()
cookiejar = six.moves.http_cookiejar.CookieJar()
self._opener = urllib2.build_opener(
urllib2.HTTPErrorProcessor(),
urllib2.HTTPCookieProcessor(cookiejar),
self._opener = urllib.request.build_opener(
urllib.request.HTTPErrorProcessor(),
urllib.request.HTTPCookieProcessor(cookiejar),
authhandler)
def set_basic_auth(self, username, password, realm):
@ -103,7 +101,7 @@ class HttpClient(object):
:param data: The data to attach to the body of the request.
:param headers: The headers to set for this request.
:return: The result of urllib2.urlopen()
:return: The result of urllib.request.urlopen()
"""
# Prepare URL and params
url = self._make_url(path, params)
@ -115,7 +113,7 @@ class HttpClient(object):
data = None
# Setup the request
request = urllib2.Request(url, data)
request = urllib.request.Request(url, data)
# Hack/workaround because urllib2 only does GET and POST
request.get_method = lambda: http_method
@ -128,7 +126,7 @@ class HttpClient(object):
url=url))
try:
return self._opener.open(request)
except urllib2.HTTPError as ex:
except urllib.error.HTTPError as ex:
message = six.text_type(ex)
try:
json_body = json.loads(message)
@ -142,7 +140,7 @@ class HttpClient(object):
if path:
res += posixpath.normpath('/' + path.lstrip('/'))
if params:
param_str = urllib.urlencode(params, True)
param_str = urllib.parse.urlencode(params, True)
res += '?' + param_str
return iri_to_uri(res)
@ -169,14 +167,14 @@ def iri_to_uri(iri):
# sub-delims = "!" / "$" / "&" / "'" / "(" / ")"
# / "*" / "+" / "," / ";" / "="
# unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
# Of the unreserved characters, urllib.quote already considers all but
# the ~ safe.
# Of the unreserved characters, urllib.parse.quote already considers all
# but the ~ safe.
# The % character is also added to the list of safe characters here, as the
# end of section 3.1 of RFC 3987 specifically mentions that % must not be
# converted.
if iri is None:
return iri
return urllib.quote(smart_str(iri), safe="/#%[]=:;$&()+,!?*@'~")
return urllib.parse.quote(smart_str(iri), safe="/#%[]=:;$&()+,!?*@'~")
#

4
sahara/plugins/cdh/client/resource.py
View File

@ -24,11 +24,11 @@
import posixpath
import socket
import urllib2
from oslo_log import log as logging
from oslo_serialization import jsonutils as json
import six
from six.moves import urllib
from sahara import context
from sahara.i18n import _
@ -111,7 +111,7 @@ class Resource(object):
context.sleep(self.retry_sleep)
try:
return self.invoke("GET", relpath, params)
except (socket.error, urllib2.URLError) as e:
except (socket.error, urllib.error.URLError) as e:
if "timed out" in six.text_type(e).lower():
if retry < self.retries:
LOG.warning(_LW("Timeout issuing GET request for "

3
sahara/plugins/cdh/v5/resources/cdh_config.py
View File

@ -14,6 +14,7 @@
# limitations under the License.
from oslo_serialization import jsonutils as json
import six
from sahara.plugins.cdh.client import api_client
@ -58,7 +59,7 @@ def process_service(service, service_name):
def parse_config(config):
cfg = []
for name, value in config.iteritems():
for name, value in six.iteritems(config):
p = {
'name': value.name,
'value': value.default,

3
sahara/plugins/cdh/v5_3_0/resources/cdh_config.py
View File

@ -14,6 +14,7 @@
# limitations under the License.
from oslo_serialization import jsonutils as json
import six
from sahara.plugins.cdh.client import api_client
@ -63,7 +64,7 @@ def process_service(service, service_name):
def parse_config(config):
cfg = []
for name, value in config.iteritems():
for name, value in six.iteritems(config):
p = {
'name': value.name,
'value': value.default,

3
sahara/plugins/hdp/versions/versionhandlerfactory.py
View File

@ -29,7 +29,8 @@ class VersionHandlerFactory(object):
src_dir = os.path.join(os.path.dirname(__file__), '')
versions = [name[8:].replace('_', '.')
for name in os.listdir(src_dir)
if os.path.isdir(os.path.join(src_dir, name))]
if os.path.isdir(os.path.join(src_dir, name))
and name.startswith('version_')]
versions.sort(key=general.natural_sort_key)
VersionHandlerFactory.versions = versions

8
sahara/plugins/spark/config_helper.py
View File

@ -186,7 +186,7 @@ PRIORITY_1_CONFS += CLUSTER_WIDE_CONFS
def _initialise_configs():
configs = []
for service, config_lists in XML_CONFS.iteritems():
for service, config_lists in six.iteritems(XML_CONFS):
for config_list in config_lists:
for config in config_list:
if config['name'] not in HIDDEN_CONFS:
@ -206,13 +206,13 @@ def _initialise_configs():
cfg.priority = 1
configs.append(cfg)
for service, config_items in ENV_CONFS.iteritems():
for name, param_format_str in config_items.iteritems():
for service, config_items in six.iteritems(ENV_CONFS):
for name, param_format_str in six.iteritems(config_items):
configs.append(p.Config(name, service, "node",
default_value=1024, priority=1,
config_type="int"))
for service, config_items in SPARK_CONFS.iteritems():
for service, config_items in six.iteritems(SPARK_CONFS):
for item in config_items['OPTIONS']:
cfg = p.Config(name=item["name"],
description=item["description"],

7
sahara/plugins/vanilla/hadoop2/config_helper.py
View File

@ -14,6 +14,7 @@
# limitations under the License.
from oslo_config import cfg
import six
from sahara import exceptions as ex
from sahara.i18n import _
@ -90,7 +91,7 @@ PRIORITY_1_CONFS += CLUSTER_WIDE_CONFS
def init_xml_configs(xml_confs):
configs = []
for service, config_lists in xml_confs.iteritems():
for service, config_lists in six.iteritems(xml_confs):
for config_list in config_lists:
for config in config_list:
if config['name'] not in HIDDEN_CONFS:
@ -150,8 +151,8 @@ DATANODES_STARTUP_TIMEOUT = p.Config(
def init_env_configs(env_confs):
configs = []
for service, config_items in env_confs.iteritems():
for name, value in config_items.iteritems():
for service, config_items in six.iteritems(env_confs):
for name, value in six.iteritems(config_items):
configs.append(p.Config(name, service, "node",
default_value=value, priority=1,
config_type="int"))

7
sahara/plugins/vanilla/v1_2_1/config_helper.py
View File

@ -15,6 +15,7 @@
from oslo_config import cfg
from oslo_log import log as logging
import six
from sahara import conductor as c
from sahara import context
@ -135,7 +136,7 @@ PRIORITY_1_CONFS += CLUSTER_WIDE_CONFS
def _initialise_configs():
configs = []
for service, config_lists in XML_CONFS.iteritems():
for service, config_lists in six.iteritems(XML_CONFS):
for config_list in config_lists:
for config in config_list:
if config['name'] not in HIDDEN_CONFS:
@ -155,8 +156,8 @@ def _initialise_configs():
cfg.priority = 1
configs.append(cfg)
for service, config_items in ENV_CONFS.iteritems():
for name, param_format_str in config_items.iteritems():
for service, config_items in six.iteritems(ENV_CONFS):
for name, param_format_str in six.iteritems(config_items):
configs.append(p.Config(name, service, "node",
default_value=1024, priority=1,
config_type="int"))

2
sahara/service/validations/base.py
View File

@ -52,7 +52,7 @@ def _get_plugin_configs(plugin_name, hadoop_version, scope=None):
def _check_duplicates(lst, message):
invalid = []
lst = collections.Counter(lst)
for (key, value) in lst.iteritems():
for key, value in six.iteritems(lst):
if value > 1:
invalid.append(key)

4
sahara/utils/resources.py
View File

@ -15,6 +15,8 @@
import inspect
import six
class BaseResource(object):
__resource_name__ = 'base'
@ -43,7 +45,7 @@ class BaseResource(object):
def to_dict(self):
dictionary = self.__dict__.copy()
return {k: v for k, v in dictionary.iteritems()
return {k: v for k, v in six.iteritems(dictionary)
if not self._filter_field(k)}
def as_resource(self):