From dc17f1903f520aefd566a3a9c66ddb406bf7cfa1 Mon Sep 17 00:00:00 2001 From: Telles Nobrega Date: Tue, 19 Feb 2019 10:20:14 -0300 Subject: [PATCH] Making Sahara Python 3 compatible Sahara fails to connect and operate on remote machines because the output from Subprocess on python 3 are bytes and that breaks follow up actions. Change-Id: Id55e6c06d3b6ead18501a0e2146af37bf493881d --- .zuul.yaml | 4 +++- sahara/cli/sahara_subprocess.py | 17 +++++++++++++---- sahara/plugins/images.py | 2 +- sahara/tests/unit/plugins/test_images.py | 2 +- sahara/utils/procutils.py | 8 +++++--- sahara/utils/ssh_remote.py | 15 ++++++++++++--- 6 files changed, 35 insertions(+), 13 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index 0786602a26..83c10e1222 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -20,6 +20,7 @@ - openstack-tox-cover: voting: false - sahara-grenade + - sahara-tests-scenario-py3 gate: queue: sahara jobs: @@ -28,6 +29,8 @@ - sahara-tests-tempest - sahara-tests-tempest-v2 - sahara-grenade + - sahara-tests-scenario-py3 + experimental: jobs: - sahara-buildimages-ambari @@ -35,7 +38,6 @@ - sahara-buildimages-mapr - sahara-buildimages-spark - sahara-tests-scenario-multinode-spark - - sahara-tests-scenario-py3 - job: name: sahara-grenade diff --git a/sahara/cli/sahara_subprocess.py b/sahara/cli/sahara_subprocess.py index 8d356d32a9..43ab26f051 100644 --- a/sahara/cli/sahara_subprocess.py +++ b/sahara/cli/sahara_subprocess.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import _io import pickle # nosec import sys import traceback @@ -32,9 +33,14 @@ def main(): # TODO(elmiko) these pickle usages should be # reinvestigated to determine a more secure manner to # deploy remote commands. - func = pickle.load(sys.stdin) # nosec - args = pickle.load(sys.stdin) # nosec - kwargs = pickle.load(sys.stdin) # nosec + if isinstance(sys.stdin, _io.TextIOWrapper): + func = pickle.load(sys.stdin.buffer) # nosec + args = pickle.load(sys.stdin.buffer) # nosec + kwargs = pickle.load(sys.stdin.buffer) # nosec + else: + func = pickle.load(sys.stdin) # nosec + args = pickle.load(sys.stdin) # nosec + kwargs = pickle.load(sys.stdin) # nosec result['output'] = func(*args, **kwargs) except BaseException as e: @@ -42,5 +48,8 @@ def main(): result['exception'] = cls_name + ': ' + str(e) result['traceback'] = traceback.format_exc() - pickle.dump(result, sys.stdout) # nosec + if isinstance(sys.stdin, _io.TextIOWrapper): + pickle.dump(result, sys.stdout.buffer, protocol=2) # nosec + else: + pickle.dump(result, sys.stdout, protocol=2) # nosec sys.stdout.flush() diff --git a/sahara/plugins/images.py b/sahara/plugins/images.py index b0e5b340c9..d84d13343e 100644 --- a/sahara/plugins/images.py +++ b/sahara/plugins/images.py @@ -684,7 +684,7 @@ class SaharaScriptValidator(SaharaImageValidatorBase): in six.iteritems(arguments) if key in self.env_vars) script = script % {"env_vars": env_vars, - "script": self.script_contents} + "script": self.script_contents.decode('utf-8')} path = '/tmp/%s.sh' % uuidutils.generate_uuid() remote.write_file_to(path, script, run_as_root=True) _sudo(remote, 'chmod +x %s' % path) diff --git a/sahara/tests/unit/plugins/test_images.py b/sahara/tests/unit/plugins/test_images.py index 90a6fb571c..3aeadfad8a 100644 --- a/sahara/tests/unit/plugins/test_images.py +++ b/sahara/tests/unit/plugins/test_images.py @@ -274,7 +274,7 @@ class TestImages(b.SaharaTestCase): uuidutils.generate_uuid.return_value = hash_value cls = images.SaharaScriptValidator image_arguments = {"distro": 'centos'} - cmd = "It's dangerous to go alone. Run this." + cmd = b"It's dangerous to go alone. Run this." validator = cls(cmd, env_vars=image_arguments.keys(), output_var="distro") diff --git a/sahara/utils/procutils.py b/sahara/utils/procutils.py index 6fe59445a4..f9a22a4be5 100644 --- a/sahara/utils/procutils.py +++ b/sahara/utils/procutils.py @@ -31,6 +31,7 @@ def _get_sub_executable(): def start_subprocess(): return subprocess.Popen((sys.executable, _get_sub_executable()), close_fds=True, + bufsize=0, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -39,12 +40,13 @@ def start_subprocess(): def run_in_subprocess(proc, func, args=None, kwargs=None, interactive=False): args = args or () kwargs = kwargs or {} + try: # TODO(elmiko) these pickle usages should be reinvestigated to # determine a more secure manner to deploy remote commands. - pickle.dump(func, proc.stdin) # nosec - pickle.dump(args, proc.stdin) # nosec - pickle.dump(kwargs, proc.stdin) # nosec + pickle.dump(func, proc.stdin, protocol=2) # nosec + pickle.dump(args, proc.stdin, protocol=2) # nosec + pickle.dump(kwargs, proc.stdin, protocol=2) # nosec proc.stdin.flush() if not interactive: diff --git a/sahara/utils/ssh_remote.py b/sahara/utils/ssh_remote.py index c734fae87d..c6da6601d3 100644 --- a/sahara/utils/ssh_remote.py +++ b/sahara/utils/ssh_remote.py @@ -152,9 +152,9 @@ def _cleanup(): def _read_paramimko_stream(recv_func): - result = '' + result = b'' buf = recv_func(1024) - while buf != '': + while buf != b'': result += buf buf = recv_func(1024) @@ -182,6 +182,12 @@ def _execute_command(cmd, run_as_root=False, get_stderr=False, stdout = _read_paramimko_stream(chan.recv) stderr = _read_paramimko_stream(chan.recv_stderr) + if type(stdout) == bytes: + stdout = stdout.decode('utf-8') + + if type(stderr) == bytes: + stderr = stderr.decode('utf-8') + ret_code = chan.recv_exit_status() if ret_code and raise_when_error: @@ -363,7 +369,10 @@ def _read_file(sftp, remote_file): fl = sftp.file(remote_file, 'r') data = fl.read() fl.close() - return data + try: + return data.decode('utf-8') + except Exception: + return data def _read_file_from(remote_file, run_as_root=False):