Making Sahara Python 3 compatible

Sahara fails to connect and operate on remote machines
because the output from Subprocess on python 3 are bytes
and that breaks follow up actions.

Change-Id: Id55e6c06d3b6ead18501a0e2146af37bf493881d

.zuul.yaml

@@ -20,6 +20,7 @@
         - openstack-tox-cover:
             voting: false
         - sahara-grenade
+        - sahara-tests-scenario-py3
     gate:
       queue: sahara
       jobs:
@@ -28,6 +29,8 @@
         - sahara-tests-tempest
         - sahara-tests-tempest-v2
         - sahara-grenade
+        - sahara-tests-scenario-py3
+
     experimental:
       jobs:
         - sahara-buildimages-ambari
@@ -35,7 +38,6 @@
         - sahara-buildimages-mapr
         - sahara-buildimages-spark
         - sahara-tests-scenario-multinode-spark
-        - sahara-tests-scenario-py3
 
 - job:
     name: sahara-grenade

sahara/cli/sahara_subprocess.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import _io
 import pickle  # nosec
 import sys
 import traceback
@@ -32,9 +33,14 @@ def main():
                 # TODO(elmiko) these pickle usages should be
                 # reinvestigated to determine a more secure manner to
                 # deploy remote commands.
-                func = pickle.load(sys.stdin)  # nosec
-                args = pickle.load(sys.stdin)  # nosec
-                kwargs = pickle.load(sys.stdin)  # nosec
+                if isinstance(sys.stdin, _io.TextIOWrapper):
+                    func = pickle.load(sys.stdin.buffer)  # nosec
+                    args = pickle.load(sys.stdin.buffer)  # nosec
+                    kwargs = pickle.load(sys.stdin.buffer)  # nosec
+                else:
+                    func = pickle.load(sys.stdin)  # nosec
+                    args = pickle.load(sys.stdin)  # nosec
+                    kwargs = pickle.load(sys.stdin)  # nosec
 
                 result['output'] = func(*args, **kwargs)
             except BaseException as e:
@@ -42,5 +48,8 @@ def main():
                 result['exception'] = cls_name + ': ' + str(e)
                 result['traceback'] = traceback.format_exc()
 
-            pickle.dump(result, sys.stdout)  # nosec
+            if isinstance(sys.stdin, _io.TextIOWrapper):
+                pickle.dump(result, sys.stdout.buffer, protocol=2)  # nosec
+            else:
+                pickle.dump(result, sys.stdout, protocol=2)  # nosec
             sys.stdout.flush()

sahara/plugins/images.py

@@ -684,7 +684,7 @@ class SaharaScriptValidator(SaharaImageValidatorBase):
                              in six.iteritems(arguments)
                              if key in self.env_vars)
         script = script % {"env_vars": env_vars,
-                           "script": self.script_contents}
+                           "script": self.script_contents.decode('utf-8')}
         path = '/tmp/%s.sh' % uuidutils.generate_uuid()
         remote.write_file_to(path, script, run_as_root=True)
         _sudo(remote, 'chmod +x %s' % path)

sahara/tests/unit/plugins/test_images.py

@@ -274,7 +274,7 @@ class TestImages(b.SaharaTestCase):
         uuidutils.generate_uuid.return_value = hash_value
         cls = images.SaharaScriptValidator
         image_arguments = {"distro": 'centos'}
-        cmd = "It's dangerous to go alone. Run this."
+        cmd = b"It's dangerous to go alone. Run this."
         validator = cls(cmd, env_vars=image_arguments.keys(),
                         output_var="distro")
 

sahara/utils/procutils.py

@@ -31,6 +31,7 @@ def _get_sub_executable():
 def start_subprocess():
     return subprocess.Popen((sys.executable, _get_sub_executable()),
                             close_fds=True,
+                            bufsize=0,
                             stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
@@ -39,12 +40,13 @@ def start_subprocess():
 def run_in_subprocess(proc, func, args=None, kwargs=None, interactive=False):
     args = args or ()
     kwargs = kwargs or {}
+
     try:
         # TODO(elmiko) these pickle usages should be reinvestigated to
         # determine a more secure manner to deploy remote commands.
-        pickle.dump(func, proc.stdin)  # nosec
-        pickle.dump(args, proc.stdin)  # nosec
-        pickle.dump(kwargs, proc.stdin)  # nosec
+        pickle.dump(func, proc.stdin, protocol=2)  # nosec
+        pickle.dump(args, proc.stdin, protocol=2)  # nosec
+        pickle.dump(kwargs, proc.stdin, protocol=2)  # nosec
         proc.stdin.flush()
 
         if not interactive:

sahara/utils/ssh_remote.py

@@ -152,9 +152,9 @@ def _cleanup():
 
 
 def _read_paramimko_stream(recv_func):
-    result = ''
+    result = b''
     buf = recv_func(1024)
-    while buf != '':
+    while buf != b'':
         result += buf
         buf = recv_func(1024)
 
@@ -182,6 +182,12 @@ def _execute_command(cmd, run_as_root=False, get_stderr=False,
     stdout = _read_paramimko_stream(chan.recv)
     stderr = _read_paramimko_stream(chan.recv_stderr)
 
+    if type(stdout) == bytes:
+        stdout = stdout.decode('utf-8')
+
+    if type(stderr) == bytes:
+        stderr = stderr.decode('utf-8')
+
     ret_code = chan.recv_exit_status()
 
     if ret_code and raise_when_error:
@@ -363,7 +369,10 @@ def _read_file(sftp, remote_file):
     fl = sftp.file(remote_file, 'r')
     data = fl.read()
     fl.close()
-    return data
+    try:
+        return data.decode('utf-8')
+    except Exception:
+        return data
 
 
 def _read_file_from(remote_file, run_as_root=False):