Browse Source

Making Sahara Python 3 compatible

Sahara fails to connect and operate on remote machines
because the output from Subprocess on python 3 are bytes
and that breaks follow up actions.

Change-Id: Id55e6c06d3b6ead18501a0e2146af37bf493881d
Telles Nobrega 2 months ago
parent
commit
dc17f1903f

+ 3
- 1
.zuul.yaml View File

@@ -20,6 +20,7 @@
20 20
         - openstack-tox-cover:
21 21
             voting: false
22 22
         - sahara-grenade
23
+        - sahara-tests-scenario-py3
23 24
     gate:
24 25
       queue: sahara
25 26
       jobs:
@@ -28,6 +29,8 @@
28 29
         - sahara-tests-tempest
29 30
         - sahara-tests-tempest-v2
30 31
         - sahara-grenade
32
+        - sahara-tests-scenario-py3
33
+
31 34
     experimental:
32 35
       jobs:
33 36
         - sahara-buildimages-ambari
@@ -35,7 +38,6 @@
35 38
         - sahara-buildimages-mapr
36 39
         - sahara-buildimages-spark
37 40
         - sahara-tests-scenario-multinode-spark
38
-        - sahara-tests-scenario-py3
39 41
 
40 42
 - job:
41 43
     name: sahara-grenade

+ 13
- 4
sahara/cli/sahara_subprocess.py View File

@@ -13,6 +13,7 @@
13 13
 # See the License for the specific language governing permissions and
14 14
 # limitations under the License.
15 15
 
16
+import _io
16 17
 import pickle  # nosec
17 18
 import sys
18 19
 import traceback
@@ -32,9 +33,14 @@ def main():
32 33
                 # TODO(elmiko) these pickle usages should be
33 34
                 # reinvestigated to determine a more secure manner to
34 35
                 # deploy remote commands.
35
-                func = pickle.load(sys.stdin)  # nosec
36
-                args = pickle.load(sys.stdin)  # nosec
37
-                kwargs = pickle.load(sys.stdin)  # nosec
36
+                if isinstance(sys.stdin, _io.TextIOWrapper):
37
+                    func = pickle.load(sys.stdin.buffer)  # nosec
38
+                    args = pickle.load(sys.stdin.buffer)  # nosec
39
+                    kwargs = pickle.load(sys.stdin.buffer)  # nosec
40
+                else:
41
+                    func = pickle.load(sys.stdin)  # nosec
42
+                    args = pickle.load(sys.stdin)  # nosec
43
+                    kwargs = pickle.load(sys.stdin)  # nosec
38 44
 
39 45
                 result['output'] = func(*args, **kwargs)
40 46
             except BaseException as e:
@@ -42,5 +48,8 @@ def main():
42 48
                 result['exception'] = cls_name + ': ' + str(e)
43 49
                 result['traceback'] = traceback.format_exc()
44 50
 
45
-            pickle.dump(result, sys.stdout)  # nosec
51
+            if isinstance(sys.stdin, _io.TextIOWrapper):
52
+                pickle.dump(result, sys.stdout.buffer, protocol=2)  # nosec
53
+            else:
54
+                pickle.dump(result, sys.stdout, protocol=2)  # nosec
46 55
             sys.stdout.flush()

+ 1
- 1
sahara/plugins/images.py View File

@@ -684,7 +684,7 @@ class SaharaScriptValidator(SaharaImageValidatorBase):
684 684
                              in six.iteritems(arguments)
685 685
                              if key in self.env_vars)
686 686
         script = script % {"env_vars": env_vars,
687
-                           "script": self.script_contents}
687
+                           "script": self.script_contents.decode('utf-8')}
688 688
         path = '/tmp/%s.sh' % uuidutils.generate_uuid()
689 689
         remote.write_file_to(path, script, run_as_root=True)
690 690
         _sudo(remote, 'chmod +x %s' % path)

+ 1
- 1
sahara/tests/unit/plugins/test_images.py View File

@@ -274,7 +274,7 @@ class TestImages(b.SaharaTestCase):
274 274
         uuidutils.generate_uuid.return_value = hash_value
275 275
         cls = images.SaharaScriptValidator
276 276
         image_arguments = {"distro": 'centos'}
277
-        cmd = "It's dangerous to go alone. Run this."
277
+        cmd = b"It's dangerous to go alone. Run this."
278 278
         validator = cls(cmd, env_vars=image_arguments.keys(),
279 279
                         output_var="distro")
280 280
 

+ 5
- 3
sahara/utils/procutils.py View File

@@ -31,6 +31,7 @@ def _get_sub_executable():
31 31
 def start_subprocess():
32 32
     return subprocess.Popen((sys.executable, _get_sub_executable()),
33 33
                             close_fds=True,
34
+                            bufsize=0,
34 35
                             stdin=subprocess.PIPE,
35 36
                             stdout=subprocess.PIPE,
36 37
                             stderr=subprocess.PIPE)
@@ -39,12 +40,13 @@ def start_subprocess():
39 40
 def run_in_subprocess(proc, func, args=None, kwargs=None, interactive=False):
40 41
     args = args or ()
41 42
     kwargs = kwargs or {}
43
+
42 44
     try:
43 45
         # TODO(elmiko) these pickle usages should be reinvestigated to
44 46
         # determine a more secure manner to deploy remote commands.
45
-        pickle.dump(func, proc.stdin)  # nosec
46
-        pickle.dump(args, proc.stdin)  # nosec
47
-        pickle.dump(kwargs, proc.stdin)  # nosec
47
+        pickle.dump(func, proc.stdin, protocol=2)  # nosec
48
+        pickle.dump(args, proc.stdin, protocol=2)  # nosec
49
+        pickle.dump(kwargs, proc.stdin, protocol=2)  # nosec
48 50
         proc.stdin.flush()
49 51
 
50 52
         if not interactive:

+ 12
- 3
sahara/utils/ssh_remote.py View File

@@ -152,9 +152,9 @@ def _cleanup():
152 152
 
153 153
 
154 154
 def _read_paramimko_stream(recv_func):
155
-    result = ''
155
+    result = b''
156 156
     buf = recv_func(1024)
157
-    while buf != '':
157
+    while buf != b'':
158 158
         result += buf
159 159
         buf = recv_func(1024)
160 160
 
@@ -182,6 +182,12 @@ def _execute_command(cmd, run_as_root=False, get_stderr=False,
182 182
     stdout = _read_paramimko_stream(chan.recv)
183 183
     stderr = _read_paramimko_stream(chan.recv_stderr)
184 184
 
185
+    if type(stdout) == bytes:
186
+        stdout = stdout.decode('utf-8')
187
+
188
+    if type(stderr) == bytes:
189
+        stderr = stderr.decode('utf-8')
190
+
185 191
     ret_code = chan.recv_exit_status()
186 192
 
187 193
     if ret_code and raise_when_error:
@@ -363,7 +369,10 @@ def _read_file(sftp, remote_file):
363 369
     fl = sftp.file(remote_file, 'r')
364 370
     data = fl.read()
365 371
     fl.close()
366
-    return data
372
+    try:
373
+        return data.decode('utf-8')
374
+    except Exception:
375
+        return data
367 376
 
368 377
 
369 378
 def _read_file_from(remote_file, run_as_root=False):

Loading…
Cancel
Save