Making Sahara Python 3 compatible

Sahara fails to connect and operate on remote machines
because the output from Subprocess on python 3 are bytes
and that breaks follow up actions.

Change-Id: Id55e6c06d3b6ead18501a0e2146af37bf493881d
This commit is contained in:
Telles Nobrega 2019-02-19 10:20:14 -03:00
parent 8010e24a56
commit dc17f1903f
6 changed files with 35 additions and 13 deletions

View File

@ -20,6 +20,7 @@
- openstack-tox-cover: - openstack-tox-cover:
voting: false voting: false
- sahara-grenade - sahara-grenade
- sahara-tests-scenario-py3
gate: gate:
queue: sahara queue: sahara
jobs: jobs:
@ -28,6 +29,8 @@
- sahara-tests-tempest - sahara-tests-tempest
- sahara-tests-tempest-v2 - sahara-tests-tempest-v2
- sahara-grenade - sahara-grenade
- sahara-tests-scenario-py3
experimental: experimental:
jobs: jobs:
- sahara-buildimages-ambari - sahara-buildimages-ambari
@ -35,7 +38,6 @@
- sahara-buildimages-mapr - sahara-buildimages-mapr
- sahara-buildimages-spark - sahara-buildimages-spark
- sahara-tests-scenario-multinode-spark - sahara-tests-scenario-multinode-spark
- sahara-tests-scenario-py3
- job: - job:
name: sahara-grenade name: sahara-grenade

View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import _io
import pickle # nosec import pickle # nosec
import sys import sys
import traceback import traceback
@ -32,9 +33,14 @@ def main():
# TODO(elmiko) these pickle usages should be # TODO(elmiko) these pickle usages should be
# reinvestigated to determine a more secure manner to # reinvestigated to determine a more secure manner to
# deploy remote commands. # deploy remote commands.
func = pickle.load(sys.stdin) # nosec if isinstance(sys.stdin, _io.TextIOWrapper):
args = pickle.load(sys.stdin) # nosec func = pickle.load(sys.stdin.buffer) # nosec
kwargs = pickle.load(sys.stdin) # nosec args = pickle.load(sys.stdin.buffer) # nosec
kwargs = pickle.load(sys.stdin.buffer) # nosec
else:
func = pickle.load(sys.stdin) # nosec
args = pickle.load(sys.stdin) # nosec
kwargs = pickle.load(sys.stdin) # nosec
result['output'] = func(*args, **kwargs) result['output'] = func(*args, **kwargs)
except BaseException as e: except BaseException as e:
@ -42,5 +48,8 @@ def main():
result['exception'] = cls_name + ': ' + str(e) result['exception'] = cls_name + ': ' + str(e)
result['traceback'] = traceback.format_exc() result['traceback'] = traceback.format_exc()
pickle.dump(result, sys.stdout) # nosec if isinstance(sys.stdin, _io.TextIOWrapper):
pickle.dump(result, sys.stdout.buffer, protocol=2) # nosec
else:
pickle.dump(result, sys.stdout, protocol=2) # nosec
sys.stdout.flush() sys.stdout.flush()

View File

@ -684,7 +684,7 @@ class SaharaScriptValidator(SaharaImageValidatorBase):
in six.iteritems(arguments) in six.iteritems(arguments)
if key in self.env_vars) if key in self.env_vars)
script = script % {"env_vars": env_vars, script = script % {"env_vars": env_vars,
"script": self.script_contents} "script": self.script_contents.decode('utf-8')}
path = '/tmp/%s.sh' % uuidutils.generate_uuid() path = '/tmp/%s.sh' % uuidutils.generate_uuid()
remote.write_file_to(path, script, run_as_root=True) remote.write_file_to(path, script, run_as_root=True)
_sudo(remote, 'chmod +x %s' % path) _sudo(remote, 'chmod +x %s' % path)

View File

@ -274,7 +274,7 @@ class TestImages(b.SaharaTestCase):
uuidutils.generate_uuid.return_value = hash_value uuidutils.generate_uuid.return_value = hash_value
cls = images.SaharaScriptValidator cls = images.SaharaScriptValidator
image_arguments = {"distro": 'centos'} image_arguments = {"distro": 'centos'}
cmd = "It's dangerous to go alone. Run this." cmd = b"It's dangerous to go alone. Run this."
validator = cls(cmd, env_vars=image_arguments.keys(), validator = cls(cmd, env_vars=image_arguments.keys(),
output_var="distro") output_var="distro")

View File

@ -31,6 +31,7 @@ def _get_sub_executable():
def start_subprocess(): def start_subprocess():
return subprocess.Popen((sys.executable, _get_sub_executable()), return subprocess.Popen((sys.executable, _get_sub_executable()),
close_fds=True, close_fds=True,
bufsize=0,
stdin=subprocess.PIPE, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE) stderr=subprocess.PIPE)
@ -39,12 +40,13 @@ def start_subprocess():
def run_in_subprocess(proc, func, args=None, kwargs=None, interactive=False): def run_in_subprocess(proc, func, args=None, kwargs=None, interactive=False):
args = args or () args = args or ()
kwargs = kwargs or {} kwargs = kwargs or {}
try: try:
# TODO(elmiko) these pickle usages should be reinvestigated to # TODO(elmiko) these pickle usages should be reinvestigated to
# determine a more secure manner to deploy remote commands. # determine a more secure manner to deploy remote commands.
pickle.dump(func, proc.stdin) # nosec pickle.dump(func, proc.stdin, protocol=2) # nosec
pickle.dump(args, proc.stdin) # nosec pickle.dump(args, proc.stdin, protocol=2) # nosec
pickle.dump(kwargs, proc.stdin) # nosec pickle.dump(kwargs, proc.stdin, protocol=2) # nosec
proc.stdin.flush() proc.stdin.flush()
if not interactive: if not interactive:

View File

@ -152,9 +152,9 @@ def _cleanup():
def _read_paramimko_stream(recv_func): def _read_paramimko_stream(recv_func):
result = '' result = b''
buf = recv_func(1024) buf = recv_func(1024)
while buf != '': while buf != b'':
result += buf result += buf
buf = recv_func(1024) buf = recv_func(1024)
@ -182,6 +182,12 @@ def _execute_command(cmd, run_as_root=False, get_stderr=False,
stdout = _read_paramimko_stream(chan.recv) stdout = _read_paramimko_stream(chan.recv)
stderr = _read_paramimko_stream(chan.recv_stderr) stderr = _read_paramimko_stream(chan.recv_stderr)
if type(stdout) == bytes:
stdout = stdout.decode('utf-8')
if type(stderr) == bytes:
stderr = stderr.decode('utf-8')
ret_code = chan.recv_exit_status() ret_code = chan.recv_exit_status()
if ret_code and raise_when_error: if ret_code and raise_when_error:
@ -363,7 +369,10 @@ def _read_file(sftp, remote_file):
fl = sftp.file(remote_file, 'r') fl = sftp.file(remote_file, 'r')
data = fl.read() data = fl.read()
fl.close() fl.close()
return data try:
return data.decode('utf-8')
except Exception:
return data
def _read_file_from(remote_file, run_as_root=False): def _read_file_from(remote_file, run_as_root=False):