Making Sahara Python 3 compatible
Sahara fails to connect and operate on remote machines because the output from Subprocess on python 3 are bytes and that breaks follow up actions. Change-Id: Id55e6c06d3b6ead18501a0e2146af37bf493881d
This commit is contained in:
parent
8010e24a56
commit
dc17f1903f
@ -20,6 +20,7 @@
|
|||||||
- openstack-tox-cover:
|
- openstack-tox-cover:
|
||||||
voting: false
|
voting: false
|
||||||
- sahara-grenade
|
- sahara-grenade
|
||||||
|
- sahara-tests-scenario-py3
|
||||||
gate:
|
gate:
|
||||||
queue: sahara
|
queue: sahara
|
||||||
jobs:
|
jobs:
|
||||||
@ -28,6 +29,8 @@
|
|||||||
- sahara-tests-tempest
|
- sahara-tests-tempest
|
||||||
- sahara-tests-tempest-v2
|
- sahara-tests-tempest-v2
|
||||||
- sahara-grenade
|
- sahara-grenade
|
||||||
|
- sahara-tests-scenario-py3
|
||||||
|
|
||||||
experimental:
|
experimental:
|
||||||
jobs:
|
jobs:
|
||||||
- sahara-buildimages-ambari
|
- sahara-buildimages-ambari
|
||||||
@ -35,7 +38,6 @@
|
|||||||
- sahara-buildimages-mapr
|
- sahara-buildimages-mapr
|
||||||
- sahara-buildimages-spark
|
- sahara-buildimages-spark
|
||||||
- sahara-tests-scenario-multinode-spark
|
- sahara-tests-scenario-multinode-spark
|
||||||
- sahara-tests-scenario-py3
|
|
||||||
|
|
||||||
- job:
|
- job:
|
||||||
name: sahara-grenade
|
name: sahara-grenade
|
||||||
|
@ -13,6 +13,7 @@
|
|||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
|
import _io
|
||||||
import pickle # nosec
|
import pickle # nosec
|
||||||
import sys
|
import sys
|
||||||
import traceback
|
import traceback
|
||||||
@ -32,9 +33,14 @@ def main():
|
|||||||
# TODO(elmiko) these pickle usages should be
|
# TODO(elmiko) these pickle usages should be
|
||||||
# reinvestigated to determine a more secure manner to
|
# reinvestigated to determine a more secure manner to
|
||||||
# deploy remote commands.
|
# deploy remote commands.
|
||||||
func = pickle.load(sys.stdin) # nosec
|
if isinstance(sys.stdin, _io.TextIOWrapper):
|
||||||
args = pickle.load(sys.stdin) # nosec
|
func = pickle.load(sys.stdin.buffer) # nosec
|
||||||
kwargs = pickle.load(sys.stdin) # nosec
|
args = pickle.load(sys.stdin.buffer) # nosec
|
||||||
|
kwargs = pickle.load(sys.stdin.buffer) # nosec
|
||||||
|
else:
|
||||||
|
func = pickle.load(sys.stdin) # nosec
|
||||||
|
args = pickle.load(sys.stdin) # nosec
|
||||||
|
kwargs = pickle.load(sys.stdin) # nosec
|
||||||
|
|
||||||
result['output'] = func(*args, **kwargs)
|
result['output'] = func(*args, **kwargs)
|
||||||
except BaseException as e:
|
except BaseException as e:
|
||||||
@ -42,5 +48,8 @@ def main():
|
|||||||
result['exception'] = cls_name + ': ' + str(e)
|
result['exception'] = cls_name + ': ' + str(e)
|
||||||
result['traceback'] = traceback.format_exc()
|
result['traceback'] = traceback.format_exc()
|
||||||
|
|
||||||
pickle.dump(result, sys.stdout) # nosec
|
if isinstance(sys.stdin, _io.TextIOWrapper):
|
||||||
|
pickle.dump(result, sys.stdout.buffer, protocol=2) # nosec
|
||||||
|
else:
|
||||||
|
pickle.dump(result, sys.stdout, protocol=2) # nosec
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
|
@ -684,7 +684,7 @@ class SaharaScriptValidator(SaharaImageValidatorBase):
|
|||||||
in six.iteritems(arguments)
|
in six.iteritems(arguments)
|
||||||
if key in self.env_vars)
|
if key in self.env_vars)
|
||||||
script = script % {"env_vars": env_vars,
|
script = script % {"env_vars": env_vars,
|
||||||
"script": self.script_contents}
|
"script": self.script_contents.decode('utf-8')}
|
||||||
path = '/tmp/%s.sh' % uuidutils.generate_uuid()
|
path = '/tmp/%s.sh' % uuidutils.generate_uuid()
|
||||||
remote.write_file_to(path, script, run_as_root=True)
|
remote.write_file_to(path, script, run_as_root=True)
|
||||||
_sudo(remote, 'chmod +x %s' % path)
|
_sudo(remote, 'chmod +x %s' % path)
|
||||||
|
@ -274,7 +274,7 @@ class TestImages(b.SaharaTestCase):
|
|||||||
uuidutils.generate_uuid.return_value = hash_value
|
uuidutils.generate_uuid.return_value = hash_value
|
||||||
cls = images.SaharaScriptValidator
|
cls = images.SaharaScriptValidator
|
||||||
image_arguments = {"distro": 'centos'}
|
image_arguments = {"distro": 'centos'}
|
||||||
cmd = "It's dangerous to go alone. Run this."
|
cmd = b"It's dangerous to go alone. Run this."
|
||||||
validator = cls(cmd, env_vars=image_arguments.keys(),
|
validator = cls(cmd, env_vars=image_arguments.keys(),
|
||||||
output_var="distro")
|
output_var="distro")
|
||||||
|
|
||||||
|
@ -31,6 +31,7 @@ def _get_sub_executable():
|
|||||||
def start_subprocess():
|
def start_subprocess():
|
||||||
return subprocess.Popen((sys.executable, _get_sub_executable()),
|
return subprocess.Popen((sys.executable, _get_sub_executable()),
|
||||||
close_fds=True,
|
close_fds=True,
|
||||||
|
bufsize=0,
|
||||||
stdin=subprocess.PIPE,
|
stdin=subprocess.PIPE,
|
||||||
stdout=subprocess.PIPE,
|
stdout=subprocess.PIPE,
|
||||||
stderr=subprocess.PIPE)
|
stderr=subprocess.PIPE)
|
||||||
@ -39,12 +40,13 @@ def start_subprocess():
|
|||||||
def run_in_subprocess(proc, func, args=None, kwargs=None, interactive=False):
|
def run_in_subprocess(proc, func, args=None, kwargs=None, interactive=False):
|
||||||
args = args or ()
|
args = args or ()
|
||||||
kwargs = kwargs or {}
|
kwargs = kwargs or {}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# TODO(elmiko) these pickle usages should be reinvestigated to
|
# TODO(elmiko) these pickle usages should be reinvestigated to
|
||||||
# determine a more secure manner to deploy remote commands.
|
# determine a more secure manner to deploy remote commands.
|
||||||
pickle.dump(func, proc.stdin) # nosec
|
pickle.dump(func, proc.stdin, protocol=2) # nosec
|
||||||
pickle.dump(args, proc.stdin) # nosec
|
pickle.dump(args, proc.stdin, protocol=2) # nosec
|
||||||
pickle.dump(kwargs, proc.stdin) # nosec
|
pickle.dump(kwargs, proc.stdin, protocol=2) # nosec
|
||||||
proc.stdin.flush()
|
proc.stdin.flush()
|
||||||
|
|
||||||
if not interactive:
|
if not interactive:
|
||||||
|
@ -152,9 +152,9 @@ def _cleanup():
|
|||||||
|
|
||||||
|
|
||||||
def _read_paramimko_stream(recv_func):
|
def _read_paramimko_stream(recv_func):
|
||||||
result = ''
|
result = b''
|
||||||
buf = recv_func(1024)
|
buf = recv_func(1024)
|
||||||
while buf != '':
|
while buf != b'':
|
||||||
result += buf
|
result += buf
|
||||||
buf = recv_func(1024)
|
buf = recv_func(1024)
|
||||||
|
|
||||||
@ -182,6 +182,12 @@ def _execute_command(cmd, run_as_root=False, get_stderr=False,
|
|||||||
stdout = _read_paramimko_stream(chan.recv)
|
stdout = _read_paramimko_stream(chan.recv)
|
||||||
stderr = _read_paramimko_stream(chan.recv_stderr)
|
stderr = _read_paramimko_stream(chan.recv_stderr)
|
||||||
|
|
||||||
|
if type(stdout) == bytes:
|
||||||
|
stdout = stdout.decode('utf-8')
|
||||||
|
|
||||||
|
if type(stderr) == bytes:
|
||||||
|
stderr = stderr.decode('utf-8')
|
||||||
|
|
||||||
ret_code = chan.recv_exit_status()
|
ret_code = chan.recv_exit_status()
|
||||||
|
|
||||||
if ret_code and raise_when_error:
|
if ret_code and raise_when_error:
|
||||||
@ -363,7 +369,10 @@ def _read_file(sftp, remote_file):
|
|||||||
fl = sftp.file(remote_file, 'r')
|
fl = sftp.file(remote_file, 'r')
|
||||||
data = fl.read()
|
data = fl.read()
|
||||||
fl.close()
|
fl.close()
|
||||||
return data
|
try:
|
||||||
|
return data.decode('utf-8')
|
||||||
|
except Exception:
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
def _read_file_from(remote_file, run_as_root=False):
|
def _read_file_from(remote_file, run_as_root=False):
|
||||||
|
Loading…
Reference in New Issue
Block a user