diff --git a/tobiko/config.py b/tobiko/config.py index f53e3801c..6c18ef554 100644 --- a/tobiko/config.py +++ b/tobiko/config.py @@ -26,7 +26,8 @@ LOG = log.getLogger(__name__) CONFIG_MODULES = ['tobiko.openstack.keystone.config', 'tobiko.openstack.neutron.config', - 'tobiko.openstack.nova.config'] + 'tobiko.openstack.nova.config', + 'tobiko.shell.sh.config'] CONFIG_DIRS = [os.getcwd(), os.path.expanduser("~/.tobiko"), @@ -88,12 +89,6 @@ def init_tobiko_config(default_config_dirs=None, product_name='tobiko', def register_tobiko_options(conf): - conf.register_opts( - group=cfg.OptGroup('shell'), - opts=[cfg.StrOpt('command', - help="Default shell command used for executing " - "local commands")]) - conf.register_opts( group=cfg.OptGroup('http'), opts=[cfg.StrOpt('http_proxy', diff --git a/tobiko/shell/__init__.py b/tobiko/shell/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tobiko/shell/sh/__init__.py b/tobiko/shell/sh/__init__.py new file mode 100644 index 000000000..20047e716 --- /dev/null +++ b/tobiko/shell/sh/__init__.py @@ -0,0 +1,27 @@ +# Copyright (c) 2019 Red Hat, Inc. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import absolute_import + +from tobiko.shell.sh import _exception +from tobiko.shell.sh import _execute + + +ShellCommandFailed = _exception.ShellCommandFailed +ShellError = _exception.ShellError +ShellTimeoutExpired = _exception.ShellTimeoutExpired + +execute = _execute.execute +ShellExecuteResult = _execute.ShellExecuteResult diff --git a/tobiko/shell/sh/_exception.py b/tobiko/shell/sh/_exception.py new file mode 100644 index 000000000..8ae98e346 --- /dev/null +++ b/tobiko/shell/sh/_exception.py @@ -0,0 +1,38 @@ +# Copyright (c) 2019 Red Hat, Inc. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import absolute_import + +import tobiko + + +class ShellError(tobiko.TobikoException): + pass + + +class ShellCommandFailed(ShellError): + """Raised when shell command exited with non-zero status + """ + message = ("Command %(command)r failed, exit status: %(exit_status)d, " + "stderr:\n%(stderr)s\n" + "stdout:\n%(stdout)s") + + +class ShellTimeoutExpired(ShellError): + """Raised when shell command timeouts and has been killed before exiting + """ + message = ("Command '%(command)s' timed out: %(timeout)d, " + "stderr:\n%(stderr)s\n" + "stdout:\n%(stdout)s") diff --git a/tobiko/shell/sh/_execute.py b/tobiko/shell/sh/_execute.py new file mode 100644 index 000000000..555c32947 --- /dev/null +++ b/tobiko/shell/sh/_execute.py @@ -0,0 +1,157 @@ +# Copyright (c) 2019 Red Hat, Inc. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import absolute_import + +import collections +import subprocess +import sys + +from oslo_log import log +import six + +from tobiko.shell.sh import _exception + + +LOG = log.getLogger(__name__) + + +def execute(command, timeout=None, shell=None, check=True, ssh_client=None): + """Execute command inside a remote or local shell + + :param command: command argument list + + :param timeout: command execution timeout in seconds + + :param check: when False it doesn't raises ShellCommandError when + exit status is not zero. True by default + + :param ssh_client: SSH client instance used for remote shell execution + + :returns: STDOUT text when command execution terminates with zero exit + status. + + :raises ShellTimeoutExpired: when timeout expires before command execution + terminates. In such case it kills the process, then it eventually would + try to read STDOUT and STDERR buffers (not fully implemented) before + raising the exception. + + :raises ShellCommandError: when command execution terminates with non-zero + exit status. + """ + if timeout: + timeout = float(timeout) + + if ssh_client: + result = execute_remote_command(command=command, timeout=timeout, + shell=shell, ssh_client=ssh_client) + else: + result = execute_local_command(command=command, timeout=timeout, + shell=shell) + + if result.exit_status == 0: + LOG.debug("Command %r succeeded:\n" + "stderr:\n%s\n" + "stdout:\n%s\n", + command, result.stderr, result.stdout) + elif result.exit_status is None: + LOG.debug("Command %r timeout expired (timeout=%s):\n" + "stderr:\n%s\n" + "stdout:\n%s\n", + command, timeout, result.stderr, result.stdout) + else: + LOG.debug("Command %r failed (exit_status=%s):\n" + "stderr:\n%s\n" + "stdout:\n%s\n", + command, result.exit_status, result.stderr, result.stdout) + if check: + result.check() + + return result + + +def execute_remote_command(command, ssh_client, timeout=None, shell=None): + """Execute command on a remote host using SSH client""" + raise NotImplementedError + + +def execute_local_command(command, timeout=None, shell=None): + """Execute command on local host using local shell""" + + LOG.debug("Executing command %r on local host (timeout=%r)...", + command, timeout) + + if not shell: + from tobiko import config + CONF = config.CONF + shell = CONF.tobiko.shell.command + + if isinstance(command, six.string_types): + command = command.split() + else: + command = [str(a) for a in command] + + if shell: + command = shell.split() + [str(subprocess.list2cmdline(command))] + process = subprocess.Popen(command, + universal_newlines=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + if timeout and sys.version_info < (3, 3): + LOG.warning("Popen.communicate method doens't support for timeout " + "on Python %r", sys.version) + timeout = None + + # Wait for process execution while reading STDERR and STDOUT streams + if timeout: + try: + stdout, stderr = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + # At this state I expect the process to be still running + # therefore it has to be kill later after calling poll() + LOG.exception("Command %r timeout expired.", command) + stdout = stderr = None + else: + stdout, stderr = process.communicate() + + # Check process termination status + exit_status = process.poll() + if exit_status is None: + # The process is still running after calling communicate(): + # let kill it + process.kill() + + return ShellExecuteResult(command=command, timeout=timeout, + stdout=stdout, stderr=stderr, + exit_status=exit_status) + + +class ShellExecuteResult(collections.namedtuple( + 'ShellExecuteResult', ['command', 'timeout', 'exit_status', 'stdout', + 'stderr'])): + + def check(self): + if self.exit_status is None: + raise _exception.ShellTimeoutExpired(command=self.command, + timeout=self.timeout, + stderr=self.stderr, + stdout=self.stdout) + + elif self.exit_status != 0: + raise _exception.ShellCommandFailed(command=self.command, + exit_status=self.exit_status, + stderr=self.stderr, + stdout=self.stdout) diff --git a/tobiko/shell/sh/config.py b/tobiko/shell/sh/config.py new file mode 100644 index 000000000..e5dec78ca --- /dev/null +++ b/tobiko/shell/sh/config.py @@ -0,0 +1,28 @@ +# Copyright (c) 2019 Red Hat, Inc. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import absolute_import + +from oslo_config import cfg + + +def register_tobiko_options(conf): + + conf.register_opts( + group=cfg.OptGroup('shell'), + opts=[cfg.StrOpt('command', + default='/bin/sh -c', + help="Default shell command used for executing " + "local commands")]) diff --git a/tobiko/tests/shell/__init__.py b/tobiko/tests/shell/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tobiko/tests/shell/test_sh.py b/tobiko/tests/shell/test_sh.py new file mode 100644 index 000000000..537dbdd0e --- /dev/null +++ b/tobiko/tests/shell/test_sh.py @@ -0,0 +1,112 @@ +# Copyright (c) 2019 Red Hat, Inc. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import absolute_import + +import sys +import unittest + + +from tobiko import config +from tobiko.shell import sh +from tobiko.tests import unit + + +CONF = config.CONF + + +class ExecuteTest(unit.TobikoUnitTest): + + def setUp(self): + super(ExecuteTest, self).setUp() + self.config = self.patch_object(CONF.tobiko, 'shell', + command='/bin/sh -c') + + def test_execute_string(self): + result = sh.execute('true') + self.assertEqual( + sh.ShellExecuteResult( + command=['/bin/sh', '-c', 'true'], + timeout=None, exit_status=0, stdout='', stderr=''), + result) + + def test_execute_list(self): + result = sh.execute(['echo', 'something']) + self.assertEqual( + sh.ShellExecuteResult( + command=['/bin/sh', '-c', 'echo something'], + timeout=None, exit_status=0, stdout='something\n', stderr=''), + result) + + def test_execute_writing_to_stdout(self): + result = sh.execute('echo something') + self.assertEqual( + sh.ShellExecuteResult( + command=['/bin/sh', '-c', 'echo something'], + timeout=None, exit_status=0, stdout='something\n', stderr=''), + result) + + def test_execute_writing_to_stderr(self): + result = sh.execute('echo something >&2') + self.assertEqual( + sh.ShellExecuteResult( + command=['/bin/sh', '-c', 'echo something >&2'], + timeout=None, exit_status=0, stdout='', stderr='something\n'), + result) + + def test_execute_failing_command(self): + ex = self.assertRaises(sh.ShellCommandFailed, sh.execute, 'exit 15') + self.assertEqual('', ex.stdout) + self.assertEqual('', ex.stderr) + self.assertEqual(15, ex.exit_status) + self.assertEqual(['/bin/sh', '-c', 'exit 15'], ex.command) + + def test_execute_failing_command_writing_to_stdout(self): + ex = self.assertRaises(sh.ShellCommandFailed, sh.execute, + 'echo something; exit 8') + self.assertEqual('something\n', ex.stdout) + self.assertEqual('', ex.stderr) + self.assertEqual(8, ex.exit_status) + self.assertEqual(['/bin/sh', '-c', 'echo something; exit 8'], + ex.command) + + def test_execute_failing_command_writing_to_stderr(self): + ex = self.assertRaises(sh.ShellCommandFailed, sh.execute, + 'echo something >&2; exit 7') + self.assertEqual('', ex.stdout) + self.assertEqual('something\n', ex.stderr) + self.assertEqual(7, ex.exit_status) + self.assertEqual(['/bin/sh', '-c', 'echo something >&2; exit 7'], + ex.command) + + def test_execute_with_timeout(self): + result = sh.execute('true', timeout=30.) + expected_timeout = None if sys.version_info < (3, 3) else 30. + self.assertEqual( + sh.ShellExecuteResult( + command=['/bin/sh', '-c', 'true'], + timeout=expected_timeout, exit_status=0, stdout='', + stderr=''), + result) + + @unittest.skipIf(sys.version_info < (3, 3), + 'not implemented for Python version < 3.3') + def test_execute_with_timeout_expired(self): + ex = self.assertRaises(sh.ShellTimeoutExpired, sh.execute, + 'echo out; echo err >&2; sleep 30', + timeout=.01) + self.assertEqual(['/bin/sh', '-c', + 'echo out; echo err >&2; sleep 30'], + ex.command)