diff --git a/neutron_tempest_plugin/common/shell.py b/neutron_tempest_plugin/common/shell.py new file mode 100644 index 000000000..bd4a7a31e --- /dev/null +++ b/neutron_tempest_plugin/common/shell.py @@ -0,0 +1,180 @@ +# Copyright (c) 2018 Red Hat, Inc. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import collections +import subprocess +import sys + +from oslo_log import log +from tempest.lib import exceptions as lib_exc + +from neutron_tempest_plugin.common import ssh +from neutron_tempest_plugin import config +from neutron_tempest_plugin import exceptions + + +LOG = log.getLogger(__name__) + +CONF = config.CONF + +if ssh.Client.proxy_jump_host: + # Perform all SSH connections passing through configured SSH server + SSH_PROXY_CLIENT = ssh.Client.create_proxy_client() +else: + SSH_PROXY_CLIENT = None + + +def execute(command, ssh_client=None, timeout=None, check=True): + """Execute command inside a remote or local shell + + :param command: command string to be executed + + :param ssh_client: SSH client instance used for remote shell execution + + :param timeout: command execution timeout in seconds + + :param check: when False it doesn't raises ShellCommandError when + exit status is not zero. True by default + + :returns: STDOUT text when command execution terminates with zero exit + status. + + :raises ShellTimeoutExpired: when timeout expires before command execution + terminates. In such case it kills the process, then it eventually would + try to read STDOUT and STDERR buffers (not fully implemented) before + raising the exception. + + :raises ShellCommandError: when command execution terminates with non-zero + exit status. + """ + ssh_client = ssh_client or SSH_PROXY_CLIENT + if timeout: + timeout = float(timeout) + + if ssh_client: + result = execute_remote_command(command=command, timeout=timeout, + ssh_client=ssh_client) + else: + result = execute_local_command(command=command, timeout=timeout) + + if result.exit_status == 0: + LOG.debug("Command %r succeeded:\n" + "stderr:\n%s\n" + "stdout:\n%s\n", + command, result.stderr, result.stdout) + elif result.exit_status is None: + LOG.debug("Command %r timeout expired (timeout=%s):\n" + "stderr:\n%s\n" + "stdout:\n%s\n", + command, timeout, result.stderr, result.stdout) + else: + LOG.debug("Command %r failed (exit_status=%s):\n" + "stderr:\n%s\n" + "stdout:\n%s\n", + command, result.exit_status, result.stderr, result.stdout) + if check: + result.check() + + return result + + +def execute_remote_command(command, ssh_client, timeout=None): + """Execute command on a remote host using SSH client""" + LOG.debug("Executing command %r on remote host %r (timeout=%r)...", + command, ssh_client.host, timeout) + + stdout = stderr = exit_status = None + + try: + # TODO(fressi): re-implement to capture stderr + stdout = ssh_client.exec_command(command, timeout=timeout) + exit_status = 0 + + except lib_exc.TimeoutException: + # TODO(fressi): re-implement to capture STDOUT and STDERR and make + # sure process is killed + pass + + except lib_exc.SSHExecCommandFailed as ex: + # Please note class SSHExecCommandFailed has been re-based on + # top of ShellCommandError + stdout = ex.stdout + stderr = ex.stderr + exit_status = ex.exit_status + + return ShellExecuteResult(command=command, timeout=timeout, + exit_status=exit_status, + stdout=stdout, stderr=stderr) + + +def execute_local_command(command, timeout=None): + """Execute command on local host using local shell""" + + LOG.debug("Executing command %r on local host (timeout=%r)...", + command, timeout) + + process = subprocess.Popen(command, shell=True, + universal_newlines=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + if timeout and sys.version_info < (3, 3): + # TODO(fressi): re-implement to timeout support on older Pythons + LOG.warning("Popen.communicate method doens't support for timeout " + "on Python %r", sys.version) + timeout = None + + # Wait for process execution while reading STDERR and STDOUT streams + if timeout: + try: + stdout, stderr = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + # At this state I expect the process to be still running + # therefore it has to be kill later after calling poll() + LOG.exception("Command %r timeout expired.", command) + stdout = stderr = None + else: + stdout, stderr = process.communicate() + + # Check process termination status + exit_status = process.poll() + if exit_status is None: + # The process is still running after calling communicate(): + # let kill it and then read buffers again + process.kill() + stdout, stderr = process.communicate() + + return ShellExecuteResult(command=command, timeout=timeout, + stdout=stdout, stderr=stderr, + exit_status=exit_status) + + +class ShellExecuteResult(collections.namedtuple( + 'ShellExecuteResult', ['command', 'timeout', 'exit_status', 'stdout', + 'stderr'])): + + def check(self): + if self.exit_status is None: + raise exceptions.ShellTimeoutExpired(command=self.command, + timeout=self.timeout, + stderr=self.stderr, + stdout=self.stdout) + + elif self.exit_status != 0: + raise exceptions.ShellCommandError(command=self.command, + exit_status=self.exit_status, + stderr=self.stderr, + stdout=self.stdout) diff --git a/neutron_tempest_plugin/common/ssh.py b/neutron_tempest_plugin/common/ssh.py index 33dffcb5b..ea30a2820 100644 --- a/neutron_tempest_plugin/common/ssh.py +++ b/neutron_tempest_plugin/common/ssh.py @@ -43,13 +43,13 @@ class Client(ssh.Client): def __init__(self, host, username, password=None, timeout=None, pkey=None, channel_timeout=10, look_for_keys=False, key_filename=None, - port=22, proxy_client=None): + port=22, proxy_client=None, create_proxy_client=True): timeout = timeout or self.timeout - if self.proxy_jump_host: + if not proxy_client and create_proxy_client and self.proxy_jump_host: # Perform all SSH connections passing through configured SSH server - proxy_client = proxy_client or self.create_proxy_client( + proxy_client = self.create_proxy_client( timeout=timeout, channel_timeout=channel_timeout) super(Client, self).__init__( @@ -115,10 +115,10 @@ class Client(ssh.Client): "set 'proxy_jump_keyfile' to provide a valid SSH key " "file.", login) - return ssh.Client( + return Client( host=host, username=username, password=password, look_for_keys=look_for_keys, key_filename=key_file, - port=port, proxy_client=None, **kwargs) + port=port, create_proxy_client=False, **kwargs) # attribute used to keep reference to opened client connection _client = None @@ -179,6 +179,16 @@ class Client(ssh.Client): user=self.username, password=self.password) + def exec_command(self, cmd, encoding="utf-8", timeout=None): + if timeout: + original_timeout = self.timeout + self.timeout = timeout + try: + return super(Client, self).exec_command(cmd=cmd, encoding=encoding) + finally: + if timeout: + self.timeout = original_timeout + def execute_script(self, script, become_root=False, combine_stderr=False, shell='sh -eux', timeout=None, **params): """Connect to remote machine and executes script. @@ -285,12 +295,12 @@ class Client(ssh.Client): stderr = _buffer_to_string(error_data, encoding) if exit_status is None: raise exc.SSHScriptTimeoutExpired( - host=self.host, script=script, stderr=stderr, stdout=stdout, - timeout=timeout) + command=shell, host=self.host, script=script, stderr=stderr, + stdout=stdout, timeout=timeout) else: raise exc.SSHScriptFailed( - host=self.host, script=script, stderr=stderr, stdout=stdout, - exit_status=exit_status) + command=shell, host=self.host, script=script, stderr=stderr, + stdout=stdout, exit_status=exit_status) def _buffer_to_string(data_buffer, encoding): diff --git a/neutron_tempest_plugin/common/utils.py b/neutron_tempest_plugin/common/utils.py index fa7bb8b16..3649cb661 100644 --- a/neutron_tempest_plugin/common/utils.py +++ b/neutron_tempest_plugin/common/utils.py @@ -88,3 +88,17 @@ def unstable_test(reason): raise self.skipTest(msg) return inner return decor + + +def override_class(overriden_class, overrider_class): + """Override class definition with a MixIn class + + If overriden_class is not a subclass of overrider_class then it creates + a new class that has as bases overrider_class and overriden_class. + """ + + if not issubclass(overriden_class, overrider_class): + name = overriden_class.__name__ + bases = (overrider_class, overriden_class) + overriden_class = type(name, bases, {}) + return overriden_class diff --git a/neutron_tempest_plugin/exceptions.py b/neutron_tempest_plugin/exceptions.py index ff5b2cfa7..895cb4078 100644 --- a/neutron_tempest_plugin/exceptions.py +++ b/neutron_tempest_plugin/exceptions.py @@ -15,18 +15,35 @@ from tempest.lib import exceptions -TempestException = exceptions.TempestException +from neutron_tempest_plugin.common import utils -class InvalidConfiguration(TempestException): +class NeutronTempestPluginException(exceptions.TempestException): + + def __init__(self, **kwargs): + super(NeutronTempestPluginException, self).__init__(**kwargs) + self._properties = kwargs + + def __getattr__(self, name): + try: + return self._properties[name] + except KeyError: + pass + + msg = ("AttributeError: {!r} object has no attribute {!r}").format( + self, name) + raise AttributeError(msg) + + +class InvalidConfiguration(NeutronTempestPluginException): message = "Invalid Configuration" -class InvalidCredentials(TempestException): +class InvalidCredentials(NeutronTempestPluginException): message = "Invalid Credentials" -class InvalidServiceTag(TempestException): +class InvalidServiceTag(NeutronTempestPluginException): message = "Invalid service tag" @@ -34,17 +51,50 @@ class SSHScriptException(exceptions.TempestException): """Base class for SSH client execute_script() exceptions""" -class SSHScriptTimeoutExpired(SSHScriptException): - message = ("Timeout expired while executing script on host %(host)r:\n" - "script:\n%(script)s\n" - "stderr:\n%(stderr)s\n" - "stdout:\n%(stdout)s\n" - "timeout: %(timeout)s") +class ShellError(NeutronTempestPluginException): + pass -class SSHScriptFailed(SSHScriptException): - message = ("Failed executing script on remote host %(host)r:\n" +class ShellCommandFailed(ShellError): + """Raised when shell command exited with non-zero status + + """ + message = ("Command %(command)r failed, exit status: %(exit_status)d, " + "stderr:\n%(stderr)s\n" + "stdout:\n%(stdout)s") + + +class SSHScriptFailed(ShellCommandFailed): + message = ("Command %(command)r failed, exit status: %(exit_status)d, " + "host: %(host)r\n" "script:\n%(script)s\n" "stderr:\n%(stderr)s\n" - "stdout:\n%(stdout)s\n" - "exit_status: %(exit_status)s") + "stdout:\n%(stdout)s") + + +class ShellTimeoutExpired(ShellError): + """Raised when shell command timeouts and has been killed before exiting + + """ + message = ("Command '%(command)s' timed out: %(timeout)d, " + "stderr:\n%(stderr)s\n" + "stdout:\n%(stdout)s") + + +class SSHScriptTimeoutExpired(ShellTimeoutExpired): + message = ("Command '%(command)s', timed out: %(timeout)d " + "host: %(host)r\n" + "script:\n%(script)s\n" + "stderr:\n%(stderr)s\n" + "stdout:\n%(stdout)s") + + +# Patch SSHExecCommandFailed exception to make sure we can access to fields +# command, exit_status, STDOUT and STDERR when SSH client reports command +# failure +exceptions.SSHExecCommandFailed = utils.override_class( + exceptions.SSHExecCommandFailed, ShellCommandFailed) + +# Above code created a new SSHExecCommandFailed class based on top +# of ShellCommandError +assert issubclass(exceptions.SSHExecCommandFailed, ShellCommandFailed)