0e04f8f2b7
Create base function called execute that takes a command and execute it using either subprocess module or an SSH client to allow implementing commands wrapper that can be executed either locally or remotelly. It also try to handle timeouts and produce good error capable of reporting whats written to stdout and stderr. Change-Id: I1a30b82338f44a4182722973e7ad3da2872295fd
309 lines
12 KiB
Python
309 lines
12 KiB
Python
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import locale
|
|
import os
|
|
import time
|
|
|
|
from oslo_log import log
|
|
import paramiko
|
|
from tempest.lib.common import ssh
|
|
from tempest.lib import exceptions
|
|
|
|
from neutron_tempest_plugin import config
|
|
from neutron_tempest_plugin import exceptions as exc
|
|
|
|
|
|
CONF = config.CONF
|
|
LOG = log.getLogger(__name__)
|
|
|
|
|
|
class Client(ssh.Client):
|
|
|
|
default_ssh_lang = 'en_US.UTF-8'
|
|
|
|
timeout = CONF.validation.ssh_timeout
|
|
|
|
proxy_jump_host = CONF.neutron_plugin_options.ssh_proxy_jump_host
|
|
proxy_jump_username = CONF.neutron_plugin_options.ssh_proxy_jump_username
|
|
proxy_jump_password = CONF.neutron_plugin_options.ssh_proxy_jump_password
|
|
proxy_jump_keyfile = CONF.neutron_plugin_options.ssh_proxy_jump_keyfile
|
|
proxy_jump_port = CONF.neutron_plugin_options.ssh_proxy_jump_port
|
|
|
|
def __init__(self, host, username, password=None, timeout=None, pkey=None,
|
|
channel_timeout=10, look_for_keys=False, key_filename=None,
|
|
port=22, proxy_client=None, create_proxy_client=True):
|
|
|
|
timeout = timeout or self.timeout
|
|
|
|
if not proxy_client and create_proxy_client and self.proxy_jump_host:
|
|
# Perform all SSH connections passing through configured SSH server
|
|
proxy_client = self.create_proxy_client(
|
|
timeout=timeout, channel_timeout=channel_timeout)
|
|
|
|
super(Client, self).__init__(
|
|
host=host, username=username, password=password, timeout=timeout,
|
|
pkey=pkey, channel_timeout=channel_timeout,
|
|
look_for_keys=look_for_keys, key_filename=key_filename, port=port,
|
|
proxy_client=proxy_client)
|
|
|
|
@classmethod
|
|
def create_proxy_client(cls, look_for_keys=True, **kwargs):
|
|
host = cls.proxy_jump_host
|
|
if not host:
|
|
# proxy_jump_host string cannot be empty or None
|
|
raise ValueError(
|
|
"'proxy_jump_host' configuration option is empty.")
|
|
|
|
# Let accept an empty string as a synonymous of default value on below
|
|
# options
|
|
password = cls.proxy_jump_password or None
|
|
key_file = cls.proxy_jump_keyfile or None
|
|
username = cls.proxy_jump_username
|
|
|
|
# Port must be a positive integer
|
|
port = cls.proxy_jump_port
|
|
if port <= 0 or port > 65535:
|
|
raise ValueError(
|
|
"Invalid value for 'proxy_jump_port' configuration option: "
|
|
"{!r}".format(port))
|
|
|
|
login = "{username}@{host}:{port}".format(username=username, host=host,
|
|
port=port)
|
|
|
|
if key_file:
|
|
# expand ~ character with user HOME directory
|
|
key_file = os.path.expanduser(key_file)
|
|
if os.path.isfile(key_file):
|
|
LOG.debug("Going to create SSH connection to %r using key "
|
|
"file: %s", login, key_file)
|
|
|
|
else:
|
|
# This message could help the user to identify a
|
|
# mis-configuration in tempest.conf
|
|
raise ValueError(
|
|
"Cannot find file specified as 'proxy_jump_keyfile' "
|
|
"option: {!r}".format(key_file))
|
|
|
|
elif password:
|
|
LOG.debug("Going to create SSH connection to %r using password.",
|
|
login)
|
|
|
|
elif look_for_keys:
|
|
# This message could help the user to identify a mis-configuration
|
|
# in tempest.conf
|
|
LOG.info("Both 'proxy_jump_password' and 'proxy_jump_keyfile' "
|
|
"options are empty. Going to create SSH connection to %r "
|
|
"looking for key file location into %r directory.",
|
|
login, os.path.expanduser('~/.ssh'))
|
|
else:
|
|
# An user that forces look_for_keys=False should really know what
|
|
# he really wants
|
|
LOG.warning("No authentication method provided to create an SSH "
|
|
"connection to %r. If it fails, then please "
|
|
"set 'proxy_jump_keyfile' to provide a valid SSH key "
|
|
"file.", login)
|
|
|
|
return Client(
|
|
host=host, username=username, password=password,
|
|
look_for_keys=look_for_keys, key_filename=key_file,
|
|
port=port, create_proxy_client=False, **kwargs)
|
|
|
|
# attribute used to keep reference to opened client connection
|
|
_client = None
|
|
|
|
def connect(self, *args, **kwargs):
|
|
"""Creates paramiko.SSHClient and connect it to remote SSH server
|
|
|
|
In case this method is called more times it returns the same client
|
|
and no new SSH connection is created until close method is called.
|
|
|
|
:returns: paramiko.Client connected to remote server.
|
|
|
|
:raises tempest.lib.exceptions.SSHTimeout: in case it fails to connect
|
|
to remote server.
|
|
"""
|
|
client = self._client
|
|
if client is None:
|
|
client = super(Client, self)._get_ssh_connection(
|
|
*args, **kwargs)
|
|
self._client = client
|
|
|
|
return client
|
|
|
|
# This overrides superclass protected method to make sure exec_command
|
|
# method is going to reuse the same SSH client and connection if called
|
|
# more times
|
|
_get_ssh_connection = connect
|
|
|
|
# This overrides superclass test_connection_auth method forbidding it to
|
|
# close connection
|
|
test_connection_auth = connect
|
|
|
|
def close(self):
|
|
"""Closes connection to SSH server and cleanup resources."""
|
|
client = self._client
|
|
if client is not None:
|
|
client.close()
|
|
self._client = None
|
|
|
|
def __exit__(self, _exception_type, _exception_value, _traceback):
|
|
self.close()
|
|
|
|
def open_session(self):
|
|
"""Gets connection to SSH server and open a new paramiko.Channel
|
|
|
|
:returns: new paramiko.Channel
|
|
"""
|
|
|
|
client = self.connect()
|
|
|
|
try:
|
|
return client.get_transport().open_session()
|
|
except paramiko.SSHException:
|
|
# the request is rejected, the session ends prematurely or
|
|
# there is a timeout opening a channel
|
|
LOG.exception("Unable to open SSH session")
|
|
raise exceptions.SSHTimeout(host=self.host,
|
|
user=self.username,
|
|
password=self.password)
|
|
|
|
def exec_command(self, cmd, encoding="utf-8", timeout=None):
|
|
if timeout:
|
|
original_timeout = self.timeout
|
|
self.timeout = timeout
|
|
try:
|
|
return super(Client, self).exec_command(cmd=cmd, encoding=encoding)
|
|
finally:
|
|
if timeout:
|
|
self.timeout = original_timeout
|
|
|
|
def execute_script(self, script, become_root=False, combine_stderr=False,
|
|
shell='sh -eux', timeout=None, **params):
|
|
"""Connect to remote machine and executes script.
|
|
|
|
Implementation note: it passes script lines to shell interpreter via
|
|
STDIN. Therefore script line number could be not available to some
|
|
script interpreters for debugging porposes.
|
|
|
|
:param script: script lines to be executed.
|
|
|
|
:param become_root: executes interpreter as root with sudo.
|
|
|
|
:param combine_stderr (bool): whenever to redirect STDERR to STDOUT so
|
|
that output from both streams are returned together. True by default.
|
|
|
|
:param shell: command line used to launch script interpreter. By
|
|
default it executes Bash with -eux options enabled. This means that
|
|
any command returning non-zero exist status or any any undefined
|
|
variable would interrupt script execution with an error and every
|
|
command executed by the script is going to be traced to STDERR.
|
|
|
|
:param timeout: time in seconds to wait before brutally aborting
|
|
script execution.
|
|
|
|
:param **params: script parameter values to be assigned at the
|
|
beginning of the script.
|
|
|
|
:returns output written by script to STDOUT.
|
|
|
|
:raises tempest.lib.exceptions.SSHTimeout: in case it fails to connect
|
|
to remote server or it fails to open a channel.
|
|
|
|
:raises tempest.lib.exceptions.SSHExecCommandFailed: in case command
|
|
script exits with non zero exit status or times out.
|
|
"""
|
|
|
|
if params:
|
|
# Append script parameters at the beginning of the script
|
|
header = ''.join(sorted(["{!s}={!s}\n".format(k, v)
|
|
for k, v in params.items()]))
|
|
script = header + '\n' + script
|
|
|
|
timeout = timeout or self.timeout
|
|
end_of_time = time.time() + timeout
|
|
output_data = b''
|
|
error_data = b''
|
|
exit_status = None
|
|
|
|
channel = self.open_session()
|
|
with channel:
|
|
|
|
# Combine STOUT and STDERR to have to handle with only one stream
|
|
channel.set_combine_stderr(combine_stderr)
|
|
|
|
# Update local environment
|
|
lang, encoding = locale.getlocale()
|
|
if not lang:
|
|
lang, encoding = locale.getdefaultlocale()
|
|
_locale = '.'.join([lang, encoding])
|
|
channel.update_environment({'LC_ALL': _locale,
|
|
'LANG': _locale})
|
|
|
|
if become_root:
|
|
shell = 'sudo ' + shell
|
|
# Spawn a Bash
|
|
channel.exec_command(shell)
|
|
|
|
end_of_script = False
|
|
lines_iterator = iter(script.splitlines())
|
|
while (not channel.exit_status_ready() and
|
|
time.time() < end_of_time):
|
|
# Drain incoming data buffers
|
|
while channel.recv_ready():
|
|
output_data += channel.recv(self.buf_size)
|
|
while channel.recv_stderr_ready():
|
|
error_data += channel.recv_stderr(self.buf_size)
|
|
|
|
if not end_of_script and channel.send_ready():
|
|
try:
|
|
line = next(lines_iterator)
|
|
except StopIteration:
|
|
# Finalize Bash script execution
|
|
channel.shutdown_write()
|
|
end_of_script = True
|
|
else:
|
|
# Send script to Bash STDIN line by line
|
|
channel.send((line + '\n').encode(encoding))
|
|
continue
|
|
|
|
time.sleep(.1)
|
|
|
|
# Get exit status and drain incoming data buffers
|
|
if channel.exit_status_ready():
|
|
exit_status = channel.recv_exit_status()
|
|
while channel.recv_ready():
|
|
output_data += channel.recv(self.buf_size)
|
|
while channel.recv_stderr_ready():
|
|
error_data += channel.recv_stderr(self.buf_size)
|
|
|
|
stdout = _buffer_to_string(output_data, encoding)
|
|
if exit_status == 0:
|
|
return stdout
|
|
|
|
stderr = _buffer_to_string(error_data, encoding)
|
|
if exit_status is None:
|
|
raise exc.SSHScriptTimeoutExpired(
|
|
command=shell, host=self.host, script=script, stderr=stderr,
|
|
stdout=stdout, timeout=timeout)
|
|
else:
|
|
raise exc.SSHScriptFailed(
|
|
command=shell, host=self.host, script=script, stderr=stderr,
|
|
stdout=stdout, exit_status=exit_status)
|
|
|
|
|
|
def _buffer_to_string(data_buffer, encoding):
|
|
return data_buffer.decode(encoding).replace("\r\n", "\n").replace(
|
|
"\r", "\n")
|