Handle socket timeout while reconnecting ssh

It's possible to catch socket timeout error that can not
be handled by devops's wait() method because it stucks at
method execution instead of analyzing ret code.
Usage of more strict RunLimit with signal mechanism inside
allow us to reconnect much faster in case of closed socket.
Also decreasing timeout for connection check - all logs shows that
response time for good connection is less than 1 second.

Change-Id: Ib39524172a6208b95edf40ad2d8f591e9b3ec551
This commit is contained in:
Vladimir Khlyunev 2017-02-13 16:09:20 +04:00
parent 96ab09c348
commit 28728949cc
1 changed files with 8 additions and 5 deletions

View File

@ -18,7 +18,6 @@ import re
import traceback
from warnings import warn
from devops.helpers.helpers import wait
from devops.helpers.metaclasses import SingletonMeta
from devops.helpers.ssh_client import SSHClient
from paramiko import RSAKey
@ -76,11 +75,15 @@ class SSHManager(six.with_metaclass(SingletonMeta, object)):
:return:
"""
try:
wait(lambda: remote.execute("cd ~")['exit_code'] == 0, timeout=20)
except Exception:
logger.info('SSHManager: Check for current '
'connection fails. Try to reconnect')
from fuelweb_test.helpers.utils import RunLimit
with RunLimit(
seconds=5,
error_message="Socket timeout! Forcing reconnection"):
remote.check_call("cd ~")
except:
logger.debug(traceback.format_exc())
logger.info('SSHManager: Check for current connection fails. '
'Trying to reconnect')
remote.reconnect()
return remote