Introduce PINGABLE and SSHABLE waiters and wait_until state support
This change introduces two new instance state waiters and uses them to extend the existing tempest.common.compute.create_test_server wait_until state support. They are being introduced in an effort to allow the guest OS time to start *before* we start attempting to interact with it either directly by connecting to the instance or indirectly by hot-plugging or hot-unplugging devices. The latter on some virt backends being an issue if the guest OS is unable to respond to the underlying ACPI requests sent to it. It should be noted that these new states rely on the instance already being ACTIVE before we begin to wait for the instance to either become pingable or accessible over SSH. This is taken into account and will always happen for these states even if validation isn't enabled in the environment and thus it isn't possible to wait until the instance is pingable or accessible over ssh. Change-Id: Ib14fa7dc5c8093eed498049cd0a56c8ac6853660
This commit is contained in:
Lee Yarwood
Martin Kopec
parent
ca03d2b504
commit
0b4bc3dcc6
@ -23,6 +23,7 @@ from urllib import parse as urlparse
|
||||
from oslo_log import log as logging
|
||||
from oslo_utils import excutils
|
||||
|
||||
from tempest.common.utils.linux import remote_client
|
||||
from tempest.common import waiters
|
||||
from tempest import config
|
||||
from tempest import exceptions
|
||||
@ -98,7 +99,9 @@ def create_test_server(clients, validatable=False, validation_resources=None,
|
||||
server. Include a keypair, a security group and an IP.
|
||||
:param tenant_network: Tenant network to be used for creating a server.
|
||||
:param wait_until: Server status to wait for the server to reach after
|
||||
its creation.
|
||||
its creation. Additionally PINGABLE and SSHABLE states are also
|
||||
accepted when the server is both validatable and has the required
|
||||
validation_resources provided.
|
||||
:param volume_backed: Whether the server is volume backed or not.
|
||||
If this is true, a volume will be created and create server will be
|
||||
requested with 'block_device_mapping_v2' populated with below values:
|
||||
@ -125,8 +128,6 @@ def create_test_server(clients, validatable=False, validation_resources=None,
|
||||
:returns: a tuple
|
||||
"""
|
||||
|
||||
# TODO(jlanoux) add support of wait_until PINGABLE/SSHABLE
|
||||
|
||||
if name is None:
|
||||
name = data_utils.rand_name(__name__ + "-instance")
|
||||
if flavor is None:
|
||||
@ -259,18 +260,50 @@ def create_test_server(clients, validatable=False, validation_resources=None,
|
||||
server_id=servers[0]['id'])
|
||||
|
||||
if wait_until:
|
||||
|
||||
# NOTE(lyarwood): PINGABLE and SSHABLE both require the instance to
|
||||
# go ACTIVE initially before we can setup the fip(s) etc so stash
|
||||
# this additional wait state for later use.
|
||||
wait_until_extra = None
|
||||
if wait_until in ['PINGABLE', 'SSHABLE']:
|
||||
wait_until_extra = wait_until
|
||||
wait_until = 'ACTIVE'
|
||||
|
||||
for server in servers:
|
||||
try:
|
||||
waiters.wait_for_server_status(
|
||||
clients.servers_client, server['id'], wait_until,
|
||||
request_id=request_id)
|
||||
|
||||
# Multiple validatable servers are not supported for now. Their
|
||||
# creation will fail with the condition above.
|
||||
if CONF.validation.run_validation and validatable:
|
||||
|
||||
if CONF.validation.connect_method == 'floating':
|
||||
_setup_validation_fip()
|
||||
|
||||
server_ip = get_server_ip(
|
||||
server, validation_resources=validation_resources)
|
||||
|
||||
if wait_until_extra == 'PINGABLE':
|
||||
waiters.wait_for_ping(
|
||||
server_ip,
|
||||
clients.servers_client.build_timeout,
|
||||
clients.servers_client.build_interval
|
||||
)
|
||||
|
||||
if wait_until_extra == 'SSHABLE':
|
||||
pkey = validation_resources['keypair']['private_key']
|
||||
ssh_client = remote_client.RemoteClient(
|
||||
server_ip,
|
||||
CONF.validation.image_ssh_user,
|
||||
pkey=pkey,
|
||||
server=server,
|
||||
servers_client=clients.servers_client
|
||||
)
|
||||
waiters.wait_for_ssh(
|
||||
ssh_client,
|
||||
clients.servers_client.build_timeout
|
||||
)
|
||||
|
||||
except Exception:
|
||||
with excutils.save_and_reraise_exception():
|
||||
for server in servers:
|
||||
|
||||
@ -10,6 +10,7 @@
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
|
||||
@ -570,3 +571,26 @@ def wait_for_server_floating_ip(servers_client, server, floating_ip,
|
||||
'in time.' % (floating_ip, server['id']))
|
||||
raise lib_exc.TimeoutException(msg)
|
||||
time.sleep(servers_client.build_interval)
|
||||
|
||||
|
||||
def wait_for_ping(server_ip, timeout=30, interval=1):
|
||||
"""Waits for an address to become pingable"""
|
||||
start_time = int(time.time())
|
||||
while int(time.time()) - start_time < timeout:
|
||||
response = os.system("ping -c 1 " + server_ip)
|
||||
if response == 0:
|
||||
return
|
||||
time.sleep(interval)
|
||||
raise lib_exc.TimeoutException()
|
||||
|
||||
|
||||
def wait_for_ssh(ssh_client, timeout=30):
|
||||
"""Waits for SSH connection to become usable"""
|
||||
start_time = int(time.time())
|
||||
while int(time.time()) - start_time < timeout:
|
||||
try:
|
||||
ssh_client.validate_authentication()
|
||||
return
|
||||
except lib_exc.SSHTimeout:
|
||||
pass
|
||||
raise lib_exc.TimeoutException()
|
||||
|
||||
@ -523,6 +523,70 @@ class TestVolumeWaiters(base.TestCase):
|
||||
mock_list_volume_attachments.assert_called_once_with(
|
||||
mock.sentinel.server_id)
|
||||
|
||||
@mock.patch('os.system')
|
||||
def test_wait_for_ping_host_alive(self, mock_ping):
|
||||
mock_ping.return_value = 0
|
||||
# Assert that nothing is raised as the host is alive
|
||||
waiters.wait_for_ping('127.0.0.1', 10, 1)
|
||||
|
||||
@mock.patch('os.system')
|
||||
def test_wait_for_ping_host_eventually_alive(self, mock_ping):
|
||||
mock_ping.side_effect = [1, 1, 0]
|
||||
# Assert that nothing is raised when the host is eventually alive
|
||||
waiters.wait_for_ping('127.0.0.1', 10, 1)
|
||||
|
||||
@mock.patch('os.system')
|
||||
def test_wait_for_ping_timeout(self, mock_ping):
|
||||
mock_ping.return_value = 1
|
||||
# Assert that TimeoutException is raised when the host is dead
|
||||
self.assertRaises(
|
||||
lib_exc.TimeoutException,
|
||||
waiters.wait_for_ping,
|
||||
'127.0.0.1',
|
||||
.1,
|
||||
.1
|
||||
)
|
||||
|
||||
def test_wait_for_ssh(self):
|
||||
mock_ssh_client = mock.Mock()
|
||||
mock_ssh_client.validate_authentication.return_value = True
|
||||
# Assert that nothing is raised when validate_authentication returns
|
||||
waiters.wait_for_ssh(mock_ssh_client, .1)
|
||||
mock_ssh_client.validate_authentication.assert_called_once()
|
||||
|
||||
def test_wait_for_ssh_eventually_up(self):
|
||||
mock_ssh_client = mock.Mock()
|
||||
timeout = lib_exc.SSHTimeout(
|
||||
host='foo',
|
||||
username='bar',
|
||||
password='fizz'
|
||||
)
|
||||
mock_ssh_client.validate_authentication.side_effect = [
|
||||
timeout,
|
||||
timeout,
|
||||
True
|
||||
]
|
||||
# Assert that nothing is raised if validate_authentication passes
|
||||
# before the timeout
|
||||
waiters.wait_for_ssh(mock_ssh_client, 10)
|
||||
|
||||
def test_wait_for_ssh_timeout(self):
|
||||
mock_ssh_client = mock.Mock()
|
||||
timeout = lib_exc.SSHTimeout(
|
||||
host='foo',
|
||||
username='bar',
|
||||
password='fizz'
|
||||
)
|
||||
mock_ssh_client.validate_authentication.side_effect = timeout
|
||||
# Assert that TimeoutException is raised when validate_authentication
|
||||
# doesn't pass in time.
|
||||
self.assertRaises(
|
||||
lib_exc.TimeoutException,
|
||||
waiters.wait_for_ssh,
|
||||
mock_ssh_client,
|
||||
.1
|
||||
)
|
||||
|
||||
|
||||
class TestServerFloatingIPWaiters(base.TestCase):
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user