Update cloud-init integration
Add method for waiting for cloud init is done before considering a server fixture ready to use Change-Id: I34c546d92e883aba998c480edede3352ca7153cb
This commit is contained in:
parent
2ef60ae638
commit
b298c8843c
|
@ -14,13 +14,15 @@
|
|||
from __future__ import absolute_import
|
||||
|
||||
import collections
|
||||
import time
|
||||
import contextlib
|
||||
import typing
|
||||
|
||||
from oslo_log import log
|
||||
import yaml
|
||||
|
||||
import tobiko
|
||||
from tobiko.shell import sh
|
||||
from tobiko.shell import ssh
|
||||
|
||||
LOG = log.getLogger(__name__)
|
||||
|
||||
|
@ -77,47 +79,128 @@ class CloudConfig(dict):
|
|||
return combine_cloud_configs([self, other])
|
||||
|
||||
|
||||
class WaitForCloudInitTimeoutError(tobiko.TobikoException):
|
||||
message = ("after {enlapsed_time} seconds cloud-init status of host "
|
||||
"{hostname!r} is still {actual!r} while it is expecting to "
|
||||
"be in {expected!r}")
|
||||
class InvalidCloudInitStatusError(tobiko.TobikoException):
|
||||
message = ("cloud-init status of host '{hostname}' is "
|
||||
"'{actual_status}' while it is expecting to "
|
||||
"be in {expected_states!r}:\n"
|
||||
"{details}")
|
||||
|
||||
|
||||
def get_cloud_init_status(ssh_client=None, timeout=None):
|
||||
output = sh.execute('cloud-init status',
|
||||
ssh_client=ssh_client,
|
||||
timeout=timeout,
|
||||
sudo=True).stdout
|
||||
return yaml.load(output)['status']
|
||||
class WaitForCloudInitTimeoutError(InvalidCloudInitStatusError):
|
||||
message = ("after {timeout} seconds cloud-init status of host "
|
||||
"'{hostname}' is still '{actual_status}' while it is "
|
||||
"expecting to be in {expected_states!r}:\n"
|
||||
"{details}")
|
||||
|
||||
|
||||
def wait_for_cloud_init_done(ssh_client=None, timeout=None,
|
||||
sleep_interval=None):
|
||||
return wait_for_cloud_init_status(expected={'done'},
|
||||
COUD_INIT_TRANSIENT_STATES = {
|
||||
'done': tuple(['running'])
|
||||
}
|
||||
|
||||
|
||||
def get_cloud_init_status(
|
||||
ssh_client: typing.Optional[ssh.SSHClientFixture] = None,
|
||||
timeout: tobiko.Seconds = None) \
|
||||
-> str:
|
||||
try:
|
||||
output = sh.execute('cloud-init status',
|
||||
ssh_client=ssh_client,
|
||||
timeout=timeout,
|
||||
sudo=True).stdout
|
||||
except sh.ShellCommandFailed as ex:
|
||||
output = ex.stdout
|
||||
if output:
|
||||
LOG.debug(f"Cloud init status error reported:\n{ex}")
|
||||
else:
|
||||
raise
|
||||
|
||||
status = yaml.load(output)
|
||||
tobiko.check_valid_type(status, dict)
|
||||
tobiko.check_valid_type(status['status'], str)
|
||||
return status['status']
|
||||
|
||||
|
||||
def wait_for_cloud_init_done(
|
||||
ssh_client: typing.Optional[ssh.SSHClientFixture] = None,
|
||||
timeout: tobiko.Seconds = None,
|
||||
sleep_interval: tobiko.Seconds = None) \
|
||||
-> str:
|
||||
return wait_for_cloud_init_status('done',
|
||||
ssh_client=ssh_client,
|
||||
timeout=timeout,
|
||||
sleep_interval=sleep_interval)
|
||||
|
||||
|
||||
def wait_for_cloud_init_status(expected, ssh_client=None, timeout=None,
|
||||
sleep_interval=None):
|
||||
expected = set(expected)
|
||||
timeout = timeout and float(timeout) or 1200.
|
||||
sleep_interval = sleep_interval and float(sleep_interval) or 5.
|
||||
start_time = time.time()
|
||||
actual = get_cloud_init_status(ssh_client=ssh_client, timeout=timeout)
|
||||
while actual not in expected:
|
||||
enlapsed_time = time.time() - start_time
|
||||
if enlapsed_time >= timeout:
|
||||
raise WaitForCloudInitTimeoutError(hostname=ssh_client.hostname,
|
||||
actual=actual,
|
||||
expected=expected,
|
||||
enlapsed_time=enlapsed_time)
|
||||
def wait_for_cloud_init_status(
|
||||
*expected_states: str,
|
||||
transient_states: typing.Optional[typing.Container[str]] = None,
|
||||
ssh_client: typing.Optional[ssh.SSHClientFixture] = None,
|
||||
timeout: tobiko.Seconds = None,
|
||||
sleep_interval: tobiko.Seconds = None) \
|
||||
-> str:
|
||||
hostname = getattr(ssh_client, 'hostname', None)
|
||||
if transient_states is None:
|
||||
transient_states = list()
|
||||
for status in expected_states:
|
||||
transient_states += COUD_INIT_TRANSIENT_STATES.get(status, [])
|
||||
|
||||
LOG.debug("Waiting cloud-init status on host %r to switch from %r to "
|
||||
"%r...",
|
||||
ssh_client.hostname, actual, expected)
|
||||
time.sleep(sleep_interval)
|
||||
actual = get_cloud_init_status(ssh_client=ssh_client,
|
||||
timeout=timeout-enlapsed_time)
|
||||
return actual
|
||||
with open_cloud_init_ouput(timeout=timeout,
|
||||
ssh_client=ssh_client) as output:
|
||||
for attempt in tobiko.retry(timeout=timeout,
|
||||
interval=sleep_interval,
|
||||
default_timeout=600.,
|
||||
default_interval=5.):
|
||||
actual_status = get_cloud_init_status(ssh_client=ssh_client,
|
||||
timeout=attempt.time_left)
|
||||
if actual_status in expected_states:
|
||||
return actual_status
|
||||
|
||||
output.readall()
|
||||
if actual_status not in transient_states:
|
||||
raise InvalidCloudInitStatusError(
|
||||
hostname=hostname,
|
||||
actual_status=actual_status,
|
||||
expected_states=expected_states,
|
||||
details=str(output))
|
||||
|
||||
try:
|
||||
attempt.check_limits()
|
||||
except tobiko.RetryTimeLimitError as ex:
|
||||
raise WaitForCloudInitTimeoutError(
|
||||
timeout=attempt.timeout,
|
||||
hostname=hostname,
|
||||
actual_status=actual_status,
|
||||
expected_states=expected_states,
|
||||
details=str(output)) from ex
|
||||
|
||||
# show only the last 10 lines
|
||||
details = '\n'.join(str(output).splitlines()[-10:])
|
||||
LOG.debug(f"Waiting cloud-init status on host '{hostname}' to "
|
||||
f"switch from '{actual_status}' to any of expected "
|
||||
f"states ({', '.join(expected_states)})\n\n"
|
||||
f"{details}\n")
|
||||
|
||||
raise RuntimeError("Retry loop ended himself")
|
||||
|
||||
|
||||
CLOUD_INIT_OUTPUT_FILE = '/var/log/cloud-init-output.log'
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def open_cloud_init_ouput(
|
||||
cloud_init_output_file: str = CLOUD_INIT_OUTPUT_FILE,
|
||||
tail=False,
|
||||
follow=False,
|
||||
**params) \
|
||||
-> typing.Generator[sh.ShellStdout, None, None]:
|
||||
command = ['tail']
|
||||
if not tail:
|
||||
# Start from the begin of the file
|
||||
command += ['-c', '+0']
|
||||
if follow:
|
||||
command += ['-F']
|
||||
|
||||
command += [cloud_init_output_file]
|
||||
process = sh.process(command, **params)
|
||||
with process:
|
||||
yield process.stdout
|
||||
|
|
|
@ -43,7 +43,7 @@ class CentosFlavorStackFixture(_nova.FlavorStackFixture):
|
|||
ram = 256
|
||||
|
||||
|
||||
class CentosServerStackFixture(_nova.ServerStackFixture):
|
||||
class CentosServerStackFixture(_nova.CloudInitServerStackFixture):
|
||||
|
||||
#: Glance image used to create a Nova server instance
|
||||
image_fixture = tobiko.required_setup_fixture(CentosImageFixture)
|
||||
|
|
|
@ -18,6 +18,7 @@ from __future__ import absolute_import
|
|||
import abc
|
||||
import os
|
||||
import typing
|
||||
from abc import ABC
|
||||
|
||||
import netaddr
|
||||
import six
|
||||
|
@ -316,30 +317,6 @@ class ServerStackFixture(heat.HeatStackFixture, abc.ABC):
|
|||
return nova.get_console_output(server=self.server_id,
|
||||
length=self.max_console_output_length)
|
||||
|
||||
@property
|
||||
def user_data(self):
|
||||
return nova.user_data(self.cloud_config)
|
||||
|
||||
#: SWAP file name
|
||||
swap_filename: str = '/swap.img'
|
||||
#: SWAP file size in bytes
|
||||
swap_size: typing.Optional[int] = None
|
||||
#: nax SWAP file size in bytes
|
||||
swap_maxsize: typing.Optional[int] = None
|
||||
|
||||
@property
|
||||
def cloud_config(self):
|
||||
cloud_config = nova.cloud_config()
|
||||
# default is to not create any swap files,
|
||||
# because 'swap_file_max_size' is set to None
|
||||
if self.swap_maxsize is not None:
|
||||
cloud_config = nova.cloud_config(
|
||||
cloud_config,
|
||||
swap={'filename': self.swap_filename,
|
||||
'size': self.swap_size or 'auto',
|
||||
'maxsize': self.swap_maxsize})
|
||||
return cloud_config
|
||||
|
||||
def ensure_server_status(
|
||||
self, status: str,
|
||||
retry_count: typing.Optional[int] = None,
|
||||
|
@ -385,6 +362,39 @@ class ServerStackFixture(heat.HeatStackFixture, abc.ABC):
|
|||
instances=1,
|
||||
cores=self.flavor_stack.vcpus or 1)
|
||||
|
||||
user_data = None
|
||||
|
||||
|
||||
class CloudInitServerStackFixture(ServerStackFixture, ABC):
|
||||
|
||||
#: SWAP file name
|
||||
swap_filename: str = '/swap.img'
|
||||
#: SWAP file size in bytes
|
||||
swap_size: typing.Optional[int] = None
|
||||
#: nax SWAP file size in bytes
|
||||
swap_maxsize: typing.Optional[int] = None
|
||||
|
||||
@property
|
||||
def user_data(self):
|
||||
return nova.user_data(self.cloud_config)
|
||||
|
||||
@property
|
||||
def cloud_config(self):
|
||||
cloud_config = nova.cloud_config()
|
||||
# default is to not create any swap files,
|
||||
# because 'swap_file_max_size' is set to None
|
||||
if self.swap_maxsize is not None:
|
||||
cloud_config = nova.cloud_config(
|
||||
cloud_config,
|
||||
swap={'filename': self.swap_filename,
|
||||
'size': self.swap_size or 'auto',
|
||||
'maxsize': self.swap_maxsize})
|
||||
return cloud_config
|
||||
|
||||
def wait_for_cloud_init_done(self, **params):
|
||||
nova.wait_for_cloud_init_done(ssh_client=self.ssh_client,
|
||||
**params)
|
||||
|
||||
|
||||
class ExternalServerStackFixture(ServerStackFixture, abc.ABC):
|
||||
# pylint: disable=abstract-method
|
||||
|
|
|
@ -57,7 +57,7 @@ class UbuntuMinimalFlavorStackFixture(_nova.FlavorStackFixture):
|
|||
ram = 128
|
||||
|
||||
|
||||
class UbuntuServerStackFixture(_nova.ServerStackFixture):
|
||||
class UbuntuServerStackFixture(_nova.CloudInitServerStackFixture):
|
||||
|
||||
#: Glance image used to create a Nova server instance
|
||||
image_fixture = tobiko.required_setup_fixture(UbuntuImageFixture)
|
||||
|
|
|
@ -49,6 +49,8 @@ HostNameError = _hostname.HostnameError
|
|||
get_hostname = _hostname.get_hostname
|
||||
|
||||
join_chunks = _io.join_chunks
|
||||
ShellStdout = _io.ShellStdout
|
||||
select_files = _io.select_files
|
||||
|
||||
local_execute = _local.local_execute
|
||||
local_process = _local.local_process
|
||||
|
|
|
@ -107,6 +107,17 @@ class ShellReadable(ShellIOBase):
|
|||
return (not self.closed and
|
||||
getattr(self.delegate, 'read_ready', False))
|
||||
|
||||
def readall(self, size=None):
|
||||
return join_chunks(self._readall(size))
|
||||
|
||||
def _readall(self, size):
|
||||
while self.read_ready:
|
||||
chunk = self.read(size=size)
|
||||
if chunk:
|
||||
yield chunk
|
||||
else:
|
||||
break
|
||||
|
||||
|
||||
class ShellWritable(ShellIOBase):
|
||||
|
||||
|
|
|
@ -81,7 +81,7 @@ class CirrosServerStackTest(testtools.TestCase):
|
|||
self.assertTrue(output)
|
||||
|
||||
def test_swap_file(self):
|
||||
if self.stack.swap_maxsize is None:
|
||||
if getattr(self.stack, 'swap_maxsize', None) is None:
|
||||
self.skipTest('Swap maxsize is None')
|
||||
|
||||
cloud_config = self.stack.cloud_config
|
||||
|
|
Loading…
Reference in New Issue