x
/
tobiko
Code Issues Proposed changes

Update cloud-init integration 

Add method for waiting for cloud init is done before considering
a server fixture ready to use

Change-Id: I34c546d92e883aba998c480edede3352ca7153cb
This commit is contained in:
Federico Ressi 2021-04-27 09:43:19 +02:00
parent 2ef60ae638
commit b298c8843c
7 changed files with 168 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

145
tobiko/openstack/nova/_cloud_init.py
View File

@ -14,13 +14,15 @@
from __future__ import absolute_import
import collections
import time
import contextlib
import typing
from oslo_log import log
import yaml
import tobiko
from tobiko.shell import sh
from tobiko.shell import ssh
LOG = log.getLogger(__name__)
@ -77,47 +79,128 @@ class CloudConfig(dict):
return combine_cloud_configs([self, other])
class WaitForCloudInitTimeoutError(tobiko.TobikoException):
message = ("after {enlapsed_time} seconds cloud-init status of host "
"{hostname!r} is still {actual!r} while it is expecting to "
"be in {expected!r}")
class InvalidCloudInitStatusError(tobiko.TobikoException):
message = ("cloud-init status of host '{hostname}' is "
"'{actual_status}' while it is expecting to "
"be in {expected_states!r}:\n"
"{details}")
def get_cloud_init_status(ssh_client=None, timeout=None):
class WaitForCloudInitTimeoutError(InvalidCloudInitStatusError):
message = ("after {timeout} seconds cloud-init status of host "
"'{hostname}' is still '{actual_status}' while it is "
"expecting to be in {expected_states!r}:\n"
"{details}")
COUD_INIT_TRANSIENT_STATES = {
'done': tuple(['running'])
}
def get_cloud_init_status(
ssh_client: typing.Optional[ssh.SSHClientFixture] = None,
timeout: tobiko.Seconds = None) \
-> str:
try:
output = sh.execute('cloud-init status',
ssh_client=ssh_client,
timeout=timeout,
sudo=True).stdout
return yaml.load(output)['status']
except sh.ShellCommandFailed as ex:
output = ex.stdout
if output:
LOG.debug(f"Cloud init status error reported:\n{ex}")
else:
raise
status = yaml.load(output)
tobiko.check_valid_type(status, dict)
tobiko.check_valid_type(status['status'], str)
return status['status']
def wait_for_cloud_init_done(ssh_client=None, timeout=None,
sleep_interval=None):
return wait_for_cloud_init_status(expected={'done'},
def wait_for_cloud_init_done(
ssh_client: typing.Optional[ssh.SSHClientFixture] = None,
timeout: tobiko.Seconds = None,
sleep_interval: tobiko.Seconds = None) \
-> str:
return wait_for_cloud_init_status('done',
ssh_client=ssh_client,
timeout=timeout,
sleep_interval=sleep_interval)
def wait_for_cloud_init_status(expected, ssh_client=None, timeout=None,
sleep_interval=None):
expected = set(expected)
timeout = timeout and float(timeout) or 1200.
sleep_interval = sleep_interval and float(sleep_interval) or 5.
start_time = time.time()
actual = get_cloud_init_status(ssh_client=ssh_client, timeout=timeout)
while actual not in expected:
enlapsed_time = time.time() - start_time
if enlapsed_time >= timeout:
raise WaitForCloudInitTimeoutError(hostname=ssh_client.hostname,
actual=actual,
expected=expected,
enlapsed_time=enlapsed_time)
def wait_for_cloud_init_status(
*expected_states: str,
transient_states: typing.Optional[typing.Container[str]] = None,
ssh_client: typing.Optional[ssh.SSHClientFixture] = None,
timeout: tobiko.Seconds = None,
sleep_interval: tobiko.Seconds = None) \
-> str:
hostname = getattr(ssh_client, 'hostname', None)
if transient_states is None:
transient_states = list()
for status in expected_states:
transient_states += COUD_INIT_TRANSIENT_STATES.get(status, [])
LOG.debug("Waiting cloud-init status on host %r to switch from %r to "
"%r...",
ssh_client.hostname, actual, expected)
time.sleep(sleep_interval)
actual = get_cloud_init_status(ssh_client=ssh_client,
timeout=timeout-enlapsed_time)
return actual
with open_cloud_init_ouput(timeout=timeout,
ssh_client=ssh_client) as output:
for attempt in tobiko.retry(timeout=timeout,
interval=sleep_interval,
default_timeout=600.,
default_interval=5.):
actual_status = get_cloud_init_status(ssh_client=ssh_client,
timeout=attempt.time_left)
if actual_status in expected_states:
return actual_status
output.readall()
if actual_status not in transient_states:
raise InvalidCloudInitStatusError(
hostname=hostname,
actual_status=actual_status,
expected_states=expected_states,
details=str(output))
try:
attempt.check_limits()
except tobiko.RetryTimeLimitError as ex:
raise WaitForCloudInitTimeoutError(
timeout=attempt.timeout,
hostname=hostname,
actual_status=actual_status,
expected_states=expected_states,
details=str(output)) from ex
# show only the last 10 lines
details = '\n'.join(str(output).splitlines()[-10:])
LOG.debug(f"Waiting cloud-init status on host '{hostname}' to "
f"switch from '{actual_status}' to any of expected "
f"states ({', '.join(expected_states)})\n\n"
f"{details}\n")
raise RuntimeError("Retry loop ended himself")
CLOUD_INIT_OUTPUT_FILE = '/var/log/cloud-init-output.log'
@contextlib.contextmanager
def open_cloud_init_ouput(
cloud_init_output_file: str = CLOUD_INIT_OUTPUT_FILE,
tail=False,
follow=False,
**params) \
-> typing.Generator[sh.ShellStdout, None, None]:
command = ['tail']
if not tail:
# Start from the begin of the file
command += ['-c', '+0']
if follow:
command += ['-F']
command += [cloud_init_output_file]
process = sh.process(command, **params)
with process:
yield process.stdout

2
tobiko/openstack/stacks/_centos.py
View File

@ -43,7 +43,7 @@ class CentosFlavorStackFixture(_nova.FlavorStackFixture):
ram = 256
class CentosServerStackFixture(_nova.ServerStackFixture):
class CentosServerStackFixture(_nova.CloudInitServerStackFixture):
#: Glance image used to create a Nova server instance
image_fixture = tobiko.required_setup_fixture(CentosImageFixture)

58
tobiko/openstack/stacks/_nova.py
View File

@ -18,6 +18,7 @@ from __future__ import absolute_import
import abc
import os
import typing
from abc import ABC
import netaddr
import six
@ -316,30 +317,6 @@ class ServerStackFixture(heat.HeatStackFixture, abc.ABC):
return nova.get_console_output(server=self.server_id,
length=self.max_console_output_length)
@property
def user_data(self):
return nova.user_data(self.cloud_config)
#: SWAP file name
swap_filename: str = '/swap.img'
#: SWAP file size in bytes
swap_size: typing.Optional[int] = None
#: nax SWAP file size in bytes
swap_maxsize: typing.Optional[int] = None
@property
def cloud_config(self):
cloud_config = nova.cloud_config()
# default is to not create any swap files,
# because 'swap_file_max_size' is set to None
if self.swap_maxsize is not None:
cloud_config = nova.cloud_config(
cloud_config,
swap={'filename': self.swap_filename,
'size': self.swap_size or 'auto',
'maxsize': self.swap_maxsize})
return cloud_config
def ensure_server_status(
self, status: str,
retry_count: typing.Optional[int] = None,
@ -385,6 +362,39 @@ class ServerStackFixture(heat.HeatStackFixture, abc.ABC):
instances=1,
cores=self.flavor_stack.vcpus or 1)
user_data = None
class CloudInitServerStackFixture(ServerStackFixture, ABC):
#: SWAP file name
swap_filename: str = '/swap.img'
#: SWAP file size in bytes
swap_size: typing.Optional[int] = None
#: nax SWAP file size in bytes
swap_maxsize: typing.Optional[int] = None
@property
def user_data(self):
return nova.user_data(self.cloud_config)
@property
def cloud_config(self):
cloud_config = nova.cloud_config()
# default is to not create any swap files,
# because 'swap_file_max_size' is set to None
if self.swap_maxsize is not None:
cloud_config = nova.cloud_config(
cloud_config,
swap={'filename': self.swap_filename,
'size': self.swap_size or 'auto',
'maxsize': self.swap_maxsize})
return cloud_config
def wait_for_cloud_init_done(self, **params):
nova.wait_for_cloud_init_done(ssh_client=self.ssh_client,
**params)
class ExternalServerStackFixture(ServerStackFixture, abc.ABC):
# pylint: disable=abstract-method

2
tobiko/openstack/stacks/_ubuntu.py
View File

@ -57,7 +57,7 @@ class UbuntuMinimalFlavorStackFixture(_nova.FlavorStackFixture):
ram = 128
class UbuntuServerStackFixture(_nova.ServerStackFixture):
class UbuntuServerStackFixture(_nova.CloudInitServerStackFixture):
#: Glance image used to create a Nova server instance
image_fixture = tobiko.required_setup_fixture(UbuntuImageFixture)

2
tobiko/shell/sh/__init__.py
View File

@ -49,6 +49,8 @@ HostNameError = _hostname.HostnameError
get_hostname = _hostname.get_hostname
join_chunks = _io.join_chunks
ShellStdout = _io.ShellStdout
select_files = _io.select_files
local_execute = _local.local_execute
local_process = _local.local_process

11
tobiko/shell/sh/_io.py
View File

@ -107,6 +107,17 @@ class ShellReadable(ShellIOBase):
return (not self.closed and
getattr(self.delegate, 'read_ready', False))
def readall(self, size=None):
return join_chunks(self._readall(size))
def _readall(self, size):
while self.read_ready:
chunk = self.read(size=size)
if chunk:
yield chunk
else:
break
class ShellWritable(ShellIOBase):

2
tobiko/tests/functional/openstack/stacks/test_cirros.py
View File

@ -81,7 +81,7 @@ class CirrosServerStackTest(testtools.TestCase):
self.assertTrue(output)
def test_swap_file(self):
if self.stack.swap_maxsize is None:
if getattr(self.stack, 'swap_maxsize', None) is None:
self.skipTest('Swap maxsize is None')
cloud_config = self.stack.cloud_config