Use Metalsmith client to discover overcloud nodes

In OSP-17 and newer, there is no Nova running as undercloud
service and instead of Nova client, Metalsmith client has to be
used to e.g. get overcloud nodes and its IP addresses.

This patch replaces Nova client with Metalsmith client to list
overcloud servers and it IP addresses

Change-Id: I0a7859ebc4e1c6d3660040100e1bf55a7218e4f2
This commit is contained in:
Federico Ressi 2022-07-05 12:52:41 +02:00
parent 1a51205f63
commit 2f69f5211d
6 changed files with 146 additions and 145 deletions

View File

@ -20,7 +20,7 @@ import pandas as pd
import testtools
from tobiko import config
from tobiko.openstack import nova
from tobiko.openstack import metalsmith
from tobiko import tripleo
from tobiko.tripleo import pacemaker
from tobiko.tripleo import services
@ -31,7 +31,7 @@ CONF = config.CONF
@tripleo.skip_if_missing_overcloud
class OvercloudSshConnectionTest(testtools.TestCase):
class OvercloudKeystoneCredentialsTest(testtools.TestCase):
def test_fetch_overcloud_credentials(self):
env = tripleo.load_overcloud_rcfile()
@ -45,20 +45,20 @@ class OvercloudSshConnectionTest(testtools.TestCase):
@tripleo.skip_if_missing_overcloud
class OvercloudNovaApiTest(testtools.TestCase):
class OvercloudMetalsmithApiTest(testtools.TestCase):
def test_list_overcloud_nodes(self):
nodes = tripleo.list_overcloud_nodes()
self.assertTrue(nodes)
for node in nodes:
node_ip = nova.find_server_ip_address(server=node,
check_connectivity=True)
node_ip = metalsmith.find_instance_ip_address(
instance=node, check_connectivity=True)
self.assertIsInstance(node_ip, netaddr.IPAddress)
def test_find_overcloud_nodes(self):
node = tripleo.find_overcloud_node()
node_ip = nova.find_server_ip_address(server=node,
check_connectivity=True)
node_ip = metalsmith.find_instance_ip_address(instance=node,
check_connectivity=True)
self.assertIsInstance(node_ip, netaddr.IPAddress)
def test_get_overcloud_node_ip_address(self):
@ -66,10 +66,13 @@ class OvercloudNovaApiTest(testtools.TestCase):
self.assertIsInstance(overcloud_node_ip, netaddr.IPAddress)
def test_overcloud_host_config(self):
hostname = tripleo.find_overcloud_node().name
instance = tripleo.find_overcloud_node()
host_config = tobiko.setup_fixture(
tripleo.overcloud_host_config(hostname=hostname))
self.assertEqual(hostname, host_config.host)
tripleo.overcloud_host_config(instance=instance))
instance_ips = set()
for ips in instance.ip_addresses().values():
instance_ips.update(ips)
self.assertIn(host_config.host, instance_ips)
self.assertIsInstance(host_config.hostname, str)
netaddr.IPAddress(host_config.hostname)
self.assertEqual(CONF.tobiko.tripleo.overcloud_ssh_port,
@ -83,8 +86,8 @@ class OvercloudNovaApiTest(testtools.TestCase):
self.assertTrue(os.path.isfile(key_filename + '.pub'))
def test_overcloud_ssh_client_connection(self):
hostname = tripleo.find_overcloud_node().name
ssh_client = tripleo.overcloud_ssh_client(hostname=hostname)
instance = tripleo.find_overcloud_node()
ssh_client = tripleo.overcloud_ssh_client(instance=instance)
ssh_client.connect()

View File

@ -39,8 +39,8 @@ class TripleoTopologyTest(test_topology.OpenStackTopologyTest):
@tripleo.skip_if_missing_overcloud
def test_overcloud_group(self):
for server in tripleo.list_overcloud_nodes():
ssh_client = tripleo.overcloud_ssh_client(server.name)
for instance in tripleo.list_overcloud_nodes():
ssh_client = tripleo.overcloud_ssh_client(instance=instance)
name = sh.get_hostname(ssh_client=ssh_client).split('.')[0]
node = self.topology.get_node(name)
self.assertIs(node.ssh_client, ssh_client)
@ -53,6 +53,6 @@ class TripleoTopologyTest(test_topology.OpenStackTopologyTest):
nodes = self.topology.get_group(group)
self.assertIn(node, nodes)
self.assertIn(group, node.groups)
host_config = tripleo.overcloud_host_config(name)
host_config = tripleo.overcloud_host_config(instance=instance)
self.assertEqual(host_config.hostname,
str(node.public_ip))

View File

@ -15,6 +15,7 @@ from __future__ import absolute_import
import io
import os
import typing
from oslo_log import log
@ -22,7 +23,7 @@ import tobiko
from tobiko import config
from tobiko.openstack import keystone
from tobiko.openstack import ironic
from tobiko.openstack import nova
from tobiko.openstack import metalsmith
from tobiko.openstack import topology
from tobiko.shell import sh
from tobiko.shell import ssh
@ -58,86 +59,74 @@ class OvercloudKeystoneCredentialsFixture(
def list_overcloud_nodes(**params):
session = _undercloud.undercloud_keystone_session()
client = nova.get_nova_client(session=session)
return nova.list_servers(client=client, **params)
client = metalsmith.get_metalsmith_client(session=session)
return metalsmith.list_instances(client=client, **params)
def find_overcloud_node(**params):
session = _undercloud.undercloud_keystone_session()
client = nova.get_nova_client(session=session)
return nova.find_server(client=client, **params)
client = metalsmith.get_metalsmith_client(session=session)
return metalsmith.find_instance(client=client, **params)
def power_on_overcloud_node(server: nova.ServerType,
def power_on_overcloud_node(instance: metalsmith.MetalsmithInstance,
timeout: tobiko.Seconds = 120.,
sleep_time: tobiko.Seconds = 5.):
session = _undercloud.undercloud_keystone_session()
node = getattr(server, 'OS-EXT-SRV-ATTR:hypervisor_hostname',
None)
if node is None:
client = nova.get_nova_client(session=session)
nova.activate_server(client=client,
server=server,
timeout=timeout,
sleep_time=sleep_time)
else:
client = ironic.get_ironic_client(session=session)
ironic.power_on_node(client=client,
node=node,
timeout=timeout,
sleep_time=sleep_time)
client = ironic.get_ironic_client(session=session)
ironic.power_on_node(client=client,
node=instance.uuid,
timeout=timeout,
sleep_time=sleep_time)
def power_off_overcloud_node(server: nova.ServerType,
def power_off_overcloud_node(instance: metalsmith.MetalsmithInstance,
timeout: tobiko.Seconds = None,
sleep_time: tobiko.Seconds = None):
session = _undercloud.undercloud_keystone_session()
node = getattr(server, 'OS-EXT-SRV-ATTR:hypervisor_hostname',
None)
if node is None:
client = nova.get_nova_client(session=session)
nova.shutoff_server(client=client,
server=server,
timeout=timeout,
sleep_time=sleep_time)
else:
client = ironic.get_ironic_client(session=session)
ironic.power_off_node(client=client,
node=node,
timeout=timeout,
sleep_time=sleep_time)
client = ironic.get_ironic_client(session=session)
ironic.power_off_node(client=client,
node=instance.uuid,
timeout=timeout,
sleep_time=sleep_time)
def overcloud_ssh_client(hostname=None, ip_version=None, network_name=None,
server=None, host_config=None):
def overcloud_ssh_client(ip_version: int = None,
network_name: str = None,
instance: metalsmith.MetalsmithInstance = None,
host_config=None):
if host_config is None:
host_config = overcloud_host_config(hostname=hostname,
ip_version=ip_version,
host_config = overcloud_host_config(ip_version=ip_version,
network_name=network_name,
server=server)
return ssh.ssh_client(host=hostname, **host_config.connect_parameters)
instance=instance)
tobiko.check_valid_type(host_config.host, str)
return ssh.ssh_client(host=host_config.host,
**host_config.connect_parameters)
def overcloud_host_config(hostname=None, ip_version=None, network_name=None,
server=None):
host_config = OvercloudHostConfig(host=hostname,
ip_version=ip_version,
def overcloud_host_config(ip_version: int = None,
network_name: str = None,
instance: metalsmith.MetalsmithInstance = None):
host_config = OvercloudHostConfig(ip_version=ip_version,
network_name=network_name,
server=server)
instance=instance)
return tobiko.setup_fixture(host_config)
def overcloud_node_ip_address(ip_version=None, network_name=None, server=None,
def overcloud_node_ip_address(ip_version: int = None,
network_name: str = None,
instance: metalsmith.MetalsmithInstance = None,
**params):
server = server or find_overcloud_node(**params)
if instance is None:
instance = find_overcloud_node(**params)
ip_version = ip_version or CONF.tobiko.tripleo.overcloud_ip_version
network_name = network_name or CONF.tobiko.tripleo.overcloud_network_name
address = nova.find_server_ip_address(server=server,
ip_version=ip_version,
network_name=network_name)
address = metalsmith.find_instance_ip_address(instance=instance,
ip_version=ip_version,
network_name=network_name)
LOG.debug(f"Got Overcloud node address '{address}' from Undercloud "
f"(ip_version={ip_version}, network_name={network_name}, "
f"server={server})")
f"instance={instance})")
return address
@ -176,41 +165,46 @@ def _get_undercloud_file(ssh_client, source, destination, mode):
class OvercloudHostConfig(tobiko.SharedFixture):
host = None
hostname = None
port = None
username = None
key_file = tobiko.required_fixture(OvercloudSshKeyFileFixture)
ip_version = None
network_name = None
key_filename = None
server = None
def __init__(self, host=None, ip_version=None, network_name=None,
server=None, **kwargs):
key_file = tobiko.required_fixture(OvercloudSshKeyFileFixture)
def __init__(self,
host: str = None,
hostname: str = None,
ip_version: int = None,
instance: metalsmith.MetalsmithInstance = None,
key_filename: str = None,
network_name: str = None,
port: int = None,
username: str = None,
**kwargs):
super(OvercloudHostConfig, self).__init__()
if host:
self.host = host
if ip_version:
self.ip_version = ip_version
if network_name:
self.network_name = network_name
if server:
self.server = server
if self.host is None:
self.host = server.name
tobiko.check_valid_type(self.host, str)
self.host = host
self.instance = instance
self.ip_version = ip_version
self.key_filename = key_filename
self.host = host
self.hostname = hostname
self.network_name = network_name
self.port = port
self.username = username
self._connect_parameters = ssh.gather_ssh_connect_parameters(**kwargs)
def setup_fixture(self):
self.hostname = str(overcloud_node_ip_address(
name=self.host, ip_version=self.ip_version,
network_name=self.network_name,
server=self.server))
self.port = self.port or CONF.tobiko.tripleo.overcloud_ssh_port
self.username = (self.username or
CONF.tobiko.tripleo.overcloud_ssh_username)
self.key_filename = self.key_filename or self.key_file.key_filename
if self.hostname is None:
self.hostname = str(overcloud_node_ip_address(
hostname=self.host,
ip_version=self.ip_version,
network_name=self.network_name,
instance=self.instance))
if self.host is None:
self.host = self.hostname
if self.port is None:
self.port = CONF.tobiko.tripleo.overcloud_ssh_port
if self.username is None:
self.username = CONF.tobiko.tripleo.overcloud_ssh_username
if self.key_filename is None:
self.key_filename = self.key_file.key_filename
@property
def connect_parameters(self):
@ -226,7 +220,9 @@ def setup_overcloud_keystone_crederntials():
OvercloudKeystoneCredentialsFixture)
def get_overcloud_nodes_dataframe(oc_node_df_function):
def get_overcloud_nodes_dataframe(
oc_node_df_function: typing.Callable[[ssh.SSHClientType],
typing.Any]):
"""
:param oc_node_df_function : a function that queries a oc node
using a cli command and returns a datraframe with an added
@ -238,10 +234,10 @@ def get_overcloud_nodes_dataframe(oc_node_df_function):
:return: dataframe of all overcloud nodes processes
"""
import pandas
oc_nodes_selection = list_overcloud_nodes()
oc_nodes_names = [node.name for node in oc_nodes_selection]
oc_nodes_dfs = [oc_node_df_function(node_name) for
node_name in oc_nodes_names]
oc_nodes_dfs = list()
for instance in list_overcloud_nodes():
ssh_client = overcloud_ssh_client(instance=instance)
oc_nodes_dfs.append(oc_node_df_function(ssh_client))
oc_procs_df = pandas.concat(oc_nodes_dfs, ignore_index=True)
return oc_procs_df

View File

@ -14,8 +14,9 @@
from __future__ import absolute_import
import re
import typing # noqa
import typing
import metalsmith
from oslo_log import log
from tobiko.openstack import neutron
@ -89,22 +90,22 @@ class TripleoTopology(topology.OpenStackTopology):
def discover_overcloud_nodes(self):
if _overcloud.has_overcloud():
for server in _overcloud.list_overcloud_nodes():
for instance in _overcloud.list_overcloud_nodes():
try:
_overcloud.power_on_overcloud_node(server)
_overcloud.power_on_overcloud_node(instance)
except Exception:
LOG.exception("Error ensuring overcloud node power "
"status is on")
host_config = _overcloud.overcloud_host_config(server=server)
host_config = _overcloud.overcloud_host_config(
instance=instance)
ssh_client = _overcloud.overcloud_ssh_client(
hostname=server.name,
instance=instance,
host_config=host_config)
node = self.add_node(address=host_config.hostname,
hostname=server.name,
group='overcloud',
ssh_client=ssh_client)
assert isinstance(node, TripleoTopologyNode)
node.overcloud_server = server
node.overcloud_instance = instance
self.discover_overcloud_node_subgroups(node)
def discover_overcloud_node_subgroups(self, node):
@ -137,68 +138,68 @@ class TripleoTopology(topology.OpenStackTopology):
class TripleoTopologyNode(topology.OpenStackTopologyNode):
overcloud_server: typing.Optional[nova.NovaServer] = None
overcloud_instance: typing.Optional[metalsmith.Instance] = None
l3_agent_conf_path = (
'/var/lib/config-data/neutron/etc/neutron/l3_agent.ini')
def reboot_overcloud_node(self, reactivate_servers=True):
def reboot_overcloud_node(self,
reactivate_servers=True):
"""Reboot overcloud node
This method reboots an overcloud node and may start every server which
changed its provisioning state to SHUTOFF because of that operation.
This method reboots an overcloud node and may start every Nova
server which is not in SHUTOFF status before restarting.
:param start_servers (bool): whether or not to start the servers which
are hosted on the node after the reboot
:param reactivate_servers: whether or not to re-start the servers which
are hosted on the compute node after the reboot
"""
running_servers: typing.List[nova.NovaServer] = []
if reactivate_servers:
servers_to_restart = self.get_running_servers()
running_servers = self.list_running_servers()
LOG.debug(f'Servers to restart after reboot: {running_servers}')
self.power_off_overcloud_node()
self.power_on_overcloud_node()
if reactivate_servers:
for server in servers_to_restart:
if running_servers:
LOG.info(f'Restart servers after rebooting overcloud compute node '
f'{self.name}...')
for server in running_servers:
nova.wait_for_server_status(server=server.id,
status='SHUTOFF')
LOG.debug(f'Server {server.name} with ID {server.id} '
f'had a SHUTOFF status before being '
f'restarted')
nova.activate_server(server)
LOG.debug(f'Server {server.name} with ID {server.id} '
f'has a {server.status} status after being '
f'restarted')
LOG.debug(f'Re-activate server {server.name} with ID '
f'{server.id}')
nova.activate_server(server=server)
LOG.debug(f'Server {server.name} with ID {server.id} has '
f'been reactivated')
def get_running_servers(self):
servers_to_reactivate = list()
def list_running_servers(self) -> typing.List[nova.NovaServer]:
running_servers = list()
for server in nova.list_servers():
server_hyp = getattr(server,
'OS-EXT-SRV-ATTR:'
'hypervisor_hostname').split('.', 1)[0]
if self.name == server_hyp and server.status != 'SHUTOFF':
servers_to_reactivate.append(server)
LOG.info(f'Servers to restart after reboot: {servers_to_reactivate}')
return servers_to_reactivate
if server.status != 'SHUTOFF':
hypervisor_name = nova.get_server_hypervisor(server,
short=True)
if self.name == hypervisor_name:
running_servers.append(server)
return running_servers
def power_on_overcloud_node(self):
server = self.overcloud_server
if server is None:
if self.overcloud_instance is None:
raise TypeError(f"Node {self.name} is not and Overcloud server")
self.ssh_client.close()
LOG.debug(f"Ensuring overcloud node {self.name} power is on...")
_overcloud.power_on_overcloud_node(server)
_overcloud.power_on_overcloud_node(instance=self.overcloud_instance)
hostname = sh.get_hostname(ssh_client=self.ssh_client)
LOG.debug(f"Overcloud node {self.name} power is on ("
f"hostname={hostname})")
def power_off_overcloud_node(self):
server = self.overcloud_server
if server is None:
if self.overcloud_instance is None:
raise TypeError(f"Node {self.name} is not and Overcloud server")
self.ssh_client.close()
LOG.debug(f"Ensuring overcloud node {self.name} power is off...")
_overcloud.power_off_overcloud_node(server)
_overcloud.power_off_overcloud_node(instance=self.overcloud_instance)
LOG.debug(f"Overcloud server node {self.name} power is off.")

View File

@ -12,6 +12,7 @@ from tobiko.openstack import neutron
from tobiko.openstack import topology
from tobiko.tripleo import overcloud
from tobiko.shell import sh
from tobiko.shell import ssh
LOG = log.getLogger(__name__)
@ -22,7 +23,7 @@ class OvercloudProcessesException(tobiko.TobikoException):
"{process_error}"
def get_overcloud_node_processes_table(hostname):
def get_overcloud_node_processes_table(ssh_client: ssh.SSHClientType):
"""
get processes tables from overcloud node
@ -47,7 +48,6 @@ root | 11| 2| 0.0| 0|00:00:05|migration/0 |[migration/0]
:return: dataframe of overcloud node processes dataframe
"""
ssh_client = overcloud.overcloud_ssh_client(hostname)
output = sh.execute(
"ps -axw -o \"%U\" -o \"DELIM%p\" -o \"DELIM%P\" -o \"DELIM%C\" -o "
"\"DELIM%z\" -o \"DELIM%x\" -o \"DELIM%c\" -o \"DELIM%a\" |grep -v "
@ -60,6 +60,7 @@ root | 11| 2| 0.0| 0|00:00:05|migration/0 |[migration/0]
table.columns = ['USER', 'PID', 'PPID', 'CPU', 'VSZ', 'TIME', 'PROCESS',
'PROCESS_ARGS']
# pylint: disable=unsupported-assignment-operation
hostname = sh.get_hostname(ssh_client=ssh_client)
table['overcloud_node'] = hostname
LOG.debug("Successfully got overcloud nodes processes status table")

View File

@ -8,6 +8,7 @@ import pandas
import tobiko
from tobiko.tripleo import overcloud
from tobiko.shell import sh
from tobiko.shell import ssh
LOG = log.getLogger(__name__)
@ -17,7 +18,7 @@ class OvercloudServiceException(tobiko.TobikoException):
message = "not all overcloud nodes services are in active state"
def get_overcloud_node_services_table(hostname):
def get_overcloud_node_services_table(ssh_client: ssh.SSHClientType):
"""
get services table from overcloud node
@ -35,7 +36,6 @@ cloud-init.service|loaded|active|exited|Initialcloud-initjob(metadataservicecr)
:return: dataframe of overcloud node services
"""
ssh_client = overcloud.overcloud_ssh_client(hostname)
units = sh.list_systemd_units(all=True,
ssh_client=ssh_client).without_attributes(
load='not-found')
@ -53,7 +53,7 @@ cloud-init.service|loaded|active|exited|Initialcloud-initjob(metadataservicecr)
data['UNIT_DESCRIPTION'].append(unit.description)
table = pandas.DataFrame.from_dict(data)
table.replace(to_replace=' ', value="", regex=True, inplace=True)
table['overcloud_node'] = hostname
table['overcloud_node'] = sh.get_hostname(ssh_client=ssh_client)
LOG.debug("Got overcloud nodes services status :\n%s", table)
return table