From 2f69f5211d8f180ac8de788dbe1b1c64efd619d3 Mon Sep 17 00:00:00 2001 From: Federico Ressi Date: Tue, 5 Jul 2022 12:52:41 +0200 Subject: [PATCH] Use Metalsmith client to discover overcloud nodes In OSP-17 and newer, there is no Nova running as undercloud service and instead of Nova client, Metalsmith client has to be used to e.g. get overcloud nodes and its IP addresses. This patch replaces Nova client with Metalsmith client to list overcloud servers and it IP addresses Change-Id: I0a7859ebc4e1c6d3660040100e1bf55a7218e4f2 --- .../functional/tripleo/test_overcloud.py | 27 +-- .../tests/functional/tripleo/test_topology.py | 6 +- tobiko/tripleo/_overcloud.py | 170 +++++++++--------- tobiko/tripleo/_topology.py | 77 ++++---- tobiko/tripleo/processes.py | 5 +- tobiko/tripleo/services.py | 6 +- 6 files changed, 146 insertions(+), 145 deletions(-) diff --git a/tobiko/tests/functional/tripleo/test_overcloud.py b/tobiko/tests/functional/tripleo/test_overcloud.py index 81f2f21db..dbd584256 100644 --- a/tobiko/tests/functional/tripleo/test_overcloud.py +++ b/tobiko/tests/functional/tripleo/test_overcloud.py @@ -20,7 +20,7 @@ import pandas as pd import testtools from tobiko import config -from tobiko.openstack import nova +from tobiko.openstack import metalsmith from tobiko import tripleo from tobiko.tripleo import pacemaker from tobiko.tripleo import services @@ -31,7 +31,7 @@ CONF = config.CONF @tripleo.skip_if_missing_overcloud -class OvercloudSshConnectionTest(testtools.TestCase): +class OvercloudKeystoneCredentialsTest(testtools.TestCase): def test_fetch_overcloud_credentials(self): env = tripleo.load_overcloud_rcfile() @@ -45,20 +45,20 @@ class OvercloudSshConnectionTest(testtools.TestCase): @tripleo.skip_if_missing_overcloud -class OvercloudNovaApiTest(testtools.TestCase): +class OvercloudMetalsmithApiTest(testtools.TestCase): def test_list_overcloud_nodes(self): nodes = tripleo.list_overcloud_nodes() self.assertTrue(nodes) for node in nodes: - node_ip = nova.find_server_ip_address(server=node, - check_connectivity=True) + node_ip = metalsmith.find_instance_ip_address( + instance=node, check_connectivity=True) self.assertIsInstance(node_ip, netaddr.IPAddress) def test_find_overcloud_nodes(self): node = tripleo.find_overcloud_node() - node_ip = nova.find_server_ip_address(server=node, - check_connectivity=True) + node_ip = metalsmith.find_instance_ip_address(instance=node, + check_connectivity=True) self.assertIsInstance(node_ip, netaddr.IPAddress) def test_get_overcloud_node_ip_address(self): @@ -66,10 +66,13 @@ class OvercloudNovaApiTest(testtools.TestCase): self.assertIsInstance(overcloud_node_ip, netaddr.IPAddress) def test_overcloud_host_config(self): - hostname = tripleo.find_overcloud_node().name + instance = tripleo.find_overcloud_node() host_config = tobiko.setup_fixture( - tripleo.overcloud_host_config(hostname=hostname)) - self.assertEqual(hostname, host_config.host) + tripleo.overcloud_host_config(instance=instance)) + instance_ips = set() + for ips in instance.ip_addresses().values(): + instance_ips.update(ips) + self.assertIn(host_config.host, instance_ips) self.assertIsInstance(host_config.hostname, str) netaddr.IPAddress(host_config.hostname) self.assertEqual(CONF.tobiko.tripleo.overcloud_ssh_port, @@ -83,8 +86,8 @@ class OvercloudNovaApiTest(testtools.TestCase): self.assertTrue(os.path.isfile(key_filename + '.pub')) def test_overcloud_ssh_client_connection(self): - hostname = tripleo.find_overcloud_node().name - ssh_client = tripleo.overcloud_ssh_client(hostname=hostname) + instance = tripleo.find_overcloud_node() + ssh_client = tripleo.overcloud_ssh_client(instance=instance) ssh_client.connect() diff --git a/tobiko/tests/functional/tripleo/test_topology.py b/tobiko/tests/functional/tripleo/test_topology.py index 641459ab4..e2d2d45b2 100644 --- a/tobiko/tests/functional/tripleo/test_topology.py +++ b/tobiko/tests/functional/tripleo/test_topology.py @@ -39,8 +39,8 @@ class TripleoTopologyTest(test_topology.OpenStackTopologyTest): @tripleo.skip_if_missing_overcloud def test_overcloud_group(self): - for server in tripleo.list_overcloud_nodes(): - ssh_client = tripleo.overcloud_ssh_client(server.name) + for instance in tripleo.list_overcloud_nodes(): + ssh_client = tripleo.overcloud_ssh_client(instance=instance) name = sh.get_hostname(ssh_client=ssh_client).split('.')[0] node = self.topology.get_node(name) self.assertIs(node.ssh_client, ssh_client) @@ -53,6 +53,6 @@ class TripleoTopologyTest(test_topology.OpenStackTopologyTest): nodes = self.topology.get_group(group) self.assertIn(node, nodes) self.assertIn(group, node.groups) - host_config = tripleo.overcloud_host_config(name) + host_config = tripleo.overcloud_host_config(instance=instance) self.assertEqual(host_config.hostname, str(node.public_ip)) diff --git a/tobiko/tripleo/_overcloud.py b/tobiko/tripleo/_overcloud.py index 9d9a9aa24..d89a3d5f6 100644 --- a/tobiko/tripleo/_overcloud.py +++ b/tobiko/tripleo/_overcloud.py @@ -15,6 +15,7 @@ from __future__ import absolute_import import io import os +import typing from oslo_log import log @@ -22,7 +23,7 @@ import tobiko from tobiko import config from tobiko.openstack import keystone from tobiko.openstack import ironic -from tobiko.openstack import nova +from tobiko.openstack import metalsmith from tobiko.openstack import topology from tobiko.shell import sh from tobiko.shell import ssh @@ -58,86 +59,74 @@ class OvercloudKeystoneCredentialsFixture( def list_overcloud_nodes(**params): session = _undercloud.undercloud_keystone_session() - client = nova.get_nova_client(session=session) - return nova.list_servers(client=client, **params) + client = metalsmith.get_metalsmith_client(session=session) + return metalsmith.list_instances(client=client, **params) def find_overcloud_node(**params): session = _undercloud.undercloud_keystone_session() - client = nova.get_nova_client(session=session) - return nova.find_server(client=client, **params) + client = metalsmith.get_metalsmith_client(session=session) + return metalsmith.find_instance(client=client, **params) -def power_on_overcloud_node(server: nova.ServerType, +def power_on_overcloud_node(instance: metalsmith.MetalsmithInstance, timeout: tobiko.Seconds = 120., sleep_time: tobiko.Seconds = 5.): session = _undercloud.undercloud_keystone_session() - node = getattr(server, 'OS-EXT-SRV-ATTR:hypervisor_hostname', - None) - if node is None: - client = nova.get_nova_client(session=session) - nova.activate_server(client=client, - server=server, - timeout=timeout, - sleep_time=sleep_time) - else: - client = ironic.get_ironic_client(session=session) - ironic.power_on_node(client=client, - node=node, - timeout=timeout, - sleep_time=sleep_time) + client = ironic.get_ironic_client(session=session) + ironic.power_on_node(client=client, + node=instance.uuid, + timeout=timeout, + sleep_time=sleep_time) -def power_off_overcloud_node(server: nova.ServerType, +def power_off_overcloud_node(instance: metalsmith.MetalsmithInstance, timeout: tobiko.Seconds = None, sleep_time: tobiko.Seconds = None): session = _undercloud.undercloud_keystone_session() - node = getattr(server, 'OS-EXT-SRV-ATTR:hypervisor_hostname', - None) - if node is None: - client = nova.get_nova_client(session=session) - nova.shutoff_server(client=client, - server=server, - timeout=timeout, - sleep_time=sleep_time) - else: - client = ironic.get_ironic_client(session=session) - ironic.power_off_node(client=client, - node=node, - timeout=timeout, - sleep_time=sleep_time) + client = ironic.get_ironic_client(session=session) + ironic.power_off_node(client=client, + node=instance.uuid, + timeout=timeout, + sleep_time=sleep_time) -def overcloud_ssh_client(hostname=None, ip_version=None, network_name=None, - server=None, host_config=None): +def overcloud_ssh_client(ip_version: int = None, + network_name: str = None, + instance: metalsmith.MetalsmithInstance = None, + host_config=None): if host_config is None: - host_config = overcloud_host_config(hostname=hostname, - ip_version=ip_version, + host_config = overcloud_host_config(ip_version=ip_version, network_name=network_name, - server=server) - return ssh.ssh_client(host=hostname, **host_config.connect_parameters) + instance=instance) + tobiko.check_valid_type(host_config.host, str) + return ssh.ssh_client(host=host_config.host, + **host_config.connect_parameters) -def overcloud_host_config(hostname=None, ip_version=None, network_name=None, - server=None): - host_config = OvercloudHostConfig(host=hostname, - ip_version=ip_version, +def overcloud_host_config(ip_version: int = None, + network_name: str = None, + instance: metalsmith.MetalsmithInstance = None): + host_config = OvercloudHostConfig(ip_version=ip_version, network_name=network_name, - server=server) + instance=instance) return tobiko.setup_fixture(host_config) -def overcloud_node_ip_address(ip_version=None, network_name=None, server=None, +def overcloud_node_ip_address(ip_version: int = None, + network_name: str = None, + instance: metalsmith.MetalsmithInstance = None, **params): - server = server or find_overcloud_node(**params) + if instance is None: + instance = find_overcloud_node(**params) ip_version = ip_version or CONF.tobiko.tripleo.overcloud_ip_version network_name = network_name or CONF.tobiko.tripleo.overcloud_network_name - address = nova.find_server_ip_address(server=server, - ip_version=ip_version, - network_name=network_name) + address = metalsmith.find_instance_ip_address(instance=instance, + ip_version=ip_version, + network_name=network_name) LOG.debug(f"Got Overcloud node address '{address}' from Undercloud " f"(ip_version={ip_version}, network_name={network_name}, " - f"server={server})") + f"instance={instance})") return address @@ -176,41 +165,46 @@ def _get_undercloud_file(ssh_client, source, destination, mode): class OvercloudHostConfig(tobiko.SharedFixture): - host = None - hostname = None - port = None - username = None - key_file = tobiko.required_fixture(OvercloudSshKeyFileFixture) - ip_version = None - network_name = None - key_filename = None - server = None - def __init__(self, host=None, ip_version=None, network_name=None, - server=None, **kwargs): + key_file = tobiko.required_fixture(OvercloudSshKeyFileFixture) + + def __init__(self, + host: str = None, + hostname: str = None, + ip_version: int = None, + instance: metalsmith.MetalsmithInstance = None, + key_filename: str = None, + network_name: str = None, + port: int = None, + username: str = None, + **kwargs): super(OvercloudHostConfig, self).__init__() - if host: - self.host = host - if ip_version: - self.ip_version = ip_version - if network_name: - self.network_name = network_name - if server: - self.server = server - if self.host is None: - self.host = server.name - tobiko.check_valid_type(self.host, str) + self.host = host + self.instance = instance + self.ip_version = ip_version + self.key_filename = key_filename + self.host = host + self.hostname = hostname + self.network_name = network_name + self.port = port + self.username = username self._connect_parameters = ssh.gather_ssh_connect_parameters(**kwargs) def setup_fixture(self): - self.hostname = str(overcloud_node_ip_address( - name=self.host, ip_version=self.ip_version, - network_name=self.network_name, - server=self.server)) - self.port = self.port or CONF.tobiko.tripleo.overcloud_ssh_port - self.username = (self.username or - CONF.tobiko.tripleo.overcloud_ssh_username) - self.key_filename = self.key_filename or self.key_file.key_filename + if self.hostname is None: + self.hostname = str(overcloud_node_ip_address( + hostname=self.host, + ip_version=self.ip_version, + network_name=self.network_name, + instance=self.instance)) + if self.host is None: + self.host = self.hostname + if self.port is None: + self.port = CONF.tobiko.tripleo.overcloud_ssh_port + if self.username is None: + self.username = CONF.tobiko.tripleo.overcloud_ssh_username + if self.key_filename is None: + self.key_filename = self.key_file.key_filename @property def connect_parameters(self): @@ -226,7 +220,9 @@ def setup_overcloud_keystone_crederntials(): OvercloudKeystoneCredentialsFixture) -def get_overcloud_nodes_dataframe(oc_node_df_function): +def get_overcloud_nodes_dataframe( + oc_node_df_function: typing.Callable[[ssh.SSHClientType], + typing.Any]): """ :param oc_node_df_function : a function that queries a oc node using a cli command and returns a datraframe with an added @@ -238,10 +234,10 @@ def get_overcloud_nodes_dataframe(oc_node_df_function): :return: dataframe of all overcloud nodes processes """ import pandas - oc_nodes_selection = list_overcloud_nodes() - oc_nodes_names = [node.name for node in oc_nodes_selection] - oc_nodes_dfs = [oc_node_df_function(node_name) for - node_name in oc_nodes_names] + oc_nodes_dfs = list() + for instance in list_overcloud_nodes(): + ssh_client = overcloud_ssh_client(instance=instance) + oc_nodes_dfs.append(oc_node_df_function(ssh_client)) oc_procs_df = pandas.concat(oc_nodes_dfs, ignore_index=True) return oc_procs_df diff --git a/tobiko/tripleo/_topology.py b/tobiko/tripleo/_topology.py index a146b129c..9239c988b 100644 --- a/tobiko/tripleo/_topology.py +++ b/tobiko/tripleo/_topology.py @@ -14,8 +14,9 @@ from __future__ import absolute_import import re -import typing # noqa +import typing +import metalsmith from oslo_log import log from tobiko.openstack import neutron @@ -89,22 +90,22 @@ class TripleoTopology(topology.OpenStackTopology): def discover_overcloud_nodes(self): if _overcloud.has_overcloud(): - for server in _overcloud.list_overcloud_nodes(): + for instance in _overcloud.list_overcloud_nodes(): try: - _overcloud.power_on_overcloud_node(server) + _overcloud.power_on_overcloud_node(instance) except Exception: LOG.exception("Error ensuring overcloud node power " "status is on") - host_config = _overcloud.overcloud_host_config(server=server) + host_config = _overcloud.overcloud_host_config( + instance=instance) ssh_client = _overcloud.overcloud_ssh_client( - hostname=server.name, + instance=instance, host_config=host_config) node = self.add_node(address=host_config.hostname, - hostname=server.name, group='overcloud', ssh_client=ssh_client) assert isinstance(node, TripleoTopologyNode) - node.overcloud_server = server + node.overcloud_instance = instance self.discover_overcloud_node_subgroups(node) def discover_overcloud_node_subgroups(self, node): @@ -137,68 +138,68 @@ class TripleoTopology(topology.OpenStackTopology): class TripleoTopologyNode(topology.OpenStackTopologyNode): - overcloud_server: typing.Optional[nova.NovaServer] = None + overcloud_instance: typing.Optional[metalsmith.Instance] = None l3_agent_conf_path = ( '/var/lib/config-data/neutron/etc/neutron/l3_agent.ini') - def reboot_overcloud_node(self, reactivate_servers=True): + def reboot_overcloud_node(self, + reactivate_servers=True): """Reboot overcloud node - This method reboots an overcloud node and may start every server which - changed its provisioning state to SHUTOFF because of that operation. + This method reboots an overcloud node and may start every Nova + server which is not in SHUTOFF status before restarting. - :param start_servers (bool): whether or not to start the servers which - are hosted on the node after the reboot + :param reactivate_servers: whether or not to re-start the servers which + are hosted on the compute node after the reboot """ + running_servers: typing.List[nova.NovaServer] = [] if reactivate_servers: - servers_to_restart = self.get_running_servers() + running_servers = self.list_running_servers() + LOG.debug(f'Servers to restart after reboot: {running_servers}') self.power_off_overcloud_node() self.power_on_overcloud_node() - if reactivate_servers: - for server in servers_to_restart: + if running_servers: + LOG.info(f'Restart servers after rebooting overcloud compute node ' + f'{self.name}...') + for server in running_servers: nova.wait_for_server_status(server=server.id, status='SHUTOFF') - LOG.debug(f'Server {server.name} with ID {server.id} ' - f'had a SHUTOFF status before being ' - f'restarted') - nova.activate_server(server) - LOG.debug(f'Server {server.name} with ID {server.id} ' - f'has a {server.status} status after being ' - f'restarted') + LOG.debug(f'Re-activate server {server.name} with ID ' + f'{server.id}') + nova.activate_server(server=server) + LOG.debug(f'Server {server.name} with ID {server.id} has ' + f'been reactivated') - def get_running_servers(self): - servers_to_reactivate = list() + def list_running_servers(self) -> typing.List[nova.NovaServer]: + running_servers = list() for server in nova.list_servers(): - server_hyp = getattr(server, - 'OS-EXT-SRV-ATTR:' - 'hypervisor_hostname').split('.', 1)[0] - if self.name == server_hyp and server.status != 'SHUTOFF': - servers_to_reactivate.append(server) - LOG.info(f'Servers to restart after reboot: {servers_to_reactivate}') - return servers_to_reactivate + if server.status != 'SHUTOFF': + hypervisor_name = nova.get_server_hypervisor(server, + short=True) + if self.name == hypervisor_name: + running_servers.append(server) + return running_servers def power_on_overcloud_node(self): - server = self.overcloud_server - if server is None: + if self.overcloud_instance is None: raise TypeError(f"Node {self.name} is not and Overcloud server") self.ssh_client.close() LOG.debug(f"Ensuring overcloud node {self.name} power is on...") - _overcloud.power_on_overcloud_node(server) + _overcloud.power_on_overcloud_node(instance=self.overcloud_instance) hostname = sh.get_hostname(ssh_client=self.ssh_client) LOG.debug(f"Overcloud node {self.name} power is on (" f"hostname={hostname})") def power_off_overcloud_node(self): - server = self.overcloud_server - if server is None: + if self.overcloud_instance is None: raise TypeError(f"Node {self.name} is not and Overcloud server") self.ssh_client.close() LOG.debug(f"Ensuring overcloud node {self.name} power is off...") - _overcloud.power_off_overcloud_node(server) + _overcloud.power_off_overcloud_node(instance=self.overcloud_instance) LOG.debug(f"Overcloud server node {self.name} power is off.") diff --git a/tobiko/tripleo/processes.py b/tobiko/tripleo/processes.py index e96282638..01d39143d 100644 --- a/tobiko/tripleo/processes.py +++ b/tobiko/tripleo/processes.py @@ -12,6 +12,7 @@ from tobiko.openstack import neutron from tobiko.openstack import topology from tobiko.tripleo import overcloud from tobiko.shell import sh +from tobiko.shell import ssh LOG = log.getLogger(__name__) @@ -22,7 +23,7 @@ class OvercloudProcessesException(tobiko.TobikoException): "{process_error}" -def get_overcloud_node_processes_table(hostname): +def get_overcloud_node_processes_table(ssh_client: ssh.SSHClientType): """ get processes tables from overcloud node @@ -47,7 +48,6 @@ root | 11| 2| 0.0| 0|00:00:05|migration/0 |[migration/0] :return: dataframe of overcloud node processes dataframe """ - ssh_client = overcloud.overcloud_ssh_client(hostname) output = sh.execute( "ps -axw -o \"%U\" -o \"DELIM%p\" -o \"DELIM%P\" -o \"DELIM%C\" -o " "\"DELIM%z\" -o \"DELIM%x\" -o \"DELIM%c\" -o \"DELIM%a\" |grep -v " @@ -60,6 +60,7 @@ root | 11| 2| 0.0| 0|00:00:05|migration/0 |[migration/0] table.columns = ['USER', 'PID', 'PPID', 'CPU', 'VSZ', 'TIME', 'PROCESS', 'PROCESS_ARGS'] # pylint: disable=unsupported-assignment-operation + hostname = sh.get_hostname(ssh_client=ssh_client) table['overcloud_node'] = hostname LOG.debug("Successfully got overcloud nodes processes status table") diff --git a/tobiko/tripleo/services.py b/tobiko/tripleo/services.py index 39ec86722..271861416 100644 --- a/tobiko/tripleo/services.py +++ b/tobiko/tripleo/services.py @@ -8,6 +8,7 @@ import pandas import tobiko from tobiko.tripleo import overcloud from tobiko.shell import sh +from tobiko.shell import ssh LOG = log.getLogger(__name__) @@ -17,7 +18,7 @@ class OvercloudServiceException(tobiko.TobikoException): message = "not all overcloud nodes services are in active state" -def get_overcloud_node_services_table(hostname): +def get_overcloud_node_services_table(ssh_client: ssh.SSHClientType): """ get services table from overcloud node @@ -35,7 +36,6 @@ cloud-init.service|loaded|active|exited|Initialcloud-initjob(metadataservicecr) :return: dataframe of overcloud node services """ - ssh_client = overcloud.overcloud_ssh_client(hostname) units = sh.list_systemd_units(all=True, ssh_client=ssh_client).without_attributes( load='not-found') @@ -53,7 +53,7 @@ cloud-init.service|loaded|active|exited|Initialcloud-initjob(metadataservicecr) data['UNIT_DESCRIPTION'].append(unit.description) table = pandas.DataFrame.from_dict(data) table.replace(to_replace=' ', value="", regex=True, inplace=True) - table['overcloud_node'] = hostname + table['overcloud_node'] = sh.get_hostname(ssh_client=ssh_client) LOG.debug("Got overcloud nodes services status :\n%s", table) return table