Use Metalsmith client to discover overcloud nodes

In OSP-17 and newer, there is no Nova running as undercloud
service and instead of Nova client, Metalsmith client has to be
used to e.g. get overcloud nodes and its IP addresses.

This patch replaces Nova client with Metalsmith client to list
overcloud servers and it IP addresses

Change-Id: I0a7859ebc4e1c6d3660040100e1bf55a7218e4f2

tobiko/tests/functional/tripleo/test_overcloud.py

@@ -20,7 +20,7 @@ import pandas as pd
 import testtools
 
 from tobiko import config
-from tobiko.openstack import nova
+from tobiko.openstack import metalsmith
 from tobiko import tripleo
 from tobiko.tripleo import pacemaker
 from tobiko.tripleo import services
@@ -31,7 +31,7 @@ CONF = config.CONF
 
 
 @tripleo.skip_if_missing_overcloud
-class OvercloudSshConnectionTest(testtools.TestCase):
+class OvercloudKeystoneCredentialsTest(testtools.TestCase):
 
     def test_fetch_overcloud_credentials(self):
         env = tripleo.load_overcloud_rcfile()
@@ -45,20 +45,20 @@ class OvercloudSshConnectionTest(testtools.TestCase):
 
 
 @tripleo.skip_if_missing_overcloud
-class OvercloudNovaApiTest(testtools.TestCase):
+class OvercloudMetalsmithApiTest(testtools.TestCase):
 
     def test_list_overcloud_nodes(self):
         nodes = tripleo.list_overcloud_nodes()
         self.assertTrue(nodes)
         for node in nodes:
-            node_ip = nova.find_server_ip_address(server=node,
+            node_ip = metalsmith.find_instance_ip_address(
-                                                  check_connectivity=True)
+                instance=node, check_connectivity=True)
             self.assertIsInstance(node_ip, netaddr.IPAddress)
 
     def test_find_overcloud_nodes(self):
         node = tripleo.find_overcloud_node()
-        node_ip = nova.find_server_ip_address(server=node,
+        node_ip = metalsmith.find_instance_ip_address(instance=node,
-                                              check_connectivity=True)
+                                                      check_connectivity=True)
         self.assertIsInstance(node_ip, netaddr.IPAddress)
 
     def test_get_overcloud_node_ip_address(self):
@@ -66,10 +66,13 @@ class OvercloudNovaApiTest(testtools.TestCase):
         self.assertIsInstance(overcloud_node_ip, netaddr.IPAddress)
 
     def test_overcloud_host_config(self):
-        hostname = tripleo.find_overcloud_node().name
+        instance = tripleo.find_overcloud_node()
         host_config = tobiko.setup_fixture(
-            tripleo.overcloud_host_config(hostname=hostname))
+            tripleo.overcloud_host_config(instance=instance))
-        self.assertEqual(hostname, host_config.host)
+        instance_ips = set()
+        for ips in instance.ip_addresses().values():
+            instance_ips.update(ips)
+        self.assertIn(host_config.host, instance_ips)
         self.assertIsInstance(host_config.hostname, str)
         netaddr.IPAddress(host_config.hostname)
         self.assertEqual(CONF.tobiko.tripleo.overcloud_ssh_port,
@@ -83,8 +86,8 @@ class OvercloudNovaApiTest(testtools.TestCase):
         self.assertTrue(os.path.isfile(key_filename + '.pub'))
 
     def test_overcloud_ssh_client_connection(self):
-        hostname = tripleo.find_overcloud_node().name
+        instance = tripleo.find_overcloud_node()
-        ssh_client = tripleo.overcloud_ssh_client(hostname=hostname)
+        ssh_client = tripleo.overcloud_ssh_client(instance=instance)
         ssh_client.connect()
 
 

tobiko/tests/functional/tripleo/test_topology.py

@@ -39,8 +39,8 @@ class TripleoTopologyTest(test_topology.OpenStackTopologyTest):
 
     @tripleo.skip_if_missing_overcloud
     def test_overcloud_group(self):
-        for server in tripleo.list_overcloud_nodes():
+        for instance in tripleo.list_overcloud_nodes():
-            ssh_client = tripleo.overcloud_ssh_client(server.name)
+            ssh_client = tripleo.overcloud_ssh_client(instance=instance)
             name = sh.get_hostname(ssh_client=ssh_client).split('.')[0]
             node = self.topology.get_node(name)
             self.assertIs(node.ssh_client, ssh_client)
@@ -53,6 +53,6 @@ class TripleoTopologyTest(test_topology.OpenStackTopologyTest):
                 nodes = self.topology.get_group(group)
                 self.assertIn(node, nodes)
                 self.assertIn(group, node.groups)
-            host_config = tripleo.overcloud_host_config(name)
+            host_config = tripleo.overcloud_host_config(instance=instance)
             self.assertEqual(host_config.hostname,
                              str(node.public_ip))

tobiko/tripleo/_overcloud.py

@@ -15,6 +15,7 @@ from __future__ import absolute_import
 
 import io
 import os
+import typing
 
 from oslo_log import log
 
@@ -22,7 +23,7 @@ import tobiko
 from tobiko import config
 from tobiko.openstack import keystone
 from tobiko.openstack import ironic
-from tobiko.openstack import nova
+from tobiko.openstack import metalsmith
 from tobiko.openstack import topology
 from tobiko.shell import sh
 from tobiko.shell import ssh
@@ -58,86 +59,74 @@ class OvercloudKeystoneCredentialsFixture(
 
 def list_overcloud_nodes(**params):
     session = _undercloud.undercloud_keystone_session()
-    client = nova.get_nova_client(session=session)
+    client = metalsmith.get_metalsmith_client(session=session)
-    return nova.list_servers(client=client, **params)
+    return metalsmith.list_instances(client=client, **params)
 
 
 def find_overcloud_node(**params):
     session = _undercloud.undercloud_keystone_session()
-    client = nova.get_nova_client(session=session)
+    client = metalsmith.get_metalsmith_client(session=session)
-    return nova.find_server(client=client, **params)
+    return metalsmith.find_instance(client=client, **params)
 
 
-def power_on_overcloud_node(server: nova.ServerType,
+def power_on_overcloud_node(instance: metalsmith.MetalsmithInstance,
                             timeout: tobiko.Seconds = 120.,
                             sleep_time: tobiko.Seconds = 5.):
     session = _undercloud.undercloud_keystone_session()
-    node = getattr(server, 'OS-EXT-SRV-ATTR:hypervisor_hostname',
+    client = ironic.get_ironic_client(session=session)
-                   None)
+    ironic.power_on_node(client=client,
-    if node is None:
+                         node=instance.uuid,
-        client = nova.get_nova_client(session=session)
+                         timeout=timeout,
-        nova.activate_server(client=client,
+                         sleep_time=sleep_time)
-                             server=server,
-                             timeout=timeout,
-                             sleep_time=sleep_time)
-    else:
-        client = ironic.get_ironic_client(session=session)
-        ironic.power_on_node(client=client,
-                             node=node,
-                             timeout=timeout,
-                             sleep_time=sleep_time)
 
 
-def power_off_overcloud_node(server: nova.ServerType,
+def power_off_overcloud_node(instance: metalsmith.MetalsmithInstance,
                              timeout: tobiko.Seconds = None,
                              sleep_time: tobiko.Seconds = None):
     session = _undercloud.undercloud_keystone_session()
-    node = getattr(server, 'OS-EXT-SRV-ATTR:hypervisor_hostname',
+    client = ironic.get_ironic_client(session=session)
-                   None)
+    ironic.power_off_node(client=client,
-    if node is None:
+                          node=instance.uuid,
-        client = nova.get_nova_client(session=session)
+                          timeout=timeout,
-        nova.shutoff_server(client=client,
+                          sleep_time=sleep_time)
-                            server=server,
-                            timeout=timeout,
-                            sleep_time=sleep_time)
-    else:
-        client = ironic.get_ironic_client(session=session)
-        ironic.power_off_node(client=client,
-                              node=node,
-                              timeout=timeout,
-                              sleep_time=sleep_time)
 
 
-def overcloud_ssh_client(hostname=None, ip_version=None, network_name=None,
+def overcloud_ssh_client(ip_version: int = None,
-                         server=None, host_config=None):
+                         network_name: str = None,
+                         instance: metalsmith.MetalsmithInstance = None,
+                         host_config=None):
     if host_config is None:
-        host_config = overcloud_host_config(hostname=hostname,
+        host_config = overcloud_host_config(ip_version=ip_version,
-                                            ip_version=ip_version,
                                             network_name=network_name,
-                                            server=server)
+                                            instance=instance)
-    return ssh.ssh_client(host=hostname, **host_config.connect_parameters)
+    tobiko.check_valid_type(host_config.host, str)
+    return ssh.ssh_client(host=host_config.host,
+                          **host_config.connect_parameters)
 
 
-def overcloud_host_config(hostname=None, ip_version=None, network_name=None,
+def overcloud_host_config(ip_version: int = None,
-                          server=None):
+                          network_name: str = None,
-    host_config = OvercloudHostConfig(host=hostname,
+                          instance: metalsmith.MetalsmithInstance = None):
-                                      ip_version=ip_version,
+    host_config = OvercloudHostConfig(ip_version=ip_version,
                                       network_name=network_name,
-                                      server=server)
+                                      instance=instance)
     return tobiko.setup_fixture(host_config)
 
 
-def overcloud_node_ip_address(ip_version=None, network_name=None, server=None,
+def overcloud_node_ip_address(ip_version: int = None,
+                              network_name: str = None,
+                              instance: metalsmith.MetalsmithInstance = None,
                               **params):
-    server = server or find_overcloud_node(**params)
+    if instance is None:
+        instance = find_overcloud_node(**params)
     ip_version = ip_version or CONF.tobiko.tripleo.overcloud_ip_version
     network_name = network_name or CONF.tobiko.tripleo.overcloud_network_name
-    address = nova.find_server_ip_address(server=server,
+    address = metalsmith.find_instance_ip_address(instance=instance,
-                                          ip_version=ip_version,
+                                                  ip_version=ip_version,
-                                          network_name=network_name)
+                                                  network_name=network_name)
     LOG.debug(f"Got Overcloud node address '{address}' from Undercloud "
               f"(ip_version={ip_version}, network_name={network_name}, "
-              f"server={server})")
+              f"instance={instance})")
     return address
 
 
@@ -176,41 +165,46 @@ def _get_undercloud_file(ssh_client, source, destination, mode):
 
 
 class OvercloudHostConfig(tobiko.SharedFixture):
-    host = None
-    hostname = None
-    port = None
-    username = None
-    key_file = tobiko.required_fixture(OvercloudSshKeyFileFixture)
-    ip_version = None
-    network_name = None
-    key_filename = None
-    server = None
 
-    def __init__(self, host=None, ip_version=None, network_name=None,
+    key_file = tobiko.required_fixture(OvercloudSshKeyFileFixture)
-                 server=None, **kwargs):
+
+    def __init__(self,
+                 host: str = None,
+                 hostname: str = None,
+                 ip_version: int = None,
+                 instance: metalsmith.MetalsmithInstance = None,
+                 key_filename: str = None,
+                 network_name: str = None,
+                 port: int = None,
+                 username: str = None,
+                 **kwargs):
         super(OvercloudHostConfig, self).__init__()
-        if host:
+        self.host = host
-            self.host = host
+        self.instance = instance
-        if ip_version:
+        self.ip_version = ip_version
-            self.ip_version = ip_version
+        self.key_filename = key_filename
-        if network_name:
+        self.host = host
-            self.network_name = network_name
+        self.hostname = hostname
-        if server:
+        self.network_name = network_name
-            self.server = server
+        self.port = port
-            if self.host is None:
+        self.username = username
-                self.host = server.name
-        tobiko.check_valid_type(self.host, str)
         self._connect_parameters = ssh.gather_ssh_connect_parameters(**kwargs)
 
     def setup_fixture(self):
-        self.hostname = str(overcloud_node_ip_address(
+        if self.hostname is None:
-            name=self.host, ip_version=self.ip_version,
+            self.hostname = str(overcloud_node_ip_address(
-            network_name=self.network_name,
+                hostname=self.host,
-            server=self.server))
+                ip_version=self.ip_version,
-        self.port = self.port or CONF.tobiko.tripleo.overcloud_ssh_port
+                network_name=self.network_name,
-        self.username = (self.username or
+                instance=self.instance))
-                         CONF.tobiko.tripleo.overcloud_ssh_username)
+        if self.host is None:
-        self.key_filename = self.key_filename or self.key_file.key_filename
+            self.host = self.hostname
+        if self.port is None:
+            self.port = CONF.tobiko.tripleo.overcloud_ssh_port
+        if self.username is None:
+            self.username = CONF.tobiko.tripleo.overcloud_ssh_username
+        if self.key_filename is None:
+            self.key_filename = self.key_file.key_filename
 
     @property
     def connect_parameters(self):
@@ -226,7 +220,9 @@ def setup_overcloud_keystone_crederntials():
         OvercloudKeystoneCredentialsFixture)
 
 
-def get_overcloud_nodes_dataframe(oc_node_df_function):
+def get_overcloud_nodes_dataframe(
+        oc_node_df_function: typing.Callable[[ssh.SSHClientType],
+                                             typing.Any]):
     """
      :param oc_node_df_function : a function that queries a oc node
      using a cli command and returns a datraframe with an added
@@ -238,10 +234,10 @@ def get_overcloud_nodes_dataframe(oc_node_df_function):
     :return: dataframe of all overcloud nodes processes
     """
     import pandas
-    oc_nodes_selection = list_overcloud_nodes()
+    oc_nodes_dfs = list()
-    oc_nodes_names = [node.name for node in oc_nodes_selection]
+    for instance in list_overcloud_nodes():
-    oc_nodes_dfs = [oc_node_df_function(node_name) for
+        ssh_client = overcloud_ssh_client(instance=instance)
-                    node_name in oc_nodes_names]
+        oc_nodes_dfs.append(oc_node_df_function(ssh_client))
     oc_procs_df = pandas.concat(oc_nodes_dfs, ignore_index=True)
     return oc_procs_df
 

tobiko/tripleo/_topology.py

@@ -14,8 +14,9 @@
 from __future__ import absolute_import
 
 import re
-import typing  # noqa
+import typing
 
+import metalsmith
 from oslo_log import log
 
 from tobiko.openstack import neutron
@@ -89,22 +90,22 @@ class TripleoTopology(topology.OpenStackTopology):
 
     def discover_overcloud_nodes(self):
         if _overcloud.has_overcloud():
-            for server in _overcloud.list_overcloud_nodes():
+            for instance in _overcloud.list_overcloud_nodes():
                 try:
-                    _overcloud.power_on_overcloud_node(server)
+                    _overcloud.power_on_overcloud_node(instance)
                 except Exception:
                     LOG.exception("Error ensuring overcloud node power "
                                   "status is on")
-                host_config = _overcloud.overcloud_host_config(server=server)
+                host_config = _overcloud.overcloud_host_config(
+                    instance=instance)
                 ssh_client = _overcloud.overcloud_ssh_client(
-                    hostname=server.name,
+                    instance=instance,
                     host_config=host_config)
                 node = self.add_node(address=host_config.hostname,
-                                     hostname=server.name,
                                      group='overcloud',
                                      ssh_client=ssh_client)
                 assert isinstance(node, TripleoTopologyNode)
-                node.overcloud_server = server
+                node.overcloud_instance = instance
                 self.discover_overcloud_node_subgroups(node)
 
     def discover_overcloud_node_subgroups(self, node):
@@ -137,68 +138,68 @@ class TripleoTopology(topology.OpenStackTopology):
 
 class TripleoTopologyNode(topology.OpenStackTopologyNode):
 
-    overcloud_server: typing.Optional[nova.NovaServer] = None
+    overcloud_instance: typing.Optional[metalsmith.Instance] = None
 
     l3_agent_conf_path = (
         '/var/lib/config-data/neutron/etc/neutron/l3_agent.ini')
 
-    def reboot_overcloud_node(self, reactivate_servers=True):
+    def reboot_overcloud_node(self,
+                              reactivate_servers=True):
         """Reboot overcloud node
 
-        This method reboots an overcloud node and may start every server which
+        This method reboots an overcloud node and may start every Nova
-        changed its provisioning state to SHUTOFF because of that operation.
+        server which is not in SHUTOFF status before restarting.
 
-        :param start_servers (bool): whether or not to start the servers which
+        :param reactivate_servers: whether or not to re-start the servers which
-            are hosted on the node after the reboot
+            are hosted on the compute node after the reboot
         """
 
+        running_servers: typing.List[nova.NovaServer] = []
         if reactivate_servers:
-            servers_to_restart = self.get_running_servers()
+            running_servers = self.list_running_servers()
+            LOG.debug(f'Servers to restart after reboot: {running_servers}')
 
         self.power_off_overcloud_node()
         self.power_on_overcloud_node()
 
-        if reactivate_servers:
+        if running_servers:
-            for server in servers_to_restart:
+            LOG.info(f'Restart servers after rebooting overcloud compute node '
+                     f'{self.name}...')
+            for server in running_servers:
                 nova.wait_for_server_status(server=server.id,
                                             status='SHUTOFF')
-                LOG.debug(f'Server {server.name} with ID {server.id} '
+                LOG.debug(f'Re-activate server {server.name} with ID '
-                          f'had a SHUTOFF status before being '
+                          f'{server.id}')
-                          f'restarted')
+                nova.activate_server(server=server)
-                nova.activate_server(server)
+                LOG.debug(f'Server {server.name} with ID {server.id} has '
-                LOG.debug(f'Server {server.name} with ID {server.id} '
+                          f'been reactivated')
-                          f'has a {server.status} status after being '
-                          f'restarted')
 
-    def get_running_servers(self):
+    def list_running_servers(self) -> typing.List[nova.NovaServer]:
-        servers_to_reactivate = list()
+        running_servers = list()
         for server in nova.list_servers():
-            server_hyp = getattr(server,
+            if server.status != 'SHUTOFF':
-                                 'OS-EXT-SRV-ATTR:'
+                hypervisor_name = nova.get_server_hypervisor(server,
-                                 'hypervisor_hostname').split('.', 1)[0]
+                                                             short=True)
-            if self.name == server_hyp and server.status != 'SHUTOFF':
+                if self.name == hypervisor_name:
-                servers_to_reactivate.append(server)
+                    running_servers.append(server)
-        LOG.info(f'Servers to restart after reboot: {servers_to_reactivate}')
+        return running_servers
-        return servers_to_reactivate
 
     def power_on_overcloud_node(self):
-        server = self.overcloud_server
+        if self.overcloud_instance is None:
-        if server is None:
             raise TypeError(f"Node {self.name} is not and Overcloud server")
         self.ssh_client.close()
         LOG.debug(f"Ensuring overcloud node {self.name} power is on...")
-        _overcloud.power_on_overcloud_node(server)
+        _overcloud.power_on_overcloud_node(instance=self.overcloud_instance)
         hostname = sh.get_hostname(ssh_client=self.ssh_client)
         LOG.debug(f"Overcloud node {self.name} power is on ("
                   f"hostname={hostname})")
 
     def power_off_overcloud_node(self):
-        server = self.overcloud_server
+        if self.overcloud_instance is None:
-        if server is None:
             raise TypeError(f"Node {self.name} is not and Overcloud server")
         self.ssh_client.close()
         LOG.debug(f"Ensuring overcloud node {self.name} power is off...")
-        _overcloud.power_off_overcloud_node(server)
+        _overcloud.power_off_overcloud_node(instance=self.overcloud_instance)
         LOG.debug(f"Overcloud server node {self.name} power is off.")
 
 

tobiko/tripleo/processes.py

@@ -12,6 +12,7 @@ from tobiko.openstack import neutron
 from tobiko.openstack import topology
 from tobiko.tripleo import overcloud
 from tobiko.shell import sh
+from tobiko.shell import ssh
 
 
 LOG = log.getLogger(__name__)
@@ -22,7 +23,7 @@ class OvercloudProcessesException(tobiko.TobikoException):
               "{process_error}"
 
 
-def get_overcloud_node_processes_table(hostname):
+def get_overcloud_node_processes_table(ssh_client: ssh.SSHClientType):
     """
     get processes tables from overcloud node
 
@@ -47,7 +48,6 @@ root    |     11|      2| 0.0|     0|00:00:05|migration/0    |[migration/0]
     :return: dataframe of overcloud node processes dataframe
     """
 
-    ssh_client = overcloud.overcloud_ssh_client(hostname)
     output = sh.execute(
         "ps -axw -o \"%U\" -o \"DELIM%p\" -o \"DELIM%P\" -o \"DELIM%C\" -o "
         "\"DELIM%z\" -o \"DELIM%x\" -o \"DELIM%c\" -o \"DELIM%a\" |grep -v "
@@ -60,6 +60,7 @@ root    |     11|      2| 0.0|     0|00:00:05|migration/0    |[migration/0]
     table.columns = ['USER', 'PID', 'PPID', 'CPU', 'VSZ', 'TIME', 'PROCESS',
                      'PROCESS_ARGS']
     # pylint: disable=unsupported-assignment-operation
+    hostname = sh.get_hostname(ssh_client=ssh_client)
     table['overcloud_node'] = hostname
 
     LOG.debug("Successfully got overcloud nodes processes status table")

tobiko/tripleo/services.py

@@ -8,6 +8,7 @@ import pandas
 import tobiko
 from tobiko.tripleo import overcloud
 from tobiko.shell import sh
+from tobiko.shell import ssh
 
 
 LOG = log.getLogger(__name__)
@@ -17,7 +18,7 @@ class OvercloudServiceException(tobiko.TobikoException):
     message = "not all overcloud nodes services are in active state"
 
 
-def get_overcloud_node_services_table(hostname):
+def get_overcloud_node_services_table(ssh_client: ssh.SSHClientType):
     """
     get services table from overcloud node
 
@@ -35,7 +36,6 @@ cloud-init.service|loaded|active|exited|Initialcloud-initjob(metadataservicecr)
 
     :return: dataframe of overcloud node services
     """
-    ssh_client = overcloud.overcloud_ssh_client(hostname)
     units = sh.list_systemd_units(all=True,
                                   ssh_client=ssh_client).without_attributes(
         load='not-found')
@@ -53,7 +53,7 @@ cloud-init.service|loaded|active|exited|Initialcloud-initjob(metadataservicecr)
         data['UNIT_DESCRIPTION'].append(unit.description)
     table = pandas.DataFrame.from_dict(data)
     table.replace(to_replace=' ', value="", regex=True, inplace=True)
-    table['overcloud_node'] = hostname
+    table['overcloud_node'] = sh.get_hostname(ssh_client=ssh_client)
 
     LOG.debug("Got overcloud nodes services status :\n%s", table)
     return table