Merge "Detect DevStack on remote nodes over SSH"

whitebox_neutron_tempest_plugin/common/utils.py

@@ -54,20 +54,25 @@ def is_regex_in_file(regex, filename, re_flags=0, fail_no_file=True):
     return result
 
 
-def is_local_service(service_name):
-    """Returns if local service exists"""
-    return bool(
-        shell.execute(
-            "systemctl list-unit-files --type service | "
-            f"grep '{service_name}'",
-            check=False
-        ).stdout.strip())
+def is_service(service_name, ssh_client=None):
+    """Return whether a systemd service unit exists.
+
+    Runs the unit-file lookup locally by default, or over SSH when an
+    ssh_client (paramiko-based) is provided. The remote variant is useful
+    when Tempest is not co-located with the deployment (e.g. it runs on a
+    separate node and reaches the control plane over SSH).
+    """
+    cmd = ("systemctl list-unit-files --type service | "
+           f"grep '{service_name}'")
+    if ssh_client is not None:
+        return bool(ssh_client.exec_command(cmd + "; true").strip())
+    return bool(shell.execute(cmd, check=False).stdout.strip())
 
 
 def is_devstack_wsgi():
     """Returns if wsgi according to neutron api service name when used"""
     # TODO(rxiao): remove this check when/if API check exists
-    return is_local_service('devstack@neutron-api')
+    return is_service('devstack@neutron-api')
 
 
 def _parse_config_content(content):

whitebox_neutron_tempest_plugin/tests/scenario/base.py

@@ -59,13 +59,84 @@ ConfigOption = collections.namedtuple(
 
 class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase):
     credentials = ['primary', 'admin']
+    # Initial DevStack detection, evaluated locally at class definition time.
+    # This is authoritative when Tempest is co-located with the deployment.
+    # When Tempest runs on a separate node (configured_hosts set and no proxy
+    # host), these values are re-checked over SSH in
+    # _detect_any_node_devstack() during resource_setup().
     if 'is_devstack' not in locals():
-        is_devstack = local_utils.is_local_service("devstack@")
+        is_devstack = local_utils.is_service("devstack@")
         is_devstack_wsgi = is_devstack and local_utils.is_devstack_wsgi()
+    # SSH client that run_on_master_controller uses to reach the control
+    # plane when Tempest is not co-located with a DevStack deployment
+    # (configured_hosts set and no proxy host, e.g. the EVPN spine). It is
+    # populated during discover_nodes(). When None, run_on_master_controller
+    # runs commands locally (co-located DevStack) or via the proxy host
+    # (podified).
+    master_controller_client = None
+
+    @classmethod
+    def _detect_any_node_devstack(cls):
+        """Refine DevStack detection by probing a remote node when needed.
+
+        The class-body check runs ``is_service("devstack@")`` on the
+        machine executing Tempest, which is authoritative when Tempest is
+        co-located with the deployment (DevStack all-in-one) or when it runs
+        on a podified Tempest pod (no local ``devstack@`` units).
+
+        It is wrong only on a remote-Tempest DevStack topology, e.g. the
+        EVPN spine: a DevStack multinode setup where Tempest runs off-box
+        and reaches the nodes only over SSH. Such setups provide
+        ``configured_hosts`` and have no proxy host (``proxy_host_address``
+        is podified-only), so in that case probe the configured hosts over
+        SSH instead.
+        """
+        # Skip the remote probe unless this is a remote-Tempest DevStack
+        # topology: already DevStack (co-located), no hosts to probe, or a
+        # proxy host is set (podified).
+        if (cls.is_devstack or
+                not WB_CONF.configured_hosts or
+                WB_CONF.proxy_host_address):
+            return
+
+        # Tempest runs on a separate DevStack node, so assume DevStack while
+        # building the node SSH clients (process_nodes_from_config selects
+        # keys based on is_devstack and honors per-host key/user overrides),
+        # then confirm by probing the nodes. Any DevStack node (controller or
+        # compute) exposes some "devstack@" systemd unit, and we do not yet
+        # know each node's role, so probe them all and treat the deployment
+        # as DevStack if any node matches.
+        cls.is_devstack = True
+        clients = [node['client']
+                   for node in cls.process_nodes_from_config()]
+        cls.is_devstack = cls._any_node_has_service(clients, "devstack@")
+        cls.is_devstack_wsgi = cls.is_devstack and cls._any_node_has_service(
+            clients, "devstack@neutron-api")
+        LOG.info("Remote DevStack detection over %d configured host(s): "
+                 "is_devstack=%s", len(clients), cls.is_devstack)
+
+    @staticmethod
+    def _any_node_has_service(clients, service_name):
+        """Return True if any node exposes the systemd service.
+
+        SSH/command failures against a node are logged and treated as
+        "service not found" so that detection never raises.
+        """
+        for client in clients:
+            try:
+                if local_utils.is_service(service_name, client):
+                    return True
+            except Exception:
+                LOG.warning(
+                    "Failed to probe node %s for service '%s'; treating it "
+                    "as absent", getattr(client, 'host', '?'), service_name,
+                    exc_info=True)
+        return False
 
     @classmethod
     def resource_setup(cls):
         super(BaseTempestWhiteboxTestCase, cls).resource_setup()
+        cls._detect_any_node_devstack()
         uri = (
             CONF.identity.uri_v3 if CONF.identity.uri_v3
             else CONF.identity.uri)
@@ -155,10 +226,16 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase):
     def run_on_master_controller(cls, cmd, check=False, timeout=10):
         LOG.debug("Command: %s", cmd)
         if cls.is_devstack:
-            output, errors = local_utils.run_local_cmd(
-                cmd, timeout=timeout, check=check)
-            LOG.debug("Stderr: %s", errors.decode())
-            output = output.decode()
+            if cls.master_controller_client:
+                # DevStack without co-location (e.g. the EVPN spine): the
+                # control plane is reachable only over SSH.
+                output = cls.master_controller_client.exec_command(cmd)
+            else:
+                # Tempest is co-located with the DevStack controller.
+                output, errors = local_utils.run_local_cmd(
+                    cmd, timeout=timeout, check=check)
+                LOG.debug("Stderr: %s", errors.decode())
+                output = output.decode()
         else:
             output = cls.proxy_host_client.exec_command(cmd)
         LOG.debug("Output: %s", output)
@@ -336,20 +413,14 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase):
         return nodes
 
     @classmethod
-    def process_nodes_from_config(cls):
-        """Process configured_hosts oslo configuration, create nodes_data.
+    def _load_configured_hosts(cls):
+        """Parse the configured_hosts option into a list of host entries.
 
-        Parses configured_hosts, applies defaults, and creates SSH clients.
-
-        Returns list of dicts with: ip, user, key_file, hostname (optional),
-        ssh client.
+        Each entry is either an IP string or a dict with at least an 'ip'
+        key. Raises ValueError if the option is not a valid list literal.
         """
-        nodes_data = []
-
-        # Load safely nested objects definition for hosts from string
         try:
-            loaded_hosts_list = ast.literal_eval(
-                WB_CONF.configured_hosts)
+            loaded_hosts_list = ast.literal_eval(WB_CONF.configured_hosts)
         except (ValueError, SyntaxError) as e:
             LOG.exception("Failed to parse configured_hosts")
             raise ValueError(
@@ -360,6 +431,20 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase):
             raise ValueError(
                 f"configured_hosts must be a list or tuple, "
                 f"got {type(loaded_hosts_list).__name__}")
+        return loaded_hosts_list
+
+    @classmethod
+    def process_nodes_from_config(cls):
+        """Process configured_hosts oslo configuration, create nodes_data.
+
+        Parses configured_hosts, applies defaults, and creates SSH clients.
+
+        Returns list of dicts with: ip, user, key_file, hostname (optional),
+        ssh client.
+        """
+        nodes_data = []
+
+        loaded_hosts_list = cls._load_configured_hosts()
 
         # Log discovery method
         LOG.info("Discovering nodes from configured_hosts configuration "
@@ -593,6 +678,21 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase):
         LOG.info(info_msg, *node_counts)
         LOG.debug(pprint.pformat(cls.nodes))
 
+        # Route run_on_master_controller through the discovered controller
+        # over SSH on DevStack. This is required when Tempest is not
+        # co-located with the control plane (e.g. the EVPN spine) and is
+        # harmless otherwise, since discover_nodes already reached these
+        # nodes over SSH.
+        if cls.is_devstack:
+            for node in cls.nodes:
+                if node['is_controller']:
+                    cls.master_controller_client = node['client']
+                    break
+            else:
+                LOG.warning(
+                    "No controller discovered; run_on_master_controller "
+                    "falls back to local execution")
+
     @classmethod
     def get_standalone_networkers(cls):
         _networkers = [node['name'] for node in cls.nodes