Wait for all instance ports to become ACTIVE

get_server_port_id_and_ip4() gets a list of neutron ports
for an instance, but it could be one or more of those have
not completed provisioning at the time of the call, so are
still marked DOWN.

Wait for all ports to become active since it could just be
neutron has not completed its work yet.

Added new waiter function and tests to verify it worked.

Closes-bug: #2079831
Change-Id: I758e5eeb8ab05e79d6bdb2b560aa0f9f38c5992c

tempest/common/waiters.py

@@ -672,6 +672,28 @@ def wait_for_port_status(client, port_id, status):
     raise lib_exc.TimeoutException
 
 
+def wait_for_server_ports_active(client, server_id, is_active, **kwargs):
+    """Wait for all server ports to reach active status
+    :param client: The network client to use when querying the port's status
+    :param server_id: The uuid of the server's ports we need to verify.
+    :param is_active: A function to call to the check port active status.
+    :param kwargs: Additional arguments, if any, to pass to list_ports()
+    """
+    start_time = time.time()
+    while (time.time() - start_time <= client.build_timeout):
+        ports = client.list_ports(device_id=server_id, **kwargs)['ports']
+        if all(is_active(port) for port in ports):
+            LOG.debug("Server ID %s ports are all ACTIVE %s: ",
+                      server_id, ports)
+            return ports
+        LOG.warning("Server ID %s has ports that are not ACTIVE, waiting "
+                    "for state to change on all: %s", server_id, ports)
+        time.sleep(client.build_interval)
+    LOG.error("Server ID %s ports have failed to transition to ACTIVE, "
+              "timing out: %s", server_id, ports)
+    raise lib_exc.TimeoutException
+
+
 def wait_for_ssh(ssh_client, timeout=30):
     """Waits for SSH connection to become usable"""
     start_time = int(time.time())

tempest/scenario/manager.py

@@ -1098,8 +1098,6 @@ class ScenarioTest(tempest.test.BaseTestCase):
 
         if ip_addr and not kwargs.get('fixed_ips'):
             kwargs['fixed_ips'] = 'ip_address=%s' % ip_addr
-        ports = self.os_admin.ports_client.list_ports(
-            device_id=server['id'], **kwargs)['ports']
 
         # A port can have more than one IP address in some cases.
         # If the network is dual-stack (IPv4 + IPv6), this port is associated
@@ -1114,6 +1112,18 @@ class ScenarioTest(tempest.test.BaseTestCase):
             return (port['status'] == 'ACTIVE' or
                     port.get('binding:vnic_type') == 'baremetal')
 
+        # Wait for all compute ports to be ACTIVE.
+        # This will raise a TimeoutException if that does not happen.
+        client = self.os_admin.ports_client
+        try:
+            ports = waiters.wait_for_server_ports_active(
+                client=client, server_id=server['id'],
+                is_active=_is_active, **kwargs)
+        except lib_exc.TimeoutException:
+            LOG.error("Server ports failed transitioning to ACTIVE for "
+                      "server: %s", server)
+            raise
+
         port_map = [(p["id"], fxip["ip_address"])
                     for p in ports
                     for fxip in p["fixed_ips"]
@@ -1121,7 +1131,8 @@ class ScenarioTest(tempest.test.BaseTestCase):
                         _is_active(p))]
         inactive = [p for p in ports if p['status'] != 'ACTIVE']
         if inactive:
-            LOG.warning("Instance has ports that are not ACTIVE: %s", inactive)
+            # This should just be Ironic ports, see _is_active() above
+            LOG.debug("Instance has ports that are not ACTIVE: %s", inactive)
 
         self.assertNotEmpty(port_map,
                             "No IPv4 addresses found in: %s" % ports)

tempest/tests/common/test_waiters.py

@@ -884,6 +884,58 @@ class TestPortCreationWaiter(base.TestCase):
                           waiters.wait_for_port_status, mock_client,
                           fake_port_id, fake_status)
 
+    def test_wait_for_server_ports_active(self):
+        """Test that the waiter replies with the ports before the timeout"""
+
+        def is_active(port):
+            return port['status'] == 'ACTIVE'
+
+        def client_response(device_id):
+            """Mock client response, replies with partial status after one
+            call and final status after 2 calls
+            """
+            if mock_client.call_count >= 2:
+                return mock_ports_active
+            else:
+                mock_client.call_count += 1
+                if mock_client.call_count > 1:
+                    return mock_ports_half_active
+                return mock_ports
+
+        mock_ports = {'ports': [{'id': '1234', 'status': 'DOWN'},
+                                {'id': '5678', 'status': 'DOWN'}]}
+        mock_ports_half_active = {'ports': [{'id': '1234', 'status': 'ACTIVE'},
+                                            {'id': '5678', 'status': 'DOWN'}]}
+        mock_ports_active = {'ports': [{'id': '1234', 'status': 'ACTIVE'},
+                                       {'id': '5678', 'status': 'ACTIVE'}]}
+        mock_client = mock.Mock(
+            spec=ports_client.PortsClient,
+            build_timeout=30, build_interval=1,
+            list_ports=client_response)
+        fake_server_id = "9876"
+        self.assertEqual(mock_ports_active['ports'],
+                         waiters.wait_for_server_ports_active(
+                             mock_client, fake_server_id, is_active))
+
+    def test_wait_for_server_ports_active_timeout(self):
+        """Negative test - checking that a timeout
+        presented by a small 'fake_timeout' and a static status of
+        'DOWN' in the mock will raise a timeout exception
+        """
+
+        def is_active(port):
+            return port['status'] == 'ACTIVE'
+
+        mock_ports = {'ports': [{'id': '1234', 'status': "DOWN"}]}
+        mock_client = mock.Mock(
+            spec=ports_client.PortsClient,
+            build_timeout=2, build_interval=1,
+            list_ports=lambda device_id: mock_ports)
+        fake_server_id = "9876"
+        self.assertRaises(lib_exc.TimeoutException,
+                          waiters.wait_for_server_ports_active,
+                          mock_client, fake_server_id, is_active)
+
 
 class TestServerFloatingIPWaiters(base.TestCase):