From 8a959ead1edcd71a55ac0570db407040a36e29b9 Mon Sep 17 00:00:00 2001 From: Artom Lifshitz Date: Mon, 27 Sep 2021 12:09:12 -0400 Subject: [PATCH] Add floating IP waiter In bug 1923194, a floating IP is associated to a server. Upon performing the operation, Neutron sends Nova a network-vif-changed external event, causing Nova to update its network info cache. Until Nova does this, the new floating IP is not reflected in `server show`. Tempest's `server show` is racing with this process, causing intermittent failures when the new floating IP does not show up in `server show` in time. This patch adds a new waiter that waits for a floating IP to either appear to disappear in the `server show` output, and modifies two tests to use the new helper. Closes bug: 1923194 Change-Id: I0f7e1c9096dc1903903fb31c5e854f07800efbfd --- tempest/common/waiters.py | 42 ++++++++++++++++++++++++++ tempest/scenario/test_minimum_basic.py | 37 +++-------------------- tempest/tests/common/test_waiters.py | 34 +++++++++++++++++++++ 3 files changed, 81 insertions(+), 32 deletions(-) diff --git a/tempest/common/waiters.py b/tempest/common/waiters.py index f6a4555fe8..ae6eec7c75 100644 --- a/tempest/common/waiters.py +++ b/tempest/common/waiters.py @@ -525,3 +525,45 @@ def wait_for_guest_os_boot(client, server_id): server_id) return time.sleep(client.build_interval) + + +def wait_for_server_floating_ip(servers_client, server, floating_ip, + wait_for_disassociate=False): + """Wait for floating IP association or disassociation. + + :param servers_client: The servers client to use when querying the server's + floating IPs. + :param server: The server JSON dict on which to wait. + :param floating_ip: The floating IP JSON dict on which to wait. + :param wait_for_disassociate: Boolean indiating whether to wait for + disassociation instead of association. + """ + + def _get_floating_ip_in_server_addresses(floating_ip, server): + for addresses in server['addresses'].values(): + for address in addresses: + if ( + address['OS-EXT-IPS:type'] == 'floating' and + address['addr'] == floating_ip['floating_ip_address'] + ): + return address + return None + + start_time = int(time.time()) + while True: + server = servers_client.show_server(server['id'])['server'] + address = _get_floating_ip_in_server_addresses(floating_ip, server) + if address is None and wait_for_disassociate: + return None + if not wait_for_disassociate and address: + return address + + if int(time.time()) - start_time >= servers_client.build_timeout: + if wait_for_disassociate: + msg = ('Floating ip %s failed to disassociate from server %s ' + 'in time.' % (floating_ip, server['id'])) + else: + msg = ('Floating ip %s failed to associate with server %s ' + 'in time.' % (floating_ip, server['id'])) + raise lib_exc.TimeoutException(msg) + time.sleep(servers_client.build_interval) diff --git a/tempest/scenario/test_minimum_basic.py b/tempest/scenario/test_minimum_basic.py index 2c981c8777..5aac19c6e0 100644 --- a/tempest/scenario/test_minimum_basic.py +++ b/tempest/scenario/test_minimum_basic.py @@ -96,13 +96,6 @@ class TestMinimumBasicScenario(manager.ScenarioTest): '%s' % (secgroup['id'], server['id'])) raise exceptions.TimeoutException(msg) - def _get_floating_ip_in_server_addresses(self, floating_ip, server): - for addresses in server['addresses'].values(): - for address in addresses: - if (address['OS-EXT-IPS:type'] == 'floating' and - address['addr'] == floating_ip['floating_ip_address']): - return address - @decorators.idempotent_id('bdbb5441-9204-419d-a225-b4fdbfb1a1a8') @utils.services('compute', 'volume', 'image', 'network') def test_minimum_basic_scenario(self): @@ -132,15 +125,8 @@ class TestMinimumBasicScenario(manager.ScenarioTest): fip = self.create_floating_ip(server) floating_ip = self.associate_floating_ip( fip, server) - # fetch the server again to make sure the addresses were refreshed - # after associating the floating IP - server = self.servers_client.show_server(server['id'])['server'] - address = self._get_floating_ip_in_server_addresses( - floating_ip, server) - self.assertIsNotNone( - address, - "Failed to find floating IP '%s' in server addresses: %s" % - (floating_ip['floating_ip_address'], server['addresses'])) + waiters.wait_for_server_floating_ip(self.servers_client, + server, floating_ip) ssh_ip = floating_ip['floating_ip_address'] else: ssh_ip = self.get_server_ip(server) @@ -165,19 +151,6 @@ class TestMinimumBasicScenario(manager.ScenarioTest): if floating_ip: # delete the floating IP, this should refresh the server addresses self.disassociate_floating_ip(floating_ip) - - def is_floating_ip_detached_from_server(): - server_info = self.servers_client.show_server( - server['id'])['server'] - address = self._get_floating_ip_in_server_addresses( - floating_ip, server_info) - return (not address) - - if not test_utils.call_until_true( - is_floating_ip_detached_from_server, - CONF.compute.build_timeout, - CONF.compute.build_interval): - msg = ("Floating IP '%s' should not be in server addresses: %s" - % (floating_ip['floating_ip_address'], - server['addresses'])) - raise exceptions.TimeoutException(msg) + waiters.wait_for_server_floating_ip( + self.servers_client, server, floating_ip, + wait_for_disassociate=True) diff --git a/tempest/tests/common/test_waiters.py b/tempest/tests/common/test_waiters.py index 5cdbfbf7c2..3092bf4661 100755 --- a/tempest/tests/common/test_waiters.py +++ b/tempest/tests/common/test_waiters.py @@ -495,3 +495,37 @@ class TestVolumeWaiters(base.TestCase): # Assert that list_volume_attachments was actually called mock_list_volume_attachments.assert_called_once_with( mock.sentinel.server_id) + + +class TestServerFloatingIPWaiters(base.TestCase): + + def test_wait_for_server_floating_ip_associate_timeout(self): + mock_server = {'server': {'id': 'fake_uuid', 'addresses': {}}} + mock_client = mock.Mock( + spec=servers_client.ServersClient, + build_timeout=1, build_interval=1, + show_server=lambda id: mock_server) + + fake_server = {'id': 'fake-uuid'} + fake_fip = {'floating_ip_address': 'fake_address'} + self.assertRaises( + lib_exc.TimeoutException, + waiters.wait_for_server_floating_ip, mock_client, fake_server, + fake_fip) + + def test_wait_for_server_floating_ip_disassociate_timeout(self): + mock_addresses = {'shared': [{'OS-EXT-IPS:type': 'floating', + 'addr': 'fake_address'}]} + mock_server = {'server': {'id': 'fake_uuid', + 'addresses': mock_addresses}} + mock_client = mock.Mock( + spec=servers_client.ServersClient, + build_timeout=1, build_interval=1, + show_server=lambda id: mock_server) + + fake_server = {'id': 'fake-uuid'} + fake_fip = {'floating_ip_address': 'fake_address'} + self.assertRaises( + lib_exc.TimeoutException, + waiters.wait_for_server_floating_ip, mock_client, fake_server, + fake_fip, wait_for_disassociate=True)