From 58dcd30dbba67464f6fd7880ce7aee543156af65 Mon Sep 17 00:00:00 2001 From: Slawek Kaplonski Date: Wed, 6 Dec 2023 12:56:30 +0100 Subject: [PATCH] [Fullstack] Double check that agent is dead when it should be dead In some fullstack tests it is expected that agent is DOWN in the Neutron DB. It could happen sometimes that in almost the same time test's client was doing GET /v2.0/agents/{agent_id} call and got result with "alive=False" and in other thread rpc worker was processing heartbeat from the agent so it was revived just after API request was finished. That was causing test failures in some cases. This patch adds second API call to get agent again after 2 seconds if it was already marked as DEAD, just to make sure that it is really dead ;) Closes-Bug: #2045757 Change-Id: I1c20c90b8abd760f3a53b24024f19ef2bd189b5a --- neutron/tests/fullstack/base.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/neutron/tests/fullstack/base.py b/neutron/tests/fullstack/base.py index a8902181fe6..19cad221f31 100644 --- a/neutron/tests/fullstack/base.py +++ b/neutron/tests/fullstack/base.py @@ -16,6 +16,7 @@ from concurrent import futures import itertools import os import random +import time import netaddr from neutron_lib.tests import tools @@ -101,6 +102,17 @@ class BaseFullStackTestCase(testlib_api.MySQLTestCaseMixin, def _wait_until_agent_down(self, agent_id): def _agent_down(): agent = self.client.show_agent(agent_id)['agent'] + if not agent.get('alive'): + # NOTE(slaweq): to avoid race between heartbeat written in the + # database and response to this API call, lets make sure that + # agent is really dead. See bug + # https://bugs.launchpad.net/neutron/+bug/2045757 + # for details. + # 2 seconds delay should be more than enough to make sure that + # all pending heartbeats are already written in the Neutron + # database + time.sleep(2) + agent = self.client.show_agent(agent_id)['agent'] return not agent.get('alive') common_utils.wait_until_true(_agent_down)