Fix ipv6 interface configuration

When configuring an ipv6 interface, wait for the tentative flag to be
off. Waiting for this flag prevents haproxy to be restarted too early
and to use invalid source ip addresses for its healthchecks.

Story 2009847
Task 44467

Change-Id: Ie859e9911dfc54c327476ee9925d0c9046fed832
(cherry picked from commit 2323797a7f8eff16b8d2ce90f41adc428438b869)

octavia/amphorae/backends/utils/interface.py

@@ -22,6 +22,8 @@ import time
 from oslo_config import cfg
 from oslo_log import log as logging
 import pyroute2
+# pylint: disable=no-name-in-module
+from pyroute2.netlink.rtnl import ifaddrmsg
 
 from octavia.amphorae.backends.utils import interface_file
 from octavia.common import constants as consts
@@ -37,6 +39,9 @@ class InterfaceController(object):
     DELETE = 'delete'
     SET = 'set'
 
+    TENTATIVE_WAIT_INTERVAL = .2
+    TENTATIVE_WAIT_TIMEOUT = 30
+
     def interface_file_list(self):
         net_dir = interface_file.InterfaceFile.get_directory()
 
@@ -134,6 +139,21 @@ class InterfaceController(object):
             LOG.debug("Running '%s'", cmd)
             subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 
+    def _wait_tentative(self, ipr, idx):
+        start = time.time()
+        while time.time() - start < self.TENTATIVE_WAIT_TIMEOUT:
+            addrs = ipr.get_addr(idx)
+            has_tentative = [
+                True
+                for addr in addrs
+                if (addr['family'] == socket.AF_INET6 and
+                    addr['flags'] & ifaddrmsg.IFA_F_TENTATIVE)]
+            if not has_tentative:
+                return
+            time.sleep(self.TENTATIVE_WAIT_INTERVAL)
+        LOG.warning("Some IPV6 addresses remain still in 'tentative' state "
+                    "after %d seconds.", self.TENTATIVE_WAIT_TIMEOUT)
+
     def up(self, interface):
         LOG.info("Setting interface %s up", interface.name)
 
@@ -158,6 +178,8 @@ class InterfaceController(object):
                 LOG.debug("%s: Adding address %s", interface.name, address)
                 self._ipr_command(ipr.addr, self.ADD, index=idx, **address)
 
+            self._wait_tentative(ipr, idx)
+
             for route in interface.routes:
                 route[consts.FAMILY] = self._family(route[consts.DST])
                 LOG.debug("%s: Adding route %s", interface.name, route)

octavia/tests/unit/amphorae/backends/utils/test_interface.py

@@ -446,8 +446,11 @@ class TestInterface(base.TestCase):
     @mock.patch('pyroute2.IPRoute.link')
     @mock.patch('pyroute2.IPRoute.link_lookup')
     @mock.patch('subprocess.check_output')
-    def test_up_auto(self, mock_check_output, mock_link_lookup, mock_link,
-                     mock_addr, mock_route, mock_rule):
+    @mock.patch('octavia.amphorae.backends.utils.interface.'
+                'InterfaceController._wait_tentative')
+    def test_up_auto(self, mock_wait_tentative, mock_check_output,
+                     mock_link_lookup, mock_link, mock_addr, mock_route,
+                     mock_rule):
         iface = interface_file.InterfaceFile(
             name="eth1",
             mtu=1450,
@@ -900,3 +903,50 @@ class TestInterface(base.TestCase):
                       stderr=-2),
             mock.call(["post-down", iface.name])
         ])
+
+    @mock.patch("time.time")
+    @mock.patch("time.sleep")
+    def test__wait_tentative(self, mock_time_sleep, mock_time_time):
+        mock_ipr = mock.MagicMock()
+        mock_ipr.get_addr.side_effect = [
+            ({'family': socket.AF_INET,
+              'flags': 0},
+             {'family': socket.AF_INET6,
+              'flags': 0x40},  # tentative
+             {'family': socket.AF_INET6,
+              'flags': 0}),
+            ({'family': socket.AF_INET,
+              'flags': 0},
+             {'family': socket.AF_INET6,
+              'flags': 0},
+             {'family': socket.AF_INET6,
+              'flags': 0})
+        ]
+
+        mock_time_time.return_value = 0
+
+        controller = interface.InterfaceController()
+        idx = 4
+
+        controller._wait_tentative(mock_ipr, idx)
+        mock_time_sleep.assert_called_once()
+
+    @mock.patch("time.time")
+    @mock.patch("time.sleep")
+    def test__wait_tentative_timeout(self, mock_time_sleep,
+                                     mock_time_time):
+        mock_ipr = mock.MagicMock()
+        mock_ipr.get_addr.return_value = (
+            {'family': socket.AF_INET6,
+             'flags': 0x40},  # tentative
+            {'family': socket.AF_INET6,
+             'flags': 0}
+        )
+
+        mock_time_time.side_effect = [0, 0, 1, 2, 29, 30, 31]
+
+        controller = interface.InterfaceController()
+        idx = 4
+
+        controller._wait_tentative(mock_ipr, idx)
+        self.assertEqual(4, len(mock_time_sleep.mock_calls))

releasenotes/notes/fix-ipv6-interface-configuration-61b1bd7d2c962cea.yaml

@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    Fix an issue with IPv6 members that could have been set in operating_status
+    ``ERROR`` just after being added.