Add retry for privsep get_link_devices

pyroute 0.6.6 introduced a new exception NetlinkDumpInterrupted which
is raised when NLM_F_DUMP_INTR is set in the flags during dump of
devices.
The suggestion from pyroute developers is to retry in case of this
exception (see [1]).

[1]: https://github.com/svinota/pyroute2/issues/874#issuecomment-1063139555

Closes-Bug: #1962608

Depends-On: https://review.opendev.org/c/openstack/requirements/+/845268
Change-Id: Ie195ad596fd148708fc30946bde964d52444afee
(cherry picked from commit 74a9e832d7)
(cherry picked from commit 541d1fc9e6)
(cherry picked from commit ea85053a87)
Conflicts: neutron/privileged/agent/linux/ip_lib.py
 	requirements.txt

neutron/privileged/agent/linux/ip_lib.py

@@ -24,6 +24,7 @@ from pyroute2.netlink import rtnl  # pylint: disable=no-name-in-module
 from pyroute2.netlink.rtnl import ifinfmsg  # pylint: disable=no-name-in-module
 from pyroute2.netlink.rtnl import ndmsg  # pylint: disable=no-name-in-module
 from pyroute2 import netns  # pylint: disable=no-name-in-module
+import tenacity
 
 from neutron._i18n import _
 from neutron import privileged
@@ -419,6 +420,12 @@ def set_link_bridge_master(device, bridge, namespace=None):
                              master=bridge_idx)
 
 
+@tenacity.retry(
+    retry=tenacity.retry_if_exception_type(
+        netlink_exceptions.NetlinkDumpInterrupted),
+    wait=tenacity.wait_exponential(multiplier=0.02, max=1),
+    stop=tenacity.stop_after_delay(8),
+    reraise=True)
 @privileged.default.entrypoint
 def get_link_attributes(device, namespace):
     link = _run_iproute_link("get", device, namespace)[0]
@@ -435,6 +442,12 @@ def get_link_attributes(device, namespace):
     }
 
 
+@tenacity.retry(
+    retry=tenacity.retry_if_exception_type(
+        netlink_exceptions.NetlinkDumpInterrupted),
+    wait=tenacity.wait_exponential(multiplier=0.02, max=1),
+    stop=tenacity.stop_after_delay(8),
+    reraise=True)
 @privileged.default.entrypoint
 def get_link_vfs(device, namespace):
     link = _run_iproute_link('get', device, namespace=namespace, ext_mask=1)[0]
@@ -506,6 +519,12 @@ def delete_neigh_entry(ip_version, ip_address, mac_address, device, namespace,
         raise
 
 
+@tenacity.retry(
+    retry=tenacity.retry_if_exception_type(
+        netlink_exceptions.NetlinkDumpInterrupted),
+    wait=tenacity.wait_exponential(multiplier=0.02, max=1),
+    stop=tenacity.stop_after_delay(8),
+    reraise=True)
 @privileged.default.entrypoint
 def dump_neigh_entries(ip_version, device, namespace, **kwargs):
     """Dump all neighbour entries.
@@ -602,6 +621,12 @@ def make_serializable(value):
     return _ensure_string(value)
 
 
+@tenacity.retry(
+    retry=tenacity.retry_if_exception_type(
+        netlink_exceptions.NetlinkDumpInterrupted),
+    wait=tenacity.wait_exponential(multiplier=0.02, max=1),
+    stop=tenacity.stop_after_delay(8),
+    reraise=True)
 @privileged.default.entrypoint
 def get_link_devices(namespace, **kwargs):
     """List interfaces in a namespace
@@ -632,6 +657,12 @@ def get_device_names(namespace, **kwargs):
     return device_names
 
 
+@tenacity.retry(
+    retry=tenacity.retry_if_exception_type(
+        netlink_exceptions.NetlinkDumpInterrupted),
+    wait=tenacity.wait_exponential(multiplier=0.02, max=1),
+    stop=tenacity.stop_after_delay(8),
+    reraise=True)
 @privileged.default.entrypoint
 def get_ip_addresses(namespace, **kwargs):
     """List of IP addresses in a namespace
@@ -647,6 +678,12 @@ def get_ip_addresses(namespace, **kwargs):
         raise
 
 
+@tenacity.retry(
+    retry=tenacity.retry_if_exception_type(
+        netlink_exceptions.NetlinkDumpInterrupted),
+    wait=tenacity.wait_exponential(multiplier=0.02, max=1),
+    stop=tenacity.stop_after_delay(8),
+    reraise=True)
 @privileged.default.entrypoint
 def list_ip_rules(namespace, ip_version, match=None, **kwargs):
     """List all IP rules"""
@@ -747,6 +784,12 @@ def add_ip_route(namespace, cidr, ip_version, device=None, via=None,
         raise
 
 
+@tenacity.retry(
+    retry=tenacity.retry_if_exception_type(
+        netlink_exceptions.NetlinkDumpInterrupted),
+    wait=tenacity.wait_exponential(multiplier=0.02, max=1),
+    stop=tenacity.stop_after_delay(8),
+    reraise=True)
 @privileged.default.entrypoint
 def list_ip_routes(namespace, ip_version, device=None, table=None, **kwargs):
     """List IP routes"""
@@ -776,6 +819,12 @@ def delete_ip_route(namespace, cidr, ip_version, device=None, via=None,
         raise
 
 
+@tenacity.retry(
+    retry=tenacity.retry_if_exception_type(
+        netlink_exceptions.NetlinkDumpInterrupted),
+    wait=tenacity.wait_exponential(multiplier=0.02, max=1),
+    stop=tenacity.stop_after_delay(8),
+    reraise=True)
 @privileged.default.entrypoint
 def list_bridge_fdb(namespace=None, **kwargs):
     """List bridge fdb table"""

neutron/tests/unit/privileged/agent/linux/test_ip_lib.py

@@ -17,6 +17,7 @@ from unittest import mock
 
 import pyroute2
 from pyroute2 import netlink
+from pyroute2.netlink import exceptions as netlink_exceptions
 from pyroute2.netlink.rtnl import ifinfmsg
 
 from neutron.privileged.agent.linux import ip_lib as priv_lib
@@ -254,7 +255,8 @@ class IpLibTestCase(base.BaseTestCase):
         priv_lib.privileged.default.client_mode = False
         self.addCleanup(self._clean, client_mode)
         with mock.patch.object(priv_lib, '_run_iproute_link') as mock_iplink:
-            mock_iplink.return_value = [value]
+            mock_iplink.side_effect = [
+                netlink_exceptions.NetlinkDumpInterrupted(), value]
             result = priv_lib.get_link_vfs('device', 'namespace')
             exp = {0: {'mac': 'mac_0', 'link_state': 0,
                        'max_tx_rate': 0, 'min_tx_rate': 0},
@@ -263,6 +265,11 @@ class IpLibTestCase(base.BaseTestCase):
                    2: {'mac': 'mac_2', 'link_state': 2,
                        'max_tx_rate': 2000, 'min_tx_rate': 1000}}
             self.assertEqual(exp, result)
+            # Check that _run_iproute_link was called twice
+            mock_iplink.assert_has_calls(
+                [mock.call('get', 'device', namespace='namespace', ext_mask=1),
+                 mock.call('get', 'device', namespace='namespace', ext_mask=1)]
+            )
 
 
 class MakeSerializableTestCase(base.BaseTestCase):

requirements.txt

@@ -48,7 +48,7 @@ ovs>=2.10.0 # Apache-2.0
 ovsdbapp>=1.9.2 # Apache-2.0
 packaging>=20.4  # Apache-2.0
 psutil>=5.3.0 # BSD
-pyroute2>=0.5.13;sys_platform!='win32' # Apache-2.0 (+ dual licensed GPL2)
+pyroute2>=0.6.6;sys_platform!='win32' # Apache-2.0 (+ dual licensed GPL2)
 pyOpenSSL>=17.1.0 # Apache-2.0
 
 python-novaclient>=9.1.0 # Apache-2.0