Do not reset FSM when ironic ports cant be queried

It can cause problems, as if a single sync fails, FSM state will be
reset and not put back to initialized state until inspector restart.

Change-Id: I24b08612c4ffc6aca60ca08f3ff5cc769c7c041d
Story: #2008971
Task: #42611

ironic_inspector/pxe_filter/dnsmasq.py

@@ -28,6 +28,7 @@ import fcntl
 import os
 import time
 
+from openstack import exceptions as os_exc
 from oslo_concurrency import processutils
 from oslo_config import cfg
 from oslo_log import log
@@ -83,11 +84,16 @@ class DnsmasqFilter(pxe_filter.BaseFilter):
         LOG.debug('Syncing the driver')
         timestamp_start = timeutils.utcnow()
 
-        # active_macs are the MACs for which introspection is active
-        active_macs = pxe_filter.get_active_macs(ironic)
+        try:
+            # active_macs are the MACs for which introspection is active
+            active_macs = pxe_filter.get_active_macs(ironic)
 
-        # ironic_macs are all the MACs know to ironic (all ironic ports)
-        ironic_macs = pxe_filter.get_ironic_macs(ironic)
+            # ironic_macs are all the MACs know to ironic (all ironic ports)
+            ironic_macs = pxe_filter.get_ironic_macs(ironic)
+        except os_exc.SDKException:
+            LOG.exception(
+                "Could not list ironic ports, can not sync dnsmasq PXE filter")
+            return
 
         denylist, allowlist = _get_deny_allow_lists()
         # removedlist are the MACs that are in either in allow or denylist,

ironic_inspector/pxe_filter/iptables.py

@@ -13,6 +13,7 @@
 
 import contextlib
 
+from openstack import exceptions as os_exc
 from oslo_concurrency import processutils
 from oslo_config import cfg
 from oslo_log import log
@@ -110,12 +111,18 @@ class IptablesFilter(pxe_filter.BaseFilter):
             self._disable_dhcp()
             return
 
-        if CONF.pxe_filter.deny_unknown_macs:
-            to_allow = pxe_filter.get_active_macs(ironic)
-            to_deny = None
-        else:
-            to_deny = pxe_filter.get_inactive_macs(ironic)
-            to_allow = None
+        try:
+            if CONF.pxe_filter.deny_unknown_macs:
+                to_allow = pxe_filter.get_active_macs(ironic)
+                to_deny = None
+            else:
+                to_deny = pxe_filter.get_inactive_macs(ironic)
+                to_allow = None
+        except os_exc.SDKException:
+            LOG.exception(
+                "Could not list ironic ports, iptables PXE filter can not "
+                "be synced")
+            return
 
         if to_deny == self.denylist_cache and to_allow == self.allowlist_cache:
             LOG.debug('Not updating iptables - no changes in MAC lists %s %s',

ironic_inspector/test/unit/test_dnsmasq_pxe_filter.py

@@ -484,6 +484,22 @@ class TestSync(DnsmasqTestBase):
                       self.timestamp_end - self.timestamp_start)
         ])
 
+    @mock.patch('time.sleep', lambda _x: None)
+    def test__sync_ironic_unavailable(self):
+        self.mock_ironic.ports.side_effect = os_exc.SDKException('boom')
+        self.driver._sync(self.mock_ironic)
+        self.mock__add_mac_to_allowlist.assert_not_called()
+        self.mock__add_mac_to_denylist.assert_not_called()
+
+        self.mock_ironic.ports.assert_called_with(fields=['address', 'extra'],
+                                                  limit=None)
+        self.mock_get_active_macs.assert_called_once_with(self.mock_ironic)
+        self.mock__get_deny_allow_lists.assert_not_called()
+        self.mock__configure_removedlist.assert_not_called()
+        self.mock_log.debug.assert_called_once_with('Syncing the driver')
+        self.mock_log.exception.assert_called_once_with(
+            "Could not list ironic ports, can not sync dnsmasq PXE filter")
+
 
 class Test_Execute(test_base.BaseTest):
     def setUp(self):

ironic_inspector/test/unit/test_iptables.py

@@ -16,6 +16,7 @@
 from unittest import mock
 
 import fixtures
+from openstack import exceptions as os_exc
 from oslo_config import cfg
 
 from ironic_inspector import node_cache
@@ -354,6 +355,21 @@ class TestIptablesDriver(test_base.NodeTest):
         self.assertRaisesRegex(utils.Error, 'Configuration error:',
                                self.driver.__init__)
 
+    def test_sync_ironic_unavailable_allowlist(self):
+        self.driver = iptables.IptablesFilter()
+        self.mock_ironic.ports.side_effect = os_exc.SDKException('boom')
+        self.driver.sync(self.mock_ironic)
+        self.mock_get_inactive_macs.assert_called_once_with(self.mock_ironic)
+        self.mock_iptables.assert_not_called()
+
+    def test_sync_ironic_unavailable_denylist(self):
+        CONF.set_override('deny_unknown_macs', True, 'pxe_filter')
+        self.driver = iptables.IptablesFilter()
+        self.mock_ironic.ports.side_effect = os_exc.SDKException('boom')
+        self.driver.sync(self.mock_ironic)
+        self.mock_get_active_macs.assert_called_once_with(self.mock_ironic)
+        self.mock_iptables.assert_not_called()
+
 
 class Test_ShouldEnableDhcp(test_base.BaseTest):
     def setUp(self):

releasenotes/notes/pxe-filter-driver-stuck-ea5844cf3eafa61f.yaml

@@ -0,0 +1,9 @@
+---
+
+fixes:
+  - |
+    Inspector now ignores failures to list ironic ports during
+    pxe filter driver sync, and just skips the sync in this case.
+    Previously such errors resulted in pxe filter driver being stuck
+    in an uninitialized state until ironic inspector was restarted.
+    See bug `2008971 <https://storyboard.openstack.org/#!/story/2008971>`_.