diff --git a/ironic_inspector/pxe_filter/dnsmasq.py b/ironic_inspector/pxe_filter/dnsmasq.py index 0935728ed..b58d79f99 100644 --- a/ironic_inspector/pxe_filter/dnsmasq.py +++ b/ironic_inspector/pxe_filter/dnsmasq.py @@ -28,6 +28,7 @@ import fcntl import os import time +from openstack import exceptions as os_exc from oslo_concurrency import processutils from oslo_config import cfg from oslo_log import log @@ -83,11 +84,16 @@ class DnsmasqFilter(pxe_filter.BaseFilter): LOG.debug('Syncing the driver') timestamp_start = timeutils.utcnow() - # active_macs are the MACs for which introspection is active - active_macs = pxe_filter.get_active_macs(ironic) + try: + # active_macs are the MACs for which introspection is active + active_macs = pxe_filter.get_active_macs(ironic) - # ironic_macs are all the MACs know to ironic (all ironic ports) - ironic_macs = pxe_filter.get_ironic_macs(ironic) + # ironic_macs are all the MACs know to ironic (all ironic ports) + ironic_macs = pxe_filter.get_ironic_macs(ironic) + except os_exc.SDKException: + LOG.exception( + "Could not list ironic ports, can not sync dnsmasq PXE filter") + return denylist, allowlist = _get_deny_allow_lists() # removedlist are the MACs that are in either in allow or denylist, diff --git a/ironic_inspector/pxe_filter/iptables.py b/ironic_inspector/pxe_filter/iptables.py index 5205882e7..cbe0bf780 100644 --- a/ironic_inspector/pxe_filter/iptables.py +++ b/ironic_inspector/pxe_filter/iptables.py @@ -13,6 +13,7 @@ import contextlib +from openstack import exceptions as os_exc from oslo_concurrency import processutils from oslo_config import cfg from oslo_log import log @@ -110,12 +111,18 @@ class IptablesFilter(pxe_filter.BaseFilter): self._disable_dhcp() return - if CONF.pxe_filter.deny_unknown_macs: - to_allow = pxe_filter.get_active_macs(ironic) - to_deny = None - else: - to_deny = pxe_filter.get_inactive_macs(ironic) - to_allow = None + try: + if CONF.pxe_filter.deny_unknown_macs: + to_allow = pxe_filter.get_active_macs(ironic) + to_deny = None + else: + to_deny = pxe_filter.get_inactive_macs(ironic) + to_allow = None + except os_exc.SDKException: + LOG.exception( + "Could not list ironic ports, iptables PXE filter can not " + "be synced") + return if to_deny == self.denylist_cache and to_allow == self.allowlist_cache: LOG.debug('Not updating iptables - no changes in MAC lists %s %s', diff --git a/ironic_inspector/test/unit/test_dnsmasq_pxe_filter.py b/ironic_inspector/test/unit/test_dnsmasq_pxe_filter.py index 8bd52d748..35a24e52f 100644 --- a/ironic_inspector/test/unit/test_dnsmasq_pxe_filter.py +++ b/ironic_inspector/test/unit/test_dnsmasq_pxe_filter.py @@ -484,6 +484,22 @@ class TestSync(DnsmasqTestBase): self.timestamp_end - self.timestamp_start) ]) + @mock.patch('time.sleep', lambda _x: None) + def test__sync_ironic_unavailable(self): + self.mock_ironic.ports.side_effect = os_exc.SDKException('boom') + self.driver._sync(self.mock_ironic) + self.mock__add_mac_to_allowlist.assert_not_called() + self.mock__add_mac_to_denylist.assert_not_called() + + self.mock_ironic.ports.assert_called_with(fields=['address', 'extra'], + limit=None) + self.mock_get_active_macs.assert_called_once_with(self.mock_ironic) + self.mock__get_deny_allow_lists.assert_not_called() + self.mock__configure_removedlist.assert_not_called() + self.mock_log.debug.assert_called_once_with('Syncing the driver') + self.mock_log.exception.assert_called_once_with( + "Could not list ironic ports, can not sync dnsmasq PXE filter") + class Test_Execute(test_base.BaseTest): def setUp(self): diff --git a/ironic_inspector/test/unit/test_iptables.py b/ironic_inspector/test/unit/test_iptables.py index 1fb7f9693..ad71b0ef6 100644 --- a/ironic_inspector/test/unit/test_iptables.py +++ b/ironic_inspector/test/unit/test_iptables.py @@ -16,6 +16,7 @@ from unittest import mock import fixtures +from openstack import exceptions as os_exc from oslo_config import cfg from ironic_inspector import node_cache @@ -354,6 +355,21 @@ class TestIptablesDriver(test_base.NodeTest): self.assertRaisesRegex(utils.Error, 'Configuration error:', self.driver.__init__) + def test_sync_ironic_unavailable_allowlist(self): + self.driver = iptables.IptablesFilter() + self.mock_ironic.ports.side_effect = os_exc.SDKException('boom') + self.driver.sync(self.mock_ironic) + self.mock_get_inactive_macs.assert_called_once_with(self.mock_ironic) + self.mock_iptables.assert_not_called() + + def test_sync_ironic_unavailable_denylist(self): + CONF.set_override('deny_unknown_macs', True, 'pxe_filter') + self.driver = iptables.IptablesFilter() + self.mock_ironic.ports.side_effect = os_exc.SDKException('boom') + self.driver.sync(self.mock_ironic) + self.mock_get_active_macs.assert_called_once_with(self.mock_ironic) + self.mock_iptables.assert_not_called() + class Test_ShouldEnableDhcp(test_base.BaseTest): def setUp(self): diff --git a/releasenotes/notes/pxe-filter-driver-stuck-ea5844cf3eafa61f.yaml b/releasenotes/notes/pxe-filter-driver-stuck-ea5844cf3eafa61f.yaml new file mode 100644 index 000000000..3cb593a31 --- /dev/null +++ b/releasenotes/notes/pxe-filter-driver-stuck-ea5844cf3eafa61f.yaml @@ -0,0 +1,9 @@ +--- + +fixes: + - | + Inspector now ignores failures to list ironic ports during + pxe filter driver sync, and just skips the sync in this case. + Previously such errors resulted in pxe filter driver being stuck + in an uninitialized state until ironic inspector was restarted. + See bug `2008971 `_.