diff --git a/neutron/notifiers/nova.py b/neutron/notifiers/nova.py index 3331f7d5f09..7a68e29b416 100644 --- a/neutron/notifiers/nova.py +++ b/neutron/notifiers/nova.py @@ -13,6 +13,8 @@ # License for the specific language governing permissions and limitations # under the License. + +from keystoneauth1 import exceptions as ks_exceptions from keystoneauth1 import loading as ks_loading from neutron_lib.callbacks import events from neutron_lib.callbacks import registry @@ -29,6 +31,7 @@ from oslo_context import context as common_context from oslo_log import log as logging from oslo_utils import uuidutils from sqlalchemy.orm import attributes as sql_attr +import tenacity from neutron.notifiers import batch_notifier @@ -242,6 +245,12 @@ class Notifier(object): 'tag': port.id}) self.send_port_status(None, None, port) + @tenacity.retry( + retry=tenacity.retry_if_exception_type( + ks_exceptions.RetriableConnectionFailure), + wait=tenacity.wait_exponential(multiplier=0.01, max=1), + stop=tenacity.stop_after_delay(1), + after=tenacity.after_log(LOG, logging.DEBUG)) def send_events(self, batched_events): LOG.debug("Sending events: %s", batched_events) novaclient = self._get_nova_client() @@ -251,6 +260,10 @@ class Notifier(object): except nova_exceptions.NotFound: LOG.debug("Nova returned NotFound for event: %s", batched_events) + except ks_exceptions.RetriableConnectionFailure: + raise + # next clause handles all exceptions + # so reraise for retry decorator except Exception: LOG.exception("Failed to notify nova on events: %s", batched_events) diff --git a/neutron/tests/unit/notifiers/test_nova.py b/neutron/tests/unit/notifiers/test_nova.py index 4b4607e3404..a1a76b8fd61 100644 --- a/neutron/tests/unit/notifiers/test_nova.py +++ b/neutron/tests/unit/notifiers/test_nova.py @@ -15,6 +15,8 @@ import mock + +from keystoneauth1 import exceptions as ks_exc from neutron_lib import constants as n_const from neutron_lib import context as n_ctx from neutron_lib import exceptions as n_exc @@ -246,6 +248,14 @@ class TestNovaNotify(base.BaseTestCase): nova_exceptions.NotFound) self.nova_notifier.send_events([]) + @mock.patch('novaclient.client.Client') + def test_nova_send_events_raises_connect_exc(self, mock_client): + create = mock_client().server_external_events.create + create.side_effect = ( + ks_exc.ConnectFailure, ks_exc.ConnectTimeout, []) + self.nova_notifier.send_events([]) + self.assertEqual(3, create.call_count) + @mock.patch('novaclient.client.Client') def test_nova_send_events_raises(self, mock_client): mock_client.server_external_events.create.return_value = Exception