Merge "Don't give up when an Exception happens in idl.run" into stable/train

This commit is contained in:
Zuul 2020-10-30 11:45:30 +00:00 committed by Gerrit Code Review
commit 11ca35873f
1 changed files with 15 additions and 9 deletions

View File

@ -15,6 +15,7 @@
import logging import logging
import os import os
import threading import threading
import time
import traceback import traceback
from ovs.db import idl from ovs.db import idl
@ -93,9 +94,9 @@ class Connection(object):
while self._is_running: while self._is_running:
# If we fail in an Idl call, we could have missed an update # If we fail in an Idl call, we could have missed an update
# from the server, leaving us out of sync with ovsdb-server. # from the server, leaving us out of sync with ovsdb-server.
# It is not safe to continue without restarting the connection, # It is not safe to continue without restarting the connection.
# though it is likely that the error is unrecoverable, so only try # Though it is likely that the error is unrecoverable, keep trying
# a few times before bailing completely. # indefinitely just in case.
try: try:
self.idl.wait(self.poller) self.idl.wait(self.poller)
self.poller.fd_wait(self.txns.alert_fileno, poller.POLLIN) self.poller.fd_wait(self.txns.alert_fileno, poller.POLLIN)
@ -107,13 +108,18 @@ class Connection(object):
# in python-ovs # in python-ovs
errors += 1 errors += 1
LOG.exception(e) LOG.exception(e)
if errors <= 3: with self.lock:
with self.lock: self.idl.force_reconnect()
self.idl.force_reconnect() try:
idlutils.wait_for_change(self.idl, self.timeout) idlutils.wait_for_change(self.idl, self.timeout)
continue except Exception as e:
self._is_running = False # This could throw the same exception as idl.run()
break # or Exception("timeout"), either way continue
LOG.exception(e)
sleep = min(2 ** errors, 60)
LOG.info("Trying to recover, sleeping %s seconds", sleep)
time.sleep(sleep)
continue
errors = 0 errors = 0
txn = self.txns.get_nowait() txn = self.txns.get_nowait()
if txn is not None: if txn is not None: