Merge "Don't give up when an Exception happens in idl.run" into stable/train
This commit is contained in:
commit
11ca35873f
|
@ -15,6 +15,7 @@
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import threading
|
import threading
|
||||||
|
import time
|
||||||
import traceback
|
import traceback
|
||||||
|
|
||||||
from ovs.db import idl
|
from ovs.db import idl
|
||||||
|
@ -93,9 +94,9 @@ class Connection(object):
|
||||||
while self._is_running:
|
while self._is_running:
|
||||||
# If we fail in an Idl call, we could have missed an update
|
# If we fail in an Idl call, we could have missed an update
|
||||||
# from the server, leaving us out of sync with ovsdb-server.
|
# from the server, leaving us out of sync with ovsdb-server.
|
||||||
# It is not safe to continue without restarting the connection,
|
# It is not safe to continue without restarting the connection.
|
||||||
# though it is likely that the error is unrecoverable, so only try
|
# Though it is likely that the error is unrecoverable, keep trying
|
||||||
# a few times before bailing completely.
|
# indefinitely just in case.
|
||||||
try:
|
try:
|
||||||
self.idl.wait(self.poller)
|
self.idl.wait(self.poller)
|
||||||
self.poller.fd_wait(self.txns.alert_fileno, poller.POLLIN)
|
self.poller.fd_wait(self.txns.alert_fileno, poller.POLLIN)
|
||||||
|
@ -107,13 +108,18 @@ class Connection(object):
|
||||||
# in python-ovs
|
# in python-ovs
|
||||||
errors += 1
|
errors += 1
|
||||||
LOG.exception(e)
|
LOG.exception(e)
|
||||||
if errors <= 3:
|
with self.lock:
|
||||||
with self.lock:
|
self.idl.force_reconnect()
|
||||||
self.idl.force_reconnect()
|
try:
|
||||||
idlutils.wait_for_change(self.idl, self.timeout)
|
idlutils.wait_for_change(self.idl, self.timeout)
|
||||||
continue
|
except Exception as e:
|
||||||
self._is_running = False
|
# This could throw the same exception as idl.run()
|
||||||
break
|
# or Exception("timeout"), either way continue
|
||||||
|
LOG.exception(e)
|
||||||
|
sleep = min(2 ** errors, 60)
|
||||||
|
LOG.info("Trying to recover, sleeping %s seconds", sleep)
|
||||||
|
time.sleep(sleep)
|
||||||
|
continue
|
||||||
errors = 0
|
errors = 0
|
||||||
txn = self.txns.get_nowait()
|
txn = self.txns.get_nowait()
|
||||||
if txn is not None:
|
if txn is not None:
|
||||||
|
|
Loading…
Reference in New Issue