openstack/kuryr-kubernetes
Code Issues Proposed changes

Add original exception to log about dead component

Browse Source 
This commit make sure we save the exception that caused a handler to be
dead and log it when printing "Component X is dead".

Change-Id: Idcc71a60b2fe448287f2e2d1a0747657c12a5719
This commit is contained in:
Michał Dulko 2021-06-22 18:28:46 +02:00
parent a17e5502c2
commit 1e2e5b1633
3 changed files with 18 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
kuryr_kubernetes/controller/managers/health.py
View File

@ -95,8 +95,14 @@ class HealthServer(base_server.BaseHealthServer):
def liveness_status(self):
for component in self._registry:
if not component.is_alive():
msg = 'Component %s is dead.' % component.__class__.__name__
LOG.error(msg)
exc = component.get_last_exception()
if not exc:
msg = f'Component {component.__class__.__name__} is dead.'
LOG.error(msg)
else:
msg = (f'Component {component.__class__.__name__} is dead.'
f' Last caught exception below')
LOG.exception(msg, exc_info=exc)
return msg, httplib.INTERNAL_SERVER_ERROR, {}
return 'ok', httplib.OK, {}

8
kuryr_kubernetes/handlers/health.py
View File

@ -37,8 +37,11 @@ class HealthHandler(object):
self._ready = True
self._manager = HealthRegister.get_instance()
self._manager.register(self)
self._last_exception = None
def set_liveness(self, alive):
def set_liveness(self, alive, exc=None):
if exc:
self._last_exception = exc
self._alive = alive
def set_readiness(self, ready):
@ -49,3 +52,6 @@ class HealthHandler(object):
def is_ready(self, *args):
return self._ready
def get_last_exception(self):
return self._last_exception

6
kuryr_kubernetes/handlers/retry.py
View File

@ -96,10 +96,10 @@ class Retry(base.EventHandler):
ex.reraise = False
else:
LOG.debug('Report handler unhealthy %s', self._handler)
self._handler.set_liveness(alive=False)
except Exception:
self._handler.set_liveness(alive=False, exc=ex.value)
except Exception as ex:
LOG.exception('Report handler unhealthy %s', self._handler)
self._handler.set_liveness(alive=False)
self._handler.set_liveness(alive=False, exc=ex)
raise
def _sleep(self, deadline, attempt, exception):