mariadb: avoid state management thread death
The mariadb container launches two threads in addition to the mysql daemon, one to mantain a configmap containing the Galera Cluster state, and the other to handle leader elections. These threads die if they suffer any exceptions talking to the kubernetes apiserver. This can happen sometimes, e.g. when a k8s control node reboots. This change logs and ignores the kubernetes.client.rest.ApiException, allowing the threads to retry and hopefully succeed once the k8s api becomes available. Change-Id: I5745a763bb07f719d83a41c1f27be2b76ce998e9
This commit is contained in:
parent
26982ca705
commit
b482b57e6e
@ -705,7 +705,10 @@ def check_if_i_lead():
|
|||||||
def monitor_cluster():
|
def monitor_cluster():
|
||||||
"""Function to kick off grastate configmap updating thread"""
|
"""Function to kick off grastate configmap updating thread"""
|
||||||
while True:
|
while True:
|
||||||
update_grastate_configmap()
|
try:
|
||||||
|
update_grastate_configmap()
|
||||||
|
except kubernetes.client.rest.ApiException as error:
|
||||||
|
logger.error("Error updating grastate configmap: {0}".format(error))
|
||||||
time.sleep(state_configmap_update_period)
|
time.sleep(state_configmap_update_period)
|
||||||
|
|
||||||
|
|
||||||
@ -723,7 +726,10 @@ def launch_cluster_monitor():
|
|||||||
def leader_election():
|
def leader_election():
|
||||||
"""Function to kick off leader election thread"""
|
"""Function to kick off leader election thread"""
|
||||||
while True:
|
while True:
|
||||||
deadmans_leader_election()
|
try:
|
||||||
|
deadmans_leader_election()
|
||||||
|
except kubernetes.client.rest.ApiException as error:
|
||||||
|
logger.error("Error electing leader: {0}".format(error))
|
||||||
time.sleep(cluster_leader_ttl / 2)
|
time.sleep(cluster_leader_ttl / 2)
|
||||||
|
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user