[etcd3gw] create new lease if expired.

It can happen membership lease can be expired
to the moment of lease refreshing.
In this case heartbeat will try to refresh
expired lease forever without success.
The patch checks if lease expired and create new one.

Closes-Bug: 1975524
Depends-On: https://review.opendev.org/c/openstack/etcd3gw/+/843003
Change-Id: I5017724fd0134f3f0a51f059640e641dd9853ff2
This commit is contained in:
Mitya_Eremeev 2022-05-23 21:53:01 +03:00
parent b21c20f79f
commit 48ee8bdaaf
1 changed files with 12 additions and 2 deletions

View File

@ -14,6 +14,7 @@
import base64
import functools
import logging
import threading
import uuid
@ -27,6 +28,8 @@ from tooz import coordination
from tooz import locking
from tooz import utils
LOG = logging.getLogger(__name__)
def _encode(data):
"""Safely encode data for consumption of the gateway."""
@ -218,6 +221,7 @@ class Etcd3Driver(coordination.CoordinationDriverWithExecutor):
self.membership_timeout = int(options.get(
'membership_timeout', timeout))
self._acquired_locks = set()
self._membership_lease = None
def _start(self):
super(Etcd3Driver, self)._start()
@ -227,11 +231,17 @@ class Etcd3Driver(coordination.CoordinationDriverWithExecutor):
return Etcd3Lock(self, name, self.lock_timeout)
def heartbeat(self):
# TODO(kaifeng) use the same lease for locks?
if self._membership_lease.refresh() == -1:
expired_lease = self._membership_lease.id
self._membership_lease = self.client.lease(self.membership_timeout)
LOG.warning(
'Created new lease %s after previous lease %s expired.',
self._membership_lease.id,
expired_lease)
# NOTE(jaypipes): Copying because set can mutate during iteration
for lock in self._acquired_locks.copy():
lock.heartbeat()
# TODO(kaifeng) use the same lease for locks?
self._membership_lease.refresh()
return min(self.lock_timeout, self.membership_timeout)
def watch_join_group(self, group_id, callback):