openstack/deb-python-cassandra-driver
Code Issues Proposed changes

ConnectionHeartbeat should mark down HostConnection on OTO

Browse Source
This commit is contained in:
Alan Boudreault 2017-05-17 11:22:36 -04:00
parent 7a29379312
commit 29ccd1564d
3 changed files with 25 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cassandra/cluster.py
View File

@ -1395,10 +1395,8 @@ class Cluster(object):
"""
if self.is_shutdown:
return
with host.lock:
was_up = host.is_up
# ignore down signals if we have open pools to the host
# this is to avoid closing pools when a control connection host became isolated
if self._discount_down_events and self.profile_manager.distance(host) != HostDistance.IGNORED:
@ -3134,7 +3132,7 @@ class ControlConnection(object):
c = getattr(self, '_connection', None)
return [c] if c else []
def return_connection(self, connection):
def return_connection(self, connection, mark_host_down=False): # noqa
if connection is self._connection and (connection.is_defunct or connection.is_closed):
self.reconnect()

2
cassandra/connection.py
View File

@ -1007,7 +1007,7 @@ class ConnectionHeartbeat(Thread):
for connection, owner, exc in failed_connections:
self._raise_if_stopped()
connection.defunct(exc)
owner.return_connection(connection)
owner.return_connection(connection, mark_host_down=True)
except self.ShutdownException:
pass
except Exception:

30
cassandra/pool.py
View File

@ -359,18 +359,29 @@ class HostConnection(object):
raise NoConnectionsAvailable("All request IDs are currently in use")
def return_connection(self, connection):
def return_connection(self, connection, mark_host_down=False):
with connection.lock:
connection.in_flight -= 1
with self._stream_available_condition:
self._stream_available_condition.notify()
if (connection.is_defunct or connection.is_closed) and not connection.signaled_error:
log.debug("Defunct or closed connection (%s) returned to pool, potentially "
"marking host %s as down", id(connection), self.host)
is_down = self._session.cluster.signal_connection_failure(
self.host, connection.last_error, is_host_addition=False)
connection.signaled_error = True
if connection.is_defunct or connection.is_closed:
if connection.signaled_error and not mark_host_down:
return
is_down = False
if not connection.signaled_error:
log.debug("Defunct or closed connection (%s) returned to pool, potentially "
"marking host %s as down", id(connection), self.host)
is_down = self._session.cluster.signal_connection_failure(
self.host, connection.last_error, is_host_addition=False)
connection.signaled_error = True
# Force mark down a host on error, used by the ConnectionHeartbeat
if mark_host_down and not is_down:
is_down = True
self._session.cluster.on_down(self.host, is_host_addition=False)
if is_down:
self.shutdown()
else:
@ -382,6 +393,11 @@ class HostConnection(object):
self._session.submit(self._replace, connection)
def _replace(self, connection):
with self._lock:
if self.is_shutdown:
self._is_replacing = False
return
log.debug("Replacing connection (%s) to %s", id(connection), self.host)
try:
conn = self._session.cluster.connection_factory(self.host.address)