db: Close ReplConnection sockets on errors/timeouts

This could happen when there was a timeout running _sync_shard_ranges()
in _choose_replication_mode() -- syncing of shard ranges failed, but we
still want to attempt to replicate. If we close the socket, the
connection will automatically spin up a new one the next time we call
request() instead of raising CannotSendRequest.

Change-Id: I242351078e26213f43c1ccc0fed534b64aa29ab6
Closes-Bug: #1968224

swift/common/db_replicator.py

@@ -174,6 +174,7 @@ class ReplConnection(BufferedHTTPConnection):
             response.data = response.read()
             return response
         except (Exception, Timeout):
+            self.close()
             self.logger.exception(
                 _('ERROR reading HTTP response from %s'), self.node)
             return None

test/unit/common/test_db_replicator.py

@@ -352,7 +352,17 @@ class TestDBReplicator(unittest.TestCase):
         def other_req(method, path, body, headers):
             raise Exception('blah')
         conn.request = other_req
+
+        class Closeable(object):
+            closed = False
+
+            def close(self):
+                self.closed = True
+
+        conn.sock = fake_sock = Closeable()
         self.assertIsNone(conn.replicate(1, 2, 3))
+        self.assertTrue(fake_sock.closed)
+        self.assertEqual(None, conn.sock)
 
     def test_rsync_file(self):
         replicator = TestReplicator({})