diff --git a/swift/obj/replicator.py b/swift/obj/replicator.py index 581bf7de2b..487724fc09 100644 --- a/swift/obj/replicator.py +++ b/swift/obj/replicator.py @@ -363,23 +363,29 @@ class ObjectReplicator(Daemon): do_listdir=(self.replication_count % 10) == 0, reclaim_age=self.reclaim_age) self.suffix_hash += hashed - successes = 0 + attempts_left = self.object_ring.replica_count nodes = itertools.chain(job['nodes'], self.object_ring.get_more_nodes(int(job['partition']))) - while successes < (self.object_ring.replica_count - 1): + while attempts_left > 0: + # If this throws StopIterator it will be caught way below node = next(nodes) + attempts_left -= 1 try: with Timeout(60): resp = http_connect(node['ip'], node['port'], node['device'], job['partition'], 'REPLICATE', '', headers={'Content-Length': '0'}).getresponse() + if resp.status == 507: + self.logger.error('%s/%s responded as unmounted' % + (node['ip'], node['device'])) + attempts_left += 1 + continue if resp.status != 200: self.logger.error("Invalid response %s from %s" % (resp.status, node['ip'])) continue remote_hash = pickle.loads(resp.read()) del resp - successes += 1 suffixes = [suffix for suffix in local_hash if local_hash[suffix] != remote_hash.get(suffix, -1)]