reconciler: Tolerate 503s on HEAD

Change-Id: Ie53ebb90db4127b798df17fc7303aefc8dcc7f2a

swift/container/reconciler.py

@@ -489,9 +489,15 @@ class ContainerReconciler(Daemon):
                           container_policy_index)
         headers = {
             'X-Backend-Storage-Policy-Index': container_policy_index}
-        dest_obj = self.swift.get_object_metadata(account, container, obj,
+        try:
-                                                  headers=headers,
+            dest_obj = self.swift.get_object_metadata(
+                account, container, obj, headers=headers,
                 acceptable_statuses=(2, 4))
+        except UnexpectedResponse:
+            self.stats_log('unavailable_destination', '%r (%f) unable to '
+                           'determine the destination timestamp, if any',
+                           path, q_ts)
+            return False
         dest_ts = Timestamp(dest_obj.get('x-backend-timestamp', 0))
         if dest_ts >= q_ts:
             self.stats_log('found_object', '%r (%f) in policy_index %s '

test/unit/container/test_reconciler.py

@@ -1281,6 +1281,44 @@ class TestReconciler(unittest.TestCase):
         self.assertEqual(deleted_container_entries, [])
         self.assertEqual(self.reconciler.stats['retry'], 1)
 
+    def test_object_move_fails_preflight(self):
+        # setup the cluster
+        self._mock_listing({
+            (None, "/.misplaced_objects/3600/1:/AUTH_bob/c/o1"): 3600.123456,
+            (1, '/AUTH_bob/c/o1'): 3600.123457,  # slightly newer
+        })
+        self._mock_oldest_spi({'c': 0})  # destination
+
+        # make the HEAD blow up
+        self.fake_swift.storage_policy[0].register(
+            'HEAD', '/v1/AUTH_bob/c/o1', swob.HTTPServiceUnavailable, {})
+        # turn the crank
+        deleted_container_entries = self._run_once()
+
+        # we did some listings...
+        self.assertEqual(
+            self.fake_swift.calls,
+            [('GET', self.current_container_path),
+             ('GET', '/v1/.misplaced_objects' + listing_qs('')),
+             ('GET', '/v1/.misplaced_objects' + listing_qs('3600')),
+             ('GET', '/v1/.misplaced_objects/3600' + listing_qs('')),
+             ('GET', '/v1/.misplaced_objects/3600' +
+              listing_qs('1:/AUTH_bob/c/o1'))])
+        # ...but we can't even tell whether anything's misplaced or not
+        self.assertEqual(self.reconciler.stats['misplaced_object'], 0)
+        self.assertEqual(self.reconciler.stats['unavailable_destination'], 1)
+        # so we don't try to do any sort of move or cleanup
+        self.assertEqual(self.reconciler.stats['copy_attempt'], 0)
+        self.assertEqual(self.reconciler.stats['cleanup_attempt'], 0)
+        self.assertEqual(self.reconciler.stats['pop_queue'], 0)
+        self.assertEqual(deleted_container_entries, [])
+        # and we'll have to try again later
+        self.assertEqual(self.reconciler.stats['retry'], 1)
+        self.assertEqual(self.fake_swift.storage_policy[1].calls, [])
+        self.assertEqual(
+            self.fake_swift.storage_policy[0].calls,
+            [('HEAD', '/v1/AUTH_bob/c/o1')])
+
     def test_object_move_fails_cleanup(self):
         # setup the cluster
         self._mock_listing({