diff --git a/swift/proxy/controllers/base.py b/swift/proxy/controllers/base.py index 74b2817bcb..b283417e97 100644 --- a/swift/proxy/controllers/base.py +++ b/swift/proxy/controllers/base.py @@ -2452,9 +2452,10 @@ class Controller(object): def _parse_listing_response(self, req, response): if not is_success(response.status_int): + record_type = req.headers.get('X-Backend-Record-Type') self.logger.warning( - 'Failed to get container listing from %s: %s', - req.path_qs, response.status_int) + 'Failed to get container %s listing from %s: %s', + record_type, req.path_qs, response.status_int) return None try: @@ -2463,9 +2464,10 @@ class Controller(object): raise ValueError('not a list') return data except ValueError as err: + record_type = response.headers.get('X-Backend-Record-Type') self.logger.error( - 'Problem with listing response from %s: %r', - req.path_qs, err) + 'Problem with container %s listing response from %s: %r', + record_type, req.path_qs, err) return None def _get_container_listing(self, req, account, container, headers=None, @@ -2495,7 +2497,7 @@ class Controller(object): self.logger.debug( 'Get listing from %s %s' % (subreq.path_qs, headers)) response = self.app.handle_request(subreq) - data = self._parse_listing_response(req, response) + data = self._parse_listing_response(subreq, response) return data, response def _parse_namespaces(self, req, listing, response): diff --git a/test/unit/proxy/controllers/test_container.py b/test/unit/proxy/controllers/test_container.py index 9bed448ed3..b94c6f4804 100644 --- a/test/unit/proxy/controllers/test_container.py +++ b/test/unit/proxy/controllers/test_container.py @@ -1541,10 +1541,21 @@ class TestGetShardedContainer(BaseTestContainerController): def test_GET_sharded_container_with_deleted_shard(self): req, resp = self._do_test_GET_sharded_container_with_deleted_shards( [404]) - warnings = self.logger.get_lines_for_level('warning') - self.assertEqual(['Failed to get container listing from ' - '%s: 404' % req.path_qs], - warnings) + warning_lines = self.app.logger.get_lines_for_level('warning') + start = 'Failed to get container auto listing from /v1/.shards_a/c_b?' + msg, _, status_txn = warning_lines[0].partition(': ') + self.assertEqual(start, msg[:len(start)]) + actual_qs = msg[len(start):] + actual_params = dict( + urllib.parse.parse_qsl(actual_qs, keep_blank_values=True)) + self.assertEqual({'format': 'json', + 'limit': '10000', + 'marker': '', + 'end_marker': 'b\x00', + 'states': 'listing'}, + actual_params) + self.assertEqual('404', status_txn[:3]) + self.assertFalse(warning_lines[1:]) self.assertEqual(resp.status_int, 503) errors = self.logger.get_lines_for_level('error') self.assertEqual( @@ -1554,9 +1565,21 @@ class TestGetShardedContainer(BaseTestContainerController): def test_GET_sharded_container_with_mix_ok_and_deleted_shard(self): req, resp = self._do_test_GET_sharded_container_with_deleted_shards( [200, 200, 404]) - warnings = self.logger.get_lines_for_level('warning') - self.assertEqual(['Failed to get container listing from ' - '%s: 404' % req.path_qs], warnings) + warning_lines = self.app.logger.get_lines_for_level('warning') + start = 'Failed to get container auto listing from /v1/.shards_a/c_?' + msg, _, status_txn = warning_lines[0].partition(': ') + self.assertEqual(start, msg[:len(start)]) + actual_qs = msg[len(start):] + actual_params = dict( + urllib.parse.parse_qsl(actual_qs, keep_blank_values=True)) + self.assertEqual({'format': 'json', + 'limit': '9998', + 'marker': 'c', + 'end_marker': '', + 'states': 'listing'}, + actual_params) + self.assertEqual('404', status_txn[:3]) + self.assertFalse(warning_lines[1:]) self.assertEqual(resp.status_int, 503) errors = self.logger.get_lines_for_level('error') self.assertEqual( @@ -1566,9 +1589,21 @@ class TestGetShardedContainer(BaseTestContainerController): def test_GET_sharded_container_mix_ok_and_unavailable_shards(self): req, resp = self._do_test_GET_sharded_container_with_deleted_shards( [200, 200, 503]) - warnings = self.logger.get_lines_for_level('warning') - self.assertEqual(['Failed to get container listing from ' - '%s: 503' % req.path_qs], warnings[-1:]) + warning_lines = self.app.logger.get_lines_for_level('warning') + start = 'Failed to get container auto listing from /v1/.shards_a/c_?' + msg, _, status_txn = warning_lines[0].partition(': ') + self.assertEqual(start, msg[:len(start)]) + actual_qs = msg[len(start):] + actual_params = dict( + urllib.parse.parse_qsl(actual_qs, keep_blank_values=True)) + self.assertEqual({'format': 'json', + 'limit': '9998', + 'marker': 'c', + 'end_marker': '', + 'states': 'listing'}, + actual_params) + self.assertEqual('503', status_txn[:3]) + self.assertFalse(warning_lines[1:]) self.assertEqual(resp.status_int, 503) errors = self.logger.get_lines_for_level('error') self.assertEqual( @@ -3940,21 +3975,42 @@ class TestGetPathNamespaceCaching(BaseTestContainerControllerGetPath): self._do_test_GET_namespaces_bad_response_body( {'bad': 'data', 'not': ' a list'}) error_lines = self.logger.get_lines_for_level('error') - self.assertEqual(1, len(error_lines), error_lines) - self.assertIn('Problem with listing response', error_lines[0]) + start = 'Problem with container shard listing response from /v1/a/c?' + msg, _, _ = error_lines[0].partition(':') + self.assertEqual(start, msg[:len(start)]) + actual_qs = msg[len(start):] + actual_params = dict( + urllib.parse.parse_qsl(actual_qs, keep_blank_values=True)) + self.assertEqual({'format': 'json', 'states': 'listing'}, + actual_params) + self.assertFalse(error_lines[1:]) self.logger.clear() self._do_test_GET_namespaces_bad_response_body( [{'not': 'a namespace'}]) error_lines = self.logger.get_lines_for_level('error') - self.assertEqual(1, len(error_lines), error_lines) - self.assertIn('Failed to get namespaces', error_lines[0]) + start = 'Failed to get namespaces from /v1/a/c?' + msg, _, _ = error_lines[0].partition(':') + self.assertEqual(start, msg[:len(start)]) + actual_qs = msg[len(start):] + actual_params = dict( + urllib.parse.parse_qsl(actual_qs, keep_blank_values=True)) + self.assertEqual({'format': 'json', 'states': 'listing'}, + actual_params) + self.assertFalse(error_lines[1:]) self.logger.clear() self._do_test_GET_namespaces_bad_response_body('not a list') error_lines = self.logger.get_lines_for_level('error') - self.assertEqual(1, len(error_lines), error_lines) - self.assertIn('Problem with listing response', error_lines[0]) + start = 'Problem with container shard listing response from /v1/a/c?' + msg, _, _ = error_lines[0].partition(':') + self.assertEqual(start, msg[:len(start)]) + actual_qs = msg[len(start):] + actual_params = dict( + urllib.parse.parse_qsl(actual_qs, keep_blank_values=True)) + self.assertEqual({'format': 'json', 'states': 'listing'}, + actual_params) + self.assertFalse(error_lines[1:]) def _do_test_GET_namespaces_cache_unused(self, sharding_state, req_params, req_hdrs=None): diff --git a/test/unit/proxy/controllers/test_obj.py b/test/unit/proxy/controllers/test_obj.py index 419651b13d..bee295e113 100644 --- a/test/unit/proxy/controllers/test_obj.py +++ b/test/unit/proxy/controllers/test_obj.py @@ -32,7 +32,7 @@ from eventlet.queue import Empty import six from six import StringIO from six.moves import range -from six.moves.urllib.parse import quote +from six.moves.urllib.parse import quote, parse_qsl if six.PY2: from email.parser import FeedParser as EmailFeedParser else: @@ -7816,9 +7816,16 @@ class TestGetUpdateShard(BaseObjectControllerMixin, unittest.TestCase): captured_hdrs.get('X-Backend-Record-Shard-Format')) self.assertIsNone(self.memcache.get('shard-updating-v2/a/c')) self.assertIsNone(actual) - lines = self.app.logger.get_lines_for_level('error') - self.assertEqual(1, len(lines)) - self.assertIn('Problem with listing response from /v1/a/c/o', lines[0]) + + error_lines = self.logger.get_lines_for_level('error') + start = 'Problem with container shard listing response from /v1/a/c?' + msg, _, _ = error_lines[0].partition(':') + self.assertEqual(start, msg[:len(start)]) + actual_qs = msg[len(start):] + actual_params = dict(parse_qsl(actual_qs, keep_blank_values=True)) + self.assertEqual({'format': 'json', 'states': 'updating'}, + actual_params) + self.assertFalse(error_lines[1:]) class TestGetUpdateShardUTF8(TestGetUpdateShard): @@ -7864,18 +7871,34 @@ class TestGetUpdatingNamespacesErrors(BaseObjectControllerMixin, def test_get_namespaces_empty_body(self): error_lines = self._check_get_namespaces_bad_data(b'') - self.assertIn('Problem with listing response', error_lines[0]) + start = 'Problem with container shard listing response from /v1/a/c?' + msg, _, err = error_lines[0].partition(':') + self.assertEqual(start, msg[:len(start)]) + actual_qs = msg[len(start):] + actual_params = dict(parse_qsl(actual_qs, keep_blank_values=True)) + self.assertEqual({'format': 'json', + 'includes': '1_test', + 'states': 'updating'}, + actual_params) if six.PY2: - self.assertIn('No JSON', error_lines[0]) + self.assertIn('No JSON', err) else: - self.assertIn('JSONDecodeError', error_lines[0]) + self.assertIn('JSONDecodeError', err) self.assertFalse(error_lines[1:]) def test_get_namespaces_not_a_list(self): body = json.dumps({}).encode('ascii') error_lines = self._check_get_namespaces_bad_data(body) - self.assertIn('Problem with listing response', error_lines[0]) - self.assertIn('not a list', error_lines[0]) + start = 'Problem with container shard listing response from /v1/a/c?' + msg, _, err = error_lines[0].partition(':') + self.assertEqual(start, msg[:len(start)]) + actual_qs = msg[len(start):] + actual_params = dict(parse_qsl(actual_qs, keep_blank_values=True)) + self.assertEqual({'format': 'json', + 'includes': '1_test', + 'states': 'updating'}, + actual_params) + self.assertIn('ValueError', err) self.assertFalse(error_lines[1:]) def test_get_namespaces_key_missing(self): @@ -7937,8 +7960,16 @@ class TestGetUpdatingNamespacesErrors(BaseObjectControllerMixin, self.assertIsNone(actual) self.assertFalse(self.app.logger.get_lines_for_level('error')) warning_lines = self.app.logger.get_lines_for_level('warning') - self.assertIn('Failed to get container listing', warning_lines[0]) - self.assertIn('/a/c', warning_lines[0]) + start = 'Failed to get container shard listing from /v1/a/c?' + msg, _, status_txn = warning_lines[0].partition(': ') + self.assertEqual(start, msg[:len(start)]) + actual_qs = msg[len(start):] + actual_params = dict(parse_qsl(actual_qs, keep_blank_values=True)) + self.assertEqual({'format': 'json', + 'includes': '1_test', + 'states': 'updating'}, + actual_params) + self.assertEqual('404', status_txn[:3]) self.assertFalse(warning_lines[1:])