From 78753987468cb6b04d0b4e06b432e22f5a7189bd Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Fri, 26 Jul 2019 22:56:08 -0700 Subject: [PATCH] Delete/overwrite symlinks better Previously, when deleting a symlink that points to an xLO, we'd clean up the xLO's segments then delete the symlink, leaving the xLO itself busted. Similar trouble would come from overwriting a symlink pointing to an xLO. Check for a Content-Location in the HEAD response and leave such segments. Co-Authored-By: Clay Gerrard Change-Id: I45b210cf380a68bd88187c91fa2d63a8b2bb709b --- swiftclient/service.py | 6 ++++-- tests/unit/test_service.py | 15 ++++++++------- tests/unit/test_shell.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/swiftclient/service.py b/swiftclient/service.py index 06de091e..5292dc5a 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -2070,7 +2070,8 @@ class SwiftService(object): 'status': 'skipped-changed' }) return res - if not options['leave_segments']: + if not options['leave_segments'] and not headers.get( + 'content-location'): old_manifest = headers.get('x-object-manifest') if is_slo: old_slo_manifest_paths.extend( @@ -2515,7 +2516,8 @@ class SwiftService(object): if not options['leave_segments']: try: headers = conn.head_object(container, obj, - headers=_headers) + headers=_headers, + query_string='symlink=get') old_manifest = headers.get('x-object-manifest') if config_true_value(headers.get('x-static-large-object')): query_string = 'multipart-manifest=delete' diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index 12fbaa00..b7603522 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -312,8 +312,8 @@ class TestServiceDelete(_TestServiceBase): s = SwiftService() r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q) - mock_conn.head_object.assert_called_once_with('test_c', 'test_o', - headers={}) + mock_conn.head_object.assert_called_once_with( + 'test_c', 'test_o', query_string='symlink=get', headers={}) mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_o', query_string=None, response_dict={}, headers={} @@ -335,7 +335,8 @@ class TestServiceDelete(_TestServiceBase): r = s._delete_object(mock_conn, 'test_c', 'test_o', opt_c, mock_q) mock_conn.head_object.assert_called_once_with( - 'test_c', 'test_o', headers={'Skip-Middleware': 'Test'}) + 'test_c', 'test_o', headers={'Skip-Middleware': 'Test'}, + query_string='symlink=get') mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_o', query_string=None, response_dict={}, headers={'Skip-Middleware': 'Test'} @@ -362,8 +363,8 @@ class TestServiceDelete(_TestServiceBase): r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q) after = time.time() - mock_conn.head_object.assert_called_once_with('test_c', 'test_o', - headers={}) + mock_conn.head_object.assert_called_once_with( + 'test_c', 'test_o', query_string='symlink=get', headers={}) mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_o', query_string=None, response_dict={}, headers={} @@ -389,8 +390,8 @@ class TestServiceDelete(_TestServiceBase): s = SwiftService() r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q) - mock_conn.head_object.assert_called_once_with('test_c', 'test_o', - headers={}) + mock_conn.head_object.assert_called_once_with( + 'test_c', 'test_o', query_string='symlink=get', headers={}) mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_o', query_string='multipart-manifest=delete', diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index d9ddb3ee..c9722815 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -832,6 +832,35 @@ class TestShell(unittest.TestCase): sorted(connection.return_value.delete_object.mock_calls) ) + @mock.patch('swiftclient.service.Connection') + def test_upload_over_symlink_to_slo(self, connection): + # Upload delete existing segments + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + connection.return_value.attempts = 0 + connection.return_value.head_object.side_effect = [ + {'x-static-large-object': 'true', + 'content-location': '/v1/a/c/manifest', + 'content-length': '2'}, + ] + connection.return_value.get_object.return_value = ( + {'content-location': '/v1/a/c/manifest'}, + b'[{"name": "container1/old_seg1"},' + b' {"name": "container2/old_seg2"}]' + ) + connection.return_value.put_object.return_value = EMPTY_ETAG + connection.return_value.delete_object.return_value = None + argv = ["", "upload", "container", self.tmpfile] + swiftclient.shell.main(argv) + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile.lstrip('/'), + mock.ANY, + content_length=0, + headers={'x-object-meta-mtime': mock.ANY}, + response_dict={}) + self.assertEqual([], connection.return_value.delete_object.mock_calls) + @mock.patch('swiftclient.service.Connection') def test_upload_leave_slo_segments(self, connection): # Test upload overwriting a manifest respects --leave-segments