From 497a268acd2a866b4180bac9318bc25823c14827 Mon Sep 17 00:00:00 2001 From: Artem Goncharov Date: Fri, 12 Aug 2022 20:28:33 +0200 Subject: [PATCH] Implement project cleanup for object-store Implement cleaning up swift object and containers with project cleanup. When server supports - use bulk-deletion. Change-Id: Ica56afb20bbafb03f43c92fac7e92556198b299d --- openstack/object_store/v1/_proxy.py | 79 +++++++++++++++++++ openstack/object_store/v1/obj.py | 5 +- .../functional/cloud/test_project_cleanup.py | 52 +++++++++++- ...roject-cleanup-swift-f67615e5c3ab8fd8.yaml | 6 ++ 4 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/project-cleanup-swift-f67615e5c3ab8fd8.yaml diff --git a/openstack/object_store/v1/_proxy.py b/openstack/object_store/v1/_proxy.py index 83be650f2..5433a45b2 100644 --- a/openstack/object_store/v1/_proxy.py +++ b/openstack/object_store/v1/_proxy.py @@ -1011,3 +1011,82 @@ class Proxy(proxy.Proxy): self.delete_object(obj, ignore_missing=True) deleted = True return deleted + + # ========== Project Cleanup ========== + def _get_cleanup_dependencies(self): + return { + 'object_store': { + 'before': [] + } + } + + def _service_cleanup( + self, + dry_run=True, + client_status_queue=None, + identified_resources=None, + filters=None, + resource_evaluation_fn=None + ): + is_bulk_delete_supported = False + bulk_delete_max_per_request = None + try: + caps = self.get_info() + except exceptions.SDKException: + pass + else: + bulk_delete = caps.swift.get("bulk_delete", {}) + is_bulk_delete_supported = bulk_delete is not None + bulk_delete_max_per_request = bulk_delete.get( + "max_deletes_per_request", 100) + + elements = [] + for cont in self.containers(): + # Iterate over objects inside container + objects_remaining = False + for obj in self.objects(cont): + need_delete = self._service_cleanup_del_res( + self.delete_object, + obj, + dry_run=True, + client_status_queue=client_status_queue, + identified_resources=identified_resources, + filters=filters, + resource_evaluation_fn=resource_evaluation_fn) + if need_delete: + if not is_bulk_delete_supported and not dry_run: + self.delete_object(obj, cont) + else: + elements.append(f"{cont.name}/{obj.name}") + if len(elements) >= bulk_delete_max_per_request: + self._bulk_delete(elements, dry_run=dry_run) + elements.clear() + else: + objects_remaining = True + + if len(elements) > 0: + self._bulk_delete(elements, dry_run=dry_run) + elements.clear() + + # Eventually delete container itself + if not objects_remaining: + self._service_cleanup_del_res( + self.delete_container, + cont, + dry_run=dry_run, + client_status_queue=client_status_queue, + identified_resources=identified_resources, + filters=filters, + resource_evaluation_fn=resource_evaluation_fn) + + def _bulk_delete(self, elements, dry_run=False): + data = "\n".join([parse.quote(x) for x in elements]) + if not dry_run: + self.delete( + "?bulk-delete", + data=data, + headers={ + 'Content-Type': 'text/plain', + 'Accept': 'application/json' + } + ) diff --git a/openstack/object_store/v1/obj.py b/openstack/object_store/v1/obj.py index 567e02094..26024cb67 100644 --- a/openstack/object_store/v1/obj.py +++ b/openstack/object_store/v1/obj.py @@ -163,7 +163,10 @@ class Object(_base.BaseResource): timestamp = resource.Header("x-timestamp") #: The date and time that the object was created or the last #: time that the metadata was changed. - last_modified_at = resource.Header("last-modified", alias='_last_modified') + last_modified_at = resource.Header( + "last-modified", + alias='_last_modified', + aka='updated_at') # Headers for PUT and POST requests #: Set to chunked to enable chunked transfer encoding. If used, diff --git a/openstack/tests/functional/cloud/test_project_cleanup.py b/openstack/tests/functional/cloud/test_project_cleanup.py index 9957e57d3..c25346cba 100644 --- a/openstack/tests/functional/cloud/test_project_cleanup.py +++ b/openstack/tests/functional/cloud/test_project_cleanup.py @@ -196,7 +196,57 @@ class TestProjectCleanup(base.BaseFunctionalTest): dry_run=False, wait_timeout=600, status_queue=status_queue) - objects = [] while not status_queue.empty(): objects.append(status_queue.get()) + + def test_cleanup_swift(self): + if not self.user_cloud.has_service('object-store'): + self.skipTest('Object service is requred, but not available') + + status_queue = queue.Queue() + self.conn.object_store.create_container('test_cleanup') + for i in range(1, 10): + self.conn.object_store.create_object( + "test_cleanup", f"test{i}", data="test{i}") + + # First round - check no resources are old enough + self.conn.project_cleanup( + dry_run=True, + wait_timeout=120, + status_queue=status_queue, + filters={'updated_at': '2000-01-01'}) + + self.assertTrue(status_queue.empty()) + + # Second round - filters set too low + self.conn.project_cleanup( + dry_run=True, + wait_timeout=120, + status_queue=status_queue, + filters={'updated_at': '2200-01-01'}) + objects = [] + while not status_queue.empty(): + objects.append(status_queue.get()) + + # At least known objects should be identified + obj_names = list(obj.name for obj in objects) + self.assertIn('test1', obj_names) + + # Ensure object still exists + obj = self.conn.object_store.get_object( + "test1", "test_cleanup") + self.assertIsNotNone(obj) + + # Last round - do a real cleanup + self.conn.project_cleanup( + dry_run=False, + wait_timeout=600, + status_queue=status_queue) + + objects.clear() + while not status_queue.empty(): + objects.append(status_queue.get()) + self.assertIsNone( + self.conn.get_container('test_container') + ) diff --git a/releasenotes/notes/project-cleanup-swift-f67615e5c3ab8fd8.yaml b/releasenotes/notes/project-cleanup-swift-f67615e5c3ab8fd8.yaml new file mode 100644 index 000000000..8e62029ed --- /dev/null +++ b/releasenotes/notes/project-cleanup-swift-f67615e5c3ab8fd8.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Project cleanup now supports cleaning Swift (object-store). If supported + by the server bulk deletion is used. Currently only filtering based on + updated_at (last_modified) is supported.