Merge "Implement project cleanup for object-store"

openstack/object_store/v1/_proxy.py

@@ -1011,3 +1011,82 @@ class Proxy(proxy.Proxy):
                 self.delete_object(obj, ignore_missing=True)
                 deleted = True
         return deleted
+
+    # ========== Project Cleanup ==========
+    def _get_cleanup_dependencies(self):
+        return {
+            'object_store': {
+                'before': []
+            }
+        }
+
+    def _service_cleanup(
+        self,
+        dry_run=True,
+        client_status_queue=None,
+        identified_resources=None,
+        filters=None,
+        resource_evaluation_fn=None
+    ):
+        is_bulk_delete_supported = False
+        bulk_delete_max_per_request = None
+        try:
+            caps = self.get_info()
+        except exceptions.SDKException:
+            pass
+        else:
+            bulk_delete = caps.swift.get("bulk_delete", {})
+            is_bulk_delete_supported = bulk_delete is not None
+            bulk_delete_max_per_request = bulk_delete.get(
+                "max_deletes_per_request", 100)
+
+        elements = []
+        for cont in self.containers():
+            # Iterate over objects inside container
+            objects_remaining = False
+            for obj in self.objects(cont):
+                need_delete = self._service_cleanup_del_res(
+                    self.delete_object,
+                    obj,
+                    dry_run=True,
+                    client_status_queue=client_status_queue,
+                    identified_resources=identified_resources,
+                    filters=filters,
+                    resource_evaluation_fn=resource_evaluation_fn)
+                if need_delete:
+                    if not is_bulk_delete_supported and not dry_run:
+                        self.delete_object(obj, cont)
+                    else:
+                        elements.append(f"{cont.name}/{obj.name}")
+                        if len(elements) >= bulk_delete_max_per_request:
+                            self._bulk_delete(elements, dry_run=dry_run)
+                            elements.clear()
+                else:
+                    objects_remaining = True
+
+            if len(elements) > 0:
+                self._bulk_delete(elements, dry_run=dry_run)
+                elements.clear()
+
+            # Eventually delete container itself
+            if not objects_remaining:
+                self._service_cleanup_del_res(
+                    self.delete_container,
+                    cont,
+                    dry_run=dry_run,
+                    client_status_queue=client_status_queue,
+                    identified_resources=identified_resources,
+                    filters=filters,
+                    resource_evaluation_fn=resource_evaluation_fn)
+
+    def _bulk_delete(self, elements, dry_run=False):
+        data = "\n".join([parse.quote(x) for x in elements])
+        if not dry_run:
+            self.delete(
+                "?bulk-delete",
+                data=data,
+                headers={
+                    'Content-Type': 'text/plain',
+                    'Accept': 'application/json'
+                }
+            )

openstack/object_store/v1/obj.py

@@ -163,7 +163,10 @@ class Object(_base.BaseResource):
     timestamp = resource.Header("x-timestamp")
     #: The date and time that the object was created or the last
     #: time that the metadata was changed.
-    last_modified_at = resource.Header("last-modified", alias='_last_modified')
+    last_modified_at = resource.Header(
+        "last-modified",
+        alias='_last_modified',
+        aka='updated_at')
 
     # Headers for PUT and POST requests
     #: Set to chunked to enable chunked transfer encoding. If used,

openstack/tests/functional/cloud/test_project_cleanup.py

@@ -196,7 +196,57 @@ class TestProjectCleanup(base.BaseFunctionalTest):
             dry_run=False,
             wait_timeout=600,
             status_queue=status_queue)
-
         objects = []
         while not status_queue.empty():
             objects.append(status_queue.get())
+
+    def test_cleanup_swift(self):
+        if not self.user_cloud.has_service('object-store'):
+            self.skipTest('Object service is requred, but not available')
+
+        status_queue = queue.Queue()
+        self.conn.object_store.create_container('test_cleanup')
+        for i in range(1, 10):
+            self.conn.object_store.create_object(
+                "test_cleanup", f"test{i}", data="test{i}")
+
+        # First round - check no resources are old enough
+        self.conn.project_cleanup(
+            dry_run=True,
+            wait_timeout=120,
+            status_queue=status_queue,
+            filters={'updated_at': '2000-01-01'})
+
+        self.assertTrue(status_queue.empty())
+
+        # Second round - filters set too low
+        self.conn.project_cleanup(
+            dry_run=True,
+            wait_timeout=120,
+            status_queue=status_queue,
+            filters={'updated_at': '2200-01-01'})
+        objects = []
+        while not status_queue.empty():
+            objects.append(status_queue.get())
+
+        # At least known objects should be identified
+        obj_names = list(obj.name for obj in objects)
+        self.assertIn('test1', obj_names)
+
+        # Ensure object still exists
+        obj = self.conn.object_store.get_object(
+            "test1", "test_cleanup")
+        self.assertIsNotNone(obj)
+
+        # Last round - do a real cleanup
+        self.conn.project_cleanup(
+            dry_run=False,
+            wait_timeout=600,
+            status_queue=status_queue)
+
+        objects.clear()
+        while not status_queue.empty():
+            objects.append(status_queue.get())
+        self.assertIsNone(
+            self.conn.get_container('test_container')
+        )

releasenotes/notes/project-cleanup-swift-f67615e5c3ab8fd8.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Project cleanup now supports cleaning Swift (object-store). If supported
+    by the server bulk deletion is used. Currently only filtering based on
+    updated_at (last_modified) is supported.