Allow resources to be skipped on project cleanup

Story: 2010370
Task: 46596
Change-Id: Id6d68e40656c92910f491fe3b66e40c69b44e352
Signed-off-by: Jan Hartkopf <jhartkopf@inovex.de>

openstack/block_storage/v3/_proxy.py

@@ -1675,6 +1675,7 @@ class Proxy(_base_proxy.BaseBlockStorageProxy):
         identified_resources=None,
         filters=None,
         resource_evaluation_fn=None,
+        skip_resources=None,
     ):
         # It is not possible to delete backup if there are dependent backups.
         # In order to be able to do cleanup those is required to have multiple
@@ -1684,6 +1685,7 @@ class Proxy(_base_proxy.BaseBlockStorageProxy):
         # required to limit amount of iterations we do (currently pick 10).  In
         # dry_run all those iterations are doing not what we want, therefore
         # only iterate in a real cleanup mode.
+        if not self.should_skip_resource_cleanup("backup", skip_resources):
             if dry_run:
                 # Just iterate and evaluate backups in dry_run mode
                 for obj in self.backups(details=False):
@@ -1735,6 +1737,7 @@ class Proxy(_base_proxy.BaseBlockStorageProxy):
                             # Well, did our best, still try further
                             pass
 
+        if not self.should_skip_resource_cleanup("snapshot", skip_resources):
             snapshots = []
             for obj in self.snapshots(details=False):
                 need_delete = self._service_cleanup_del_res(
@@ -1757,6 +1760,7 @@ class Proxy(_base_proxy.BaseBlockStorageProxy):
                     # Well, did our best, still try further
                     pass
 
+        if not self.should_skip_resource_cleanup("volume", skip_resources):
             for obj in self.volumes(details=True):
                 self._service_cleanup_del_res(
                     self.delete_volume,

openstack/cloud/openstackcloud.py

@@ -813,6 +813,7 @@ class _OpenStackCloudMixin:
         status_queue=None,
         filters=None,
         resource_evaluation_fn=None,
+        skip_resources=None,
     ):
         """Cleanup the project resources.
 
@@ -829,6 +830,8 @@ class _OpenStackCloudMixin:
         :param resource_evaluation_fn: A callback function, which will be
             invoked for each resurce and must return True/False depending on
             whether resource need to be deleted or not.
+        :param skip_resources: List of specific resources whose cleanup should
+            be skipped.
         """
         dependencies = {}
         get_dep_fn_name = '_get_cleanup_dependencies'
@@ -879,6 +882,7 @@ class _OpenStackCloudMixin:
                             identified_resources=cleanup_resources,
                             filters=filters,
                             resource_evaluation_fn=resource_evaluation_fn,
+                            skip_resources=skip_resources,
                         )
             except exceptions.ServiceDisabledException:
                 # same reason as above

openstack/compute/v2/_proxy.py

@@ -2548,7 +2548,11 @@ class Proxy(proxy.Proxy):
         identified_resources=None,
         filters=None,
         resource_evaluation_fn=None,
+        skip_resources=None,
     ):
+        if self.should_skip_resource_cleanup("server", skip_resources):
+            return
+
         servers = []
         for obj in self.servers():
             need_delete = self._service_cleanup_del_res(

openstack/dns/v2/_proxy.py

@@ -667,7 +667,9 @@ class Proxy(proxy.Proxy):
         identified_resources=None,
         filters=None,
         resource_evaluation_fn=None,
+        skip_resources=None,
     ):
+        if not self.should_skip_resource_cleanup("zone", skip_resources):
             # Delete all zones
             for obj in self.zones():
                 self._service_cleanup_del_res(
@@ -679,6 +681,10 @@ class Proxy(proxy.Proxy):
                     filters=filters,
                     resource_evaluation_fn=resource_evaluation_fn,
                 )
+
+        if not self.should_skip_resource_cleanup(
+            "floating_ip", skip_resources
+        ):
             # Unset all floatingIPs
             # NOTE: FloatingIPs are not cleaned when filters are set
             for obj in self.floating_ips():

openstack/network/v2/_proxy.py

@@ -6212,8 +6212,13 @@ class Proxy(proxy.Proxy, Generic[T]):
         identified_resources=None,
         filters=None,
         resource_evaluation_fn=None,
+        skip_resources=None,
     ):
         project_id = self.get_project_id()
+
+        if not self.should_skip_resource_cleanup(
+            "floating_ip", skip_resources
+        ):
             # Delete floating_ips in the project if no filters defined OR all
             # filters are matching and port_id is empty
             for obj in self.ips(project_id=project_id):
@@ -6227,6 +6232,9 @@ class Proxy(proxy.Proxy, Generic[T]):
                     resource_evaluation_fn=fip_cleanup_evaluation,
                 )
 
+        if not self.should_skip_resource_cleanup(
+            "security_group", skip_resources
+        ):
             # Delete (try to delete) all security groups in the project
             # Let's hope we can't drop SG in use
             for obj in self.security_groups(project_id=project_id):
@@ -6241,12 +6249,19 @@ class Proxy(proxy.Proxy, Generic[T]):
                         resource_evaluation_fn=resource_evaluation_fn,
                     )
 
+        if (
+            not self.should_skip_resource_cleanup("network", skip_resources)
+            and not self.should_skip_resource_cleanup("port", skip_resources)
+            and not self.should_skip_resource_cleanup("subnet", skip_resources)
+        ):
             # Networks are crazy, try to delete router+net+subnet
             # if there are no "other" ports allocated on the net
             for net in self.networks(project_id=project_id):
                 network_has_ports_allocated = False
                 router_if = list()
-            for port in self.ports(project_id=project_id, network_id=net.id):
+                for port in self.ports(
+                    project_id=project_id, network_id=net.id
+                ):
                     self.log.debug('Looking at port %s' % port)
                     if port.device_owner in [
                         'network:router_interface',
@@ -6307,7 +6322,9 @@ class Proxy(proxy.Proxy, Generic[T]):
                         resource_evaluation_fn=None,
                     )
                 # Drop ports not belonging to anybody
-            for port in self.ports(project_id=project_id, network_id=net.id):
+                for port in self.ports(
+                    project_id=project_id, network_id=net.id
+                ):
                     if port.device_owner is None or port.device_owner == '':
                         self._service_cleanup_del_res(
                             self.delete_port,
@@ -6320,7 +6337,9 @@ class Proxy(proxy.Proxy, Generic[T]):
                         )
 
                 # Drop all subnets in the net (no further conditions)
-            for obj in self.subnets(project_id=project_id, network_id=net.id):
+                for obj in self.subnets(
+                    project_id=project_id, network_id=net.id
+                ):
                     self._service_cleanup_del_res(
                         self.delete_subnet,
                         obj,
@@ -6342,7 +6361,14 @@ class Proxy(proxy.Proxy, Generic[T]):
                     filters=None,
                     resource_evaluation_fn=None,
                 )
+        else:
+            self.log.debug(
+                "Skipping cleanup of networks, ports and subnets "
+                "as those resources require none of them to be "
+                "excluded, but at least one should be kept"
+            )
 
+        if not self.should_skip_resource_cleanup("router", skip_resources):
             # It might happen, that we have routers not attached to anything
             for obj in self.routers():
                 ports = list(self.ports(device_id=obj.id))

openstack/object_store/v1/_proxy.py

@@ -1130,7 +1130,13 @@ class Proxy(proxy.Proxy):
         identified_resources=None,
         filters=None,
         resource_evaluation_fn=None,
+        skip_resources=None,
     ):
+        if self.should_skip_resource_cleanup(
+            "container", skip_resources
+        ) or self.should_skip_resource_cleanup("object", skip_resources):
+            return
+
         is_bulk_delete_supported = False
         bulk_delete_max_per_request = None
         try:

openstack/orchestration/v1/_proxy.py

@@ -549,7 +549,11 @@ class Proxy(proxy.Proxy):
         identified_resources=None,
         filters=None,
         resource_evaluation_fn=None,
+        skip_resources=None,
     ):
+        if self.should_skip_resource_cleanup("stack", skip_resources):
+            return
+
         stacks = []
         for obj in self.stacks():
             need_delete = self._service_cleanup_del_res(

openstack/proxy.py

@@ -100,7 +100,7 @@ class Proxy(adapter.Adapter, Generic[T]):
         influxdb_config=None,
         influxdb_client=None,
         *args,
-        **kwargs
+        **kwargs,
     ):
         # NOTE(dtantsur): keystoneauth defaults retriable_status_codes to None,
         # override it with a class-level value.
@@ -144,7 +144,7 @@ class Proxy(adapter.Adapter, Generic[T]):
         connect_retries=1,
         global_request_id=None,
         *args,
-        **kwargs
+        **kwargs,
     ):
         conn = self._get_connection()
         if not global_request_id:
@@ -180,7 +180,7 @@ class Proxy(adapter.Adapter, Generic[T]):
                             connect_retries=connect_retries,
                             raise_exc=raise_exc,
                             global_request_id=global_request_id,
-                            **kwargs
+                            **kwargs,
                         ),
                     ),
                     expiration_time=expiration_time,
@@ -196,7 +196,7 @@ class Proxy(adapter.Adapter, Generic[T]):
                     connect_retries=connect_retries,
                     raise_exc=raise_exc,
                     global_request_id=global_request_id,
-                    **kwargs
+                    **kwargs,
                 )
 
             for h in response.history:
@@ -623,7 +623,7 @@ class Proxy(adapter.Adapter, Generic[T]):
         requires_id=True,
         base_path=None,
         skip_cache=False,
-        **attrs
+        **attrs,
     ):
         """Fetch a resource
 
@@ -665,7 +665,7 @@ class Proxy(adapter.Adapter, Generic[T]):
         paginated=True,
         base_path=None,
         jmespath_filters=None,
-        **attrs
+        **attrs,
     ) -> Generator[T, None, None]:
         """List a resource
 
@@ -736,6 +736,7 @@ class Proxy(adapter.Adapter, Generic[T]):
         identified_resources=None,
         filters=None,
         resource_evaluation_fn=None,
+        skip_resources=None,
     ):
         return None
 
@@ -814,6 +815,20 @@ class Proxy(adapter.Adapter, Generic[T]):
         else:
             return False
 
+    def should_skip_resource_cleanup(self, resource=None, skip_resources=None):
+        if resource is None or skip_resources is None:
+            return False
+
+        resource_name = f"{self.service_type.replace('-', '_')}.{resource}"
+
+        if resource_name in skip_resources:
+            self.log.debug(
+                f"Skipping resource {resource_name} " "in project cleanup"
+            )
+            return True
+
+        return False
+
 
 # TODO(stephenfin): Remove this and all users. Use of this generally indicates
 # a missing Resource type.

openstack/tests/unit/test_proxy.py

@@ -728,6 +728,7 @@ class TestProxyCleanup(base.TestCase):
         self.res_no_updated.created_at = '2020-01-02T03:04:05'
 
         self.sot = proxy.Proxy(self.session)
+        self.sot.service_type = "block-storage"
 
         self.delete_mock = mock.Mock()
 
@@ -867,3 +868,12 @@ class TestProxyCleanup(base.TestCase):
             )
         )
         self.assertEqual(self.res, q.get_nowait())
+
+    def test_should_skip_resource_cleanup(self):
+        excluded = ["block_storage.backup"]
+        self.assertTrue(
+            self.sot.should_skip_resource_cleanup("backup", excluded)
+        )
+        self.assertFalse(
+            self.sot.should_skip_resource_cleanup("volume", excluded)
+        )

releasenotes/notes/project-cleanup-exclude-option-65cba962eaa5b61a.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Project cleanup now supports skipping specific resources,
+    which will be kept as-is. Resource names are based on the
+    resource registry names, e. g. "block_storage.volume".