Add wait_for_resource_deletion for swift api clients

Currently today we dont have any way in swift to verify that resources were deleted
before moving to the next command
In current code there was hardcoded sleep for 2 seconds instead of checking if
resource really deleted.

Added to the current cleanup :
Implement is_resource_deleted for object_client and container_client
After remove action we wait/ verify till resource really deleted
Remove hardcoded sleep for 2 seconds
Remove ignore for not found in reomval , if we hit on it means something
wrong in our code.

Change-Id: I32f37f8e874a3510bb1af6db45a1b9a8d2fed543

tempest/api/object_storage/base.py

@@ -13,12 +13,9 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import time
-
 from tempest.common import custom_matchers
 from tempest import config
 from tempest.lib.common.utils import data_utils
-from tempest.lib.common.utils import test_utils
 from tempest.lib import exceptions as lib_exc
 import tempest.test
 
@@ -50,12 +47,11 @@ def delete_containers(containers, container_client, object_client):
             _, objlist = container_client.list_container_objects(cont, params)
             # delete every object in the container
             for obj in objlist:
-                test_utils.call_and_ignore_notfound_exc(
+                object_client.delete_object(cont, obj['name'])
-                    object_client.delete_object, cont, obj['name'])
+                object_client.wait_for_resource_deletion(obj['name'], cont)
-            # sleep 2 seconds to sync the deletion of the objects
+            # Verify resource deletion
-            # in HA deployment
-            time.sleep(2)
             container_client.delete_container(cont)
+            container_client.wait_for_resource_deletion(cont)
         except lib_exc.NotFound:
             pass
 

tempest/lib/common/rest_client.py

@@ -890,7 +890,7 @@ class RestClient(object):
             return True
         return 'exceed' in resp_body.get('message', 'blabla')
 
-    def wait_for_resource_deletion(self, id):
+    def wait_for_resource_deletion(self, id, *args, **kwargs):
         """Waits for a resource to be deleted
 
         This method will loop over is_resource_deleted until either
@@ -903,7 +903,7 @@ class RestClient(object):
         """
         start_time = int(time.time())
         while True:
-            if self.is_resource_deleted(id):
+            if self.is_resource_deleted(id, *args, **kwargs):
                 return
             if int(time.time()) - start_time >= self.build_timeout:
                 message = ('Failed to delete %(resource_type)s %(id)s within '

tempest/lib/services/object_storage/container_client.py

@@ -20,10 +20,18 @@ from oslo_serialization import jsonutils as json
 from six.moves.urllib import parse as urllib
 
 from tempest.lib.common import rest_client
+from tempest.lib import exceptions
 
 
 class ContainerClient(rest_client.RestClient):
 
+    def is_resource_deleted(self, container):
+        try:
+            self.list_container_metadata(container)
+        except exceptions.NotFound:
+            return True
+        return False
+
     def update_container(self, container_name, **headers):
         """Creates or Updates a container
 

tempest/lib/services/object_storage/object_client.py

@@ -23,6 +23,13 @@ from tempest.lib import exceptions
 
 class ObjectClient(rest_client.RestClient):
 
+    def is_resource_deleted(self, object_name, container):
+        try:
+            self.get_object(container, object_name)
+        except exceptions.NotFound:
+            return True
+        return False
+
     def create_object(self, container, object_name, data,
                       params=None, metadata=None, headers=None,
                       chunked=False):