Refactored cleaner and added cleanup/utils

Moved context/cleaner to its own package, called cleanup, under
benchmark.context and renamed it to cleanup.py. To improve cohesion,
splitted benchmark/utils.py into two, and all cleanup related functions
were moved to benchmark.context.cleanup.utils. Removed cleanup function
in cleanup.py and references to it, since it's obsolete. Finally updated
all references and unit tests accordingly to reflect the changes.

bp benchmark-context-cleanup-refactor
Change-Id: Iaffc4e35e21b734a7b4cb77ffaf810112054ebae

rally/benchmark/context/cleanup/cleanup.py

@@ -17,7 +17,7 @@ import functools
 import sys
 
 from rally.benchmark.context import base
-from rally.benchmark import utils
+from rally.benchmark.context.cleanup import utils
 from rally.openstack.common.gettextutils import _
 from rally.openstack.common import log as logging
 from rally import osclients
@@ -100,17 +100,3 @@ class ResourceCleaner(base.Context):
             self._cleanup_users_resources()
         if self.admin:
             self._cleanup_admin_resources()
-
-
-def cleanup(services):
-    """Decorates scenario methods requiring a cleanup of resources.
-
-    If a scenario method is not decorated by @cleanup all the resources
-    (nova, glance and cinder) will be cleaned.
-
-    :param services: list of services which will be cleaned.
-    """
-    def wrap(func):
-        func.cleanup_services = services
-        return func
-    return wrap

rally/benchmark/context/cleanup/utils.py

@@ -0,0 +1,123 @@
+# Copyright 2013: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import logging
+
+from rally.benchmark.scenarios.keystone import utils as kutils
+from rally.benchmark import utils as bench_utils
+
+LOG = logging.getLogger(__name__)
+
+
+def delete_cinder_resources(cinder):
+    delete_volume_transfers(cinder)
+    delete_volumes(cinder)
+    delete_volume_snapshots(cinder)
+    delete_volume_backups(cinder)
+
+
+def delete_glance_resources(glance, project_uuid):
+    delete_images(glance, project_uuid)
+
+
+def delete_keystone_resources(keystone):
+    for resource in ["users", "tenants", "services", "roles"]:
+        _delete_single_keystone_resource_type(keystone, resource)
+
+
+def _delete_single_keystone_resource_type(keystone, resource_name):
+    for resource in getattr(keystone, resource_name).list():
+        if kutils.is_temporary(resource):
+            resource.delete()
+
+
+def delete_images(glance, project_uuid):
+    for image in glance.images.list(owner=project_uuid):
+        image.delete()
+    _wait_for_list_statuses(glance.images, statuses=["DELETED"],
+                            list_query={'owner': project_uuid},
+                            timeout=600, check_interval=3)
+
+
+def delete_quotas(admin_clients, project_uuid):
+    # TODO(yingjun): We need to add the cinder part for deleting
+    #                quotas when the new cinderclient released.
+    admin_clients.nova().quotas.delete(project_uuid)
+
+
+def delete_volumes(cinder):
+    for vol in cinder.volumes.list():
+        vol.delete()
+    _wait_for_empty_list(cinder.volumes, timeout=120)
+
+
+def delete_volume_transfers(cinder):
+    for transfer in cinder.transfers.list():
+        transfer.delete()
+    _wait_for_empty_list(cinder.transfers)
+
+
+def delete_volume_snapshots(cinder):
+    for snapshot in cinder.volume_snapshots.list():
+        snapshot.delete()
+    _wait_for_empty_list(cinder.volume_snapshots, timeout=240)
+
+
+def delete_volume_backups(cinder):
+    for backup in cinder.backups.list():
+        backup.delete()
+    _wait_for_empty_list(cinder.backups, timeout=240)
+
+
+def delete_nova_resources(nova):
+    delete_servers(nova)
+    delete_keypairs(nova)
+
+
+def delete_servers(nova):
+    for server in nova.servers.list():
+        server.delete()
+    _wait_for_empty_list(nova.servers, timeout=600, check_interval=3)
+
+
+def delete_keypairs(nova):
+    for keypair in nova.keypairs.list():
+        keypair.delete()
+    _wait_for_empty_list(nova.keypairs)
+
+
+def _wait_for_empty_list(mgr, timeout=10, check_interval=1):
+    _wait_for_list_size(mgr, sizes=[0], timeout=timeout,
+                        check_interval=check_interval)
+
+
+def _wait_for_list_size(mgr, sizes=[0], timeout=10, check_interval=1):
+    bench_utils.wait_for(mgr, is_ready=bench_utils.manager_list_size(sizes),
+                         update_resource=None, timeout=timeout,
+                         check_interval=check_interval)
+
+
+def _wait_for_list_statuses(mgr, statuses, list_query=None,
+                            timeout=10, check_interval=1):
+    list_query = list_query or {}
+
+    def _list_statuses(mgr):
+        for resource in mgr.list(**list_query):
+            if resource.status not in statuses:
+                return False
+        return True
+
+    bench_utils.wait_for(mgr, is_ready=_list_statuses, update_resource=None,
+                         timeout=timeout, check_interval=check_interval)

rally/benchmark/scenarios/vm/vmtasks.py

@@ -15,7 +15,6 @@
 
 import json
 
-from rally.benchmark.context import cleaner as context_cleaner
 from rally.benchmark.scenarios import base
 from rally.benchmark.scenarios.nova import utils as nova_utils
 from rally.benchmark.scenarios.vm import utils as vm_utils
@@ -32,7 +31,6 @@ class VMTasks(nova_utils.NovaScenario, vm_utils.VMScenario):
     def __init__(self, *args, **kwargs):
         super(VMTasks, self).__init__(*args, **kwargs)
 
-    @context_cleaner.cleanup(['nova'])
     @valid.add_validator(valid.image_valid_on_flavor("flavor_id", "image_id"))
     @valid.add_validator(valid.file_exists("script"))
     @valid.add_validator(valid.number("port", minval=1, maxval=65535,

rally/benchmark/utils.py

@@ -20,7 +20,6 @@ import time
 import traceback
 
 from novaclient.v1_1 import servers
-from rally.benchmark.scenarios.keystone import utils as kutils
 from rally import exceptions
 
 
@@ -113,33 +112,9 @@ def wait_for(resource, is_ready, update_resource=None, timeout=60,
     return resource
 
 
-def _wait_for_list_statuses(mgr, statuses, list_query=None,
-                            timeout=10, check_interval=1):
-    list_query = list_query or {}
-
-    def _list_statuses(mgr):
-        for resource in mgr.list(**list_query):
-            if resource.status not in statuses:
-                return False
-        return True
-
-    wait_for(mgr, is_ready=_list_statuses, update_resource=None,
-             timeout=timeout, check_interval=check_interval)
-
-
-def _wait_for_empty_list(mgr, timeout=10, check_interval=1):
-    _wait_for_list_size(mgr, sizes=[0], timeout=timeout,
-                        check_interval=check_interval)
-
-
-def _wait_for_list_size(mgr, sizes=[0], timeout=10, check_interval=1):
-    wait_for(mgr, is_ready=manager_list_size(sizes), update_resource=None,
-             timeout=timeout, check_interval=check_interval)
-
-
 def wait_for_delete(resource, update_resource=None, timeout=60,
                     check_interval=1):
-    """Waits for the full deletion of resource.
+    """Wait for the full deletion of resource.
 
     :param update_resource: Function that should take the resource object
                             and return an 'updated' resource, or raise
@@ -186,80 +161,3 @@ def run_concurrent(concurrent, fn, fn_args):
     pool.join()
 
     return iterator
-
-
-def delete_servers(nova):
-    for server in nova.servers.list():
-        server.delete()
-    _wait_for_empty_list(nova.servers, timeout=600, check_interval=3)
-
-
-def delete_keypairs(nova):
-    for keypair in nova.keypairs.list():
-        keypair.delete()
-    _wait_for_empty_list(nova.keypairs)
-
-
-def delete_images(glance, project_uuid):
-    for image in glance.images.list(owner=project_uuid):
-        image.delete()
-    _wait_for_list_statuses(glance.images, statuses=["DELETED"],
-                            list_query={'owner': project_uuid},
-                            timeout=600, check_interval=3)
-
-
-def delete_volumes(cinder):
-    for vol in cinder.volumes.list():
-        vol.delete()
-    _wait_for_empty_list(cinder.volumes, timeout=120)
-
-
-def delete_volume_transfers(cinder):
-    for transfer in cinder.transfers.list():
-        transfer.delete()
-    _wait_for_empty_list(cinder.transfers)
-
-
-def delete_volume_snapshots(cinder):
-    for snapshot in cinder.volume_snapshots.list():
-        snapshot.delete()
-    _wait_for_empty_list(cinder.volume_snapshots, timeout=240)
-
-
-def delete_volume_backups(cinder):
-    for backup in cinder.backups.list():
-        backup.delete()
-    _wait_for_empty_list(cinder.backups, timeout=240)
-
-
-def delete_keystone_resources(keystone):
-    for resource in ["users", "tenants", "services", "roles"]:
-        _delete_single_keystone_resource_type(keystone, resource)
-
-
-def _delete_single_keystone_resource_type(keystone, resource_name):
-    for resource in getattr(keystone, resource_name).list():
-        if kutils.is_temporary(resource):
-            resource.delete()
-
-
-def delete_nova_resources(nova):
-    delete_servers(nova)
-    delete_keypairs(nova)
-
-
-def delete_cinder_resources(cinder):
-    delete_volume_transfers(cinder)
-    delete_volumes(cinder)
-    delete_volume_snapshots(cinder)
-    delete_volume_backups(cinder)
-
-
-def delete_glance_resources(glance, project_uuid):
-    delete_images(glance, project_uuid)
-
-
-def delete_quotas(admin_clients, project_uuid):
-    # TODO(yingjun): We need to add the cinder part for deleting
-    #                quotas when the new cinderclient released.
-    admin_clients.nova().quotas.delete(project_uuid)

tests/benchmark/context/cleanup/test_cleanup.py

@@ -15,13 +15,13 @@
 
 import mock
 
-from rally.benchmark.context import cleaner as cleaner_ctx
+from rally.benchmark.context.cleanup import cleanup as cleanup_ctx
 
 from tests import fakes
 from tests import test
 
 
-BASE = "rally.benchmark.context.cleaner"
+BASE = "rally.benchmark.context.cleanup.cleanup"
 
 
 class ResourceCleanerTestCase(test.TestCase):
@@ -33,14 +33,14 @@ class ResourceCleanerTestCase(test.TestCase):
             "users": [],
             "tenants": [],
         }
-        resource_cleaner = cleaner_ctx.ResourceCleaner(context)
+        resource_cleaner = cleanup_ctx.ResourceCleaner(context)
         with resource_cleaner:
             resource_cleaner.setup()
 
     def test_with_statement(self):
         fake_user_ctx = fakes.FakeUserContext({}).context
         fake_user_ctx["config"] = {"cleanup": ["nova"]}
-        res_cleaner = cleaner_ctx.ResourceCleaner(fake_user_ctx)
+        res_cleaner = cleanup_ctx.ResourceCleaner(fake_user_ctx)
         res_cleaner.setup()
 
         res_cleaner._cleanup_users_resources = mock.MagicMock()
@@ -60,7 +60,7 @@ class ResourceCleanerTestCase(test.TestCase):
             "config": {"cleanup": ["cinder", "nova"]},
             "admin": {"endpoint": mock.MagicMock()},
         }
-        res_cleaner = cleaner_ctx.ResourceCleaner(context)
+        res_cleaner = cleanup_ctx.ResourceCleaner(context)
 
         mock_clients.return_value.keystone.return_value = 'keystone'
 
@@ -85,7 +85,7 @@ class ResourceCleanerTestCase(test.TestCase):
             "config": {"cleanup": ["cinder", "nova", "glance"]},
             "tenants": [mock.MagicMock()]
         }
-        res_cleaner = cleaner_ctx.ResourceCleaner(context)
+        res_cleaner = cleanup_ctx.ResourceCleaner(context)
 
         with res_cleaner:
             res_cleaner.setup()
@@ -105,7 +105,7 @@ class ResourceCleanerTestCase(test.TestCase):
             "users": [{"endpoint": mock.MagicMock()},
                       {"endpoint": mock.MagicMock()}],
         }
-        res_cleaner = cleaner_ctx.ResourceCleaner(context)
+        res_cleaner = cleanup_ctx.ResourceCleaner(context)
 
         with res_cleaner:
             res_cleaner.setup()
@@ -125,7 +125,7 @@ class ResourceCleanerTestCase(test.TestCase):
             "config": {"cleanup": ["cinder", "nova"]},
             "tenants": [mock.MagicMock()]
         }
-        res_cleaner = cleaner_ctx.ResourceCleaner(context)
+        res_cleaner = cleanup_ctx.ResourceCleaner(context)
 
         with res_cleaner:
             res_cleaner.setup()

tests/benchmark/test_utils.py

@@ -114,7 +114,7 @@ class WaitForTestCase(test.TestCase):
         self.resource = object()
         self.load_secs = 0.01
         self.fake_checker_delayed = self.get_fake_checker_delayed(
-                                                        seconds=self.load_secs)
+            seconds=self.load_secs)
 
     def get_fake_checker_delayed(self, **delay):
         deadline = datetime.datetime.now() + datetime.timedelta(**delay)