Improved wait_for benchmark helper

wait_for helper improved to handle and log the status changes of the
resource. wait_for now accepts active_statuses, failure_statuses and attribute
name to perform status checks on the resource.

The wait_for method change is backward compatible so the current usages
are not affected.

The new methods added to handle specific wait cases:
* wait_is_ready - old wait_for symantics. Checks a resource with a
user-provided is_ready callable
* wait_for_status - implements waiting for specific resource status.
Logs the status changes.

Change-Id: I04f917033a02ca21fa1ad6eb55fcdb727ce82090

rally/benchmark/utils.py

@@ -30,16 +30,28 @@ from rally import exceptions
 LOG = logging.getLogger(__name__)
 
 
-def get_status(resource):
+def get_status(resource, status_attr="status"):
-    # workaround for heat resources - using stack_status instead of status
+    """Get the status of a given resource object.
-    if ((hasattr(resource, "stack_status") and
+
-         isinstance(resource.stack_status, six.string_types))):
+    The status is returned in upper case. The status is checked for the
-        return resource.stack_status.upper()
+    standard field names with special cases for Heat and Ceilometer.
-    # workaround for ceilometer alarms - using state instead of status
+
-    if ((hasattr(resource, "state") and
+    :param resource: The resource object or dict.
-         isinstance(resource.state, six.string_types))):
+    :param status_attr: Allows to specify non-standard status fields.
-        return resource.state.upper()
+    :return: The status or "NONE" if it is not available.
-    return getattr(resource, "status", "NONE").upper()
+    """
+
+    for s_attr in ["stack_status", "state", status_attr]:
+        status = getattr(resource, s_attr, None)
+        if isinstance(status, six.string_types):
+            return status.upper()
+
+    # Dict case
+    if ((isinstance(resource, dict) and status_attr in resource.keys() and
+         isinstance(resource[status_attr], six.string_types))):
+        return resource[status_attr].upper()
+
+    return "NONE"
 
 
 class resource_is(object):
@@ -87,15 +99,27 @@ def manager_list_size(sizes):
     return _list
 
 
-def wait_for(resource, is_ready, update_resource=None, timeout=60,
+def wait_for(resource, is_ready=None, ready_statuses=None,
-             check_interval=1):
+             failure_statuses=None, status_attr="status", update_resource=None,
-    """Waits for the given resource to come into the desired state.
+             timeout=60, check_interval=1):
+    """Waits for the given resource to come into the one of the given statuses.
 
-    Uses the readiness check function passed as a parameter and (optionally)
+    The method can be used to check resource for status with a `is_ready`
-    a function that updates the resource being waited for.
+    function or with a list of expected statuses and the status attribute
+
+    In case when the is_ready checker is not provided the resource should have
+    status_attr. It may be an object attribute or a dictionary key. The value
+    of the attribute is checked against ready statuses list and failure
+    statuses. In case of a failure the wait exits with an exception. The
+    resource is updated between iterations with an update_resource call.
 
     :param is_ready: A predicate that should take the resource object and
                      return True iff it is ready to be returned
+    :param ready_statuses: List of statuses which mean that the resource is
+                         ready
+    :param failure_statuses: List of statuses which mean that an error has
+                           occurred while waiting for the resource
+    :param status_attr: The name of the status attribute of the resource
     :param update_resource: Function that should take the resource object
                           and return an 'updated' resource. If set to
                           None, no result updating is performed
@@ -107,23 +131,110 @@ def wait_for(resource, is_ready, update_resource=None, timeout=60,
     :returns: The "ready" resource object
     """
 
+    if is_ready is not None:
+        return wait_is_ready(resource=resource, is_ready=is_ready,
+                             update_resource=update_resource, timeout=timeout,
+                             check_interval=check_interval)
+    else:
+        return wait_for_status(resource=resource,
+                               ready_statuses=ready_statuses,
+                               failure_statuses=failure_statuses,
+                               status_attr=status_attr,
+                               update_resource=update_resource,
+                               timeout=timeout,
+                               check_interval=check_interval)
+
+
+def wait_is_ready(resource, is_ready, update_resource=None,
+                  timeout=60, check_interval=1):
+
+    resource_repr = getattr(resource, "name", repr(resource))
     start = time.time()
+
     while True:
-        # NOTE(boden): mitigate 1st iteration waits by updating immediately
+        if update_resource is not None:
-        if update_resource:
             resource = update_resource(resource)
+
         if is_ready(resource):
-            break
+            return resource
+
         time.sleep(check_interval)
         if time.time() - start > timeout:
             raise exceptions.TimeoutException(
                 desired_status=str(is_ready),
-                resource_name=getattr(resource, "name", repr(resource)),
+                resource_name=resource_repr,
                 resource_type=resource.__class__.__name__,
                 resource_id=getattr(resource, "id", "<no id>"),
                 resource_status=get_status(resource))
 
-    return resource
+
+def wait_for_status(resource, ready_statuses, failure_statuses=None,
+                    status_attr="status", update_resource=None,
+                    timeout=60, check_interval=1):
+
+    resource_repr = getattr(resource, "name", repr(resource))
+    if not isinstance(ready_statuses, (set, list, tuple)):
+        raise ValueError("Ready statuses should be supplied as set, list or "
+                         "tuple")
+    if failure_statuses and not isinstance(failure_statuses,
+                                           (set, list, tuple)):
+        raise ValueError("Failure statuses should be supplied as set, list or "
+                         "tuple")
+
+    # make all statuses upper case
+    ready_statuses = set([s.upper() for s in ready_statuses or []])
+    failure_statuses = set([s.upper() for s in failure_statuses or []])
+
+    if len(ready_statuses & failure_statuses) > 0:
+        raise ValueError(
+            "Can't wait for resource's %s status. Ready and Failure"
+            "statuses conflict." % resource_repr)
+    if not ready_statuses:
+        raise ValueError(
+            "Can't wait for resource's %s status. No ready "
+            "statuses provided" % resource_repr)
+    if not update_resource:
+        raise ValueError(
+            "Can't wait for resource's %s status. No update method."
+            % resource_repr)
+
+    start = time.time()
+
+    latest_status = get_status(resource, status_attr)
+    latest_status_update = start
+
+    while True:
+        resource = update_resource(resource)
+        status = get_status(resource, status_attr)
+
+        if status != latest_status:
+            current_time = time.time()
+            delta = current_time - latest_status_update
+            LOG.debug(
+                "Waiting for resource %(resource)s. Status changed: "
+                "%(latest)s => %(current)s in %(delta)s" %
+                {"resource": resource_repr, "latest": latest_status,
+                 "current": status, "delta": delta})
+
+            latest_status = status
+            latest_status_update = current_time
+
+        if status in ready_statuses:
+            return resource
+        if status in failure_statuses:
+            raise exceptions.GetResourceErrorStatus(
+                resource=resource,
+                status=status,
+                fault="Status in failure list %s" % str(failure_statuses))
+
+        time.sleep(check_interval)
+        if time.time() - start > timeout:
+            raise exceptions.TimeoutException(
+                desired_status=ready_statuses,
+                resource_name=resource_repr,
+                resource_type=resource.__class__.__name__,
+                resource_id=getattr(resource, "id", "<no id>"),
+                resource_status=get_status(resource))
 
 
 def wait_for_delete(resource, update_resource=None, timeout=60,

tests/unit/benchmark/test_utils.py

@@ -203,15 +203,17 @@ class WaitForTestCase(test.TestCase):
 
     def test_wait_for_with_updater(self):
         loaded_resource = utils.wait_for(self.resource,
-                                         self.fake_checker_delayed,
+                                         is_ready=self.fake_checker_delayed,
-                                         self.fake_updater,
+                                         update_resource=self.fake_updater,
-                                         1, self.load_secs / 3)
+                                         timeout=1,
+                                         check_interval=self.load_secs / 3)
         self.assertEqual(loaded_resource, self.resource)
 
     def test_wait_for_no_updater(self):
         loaded_resource = utils.wait_for(self.resource,
-                                         self.fake_checker_delayed,
+                                         is_ready=self.fake_checker_delayed,
-                                         None, 1, self.load_secs / 3)
+                                         update_resource=None, timeout=1,
+                                         check_interval=self.load_secs / 3)
         self.assertEqual(loaded_resource, self.resource)
 
     def test_wait_for_timeout_failure(self):
@@ -222,9 +224,9 @@ class WaitForTestCase(test.TestCase):
         is_ready = utils.resource_is("fake_new_status")
         exc = self.assertRaises(
             exceptions.TimeoutException, utils.wait_for,
-            self.resource, is_ready,
+            self.resource, is_ready=is_ready,
-            self.fake_updater, self.load_secs,
+            update_resource=self.fake_updater, timeout=self.load_secs,
-            self.load_secs / 3)
+            check_interval=self.load_secs / 3)
 
         self.assertEqual(exc.kwargs["resource_name"], "fake_name")
         self.assertEqual(exc.kwargs["resource_id"], "fake_id")
@@ -398,3 +400,61 @@ class ActionBuilderTestCase(test.TestCase):
         for i in range(3):
             mock_calls.append(mock.call("two", "three", c=3, d=4))
         mock_action_two.assert_has_calls(mock_calls)
+
+
+class WaitForStatusTestCase(test.TestCase):
+
+    def test_wrong_ready_statuses_type(self):
+        self.assertRaises(ValueError,
+                          utils.wait_for, {}, ready_statuses="abc")
+
+    def test_wrong_failure_statuses_type(self):
+        self.assertRaises(ValueError,
+                          utils.wait_for, {}, ready_statuses=["abc"],
+                          failure_statuses="abc")
+
+    def test_no_ready_statuses(self):
+        self.assertRaises(ValueError,
+                          utils.wait_for, {}, ready_statuses=[])
+
+    def test_no_update(self):
+        self.assertRaises(ValueError,
+                          utils.wait_for, {}, ready_statuses=["ready"])
+
+    @mock.patch("rally.benchmark.utils.time.sleep")
+    def test_exit_instantly(self, mock_sleep):
+        res = {"status": "ready"}
+        upd = mock.MagicMock(return_value=res)
+
+        utils.wait_for(resource=res, ready_statuses=["ready"],
+                       update_resource=upd)
+
+        upd.assert_called_once_with(res)
+        self.assertFalse(mock_sleep.called)
+
+    @mock.patch("rally.benchmark.utils.time.sleep")
+    @mock.patch("rally.benchmark.utils.time.time", return_value=1)
+    def test_wait_successful(self, mock_time, mock_sleep):
+        res = {"status": "not_ready"}
+        upd = mock.MagicMock(side_effect=[{"status": "not_ready"},
+                                          {"status": "not_ready_yet"},
+                                          {"status": "still_not_ready"},
+                                          {"status": "almost_ready"},
+                                          {"status": "ready"}])
+        utils.wait_for(resource=res, ready_statuses=["ready"],
+                       update_resource=upd)
+        upd.assert_has_calls([mock.call({"status": "not_ready"}),
+                              mock.call({"status": "not_ready"}),
+                              mock.call({"status": "not_ready_yet"}),
+                              mock.call({"status": "still_not_ready"}),
+                              mock.call({"status": "almost_ready"})])
+
+    @mock.patch("rally.benchmark.utils.time.sleep")
+    @mock.patch("rally.benchmark.utils.time.time", return_value=1)
+    def test_wait_failure(self, mock_time, mock_sleep):
+        res = {"status": "not_ready"}
+        upd = mock.MagicMock(side_effect=[{"status": "not_ready"},
+                                          {"status": "fail"}])
+        self.assertRaises(exceptions.GetResourceErrorStatus, utils.wait_for,
+                          resource=res, ready_statuses=["ready"],
+                          failure_statuses=["fail"], update_resource=upd)