From 1bd6977194df45c556a26c0fc80d2e1cfdfe1f26 Mon Sep 17 00:00:00 2001 From: Nikita Konovalov Date: Fri, 10 Apr 2015 17:30:43 +0300 Subject: [PATCH] Improved wait_for benchmark helper wait_for helper improved to handle and log the status changes of the resource. wait_for now accepts active_statuses, failure_statuses and attribute name to perform status checks on the resource. The wait_for method change is backward compatible so the current usages are not affected. The new methods added to handle specific wait cases: * wait_is_ready - old wait_for symantics. Checks a resource with a user-provided is_ready callable * wait_for_status - implements waiting for specific resource status. Logs the status changes. Change-Id: I04f917033a02ca21fa1ad6eb55fcdb727ce82090 --- rally/benchmark/utils.py | 151 +++++++++++++++++++++++++---- tests/unit/benchmark/test_utils.py | 76 +++++++++++++-- 2 files changed, 199 insertions(+), 28 deletions(-) diff --git a/rally/benchmark/utils.py b/rally/benchmark/utils.py index f403ea2da6..9eaa5b5271 100644 --- a/rally/benchmark/utils.py +++ b/rally/benchmark/utils.py @@ -30,16 +30,28 @@ from rally import exceptions LOG = logging.getLogger(__name__) -def get_status(resource): - # workaround for heat resources - using stack_status instead of status - if ((hasattr(resource, "stack_status") and - isinstance(resource.stack_status, six.string_types))): - return resource.stack_status.upper() - # workaround for ceilometer alarms - using state instead of status - if ((hasattr(resource, "state") and - isinstance(resource.state, six.string_types))): - return resource.state.upper() - return getattr(resource, "status", "NONE").upper() +def get_status(resource, status_attr="status"): + """Get the status of a given resource object. + + The status is returned in upper case. The status is checked for the + standard field names with special cases for Heat and Ceilometer. + + :param resource: The resource object or dict. + :param status_attr: Allows to specify non-standard status fields. + :return: The status or "NONE" if it is not available. + """ + + for s_attr in ["stack_status", "state", status_attr]: + status = getattr(resource, s_attr, None) + if isinstance(status, six.string_types): + return status.upper() + + # Dict case + if ((isinstance(resource, dict) and status_attr in resource.keys() and + isinstance(resource[status_attr], six.string_types))): + return resource[status_attr].upper() + + return "NONE" class resource_is(object): @@ -87,15 +99,27 @@ def manager_list_size(sizes): return _list -def wait_for(resource, is_ready, update_resource=None, timeout=60, - check_interval=1): - """Waits for the given resource to come into the desired state. +def wait_for(resource, is_ready=None, ready_statuses=None, + failure_statuses=None, status_attr="status", update_resource=None, + timeout=60, check_interval=1): + """Waits for the given resource to come into the one of the given statuses. - Uses the readiness check function passed as a parameter and (optionally) - a function that updates the resource being waited for. + The method can be used to check resource for status with a `is_ready` + function or with a list of expected statuses and the status attribute + + In case when the is_ready checker is not provided the resource should have + status_attr. It may be an object attribute or a dictionary key. The value + of the attribute is checked against ready statuses list and failure + statuses. In case of a failure the wait exits with an exception. The + resource is updated between iterations with an update_resource call. :param is_ready: A predicate that should take the resource object and return True iff it is ready to be returned + :param ready_statuses: List of statuses which mean that the resource is + ready + :param failure_statuses: List of statuses which mean that an error has + occurred while waiting for the resource + :param status_attr: The name of the status attribute of the resource :param update_resource: Function that should take the resource object and return an 'updated' resource. If set to None, no result updating is performed @@ -107,23 +131,110 @@ def wait_for(resource, is_ready, update_resource=None, timeout=60, :returns: The "ready" resource object """ + if is_ready is not None: + return wait_is_ready(resource=resource, is_ready=is_ready, + update_resource=update_resource, timeout=timeout, + check_interval=check_interval) + else: + return wait_for_status(resource=resource, + ready_statuses=ready_statuses, + failure_statuses=failure_statuses, + status_attr=status_attr, + update_resource=update_resource, + timeout=timeout, + check_interval=check_interval) + + +def wait_is_ready(resource, is_ready, update_resource=None, + timeout=60, check_interval=1): + + resource_repr = getattr(resource, "name", repr(resource)) start = time.time() + while True: - # NOTE(boden): mitigate 1st iteration waits by updating immediately - if update_resource: + if update_resource is not None: resource = update_resource(resource) + if is_ready(resource): - break + return resource + time.sleep(check_interval) if time.time() - start > timeout: raise exceptions.TimeoutException( desired_status=str(is_ready), - resource_name=getattr(resource, "name", repr(resource)), + resource_name=resource_repr, resource_type=resource.__class__.__name__, resource_id=getattr(resource, "id", ""), resource_status=get_status(resource)) - return resource + +def wait_for_status(resource, ready_statuses, failure_statuses=None, + status_attr="status", update_resource=None, + timeout=60, check_interval=1): + + resource_repr = getattr(resource, "name", repr(resource)) + if not isinstance(ready_statuses, (set, list, tuple)): + raise ValueError("Ready statuses should be supplied as set, list or " + "tuple") + if failure_statuses and not isinstance(failure_statuses, + (set, list, tuple)): + raise ValueError("Failure statuses should be supplied as set, list or " + "tuple") + + # make all statuses upper case + ready_statuses = set([s.upper() for s in ready_statuses or []]) + failure_statuses = set([s.upper() for s in failure_statuses or []]) + + if len(ready_statuses & failure_statuses) > 0: + raise ValueError( + "Can't wait for resource's %s status. Ready and Failure" + "statuses conflict." % resource_repr) + if not ready_statuses: + raise ValueError( + "Can't wait for resource's %s status. No ready " + "statuses provided" % resource_repr) + if not update_resource: + raise ValueError( + "Can't wait for resource's %s status. No update method." + % resource_repr) + + start = time.time() + + latest_status = get_status(resource, status_attr) + latest_status_update = start + + while True: + resource = update_resource(resource) + status = get_status(resource, status_attr) + + if status != latest_status: + current_time = time.time() + delta = current_time - latest_status_update + LOG.debug( + "Waiting for resource %(resource)s. Status changed: " + "%(latest)s => %(current)s in %(delta)s" % + {"resource": resource_repr, "latest": latest_status, + "current": status, "delta": delta}) + + latest_status = status + latest_status_update = current_time + + if status in ready_statuses: + return resource + if status in failure_statuses: + raise exceptions.GetResourceErrorStatus( + resource=resource, + status=status, + fault="Status in failure list %s" % str(failure_statuses)) + + time.sleep(check_interval) + if time.time() - start > timeout: + raise exceptions.TimeoutException( + desired_status=ready_statuses, + resource_name=resource_repr, + resource_type=resource.__class__.__name__, + resource_id=getattr(resource, "id", ""), + resource_status=get_status(resource)) def wait_for_delete(resource, update_resource=None, timeout=60, diff --git a/tests/unit/benchmark/test_utils.py b/tests/unit/benchmark/test_utils.py index 11fa86f137..42452a2c8f 100644 --- a/tests/unit/benchmark/test_utils.py +++ b/tests/unit/benchmark/test_utils.py @@ -203,15 +203,17 @@ class WaitForTestCase(test.TestCase): def test_wait_for_with_updater(self): loaded_resource = utils.wait_for(self.resource, - self.fake_checker_delayed, - self.fake_updater, - 1, self.load_secs / 3) + is_ready=self.fake_checker_delayed, + update_resource=self.fake_updater, + timeout=1, + check_interval=self.load_secs / 3) self.assertEqual(loaded_resource, self.resource) def test_wait_for_no_updater(self): loaded_resource = utils.wait_for(self.resource, - self.fake_checker_delayed, - None, 1, self.load_secs / 3) + is_ready=self.fake_checker_delayed, + update_resource=None, timeout=1, + check_interval=self.load_secs / 3) self.assertEqual(loaded_resource, self.resource) def test_wait_for_timeout_failure(self): @@ -222,9 +224,9 @@ class WaitForTestCase(test.TestCase): is_ready = utils.resource_is("fake_new_status") exc = self.assertRaises( exceptions.TimeoutException, utils.wait_for, - self.resource, is_ready, - self.fake_updater, self.load_secs, - self.load_secs / 3) + self.resource, is_ready=is_ready, + update_resource=self.fake_updater, timeout=self.load_secs, + check_interval=self.load_secs / 3) self.assertEqual(exc.kwargs["resource_name"], "fake_name") self.assertEqual(exc.kwargs["resource_id"], "fake_id") @@ -398,3 +400,61 @@ class ActionBuilderTestCase(test.TestCase): for i in range(3): mock_calls.append(mock.call("two", "three", c=3, d=4)) mock_action_two.assert_has_calls(mock_calls) + + +class WaitForStatusTestCase(test.TestCase): + + def test_wrong_ready_statuses_type(self): + self.assertRaises(ValueError, + utils.wait_for, {}, ready_statuses="abc") + + def test_wrong_failure_statuses_type(self): + self.assertRaises(ValueError, + utils.wait_for, {}, ready_statuses=["abc"], + failure_statuses="abc") + + def test_no_ready_statuses(self): + self.assertRaises(ValueError, + utils.wait_for, {}, ready_statuses=[]) + + def test_no_update(self): + self.assertRaises(ValueError, + utils.wait_for, {}, ready_statuses=["ready"]) + + @mock.patch("rally.benchmark.utils.time.sleep") + def test_exit_instantly(self, mock_sleep): + res = {"status": "ready"} + upd = mock.MagicMock(return_value=res) + + utils.wait_for(resource=res, ready_statuses=["ready"], + update_resource=upd) + + upd.assert_called_once_with(res) + self.assertFalse(mock_sleep.called) + + @mock.patch("rally.benchmark.utils.time.sleep") + @mock.patch("rally.benchmark.utils.time.time", return_value=1) + def test_wait_successful(self, mock_time, mock_sleep): + res = {"status": "not_ready"} + upd = mock.MagicMock(side_effect=[{"status": "not_ready"}, + {"status": "not_ready_yet"}, + {"status": "still_not_ready"}, + {"status": "almost_ready"}, + {"status": "ready"}]) + utils.wait_for(resource=res, ready_statuses=["ready"], + update_resource=upd) + upd.assert_has_calls([mock.call({"status": "not_ready"}), + mock.call({"status": "not_ready"}), + mock.call({"status": "not_ready_yet"}), + mock.call({"status": "still_not_ready"}), + mock.call({"status": "almost_ready"})]) + + @mock.patch("rally.benchmark.utils.time.sleep") + @mock.patch("rally.benchmark.utils.time.time", return_value=1) + def test_wait_failure(self, mock_time, mock_sleep): + res = {"status": "not_ready"} + upd = mock.MagicMock(side_effect=[{"status": "not_ready"}, + {"status": "fail"}]) + self.assertRaises(exceptions.GetResourceErrorStatus, utils.wait_for, + resource=res, ready_statuses=["ready"], + failure_statuses=["fail"], update_resource=upd)