diff --git a/rally/deploy/engine.py b/rally/deploy/engine.py
index 9923c6b6d5..70e87fb693 100644
--- a/rally/deploy/engine.py
+++ b/rally/deploy/engine.py
@@ -72,13 +72,23 @@ class EngineFactory(object):
     def cleanup(self):
         """Cleanup OpenStack deployment."""
 
-    def __enter__(self):
+    def make(self):
         self.task.update_status(consts.TaskStatus.DEPLOY_STARTED)
-        deploy = self.deploy()
+        endpoints = self.deploy()
         self.task.update_status(consts.TaskStatus.DEPLOY_FINISHED)
-        return deploy
+        return endpoints
+
+    def __enter__(self):
+        return self
 
     def __exit__(self, type, value, traceback):
+        if type:
+            self.task.set_failed()
         self.task.update_status(consts.TaskStatus.CLEANUP)
-        self.cleanup()
-        self.task.update_status(consts.TaskStatus.FINISHED)
+        try:
+            self.cleanup()
+        except Exception:
+            self.task.set_failed()
+            raise
+        finally:
+            self.task.update_status(consts.TaskStatus.FINISHED)
diff --git a/rally/orchestrator/api.py b/rally/orchestrator/api.py
index 833459b2e5..90b645241f 100644
--- a/rally/orchestrator/api.py
+++ b/rally/orchestrator/api.py
@@ -37,8 +37,9 @@ def start_task(config):
                                                deploy_conf)
     tester = engine.TestEngine(config['tests'], task_object)
 
-    with deployer as deployment:
-        with tester.bind(deployment):
+    with deployer:
+        endpoints = deployer.make()
+        with tester.bind(endpoints):
             tester.verify()
             tester.benchmark()
 
diff --git a/rally/task.py b/rally/task.py
index eb325f8b68..cee5f9c01e 100644
--- a/rally/task.py
+++ b/rally/task.py
@@ -31,3 +31,6 @@ class Task(object):
 
     def update_status(self, status):
         db.task_update(self.task['uuid'], {'status': status})
+
+    def set_failed(self):
+        db.task_update(self.task['uuid'], {'failed': True})
diff --git a/tests/benchmark/test_test_engine.py b/tests/benchmark/test_test_engine.py
index 4c9c358be6..31f5a9fb79 100644
--- a/tests/benchmark/test_test_engine.py
+++ b/tests/benchmark/test_test_engine.py
@@ -120,3 +120,12 @@ class TestEngineTestCase(test.NoDBTestCase):
             mock.call.update_status(s.TEST_TOOL_BENCHMARKING),
         ]
         self.assertEqual(fake_task.mock_calls, expected)
+
+    def test_task_status_invalid_config(self):
+        fake_task = mock.MagicMock()
+        try:
+            engine.TestEngine(self.invalid_test_config_bad_key, fake_task)
+        except exceptions.InvalidConfigException:
+            pass
+        expected = []
+        self.assertEqual(fake_task.mock_calls, expected)
diff --git a/tests/deploy/test_engine_factory.py b/tests/deploy/test_engine_factory.py
index b67b6322ff..4e68663c7d 100644
--- a/tests/deploy/test_engine_factory.py
+++ b/tests/deploy/test_engine_factory.py
@@ -86,7 +86,8 @@ class EngineFactoryTestCase(test.NoDBTestCase):
                 self.cleanuped = True
 
         with deploy.EngineFactory.get_engine('A', mock.Mock(),
-                                             None) as deployment:
-            self.assertTrue(deployment.deployed)
+                                             None) as deployer:
+            endpoints = deployer.make()
+            self.assertTrue(endpoints.deployed)
 
-        self.assertTrue(deployment.cleanuped)
+        self.assertTrue(endpoints.cleanuped)
diff --git a/tests/deploy/test_task_status.py b/tests/deploy/test_task_status.py
index e81e0dd944..9128a44e0b 100644
--- a/tests/deploy/test_task_status.py
+++ b/tests/deploy/test_task_status.py
@@ -47,8 +47,8 @@ class DeployEngineTaskStatusTestCase(test.NoDBTestCase):
 
     def test_task_status_basic_chain(self):
         fake_task = mock.MagicMock()
-        with get_engine('FakeEngine', fake_task, {}):
-            pass
+        with get_engine('FakeEngine', fake_task, {}) as deployer:
+            deployer.make()
         s = consts.TaskStatus
         expected = [
             mock.call.update_status(s.DEPLOY_STARTED),
@@ -56,22 +56,24 @@ class DeployEngineTaskStatusTestCase(test.NoDBTestCase):
             mock.call.update_status(s.CLEANUP),
             mock.call.update_status(s.FINISHED),
         ]
-        self.assertEqual(fake_task.mock_calls, expected)
+        self.assertEqual(expected, fake_task.mock_calls)
 
     def _test_failure(self, engine, expected_calls):
         fake_task = mock.MagicMock()
-        engine = get_engine(engine, fake_task, {})
         try:
-            with engine:
-                pass
+            with get_engine(engine, fake_task, {}) as deployer:
+                deployer.make()
         except FakeFailure:
             pass
-        self.assertEqual(fake_task.mock_calls, expected_calls)
+        self.assertEqual(expected_calls, fake_task.mock_calls)
 
     def test_task_status_failed_deploy(self):
         s = consts.TaskStatus
         expected = [
             mock.call.update_status(s.DEPLOY_STARTED),
+            mock.call.set_failed(),
+            mock.call.update_status(s.CLEANUP),
+            mock.call.update_status(s.FINISHED),
         ]
         self._test_failure('EngineFailedDeploy', expected)
 
@@ -81,5 +83,7 @@ class DeployEngineTaskStatusTestCase(test.NoDBTestCase):
             mock.call.update_status(s.DEPLOY_STARTED),
             mock.call.update_status(s.DEPLOY_FINISHED),
             mock.call.update_status(s.CLEANUP),
+            mock.call.set_failed(),
+            mock.call.update_status(s.FINISHED),
         ]
         self._test_failure('EngineFailedCleanup', expected)