diff --git a/rally/benchmark/engine.py b/rally/benchmark/engine.py
index a069f516bc..c41e4afaf8 100644
--- a/rally/benchmark/engine.py
+++ b/rally/benchmark/engine.py
@@ -146,6 +146,7 @@ class TestEngine(object):
                                                    "validation":
                                                    {"is_valid": False,
                                                     "exc_msg": e.message}})
+                    self.task.set_failed()
                     LOG.error(_("Scenario (%(pos)s, %(name)s) input arguments "
                                 "validation error: %(msg)s") %
                               {"pos": n, "name": name, "msg": e.message})
diff --git a/tests/benchmark/test_engine.py b/tests/benchmark/test_engine.py
index 1bebc58bbc..da335c18cc 100644
--- a/tests/benchmark/test_engine.py
+++ b/tests/benchmark/test_engine.py
@@ -194,6 +194,7 @@ class TestEngineTestCase(test.TestCase):
                                      {'raw': [],
                                       'validation': {'is_valid': False,
                                       'exc_msg': validation_exc.message}}),
+            mock.call.set_failed(),
             mock.call.update_status(s.FINISHED)
         ]
         # NOTE(msdubov): Ignore task['uuid'] calls which are used for logging