Improve error messages of validators

The current error message from validation step doesn't include what
plugin fails and by which validator. This information is available only
in debug mode.

Without this information it is hard to understand what is going wrong.

Change-Id: I4e0bc3eb36c3d8116a2e867b48604b4cc514580a

CHANGELOG.rst

@@ -28,7 +28,7 @@ Added
 Changed
 ~~~~~~~
 
-* Improved validation errors for task config.
+* Improved validation errors for task component.
 
 [1.1.0] - 2018-08-07
 --------------------

rally/common/validation.py

@@ -271,10 +271,15 @@ class ValidatablePluginMixin(object):
                     # Unexpected error is returned. save traceback as well
                     result = traceback.format_exc()
                 if result:
-                    LOG.debug("Result of validator '%s' is not successful for "
-                              "plugin %s." % (validator_cls.get_name(), name))
-                    results.append(result)
-
+                    results.append(
+                        "%(base)s plugin '%(pname)s' doesn't pass %(vname)s "
+                        "validation. Details: %(error)s" % {
+                            "base": cls.__name__,
+                            "pname": name,
+                            "vname": validator_cls.get_fullname(),
+                            "error": result
+                        }
+                    )
             if results:
                 break
 

tests/unit/common/test_validation.py

@@ -43,17 +43,27 @@ class ValidationHelpersTestCase(test.TestCase):
 @plugin.configure(name="dummy_validator")
 class DummyValidator(validation.Validator):
 
-    def __init__(self, foo):
+    def __init__(self, foo, exc=False):
         """Dummy validator
 
         :param foo: additional parameter for Dummy validator
+        :param exc: whether to raise expected on unexpected error
         """
         super(DummyValidator, self).__init__()
         self.foo = foo
+        self.exc = exc
 
     def validate(self, context, config, plugin_cls, plugin_cfg):
         if self.foo not in config:
-            raise Exception("foo")
+            if self.exc:
+                raise Exception("foo")
+            self.fail("oops")
+
+
+@plugin.base()
+class DummyPluginBase(plugin.Plugin,
+                      validation.ValidatablePluginMixin):
+    pass
 
 
 class ValidatorTestCase(test.TestCase):
@@ -64,7 +74,7 @@ class ValidatorTestCase(test.TestCase):
                               validation.ValidatablePluginMixin):
             pass
 
-        @validation.add(name="dummy_validator", foo="bar")
+        @validation.add(name="dummy_validator", foo="bar", exc=True)
         @validation.add(name="required_platform", platform="foo", users=True)
         @plugin.configure(name="dummy_plugin")
         class DummyPlugin(DummyPluginBase):
@@ -84,17 +94,28 @@ class ValidatorTestCase(test.TestCase):
 
         DummyPlugin.unregister()
 
-    def test_failures(self):
-        @plugin.base()
-        class DummyPluginBase(plugin.Plugin,
-                              validation.ValidatablePluginMixin):
-            pass
+    def test_simple_failure(self):
 
         result = DummyPluginBase.validate("dummy_plugin", None, None, None)
         self.assertEqual(1, len(result))
         self.assertIn("There is no DummyPluginBase plugin "
                       "with name: 'dummy_plugin'", result[0])
 
+    def test_failure_includes_detailed_info(self):
+
+        @validation.add("dummy_validator", foo="bar")
+        @plugin.configure(name=self.id())
+        class Foo(DummyPluginBase):
+            pass
+
+        result = DummyPluginBase.validate(self.id(), {}, {}, None,
+                                          vtype="semantic")
+        self.assertEqual(1, len(result))
+        self.assertEqual(
+            "DummyPluginBase plugin '%s' doesn't pass dummy_validator@default "
+            "validation. Details: oops" % self.id(),
+            result[0])
+
 
 @ddt.ddt
 class RequiredPlatformValidatorTestCase(test.TestCase):