improve rally output (exception part)

This patch does a lot of trivial changes. But all changes belong to two
things.

The first thing is to make clear that when we could put traceback in rally
output. There is a strategy:

----------------------------------------
|  err type    |  [0]  |  [1]  |  [2]  |
| normal mode  |   N   |   N   |   N   |
| verbose mode |   N   |   N*  |   Y   |
|  debug mode  |   N   |   Y   |   Y   |
----------------------------------------

Why need the [1] between [0] and [2]? The reason is that OpenStack APIs
always return something hard to explain[3]. In another word, we need some
*hacking* for it in rally codes. At least we have to parse it in a way
different with [0] and [2].

The second thing is improving error message so that we could understand
output better. An easy example is that when updating resource status,
the old output always said "resource is in ERROR status". Who is it?
What's the reason for it? This is what this patch is trying to tell you.

Firstly some exception message is improved:

GetResourceFailture: telling users *which resource* failed and *why*
GetResourceNotFound: telling users *which resource* is not found
GetResourceErrorStatus: telling users *which resource* is in a *error
                        status* and *why* (if possible)

And print fault message of nova servers.

[0] Server side fault
[1] Client side fault caused by server side
[2] Client side fault
[3] bug #1280453

Change-Id: I3082a72b6547493ed19def83847fe8f6e9ff0e2f

rally/benchmark/utils.py

@@ -18,8 +18,9 @@ import logging
 import time
 import traceback
 
+from novaclient.v1_1 import servers
 from rally.benchmark.scenarios.keystone import utils as kutils
-from rally import exceptions as rally_exceptions
+from rally import exceptions
 
 
 LOG = logging.getLogger(__name__)
@@ -34,18 +35,36 @@ def get_from_manager(error_statuses=None):
     error_statuses = map(lambda str: str.upper(), error_statuses)
 
     def _get_from_manager(resource):
+        # catch client side errors
         try:
-            resource = resource.manager.get(resource.id)
+            res = resource.manager.get(resource.id)
         except Exception as e:
             if getattr(e, 'code', 400) == 404:
-                raise rally_exceptions.GetResourceNotFound(status="404")
-            raise rally_exceptions.GetResourceFailure(status=e)
-        status = resource.status.upper()
+                raise exceptions.GetResourceNotFound(resource=resource)
+            raise exceptions.GetResourceFailure(resource=resource, err=e)
+
+        # catch client side errors caused by server side
+        if not hasattr(res, 'status') or not hasattr(res, 'name'):
+            # In many test cases, nova 'get-server' api maybe return an
+            # incomplete response which doesn't have some common key-value
+            # like 'status' or 'name'. Fortunately, it just happened in
+            # pre 'BUILD' state. So an easy solution is to pass this update
+            # cycle and request api in next one.
+            return res
+
+        # catch abnormal status, such as "no valid host" for servers
+        status = res.status.upper()
         if status == "DELETED":
-            raise rally_exceptions.GetResourceNotFound(status="404")
+            raise exceptions.GetResourceNotFound(resource=res)
         if status in error_statuses:
-            raise rally_exceptions.GetResourceErrorStatus(status=status)
-        return resource
+            if isinstance(res.manager, servers.ServerManager):
+                msg = res.fault['message']
+            else:
+                msg = ''
+            raise exceptions.GetResourceErrorStatus(resource=res,
+                                                    status=status, fault=msg)
+
+        return res
 
     return _get_from_manager
 
@@ -85,7 +104,7 @@ def wait_for(resource, is_ready, update_resource=None, timeout=60,
             break
         time.sleep(check_interval)
         if time.time() - start > timeout:
-            raise rally_exceptions.TimeoutException()
+            raise exceptions.TimeoutException()
     return resource
 
 
@@ -131,11 +150,11 @@ def wait_for_delete(resource, update_resource=None, timeout=60,
     while True:
         try:
             resource = update_resource(resource)
-        except rally_exceptions.GetResourceNotFound:
+        except exceptions.GetResourceNotFound:
             break
         time.sleep(check_interval)
         if time.time() - start > timeout:
-            raise rally_exceptions.TimeoutException()
+            raise exceptions.TimeoutException()
 
 
 def format_exc(exc):

rally/exceptions.py

@@ -54,14 +54,12 @@ class RallyException(Exception):
         if not message:
             try:
                 message = self.msg_fmt % kwargs
-
-            except Exception:
+            except KeyError:
                 exc_info = sys.exc_info()
                 # kwargs doesn't match a variable in the message
                 # log the issue and the kwargs
-                LOG.exception(_('Exception in string format operation'))
-                for name, value in kwargs.iteritems():
-                    LOG.error("%s: %s" % (name, value))
+                msg = "kwargs don't match in string format operation: %s"
+                LOG.debug(msg % kwargs, exc_info=exc_info)
 
                 if CONF.fatal_exception_format_errors:
                     raise exc_info[0], exc_info[1], exc_info[2]
@@ -163,15 +161,15 @@ class TimeoutException(RallyException):
 
 
 class GetResourceFailure(RallyException):
-    msg_fmt = _("Failed to get the resource: '`%(status)s`'")
+    msg_fmt = _("Failed to get the resource %(resource)s: %(err)s")
 
 
 class GetResourceNotFound(GetResourceFailure):
-    msg_fmt = _("Resource not found: `%(status)s`")
+    msg_fmt = _("Resource %(resource)s is not found.")
 
 
 class GetResourceErrorStatus(GetResourceFailure):
-    msg_fmt = _("Resouce has invalid status: `%(status)s`")
+    msg_fmt = _("Resouce %(resource)s has %(status)s status: %(fault)s")
 
 
 class SSHError(RallyException):

tests/benchmark/test_utils.py

@@ -66,6 +66,14 @@ class BenchmarkUtilsTestCase(test.TestCase):
         self.assertRaises(exceptions.GetResourceFailure,
                           get_from_manager, resource)
 
+    def test_get_from_manager_in_deleted_state(self):
+        get_from_manager = utils.get_from_manager()
+        manager = fakes.FakeManager()
+        resource = fakes.FakeResource(manager=manager, status="DELETED")
+        manager._cache(resource)
+        self.assertRaises(exceptions.GetResourceNotFound,
+                          get_from_manager, resource)
+
     def test_get_from_manager_not_found(self):
         get_from_manager = utils.get_from_manager()
         manager = mock.MagicMock()