Increase usefulness of Agent diagnostic commands

This update adds the following:
 - Added color and error-resolving hints to 'monasca-forwarder info'
 - Made 'monasca-collector check <checkname>' output more human-readable
 - Added 'monasca-collector check_all' parameter to run all configured checks

Change-Id: I752eeddebc0bdd47a515d4af5affb6d850b44b0d

monagent/collector/checks/check.py

@@ -3,6 +3,9 @@
 If you are writing your own checks you should subclass the AgentCheck class.
 The Check class is being deprecated so don't write new checks with it.
 """
+# This file uses 'print' as a function rather than a statement, a la Python3
+from __future__ import print_function
+
 import logging
 import os
 from pprint import pprint
@@ -19,7 +22,8 @@ from monagent.common.util import LaconicFilter, get_hostname, get_os
 log = logging.getLogger(__name__)
 
 
-# todo convert all checks to the new interface then remove this. Is the LaconicFilter on logs used elsewhere?
+# todo convert all checks to the new interface then remove this.
+#      Is the LaconicFilter on logs used elsewhere?
 # ==============================================================================
 # DEPRECATED
 # ------------------------------
@@ -235,7 +239,8 @@ class Check(object):
         This is the preferred method to retrieve metrics
 
         @return the list of samples
-        @rtype [(metric_name, timestamp, value, {"dimensions": {"name1": "key1", "name2": "key2"}}), ...]
+        @rtype [(metric_name, timestamp, value,
+                {"dimensions": {"name1": "key1", "name2": "key2"}}), ...]
         """
         metrics = []
         for m in self._sample_store:
@@ -258,6 +263,8 @@ class Check(object):
                     metrics.append((m, int(ts), val, attributes))
             except Exception:
                 pass
+            if prettyprint:
+                print("Metrics: {}".format(metrics))
         return metrics
 
 
@@ -413,13 +420,27 @@ class AgentCheck(object):
         """
         return len(self.events) > 0
 
-    def get_metrics(self):
+    def get_metrics(self, prettyprint=False):
         """
         Get all metrics, including the ones that are tagged.
 
         @return the list of samples
         @rtype list of Measurement objects from monagent.common.metrics
         """
+        if prettyprint:
+            metrics = self.aggregator.flush()
+            for metric in metrics:
+                print(" Timestamp:  {}".format(metric.timestamp))
+                print(" Name:       {}".format(metric.name))
+                print(" Value:      {}".format(metric.value))
+                print(" Dimensions: ", end='')
+                line = 0
+                for name in metric.dimensions:
+                    if line != 0:
+                        print(" " * 13, end='')
+                    print("{0}={1}".format(name, metric.dimensions[name]))
+                    line += 1
+                print("-" * 24)
         return self.aggregator.flush()
 
     def get_events(self):

monagent/collector/daemon.py

@@ -15,15 +15,15 @@ import sys
 import time
 import glob
 
-# Check we're not using an old version of Python. We need 2.4 above because some modules (like subprocess)
-# were only introduced in 2.4.
+# Check we're not using an old version of Python. We need 2.4 above because
+# some modules (like subprocess) were only introduced in 2.4.
 if int(sys.version_info[1]) <= 3:
     sys.stderr.write("Monasca Agent requires python 2.4 or later.\n")
     sys.exit(2)
 
 # Custom modules
 from checks.collector import Collector
-from monagent.common.check_status import CollectorStatus
+from monagent.common.check_status import CollectorStatus, ForwarderStatus
 from monagent.common.config import get_config, get_parsed_args, load_check_directory, get_confd_path, check_yaml, get_logging_config
 from monagent.common.daemon import Daemon, AgentSupervisor
 from monagent.common.emitter import http_emitter
@@ -193,6 +193,7 @@ def main():
         'status',
         'info',
         'check',
+        'check_all',
         'configcheck',
         'jmx',
     ]
@@ -262,14 +263,26 @@ def main():
             for check in checks['initialized_checks']:
                 if check.name == check_name:
                     check.run()
-                    print(check.get_metrics())
-                    print(check.get_events())
+                    print("Metrics: ")
+                    check.get_metrics(prettyprint=True)
                     if len(args) == 3 and args[2] == 'check_rate':
                         print("Running 2nd iteration to capture rate metrics")
                         time.sleep(1)
                         check.run()
-                        print(check.get_metrics())
-                        print(check.get_events())
+                        print("Metrics: ")
+                        check.get_metrics(prettyprint=True)
+
+    elif 'check_all' == command:
+        print("Loading check directory...")
+        checks = load_check_directory(agentConfig)
+        print("...directory loaded.\n")
+        for check in checks['initialized_checks']:
+            print("#" * 80)
+            print("Check name: '{}'\n".format(check.name))
+            check.run()
+            print("Metrics: ")
+            check.get_metrics(prettyprint=True)
+            print("#" * 80 + "\n\n")
 
     elif 'configcheck' == command or 'configtest' == command:
         osname = get_os()

monagent/common/check_status.py

@@ -571,6 +571,18 @@ class ForwarderStatus(AgentStatus):
             "Transactions received: %s" % self.transactions_received,
             "Transactions flushed: %s" % self.transactions_flushed
         ]
+        if self.transactions_flushed == 0:
+            lines.append("[%s]: Unable to flush transactions\n           %s" %
+                         (style(STATUS_ERROR, 'red'),
+                          "Please verify monasca-api is running as configured"))
+        elif self.transactions_flushed != self.transactions_received:
+            lines.append("[%s]: Transactions out of sync\n             %s" %
+                         (style(STATUS_WARNING, 'yellow'),
+                          "Likely contact interruption with monasca-api"))
+        else:
+            lines.append("[%s]: Transactions up to date" %
+                         style(STATUS_OK, 'green'))
+
         return lines
 
     def has_error(self):