Test the puppet logs for errors

If an unecpected error occurs then raise an exception and quit
https://bugzilla.redhat.com/show_bug.cgi?id=887956

packstack/modules/ospluginutils.py

@@ -1,5 +1,7 @@
 
+import logging
 import os
+import re
 
 from packstack.installer import basedefs
 from packstack.installer.setup_controller import Controller
@@ -9,6 +11,11 @@ controller = Controller()
 PUPPET_DIR = os.path.join(basedefs.DIR_PROJECT_DIR, "puppet")
 PUPPET_TEMPLATE_DIR = os.path.join(PUPPET_DIR, "templates")
 
+
+class PackStackError(Exception):
+    pass
+
+
 class NovaConfig(object):
     """
     Helper class to create puppet manifest entries for nova_config
@@ -77,3 +84,47 @@ def gethostlist(CONF):
                 if host not in hosts:
                     hosts.append(host)
     return hosts
+
+
+_error_exceptions = [
+    # puppet preloads a provider using the mysql command before it is installed
+    re.compile('Command mysql is missing'),
+    # swift puppet module tries to install swift-plugin-s3, there is no such
+    # pakage on RHEL, fixed in the upstream puppet module
+    re.compile('yum.*?install swift-plugin-s3'),
+]
+
+
+def isErrorException(line):
+    for ee in _error_exceptions:
+        if ee.search(line):
+            return True
+    return False
+
+
+_re_errorline = re.compile('err: | Syntax error at')
+_re_color = re.compile('\x1b.*?\d\dm')
+def validate_puppet_logfile(logfile):
+    """
+    Check a puppet log file for errors and raise an error if we find any
+    """
+    fp = open(logfile)
+    data = fp.read()
+    fp.close()
+    manifestfile = os.path.splitext(logfile)[0]
+    for line in data.split('\n'):
+        line = line.strip()
+
+        if _re_errorline.search(line) == None:
+            continue
+
+        message = _re_color.sub('', line)  # remove colors
+        if isErrorException(line):
+            logging.info("Ignoring expected error during puppet run %s : %s" %
+                (manifestfile, message))
+            continue
+
+        message = "Error during puppet run : " + message
+        logging.error("Error  during remote puppet apply of " + manifestfile)
+        logging.error(data)
+        raise PackStackError(message)

packstack/plugins/puppet_950.py

@@ -9,7 +9,9 @@ import time
 from packstack.installer import basedefs
 import packstack.installer.common_utils as utils
 
-from packstack.modules.ospluginutils import gethostlist, manifestfiles
+from packstack.modules.ospluginutils import gethostlist,\
+                                            manifestfiles,\
+                                            validate_puppet_logfile
 
 # Controller object will be initialized from main flow
 controller = None
@@ -84,19 +86,28 @@ def copyPuppetModules():
 def waitforpuppet(currently_running):
     while currently_running:
         for hostname, log in currently_running:
-            server = utils.ScriptRunner(hostname)
+            print "Testing if puppet apply is finished : %s" % os.path.split(log)[1],
-            server.append("test -e %s"%log)
-            server.append("cat %s"%log)
-            print "Testing if puppet apply is finished : %s"%os.path.split(log)[1],
             try:
-                # Errors are expected here if the puppet run isn't finished so we suppress their logging
+                # Once a remote puppet run has finished, we retrieve the log
-                server.execute(logerrors=False)
+                # file and check it for errors
-                currently_running.remove((hostname,log))
+                local_server = utils.ScriptRunner()
-                print "OK"
+                local_server.append('scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@%s:%s %s' % (hostname, log, log))
+                # Errors are expected here if the puppet run isn't finished so we suppress logging them
+                local_server.execute(logerrors=False)
+
+                # If we got to this point the puppet apply has finished
+                currently_running.remove((hostname, log))
+
             except Exception, e:
                 # the test raises an exception if the file doesn't exist yet
                 time.sleep(3)
                 print
+                continue
+
+            # check the log file for errors
+            validate_puppet_logfile(log)
+            print "OK"
+
 
 def applyPuppetManifest():
     print