Now applying some puppet manifests in parallel

This will speed up the case where we are installing on
multiple machines, this commit makes use of flock to wrap
puppet apply so that packstack can ssh to a machine and apply
puppet in the background moving on to the next manifest. flock
ensures they are not run simultaniously if on the same machine.

packstack/modules/ospluginutils.py

@@ -28,16 +28,31 @@ class NovaConfig(object):
         entry += "}"
         return entry
 
+class ManifestFiles(object):
+    def __init__(self):
+        self.filelist = []
+
+    # continuous manifest file that have the same marker can be 
+    # installed in parallel, if on different servers
+    def addFile(self, filename, marker):
+        for f,p in self.filelist:
+            if f == filename:
+                return
+        self.filelist.append((filename, marker,))
+
+    def getFiles(self):
+        return [f for f in self.filelist]
+manifestfiles = ManifestFiles()
+
 def getManifestTemplate(template_name):
     with open(os.path.join(PUPPET_TEMPLATE_DIR, template_name)) as fp:
         return fp.read()%controller.CONF
 
-def appendManifestFile(manifest_name, data):
+def appendManifestFile(manifest_name, data, marker=''):
     if not os.path.exists(basedefs.PUPPET_MANIFEST_DIR):
         os.mkdir(basedefs.PUPPET_MANIFEST_DIR)
     manifestfile = os.path.join(basedefs.PUPPET_MANIFEST_DIR, manifest_name)
-    if manifestfile not in controller.CONF['CONFIG_MANIFESTFILES']:
-        controller.CONF['CONFIG_MANIFESTFILES'].append(manifestfile)
+    manifestfiles.addFile(manifestfile, marker)
     with open(manifestfile, 'a') as fp:
         fp.write("\n")
         fp.write(data)
@@ -56,5 +71,3 @@ def gethostlist(CONF):
                 if host not in hosts:
                     hosts.append(host)
     return hosts
-
-                                         

packstack/plugins/mysql_001.py

@@ -82,5 +82,5 @@ def initSequences(controller):
 def createmanifest():
     manifestfile = "%s_mysql.pp"%controller.CONF['CONFIG_MYSQL_HOST']
     manifestdata = getManifestTemplate("mysql.pp")
-    appendManifestFile(manifestfile, manifestdata)
+    appendManifestFile(manifestfile, manifestdata, 'pre')
 

packstack/plugins/nova_300.py

@@ -8,7 +8,7 @@ import os
 import packstack.installer.engine_validators as validate
 import packstack.installer.common_utils as utils
 
-from packstack.modules.ospluginutils import NovaConfig, getManifestTemplate, appendManifestFile
+from packstack.modules.ospluginutils import NovaConfig, getManifestTemplate, appendManifestFile, manifestfiles
 
 # Controller object will be initialized from main flow
 controller = None
@@ -181,7 +181,7 @@ def initSequences(controller):
 def createapimanifest():
     manifestfile = "%s_api_nova.pp"%controller.CONF['CONFIG_NOVA_API_HOST']
     manifestdata = getManifestTemplate("nova_api.pp")
-    appendManifestFile(manifestfile, manifestdata)
+    appendManifestFile(manifestfile, manifestdata, 'novaapi')
 
 def createkeystonemanifest():
     manifestfile = "%s_keystone.pp"%controller.CONF['CONFIG_KEYSTONE_HOST']
@@ -231,7 +231,7 @@ def createschedmanifest():
     appendManifestFile(manifestfile, manifestdata)
 
 def createcommonmanifest():
-    for manifestfile in controller.CONF['CONFIG_MANIFESTFILES']:
+    for manifestfile, marker in manifestfiles.getFiles():
         if manifestfile.endswith("_nova.pp"):
             data = getManifestTemplate("nova_common.pp")
             appendManifestFile(os.path.split(manifestfile)[1], data)

packstack/plugins/puppet_950.py

@@ -4,11 +4,12 @@ Installs and configures puppet
 import logging
 import os
 import platform
+import time
 
 from packstack.installer import basedefs
 import packstack.installer.common_utils as utils
 
-from packstack.modules.ospluginutils import gethostlist
+from packstack.modules.ospluginutils import gethostlist, manifestfiles
 
 # Controller object will be initialized from main flow
 controller = None
@@ -52,8 +53,6 @@ def initSequences(controller):
     ]
     controller.addSequence("Puppet", [], [], puppetsteps)
 
-    controller.CONF.setdefault('CONFIG_MANIFESTFILES', [])
-
 def runCleanup():
     localserver = utils.ScriptRunner()
     localserver.append("rm -rf %s/*pp"%basedefs.PUPPET_MANIFEST_DIR)
@@ -80,14 +79,40 @@ def copyPuppetModules():
         server.append("tar %s --dereference -czf - ../manifests | ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@%s tar -C %s -xzf -"%(tar_opts, hostname, basedefs.VAR_DIR))
     server.execute()
 
+def waitforpuppet(currently_running):
+    while currently_running:
+        for hostname, log in currently_running:
+            server = utils.ScriptRunner(hostname)
+            server.append("test -e %s"%log)
+            print "Testing if puppet apply is finished : %s"%os.path.split(log)[1],
+            try:
+                server.execute()
+                currently_running.remove((hostname,log))
+                print "OK"
+            except Exception, e:
+                # the test raises an exception if the file doesn't exist yet
+                time.sleep(3)
+                print
+
 def applyPuppetManifest():
     print
-    for manifest in controller.CONF['CONFIG_MANIFESTFILES']:
+    currently_running = []
+    lastmarker = None
+    for manifest, marker in manifestfiles.getFiles():
+        # if the marker has changed then we don't want to proceed until
+        # all of the previous puppet runs have finished
+        if lastmarker != None and lastmarker != marker:
+            waitforpuppet(currently_running)
+        lastmarker = marker
+        
         for hostname in gethostlist(controller.CONF):
             if "/%s_"%hostname not in manifest: continue
 
             print "Applying "+ manifest
             server = utils.ScriptRunner(hostname)
-            server.append("puppet apply --modulepath %s/modules %s"%(basedefs.VAR_DIR, manifest))
-            server.execute()
 
+            logfile = "%s.log"%manifest
+            currently_running.append((hostname, logfile))
+            command = "( flock %s/ps.lock puppet apply --modulepath %s/modules %s > %s_ 2>&1 < /dev/null ; mv %s_ %s ) > /dev/null 2>&1 < /dev/null &"%(basedefs.VAR_DIR, basedefs.VAR_DIR, manifest, logfile, logfile, logfile)
+            server.append(command)
+            server.execute()

packstack/plugins/qpid_002.py

@@ -60,4 +60,4 @@ def initSequences(controller):
 def createmanifest():
     manifestfile = "%s_qpid.pp"%controller.CONF['CONFIG_QPID_HOST']
     manifestdata = getManifestTemplate("qpid.pp")
-    appendManifestFile(manifestfile, manifestdata)
+    appendManifestFile(manifestfile, manifestdata, 'pre')

packstack/plugins/swift_600.py

@@ -9,7 +9,7 @@ import packstack.installer.engine_validators as validate
 from packstack.installer import basedefs
 import packstack.installer.common_utils as utils
 
-from packstack.modules.ospluginutils import getManifestTemplate, appendManifestFile
+from packstack.modules.ospluginutils import getManifestTemplate, appendManifestFile, manifestfiles
 
 # Controller object will be initialized from main flow
 controller = None
@@ -147,7 +147,7 @@ def createbuildermanifest():
         manifestdata = manifestdata + '\n@@ring_container_device { "%s:6001/%s":\n zone        => %s,\n weight      => 10, }'%(host, devicename, zone)
         manifestdata = manifestdata + '\n@@ring_account_device { "%s:6002/%s":\n zone        => %s,\n weight      => 10, }'%(host, devicename, zone)
 
-    appendManifestFile(manifestfile, manifestdata)
+    appendManifestFile(manifestfile, manifestdata, 'swiftbuilder')
 
 def createproxymanifest():
     manifestfile = "%s_swift.pp"%controller.CONF['CONFIG_SWIFT_PROXY_HOSTS']
@@ -185,7 +185,7 @@ def createstoragemanifest():
         appendManifestFile(manifestfile, manifestdata)
 
 def createcommonmanifest():
-    for manifestfile in controller.CONF['CONFIG_MANIFESTFILES']:
+    for manifestfile, marker in manifestfiles.getFiles():
         if manifestfile.endswith("_swift.pp"):
             data = getManifestTemplate("swift_common.pp")
             appendManifestFile(os.path.split(manifestfile)[1], data)