Merge pull request #889 from Mirantis/fix_corosync_cleanup

Fix start/cleanup race condition

deployment/puppet/corosync/lib/facter/pacemaker_hostname.rb

@@ -0,0 +1,9 @@
+# Fact: pacemaker_hostname
+#
+# Purpose: Return name of the node used by Pacemaker
+#
+Facter.add(:pacemaker_hostname) do
+  setcode do
+    rv = Facter::Util::Resolution.exec('uname -n')
+  end
+end

deployment/puppet/corosync/lib/puppet/provider/service/pacemaker.rb

@@ -10,6 +10,7 @@ Puppet::Type.type(:service).provide :pacemaker, :parent => Puppet::Provider::Cor
   commands :crm => 'crm'
   commands :cibadmin => 'cibadmin'
   commands :crm_attribute => 'crm_attribute'
+  commands :crm_resource => 'crm_resource'
 
   desc "Pacemaker service management."
 
@@ -208,6 +209,12 @@ Puppet::Type.type(:service).provide :pacemaker, :parent => Puppet::Provider::Cor
           last_successful_op = 'start'
         else
           last_successful_op = 'stop'
+          if last_op.attributes['rc-code'].to_i == 5 and node[:uname] == Facter.value(:pacemaker_hostname)
+            crm_resource('--cleanup','--resource',get_service_name,'--node',Facter.value(:pacemaker_hostname))
+            sleep 15
+            self.class.get_cib
+            retry
+          end
         end
       end
       debug("LAST SUCCESSFUL OP :\n\n #{last_successful_op.inspect}")

deployment/puppet/corosync/manifests/cleanup.pp

@@ -14,16 +14,21 @@
 #
 # Copyright 2012 Puppet Labs, LLC.
 #
-define corosync::cleanup () {
-  Cs_resource <| name == $name |> ~> Exec["crm resource cleanup $name"]
+define corosync::cleanup (
+  $force = false,
+  $wait_before = 15
+) {
+  if $force   {
+    Cs_resource <| name == $name |> ~> Exec["crm resource cleanup $name"]
 
-  ##FIXME: we need to create a better way to workaround crm commit <-> cleanup race condition than a simple sleep 
-  #Workaround for hostname bugs with FQDN vs short hostname
-  exec { "crm resource cleanup $name":
-    command     => "bash -c \"(sleep 5 && crm_resource --resource ${name}  --cleanup --node `uname -n`) || :\"",
-    path        => ['/bin', '/usr/bin', '/sbin', '/usr/sbin'],
-    returns     => [0,""],
-    refreshonly => true,
-    timeout     => 600,
-  }
-}  
+    ##FIXME: we need to create a better way to workaround crm commit <-> cleanup race condition than a simple sleep 
+    #Workaround for hostname bugs with FQDN vs short hostname
+    exec { "crm resource cleanup $name":
+      command     => "bash -c \"(sleep ${wait_before} && crm_resource --resource ${name}  --cleanup --node `uname -n`) || :\"",
+      path        => ['/bin', '/usr/bin', '/sbin', '/usr/sbin'],
+      returns     => [0,""],
+      refreshonly => true,
+      timeout     => 600,
+    }
+  } 
+}