Add $verify_on_create ability for all resources.

Implements a workaround for the rare condidtion where "pcs resource
create" may succeed on the command line, but pacemaker is not updated
with the new resource.

When $verify_on_create is true, make sure the resource exists after
$try_sleep with an additional "pcs" call.  Retry up to $tries per the
usual behaviour.

When $verify_on_create is false (the default), behaviour is exactly
the same as before, i.e. only the return value from "pcs resource
create" is checked to determine whether the resource creation was
successful or not.

lib/puppet/provider/pcmk_resource/default.rb

@@ -113,6 +113,9 @@ Puppet::Type.type(:pcmk_resource).provide(:default) do
   end
 
   def pcs(name, cmd)
+    if name.start_with?("create") && @resource[:verify_on_create]
+      return pcs_create_with_verify(name, cmd)
+    end
     try_sleep =  @resource[:try_sleep]
     max_tries = name.include?('show') ? 1 : @resource[:tries]
     max_tries.times do |try|
@@ -139,4 +142,32 @@ Puppet::Type.type(:pcmk_resource).provide(:default) do
     end
   end
 
+  def pcs_create_with_verify(name, cmd)
+    try_sleep = @resource[:try_sleep]
+    max_tries = @resource[:tries]
+    max_tries.times do |try|
+      try_text = max_tries > 1 ? "try #{try+1}/#{max_tries}: " : ''
+      Puppet.debug("#{try_text}/usr/sbin/pcs #{cmd}")
+      pcs_out = `/usr/sbin/pcs #{cmd} 2>&1`
+      if $?.exitstatus == 0
+        sleep try_sleep
+        cmd_show = "/usr/sbin/pcs resource show "+ @resource[:name]
+        Puppet.debug("Verifying with: "+cmd_show)
+        `#{cmd_show}`
+        if $?.exitstatus == 0
+          return pcs_out
+        else
+          Puppet.debug("Warning: verification of pcs resource creation failed")
+        end
+      else
+        Puppet.debug("Error: #{pcs_out}")
+        sleep try_sleep
+      end
+      if try == max_tries-1
+        pcs_out_line = pcs_out.lines.first ? pcs_out.lines.first.chomp! : ''
+        raise Puppet::Error, "pcs #{name} failed: #{pcs_out_line}"
+      end
+    end
+  end
+
 end

lib/puppet/type/pcmk_resource.rb

@@ -1,3 +1,5 @@
+require 'puppet/parameter/boolean'
+
 Puppet::Type.newtype(:pcmk_resource) do
   @doc = "Base resource definition for a pacemaker resource"
 
@@ -65,6 +67,16 @@ Puppet::Type.newtype(:pcmk_resource) do
     defaultto 0
   end
 
+  newparam(:verify_on_create, :boolean => true, :parent => Puppet::Parameter::Boolean) do
+     desc "Whether to verify pcs resource creation with an additional
+     call to 'pcs resource show' rather than just relying on the exit
+     status of 'pcs resource create'.  When true, $try_sleep
+     determines how long to wait to verify and $post_success_sleep is
+     ignored.  Defaults to `false`."
+
+     defaultto false
+   end
+
   newproperty(:op_params) do
     desc "op parameters"
   end

manifests/resource/filesystem.pp

@@ -45,6 +45,7 @@ define pacemaker::resource::filesystem(
   $post_success_sleep = 0,
   $tries              = 1,
   $try_sleep          = 0,
+  $verify_on_create   = false,
 ) {
   $resource_id = delete("fs-${directory}", '/')
 
@@ -64,5 +65,6 @@ define pacemaker::resource::filesystem(
     post_success_sleep => $post_success_sleep,
     tries              => $tries,
     try_sleep          => $try_sleep,
+    verify_on_create   => $verify_on_create,
   }
 }

manifests/resource/ip.pp

@@ -32,6 +32,7 @@ define pacemaker::resource::ip(
   $post_success_sleep = 0,
   $tries              = 1,
   $try_sleep          = 0,
+  $verify_on_create   = false,
   ) {
 
   $cidr_option = $cidr_netmask ? {
@@ -51,6 +52,7 @@ define pacemaker::resource::ip(
     post_success_sleep => $post_success_sleep,
     tries              => $tries,
     try_sleep          => $try_sleep,
+    verify_on_create   => $verify_on_create,
   }
 
 }

manifests/resource/lsb.pp

@@ -14,6 +14,7 @@ define pacemaker::resource::lsb(
   $post_success_sleep = 0,
   $tries              = 1,
   $try_sleep          = 0,
+  $verify_on_create   = false,
 ) {
   pcmk_resource { $name:
     ensure             => $ensure,
@@ -26,5 +27,6 @@ define pacemaker::resource::lsb(
     post_success_sleep => $post_success_sleep,
     tries              => $tries,
     try_sleep          => $try_sleep,
+    verify_on_create   => $verify_on_create,
   }
 }

manifests/resource/ocf.pp

@@ -44,6 +44,7 @@ define pacemaker::resource::ocf(
   $post_success_sleep = 0,
   $tries              = 1,
   $try_sleep          = 0,
+  $verify_on_create   = false,
 ) {
   pcmk_resource { $name:
     ensure             => $ensure,
@@ -57,5 +58,6 @@ define pacemaker::resource::ocf(
     post_success_sleep => $post_success_sleep,
     tries              => $tries,
     try_sleep          => $try_sleep,
+    verify_on_create   => $verify_on_create,
   }
 }

manifests/resource/route.pp

@@ -9,6 +9,7 @@ define pacemaker::resource::route(
   $post_success_sleep = 0,
   $tries              = 1,
   $try_sleep          = 0,
+  $verify_on_create   = false,
 ) {
 
   $nic_option = $nic ? {
@@ -40,6 +41,7 @@ define pacemaker::resource::route(
     post_success_sleep => $post_success_sleep,
     tries              => $tries,
     try_sleep          => $try_sleep,
+    verify_on_create   => $verify_on_create,
   }
 
 }

manifests/resource/service.pp

@@ -40,6 +40,7 @@ define pacemaker::resource::service(
   $post_success_sleep = 0,
   $tries              = 1,
   $try_sleep          = 0,
+  $verify_on_create   = false,
 ) {
   include ::pacemaker::params
   $res = "pacemaker::resource::${::pacemaker::params::services_manager}"
@@ -56,6 +57,7 @@ define pacemaker::resource::service(
       post_success_sleep => $post_success_sleep,
       tries              => $tries,
       try_sleep          => $try_sleep,
+      verify_on_create   => $verify_on_create,
     }
   })
 }

manifests/resource/systemd.pp

@@ -14,6 +14,7 @@ define pacemaker::resource::systemd(
   $post_success_sleep = 0,
   $tries              = 1,
   $try_sleep          = 0,
+  $verify_on_create   = false,
 ) {
   pcmk_resource { $name:
     ensure             => $ensure,
@@ -26,5 +27,6 @@ define pacemaker::resource::systemd(
     post_success_sleep => $post_success_sleep,
     tries              => $tries,
     try_sleep          => $try_sleep,
+    verify_on_create   => $verify_on_create,
   }
 }