puppet-pacemaker/lib/pacemaker/wait.rb

module Pacemaker
  # functions that can wait for something repeatedly
  # polling the system status until the condition is met
  module Wait
    # retry the given command until it runs without errors
    # or for RETRY_COUNT times with RETRY_STEP sec step
    # print cluster status report on fail
    # @param options [Hash]
    def retry_block(options = {})
      options = pacemaker_options.merge options

      options[:retry_count].times do
        begin
          out = Timeout.timeout(options[:retry_timeout]) { yield }
          if options[:retry_false_is_failure]
            return out if out
          else
            return out
          end
        rescue => e
          debug "Execution failure: #{e.message}"
        end
        sleep options[:retry_step]
      end
      raise "Execution timeout after #{options[:retry_count] * options[:retry_step]} seconds!" if options[:retry_fail_on_timeout]
    end

    # wait for pacemaker to become online
    # @param comment [String] log tag comment to trace calls
    def wait_for_online(comment = nil)
      message = "Waiting #{max_wait_time} seconds for Pacemaker to become online"
      message += " (#{comment})" if comment
      debug message
      retry_block do
        cib_reset 'wait_for_online'
        online?
      end
      debug 'Pacemaker is online'
    end

    # wait until a primitive has known status
    # @param primitive [String] primitive name
    # @param node [String] on this node if given
    def wait_for_status(primitive, node = nil)
      message = "Waiting #{max_wait_time} seconds for a known status of '#{primitive}'"
      message += " on node '#{node}'" if node
      debug message
      retry_block do
        cib_reset 'wait_for_status'
        !primitive_status(primitive).nil?
      end
      message = "Primitive '#{primitive}' has status '#{primitive_status primitive}'"
      message += " on node '#{node}'" if node
      debug message
    end

    # wait for primitive to start
    # if node is given then start on this node
    # @param primitive [String] primitive id
    # @param node [String] on this node if given
    def wait_for_start(primitive, node = nil)
      message = "Waiting #{max_wait_time} seconds for the service '#{primitive}' to start"
      message += " on node '#{node}'" if node
      debug message
      retry_block do
        cib_reset 'wait_for_start'
        primitive_is_running? primitive, node
      end
      message = "Service '#{primitive}' have started"
      message += " on node '#{node}'" if node
      debug message
    end

    # wait for primitive to start as a master
    # if node is given then start as a master on this node
    # @param primitive [String] primitive id
    # @param node [String] on this node if given
    def wait_for_master(primitive, node = nil)
      message = "Waiting #{max_wait_time} seconds for the service '#{primitive}' to start master"
      message += " on node '#{node}'" if node
      debug message
      retry_block do
        cib_reset 'wait_for_master'
        primitive_has_master_running? primitive, node
      end
      message = "Service '#{primitive}' have started master"
      message += " on node '#{node}'" if node
      debug message
    end

    # wait for primitive to stop
    # if node is given then start on this node
    # @param primitive [String] primitive id
    # @param node [String] on this node if given
    def wait_for_stop(primitive, node = nil)
      message = "Waiting #{max_wait_time} seconds for the service '#{primitive}' to stop"
      message += " on node '#{node}'" if node
      debug message
      retry_block do
        cib_reset 'wait_for_stop'
        result = primitive_is_running? primitive, node
        result.is_a? FalseClass
      end
      message = "Service '#{primitive}' was stopped"
      message += " on node '#{node}'" if node
      debug message
    end

    # add a new primitive to CIB
    # and wait for it to be actually created
    # @param xml [String, REXML::Element] XML block to add
    # @param primitive [String] the id of the new primitive
    # @param scope [String] XML root scope
    def wait_for_primitive_create(xml, primitive, scope = 'resources')
      message = "Waiting #{max_wait_time} seconds for the primitive '#{primitive}' to be created"
      debug message
      retry_block do
        if pacemaker_options[:cibadmin_idempotency_checks]
          cib_reset 'wait_for_primitive_create'
          break true if primitive_exists? primitive
        end
        cibadmin_create xml, scope
      end
      message = "Primitive '#{primitive}' was created"
      debug message
    end

    # remove a primitive from CIB
    # and wait for it to be actually removed
    # @param xml [String, REXML::Element] XML block to remove
    # @param primitive [String] the id of the removed primitive
    # @param scope [String] XML root scope
    def wait_for_primitive_remove(xml, primitive, scope = 'resources')
      message = "Waiting #{max_wait_time} seconds for the primitive '#{primitive}' to be removed"
      debug message
      retry_block do
        if pacemaker_options[:cibadmin_idempotency_checks]
          cib_reset 'wait_for_primitive_remove'
          break true unless primitive_exists? primitive
        end
        cibadmin_delete xml, scope
      end
      message = "Primitive '#{primitive}' was removed"
      debug message
    end

    # update a primitive in CIB
    # and wait for it to be actually updated
    # @param xml [String, REXML::Element] XML block to update
    # @param primitive [String] the id of the updated primitive
    # @param scope [String] XML root scope
    def wait_for_primitive_update(xml, primitive, scope = 'resources')
      message = "Waiting #{max_wait_time} seconds for the primitive '#{primitive}' to be updated"
      debug message
      retry_block do
        # replace action is already idempotent
        cibadmin_replace xml, scope
      end
      message = "Primitive '#{primitive}' was updated"
      debug message
    end

    # add a new constraint to CIB
    # and wait for it to be actually created
    # @param xml [String, REXML::Element] XML block to add
    # @param constraint [String] the id of the new constraint
    # @param scope [String] XML root scope
    def wait_for_constraint_create(xml, constraint, scope = 'constraints')
      message = "Waiting #{max_wait_time} seconds for the constraint '#{constraint}' to be created"
      debug message
      retry_block do
        if pacemaker_options[:cibadmin_idempotency_checks]
          cib_reset 'wait_for_constraint_create'
          break true if constraint_exists? constraint
        end
        cibadmin_create xml, scope
      end
      message = "Constraint '#{constraint}' was created"
      debug message
    end

    # remove a constraint from CIB
    # and wait for it to be actually removed
    # @param xml [String, REXML::Element] XML block to remove
    # @param constraint [String] the id of the removed constraint
    # @param scope [String] XML root scope
    def wait_for_constraint_remove(xml, constraint, scope = 'constraints')
      message = "Waiting #{max_wait_time} seconds for the constraint '#{constraint}' to be removed"
      debug message
      retry_block do
        if pacemaker_options[:cibadmin_idempotency_checks]
          cib_reset 'wait_for_constraint_remove'
          break true unless constraint_exists? constraint
        end
        cibadmin_delete xml, scope
      end
      message = "Constraint '#{constraint}' was removed"
      debug message
    end

    # update a constraint in CIB
    # and wait for it to be actually updated
    # @param xml [String, REXML::Element] XML block to update
    # @param constraint [String] the id of the updated constraint
    # @param scope [String] XML root scope
    def wait_for_constraint_update(xml, constraint, scope = 'constraints')
      message = "Waiting #{max_wait_time} seconds for the constraint '#{constraint}' to be updated"
      debug message
      retry_block do
        # replace action is already idempotent
        cibadmin_replace xml, scope
      end
      message = "Constraint '#{constraint}' was updated"
      debug message
    end

    # check if pacemaker is online and we can work with it
    # * pacemaker is online if cib can be downloaded
    # * dc_version attribute can be obtained
    # * DC have been designated
    # Times out a stuck command calls and catches failed command calls
    # @return [TrueClass,FalseClass]
    def online?
      Timeout.timeout(pacemaker_options[:retry_timeout]) do
        return false unless dc_version
        return false unless dc
        return false unless cib_section_node_state
        true
      end
    rescue Puppet::ExecutionFailure => e
      debug "Cluster is offline: #{e.message}"
      false
    rescue Timeout::Error
      debug 'Online check timeout!'
      false
    end
  end
end