From d5b4999b8cdde5342a4a0299b4155992ae642912 Mon Sep 17 00:00:00 2001 From: Bogdan Dobrelya Date: Mon, 24 Nov 2014 10:25:54 +0100 Subject: [PATCH] Add pacemaker provider for HA services * Add pacemaker service provider and unit tests * Add generic OCF handler script for RA scripts * Add service class and docs for pacemaker primitives creation with default OCF type = pacemaker * Add specs for pacemaker service define Implements step 1 of blueprint pacemaker-provider-for-openstack Change-Id: I5d98d8f9494bb7df4466022b3d49ac6392deb1a5 Co-Author: Dmitry Ilyin (idv1985 ) Signed-off-by: Bogdan Dobrelya --- .fixtures.yml | 1 + .gitignore | 1 + lib/puppet/provider/pacemaker_common.rb | 709 ++++++++++++++++++ lib/puppet/provider/service/pacemaker.rb | 199 +++++ manifests/pacemaker/service.pp | 192 +++++ metadata.json | 1 + ...openstack_extras_pacemaker_service_spec.rb | 139 ++++ spec/fixtures/manifests/site.pp | 1 + .../modules/foo/files/scripts/foo.ocf | 0 spec/fixtures/modules/foo/manifests/init.pp | 3 + .../modules/foo/templates/foo.ocf.erb | 1 + spec/init_spec.rb | 1 - spec/spec_helper.rb | 2 +- spec/unit/puppet/provider/cib.xml | 483 ++++++++++++ .../puppet/provider/pacemaker_common_spec.rb | 222 ++++++ .../puppet/provider/service/pacemaker_spec.rb | 227 ++++++ templates/ocf_handler.erb | 118 +++ 17 files changed, 2298 insertions(+), 2 deletions(-) create mode 100644 lib/puppet/provider/pacemaker_common.rb create mode 100644 lib/puppet/provider/service/pacemaker.rb create mode 100644 manifests/pacemaker/service.pp create mode 100644 spec/defines/openstack_extras_pacemaker_service_spec.rb create mode 100644 spec/fixtures/manifests/site.pp create mode 100644 spec/fixtures/modules/foo/files/scripts/foo.ocf create mode 100644 spec/fixtures/modules/foo/manifests/init.pp create mode 100644 spec/fixtures/modules/foo/templates/foo.ocf.erb delete mode 100644 spec/init_spec.rb create mode 100644 spec/unit/puppet/provider/cib.xml create mode 100644 spec/unit/puppet/provider/pacemaker_common_spec.rb create mode 100644 spec/unit/puppet/provider/service/pacemaker_spec.rb create mode 100644 templates/ocf_handler.erb diff --git a/.fixtures.yml b/.fixtures.yml index 5c237af..22f2f08 100644 --- a/.fixtures.yml +++ b/.fixtures.yml @@ -1,5 +1,6 @@ fixtures: repositories: + 'corosync': 'https://github.com/puppetlabs/puppetlabs-corosync' 'apt' : 'git://github.com/puppetlabs/puppetlabs-apt' 'stdlib' : 'git://github.com/puppetlabs/puppetlabs-stdlib' symlinks: diff --git a/.gitignore b/.gitignore index cce0112..8dee882 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ spec/fixtures/ pkg Gemfile.lock *.swp +.idea diff --git a/lib/puppet/provider/pacemaker_common.rb b/lib/puppet/provider/pacemaker_common.rb new file mode 100644 index 0000000..c9e45e2 --- /dev/null +++ b/lib/puppet/provider/pacemaker_common.rb @@ -0,0 +1,709 @@ +require 'rexml/document' + +class Puppet::Provider::Pacemaker_common < Puppet::Provider + + @raw_cib = nil + @cib = nil + @primitives = nil + @primitives_structure = nil + + RETRY_COUNT = 100 + RETRY_STEP = 6 + + # get a raw CIB from cibadmin + # or from a debug file if raw_cib_file is set + # @return [String] cib xml + def raw_cib + @raw_cib = cibadmin '-Q' + if @raw_cib == '' or not @raw_cib + fail 'Could not dump CIB XML using "cibadmin -Q" command!' + end + @raw_cib + end + + # create a new REXML CIB document + # @return [REXML::Document] at '/' + def cib + return @cib if @cib + @cib = REXML::Document.new(raw_cib) + end + + # reset all saved variables to obtain new data + def cib_reset + # Puppet.debug 'Reset CIB memoization' + @raw_cib = nil + @cib = nil + @primitives = nil + @primitives_structure = nil + @nodes_structure = nil + end + + # get status CIB section + # @return [REXML::Element] at /cib/status + def cib_section_status + REXML::XPath.match cib, '/cib/status' + end + + # get lrm_rsc_ops section from lrm_resource section CIB section + # @param lrm_resource [REXML::Element] + # at /cib/status/node_state/lrm[@id="node-name"]/lrm_resources/lrm_resource[@id="resource-name"]/lrm_rsc_op + # @return [REXML::Element] + def cib_section_lrm_rsc_ops(lrm_resource) + REXML::XPath.match lrm_resource, 'lrm_rsc_op' + end + + # get node_state CIB section + # @return [REXML::Element] at /cib/status/node_state + def cib_section_nodes_state + REXML::XPath.match cib_section_status, 'node_state' + end + + # get primitives CIB section + # @return [REXML::Element] at /cib/configuration/resources/primitive + def cib_section_primitives + REXML::XPath.match cib, '//primitive' + end + + # get lrm_rsc_ops section from lrm_resource section CIB section + # @param lrm [REXML::Element] + # at /cib/status/node_state/lrm[@id="node-name"]/lrm_resources/lrm_resource + # @return [REXML::Element] + def cib_section_lrm_resources(lrm) + REXML::XPath.match lrm, 'lrm_resources/lrm_resource' + end + + # determine the status of a single operation + # @param op [Hash String>] + # @return ['start','stop','master',nil] + def operation_status(op) + # skip incomplete ops + return unless op['op-status'] == '0' + + if op['operation'] == 'monitor' + # for monitor operation status is determined by its rc-code + # 0 - start, 8 - master, 7 - stop, else - error + case op['rc-code'] + when '0' + 'start' + when '7' + 'stop' + when '8' + 'master' + else + # not entirely correct but count failed monitor as 'stop' + 'stop' + end + elsif %w(start stop promote).include? op['operation'] + # for start/stop/promote status is set if op was successful + # use master instead of promote + return unless %w(0 7 8).include? op['rc-code'] + if op['operation'] == 'promote' + 'master' + else + op['operation'] + end + else + # other operations are irrelevant + nil + end + end + + # determine resource status by parsing last operations + # @param ops [Array] + # @return ['start','stop','master',nil] + # nil means that status is unknown + def determine_primitive_status(ops) + status = nil + ops.each do |op| + op_status = operation_status op + status = op_status if op_status + end + status + end + + # check if operations have same failed operations + # that should be cleaned up later + # @param ops [Array] + # @return [TrueClass,FalseClass] + def failed_operations_found?(ops) + ops.each do |op| + # skip incompleate ops + next unless op['op-status'] == '0' + # skip useless ops + next unless %w(start stop monitor promote).include? op['operation'] + + # are there failed start, stop + if %w(start stop promote).include? op['operation'] + return true if op['rc-code'] != '0' + end + + # are there failed monitors + if op['operation'] == 'monitor' + return true unless %w(0 7 8).include? op['rc-code'] + end + end + false + end + + # convert elements's attributes to hash + # @param element [REXML::Element] + # @return [Hash String>] + def attributes_to_hash(element) + hash = {} + element.attributes.each do |a, v| + hash.store a.to_s, v.to_s + end + hash + end + + # convert element's children to hash + # of their attributes using key and hash key + # @param element [REXML::Element] + # @param key + # @return [Hash String>] + def elements_to_hash(element, key, tag = nil) + elements = {} + children = element.get_elements tag + return elements unless children + children.each do |child| + child_structure = attributes_to_hash child + name = child_structure[key] + next unless name + elements.store name, child_structure + end + elements + end + + # decode lrm_resources section of CIB + # @param lrm_resources [REXML::Element] + # @return [Hash Hash>] + def decode_lrm_resources(lrm_resources) + resources = {} + lrm_resources.each do |lrm_resource| + resource = attributes_to_hash lrm_resource + id = resource['id'] + next unless id + lrm_rsc_ops = cib_section_lrm_rsc_ops lrm_resource + ops = decode_lrm_rsc_ops lrm_rsc_ops + resource.store 'ops', ops + resource.store 'status', determine_primitive_status(ops) + resource.store 'failed', failed_operations_found?(ops) + resources.store id, resource + end + resources + end + + # decode lrm_rsc_ops section of the resource's CIB + # @param lrm_rsc_ops [REXML::Element] + # @return [Array] + def decode_lrm_rsc_ops(lrm_rsc_ops) + ops = [] + lrm_rsc_ops.each do |lrm_rsc_op| + op = attributes_to_hash lrm_rsc_op + next unless op['call-id'] + ops << op + end + ops.sort { |a,b| a['call-id'].to_i <=> b['call-id'].to_i } + end + + # get nodes structure with resources and their statuses + # @return [Hash Hash>] + def nodes + return @nodes_structure if @nodes_structure + @nodes_structure = {} + cib_section_nodes_state.each do |node_state| + node = attributes_to_hash node_state + id = node['id'] + next unless id + lrm = node_state.elements['lrm'] + lrm_resources = cib_section_lrm_resources lrm + resources = decode_lrm_resources lrm_resources + node.store 'primitives', resources + @nodes_structure.store id, node + end + @nodes_structure + end + + # get primitives configuration structure with primitives and their attributes + # @return [Hash Hash>] + def primitives + return @primitives_structure if @primitives_structure + @primitives_structure = {} + cib_section_primitives.each do |primitive| + primitive_structure = {} + id = primitive.attributes['id'] + next unless id + primitive_structure.store 'name', id + primitive.attributes.each do |k, v| + primitive_structure.store k.to_s, v + end + + if primitive.parent.name and primitive.parent.attributes['id'] + parent_structure = { + 'id' => primitive.parent.attributes['id'], + 'type' => primitive.parent.name + } + primitive_structure.store 'name', parent_structure['id'] + primitive_structure.store 'parent', parent_structure + end + + instance_attributes = primitive.elements['instance_attributes'] + if instance_attributes + instance_attributes_structure = elements_to_hash instance_attributes, 'name', 'nvpair' + primitive_structure.store 'instance_attributes', instance_attributes_structure + end + + meta_attributes = primitive.elements['meta_attributes'] + if meta_attributes + meta_attributes_structure = elements_to_hash meta_attributes, 'name', 'nvpair' + primitive_structure.store 'meta_attributes', meta_attributes_structure + end + + operations = primitive.elements['operations'] + if operations + operations_structure = elements_to_hash operations, 'id', 'op' + primitive_structure.store 'operations', operations_structure + end + + @primitives_structure.store id, primitive_structure + end + @primitives_structure + end + + # check if primitive is clone or multistate + # @param primitive [String] primitive id + # @return [TrueClass,FalseClass] + def primitive_is_complex?(primitive) + return unless primitive_exists? primitive + primitives[primitive].key? 'parent' + end + + # check if primitive is clone + # @param primitive [String] primitive id + # @return [TrueClass,FalseClass] + def primitive_is_clone?(primitive) + is_complex = primitive_is_complex? primitive + return is_complex unless is_complex + primitives[primitive]['parent']['type'] == 'clone' + end + + # check if primitive is multistate + # @param primitive [String] primitive id + # @return [TrueClass,FalseClass] + def primitive_is_multistate?(primitive) + is_complex = primitive_is_complex? primitive + return is_complex unless is_complex + primitives[primitive]['parent']['type'] == 'master' + end + + # disable this primitive + # @param primitive [String] + def disable_primitive(primitive) + retry_command { + pcs 'resource', 'disable', primitive + } + end + alias :stop_primitive :disable_primitive + + # enable this primitive + # @param primitive [String] + def enable_primitive(primitive) + retry_command { + pcs 'resource', 'enable', primitive + } + end + alias :start_primitive :enable_primitive + + # ban this primitive + # @param primitive [String] + def ban_primitive(primitive, node = '') + retry_command { + pcs 'resource', 'ban', primitive, node + } + end + + # move this primitive + # @param primitive [String] + def move_primitive(primitive, node = '') + retry_command { + pcs 'resource', 'move', primitive, node + } + end + + # unban/unmove this primitive + # @param primitive [String] + def unban_primitive(primitive, node = '') + retry_command { + pcs 'resource', 'clear', primitive, node + } + end + alias :clear_primitive :unban_primitive + alias :unmove_primitive :unban_primitive + + # cleanup this primitive + # @param primitive [String] + def cleanup_primitive(primitive, node = '') + opts = ['--cleanup', "--resource=#{primitive}"] + opts << "--node=#{node}" if ! node.empty? + retry_command { + crm_resource opts + } + end + + # manage this primitive + # @param primitive [String] + def manage_primitive(primitive) + retry_command { + pcs 'resource', 'manage', primitive + } + end + + # unamanage this primitive + # @param primitive [String] + def unmanage_primitive(primitive) + retry_command { + pcs 'resource', 'unmanage', primitive + } + end + + # set quorum_policy of the cluster + # @param primitive [String] + def no_quorum_policy(primitive) + retry_command { + pcs 'property', 'set', "no-quorum-policy=#{primitive}" + } + end + + # set maintenance_mode of the cluster + # @param primitive [TrueClass,FalseClass] + def maintenance_mode(primitive) + retry_command { + pcs 'property', 'set', "maintenance-mode=#{primitive}" + } + end + + # add a location constraint + # @param primitive [String] the primitive's name + # @param node [String] the node's name + # @param score [Numeric,String] score value + def constraint_location_add(primitive, node, score = 100) + id = "#{primitive}_on_#{node}" + retry_command { + pcs 'constraint', 'location', 'add', id, primitive, node, score + } + end + + # remove a location constraint + # @param primitive [String] the primitive's name + # @param node [String] the node's name + def constraint_location_remove(primitive, node) + id = "#{primitive}_on_#{node}" + retry_command { + pcs 'constraint', 'location', 'remove', id + } + end + + # get a status of a primitive on the entire cluster + # of on a node if node name param given + # @param primitive [String] + # @param node [String] + # @return [String] + def primitive_status(primitive, node = nil) + if node + nodes. + fetch(node, {}). + fetch('primitives',{}). + fetch(primitive, {}). + fetch('status', nil) + else + statuses = [] + nodes.each do |k,v| + status = v.fetch('primitives',{}). + fetch(primitive, {}). + fetch('status', nil) + statuses << status + end + status_values = { + 'stop' => 0, + 'start' => 1, + 'master' => 2, + } + statuses.max_by do |status| + return unless status + status_values[status] + end + end + end + + # generate report of primitive statuses by node + # mostly for debugging + # @return [Hash] + def primitives_status_by_node + report = {} + return unless nodes.is_a? Hash + nodes.each do |node_name, node_data| + primitives_of_node = node_data['primitives'] + next unless primitives_of_node.is_a? Hash + primitives_of_node.each do |primitive, primitive_data| + primitive_status = primitive_data['status'] + report[primitive] = {} unless report[primitive].is_a? Hash + report[primitive][node_name] = primitive_status + end + end + report + end + + # form a cluster status report for debugging + # @return [String] + def get_cluster_debug_report + report = "\n" + primitives_status_by_node.each do |primitive, data| + primitive_name = primitive + primitive_name = primitives[primitive]['name'] if primitives[primitive]['name'] + primitive_type = 'Simple' + primitive_type = 'Cloned' if primitive_is_clone? primitive + primitive_type = 'Multistate' if primitive_is_multistate? primitive + primitive_status = primitive_status primitive + + report += "-> #{primitive_type} primitive '#{primitive_name}' global status: #{primitive_status}" + report += ' (UNMANAGE)' unless primitive_is_managed? primitive + report += "\n" + report += ' ' if data.any? + nodes = [] + data.keys.sort.each do |node_name| + node_status = data.fetch node_name + node_block = "#{node_name}: #{node_status}" + node_block += ' (FAIL)' if primitive_has_failures? primitive, node_name + nodes << node_block + end + report += nodes.join ' | ' + report += "\n" + end + report + end + + # does this primitive have failed operations? + # @param primitive [String] primitive name + # @param node [String] on this node if given + # @return [TrueClass,FalseClass] + def primitive_has_failures?(primitive, node = nil) + return unless primitive_exists? primitive + if node + nodes. + fetch(node, {}). + fetch('primitives',{}). + fetch(primitive, {}). + fetch('failed', nil) + else + nodes.each do |k,v| + failed = v.fetch('primitives',{}). + fetch(primitive, {}). + fetch('failed', nil) + return true if failed + end + false + end + end + + # determine if a primitive is running on the entire cluster + # of on a node if node name param given + # @param primitive [String] primitive id + # @param node [String] on this node if given + # @return [TrueClass,FalseClass] + def primitive_is_running?(primitive, node = nil) + return unless primitive_exists? primitive + status = primitive_status primitive, node + return status unless status + %w(start master).include? status + end + + # check if primitive is running as a master + # either anywhere or on the give node + # @param primitive [String] primitive id + # @param node [String] on this node if given + # @return [TrueClass,FalseClass] + def primitive_has_master_running?(primitive, node = nil) + is_multistate = primitive_is_multistate? primitive + return is_multistate unless is_multistate + status = primitive_status primitive, node + return status unless status + status == 'master' + end + + # return service status value expected by Puppet + # puppet wants :running or :stopped symbol + # @param primitive [String] primitive id + # @param node [String] on this node if given + # @return [:running,:stopped] + def get_primitive_puppet_status(primitive, node = nil) + if primitive_is_running? primitive, node + :running + else + :stopped + end + end + + # return service enabled status value expected by Puppet + # puppet wants :true or :false symbols + # @param primitive [String] + # @return [:true,:false] + def get_primitive_puppet_enable(primitive) + if primitive_is_managed? primitive + :true + else + :false + end + end + + # check if primitive exists in the confiuguration + # @param primitive primitive id or name + def primitive_exists?(primitive) + primitives.key? primitive + end + + # determine if primitive is managed + # @param primitive [String] primitive id + # @return [TrueClass,FalseClass] + # TODO: will not work correctly if cluster is in management mode + def primitive_is_managed?(primitive) + return unless primitive_exists? primitive + is_managed = primitives.fetch(primitive).fetch('meta_attributes', {}).fetch('is-managed', {}).fetch('value', 'true') + is_managed == 'true' + end + + # determine if primitive has target-state started + # @param primitive [String] primitive id + # @return [TrueClass,FalseClass] + # TODO: will not work correctly if target state is set globally to stopped + def primitive_is_started?(primitive) + return unless primitive_exists? primitive + target_role = primitives.fetch(primitive).fetch('meta_attributes', {}).fetch('target-role', {}).fetch('value', 'Started') + target_role == 'Started' + end + + # check if pacemaker is online + # and we can work with it + # @return [TrueClass,FalseClass] + def is_online? + begin + cibadmin '-Q' + true + rescue Puppet::ExecutionFailure + false + else + true + end + end + + # retry the given command until it runs without errors + # or for RETRY_COUNT times with RETRY_STEP sec step + # print cluster status report on fail + # returns normal command output on success + # @return [String] + def retry_command + (0..RETRY_COUNT).each do + begin + out = yield + rescue Puppet::ExecutionFailure => e + Puppet.debug "Command failed: #{e.message}" + sleep RETRY_STEP + else + return out + end + end + Puppet.debug get_cluster_debug_report if is_online? + fail "Execution timeout after #{RETRY_COUNT * RETRY_STEP} seconds!" + end + + # retry the given block until it returns true + # or for RETRY_COUNT times with RETRY_STEP sec step + # print cluster status report on fail + def retry_block_until_true + (0..RETRY_COUNT).each do + return if yield + sleep RETRY_STEP + end + Puppet.debug get_cluster_debug_report if is_online? + fail "Execution timeout after #{RETRY_COUNT * RETRY_STEP} seconds!" + end + + # wait for pacemaker to become online + def wait_for_online + Puppet.debug "Waiting #{RETRY_COUNT * RETRY_STEP} seconds for Pacemaker to become online" + retry_block_until_true do + is_online? + end + Puppet.debug 'Pacemaker is online' + end + + # cleanup a primitive and then wait until + # we can get it's status again because + # cleanup blocks operations sections for a while + # @param primitive [String] primitive name + def cleanup_with_wait(primitive, node = '') + node_msgpart = node.empty? ? '' : " on node '#{node}'" + Puppet.debug "Cleanup primitive '#{primitive}'#{node_msgpart} and wait until cleanup finishes" + cleanup_primitive(primitive, node) + retry_block_until_true do + cib_reset + primitive_status(primitive) != nil + end + Puppet.debug "Primitive '#{primitive}' have been cleaned up#{node_msgpart} and is online again" + end + + # wait for primitive to start + # if node is given then start on this node + # @param primitive [String] primitive id + # @param node [String] on this node if given + def wait_for_start(primitive, node = nil) + message = "Waiting #{RETRY_COUNT * RETRY_STEP} seconds for service '#{primitive}' to start" + message += " on node '#{node}'" if node + Puppet.debug message + retry_block_until_true do + cib_reset + primitive_is_running? primitive, node + end + Puppet.debug get_cluster_debug_report + message = "Service '#{primitive}' have started" + message += " on node '#{node}'" if node + Puppet.debug message + end + + # wait for primitive to start as a master + # if node is given then start as a master on this node + # @param primitive [String] primitive id + # @param node [String] on this node if given + def wait_for_master(primitive, node = nil) + message = "Waiting #{RETRY_COUNT * RETRY_STEP} seconds for service '#{primitive}' to start master" + message += " on node '#{node}'" if node + Puppet.debug message + retry_block_until_true do + cib_reset + primitive_has_master_running? primitive, node + end + Puppet.debug get_cluster_debug_report + message = "Service '#{primitive}' have started master" + message += " on node '#{node}'" if node + Puppet.debug message + end + + # wait for primitive to stop + # if node is given then start on this node + # @param primitive [String] primitive id + # @param node [String] on this node if given + def wait_for_stop(primitive, node = nil) + message = "Waiting #{RETRY_COUNT * RETRY_STEP} seconds for service '#{primitive}' to stop" + message += " on node '#{node}'" if node + Puppet.debug message + retry_block_until_true do + cib_reset + result = primitive_is_running? primitive, node + result.is_a? FalseClass + end + Puppet.debug get_cluster_debug_report + message = "Service '#{primitive}' was stopped" + message += " on node '#{node}'" if node + Puppet.debug message + end + +end diff --git a/lib/puppet/provider/service/pacemaker.rb b/lib/puppet/provider/service/pacemaker.rb new file mode 100644 index 0000000..5a6a0f2 --- /dev/null +++ b/lib/puppet/provider/service/pacemaker.rb @@ -0,0 +1,199 @@ +require File.join File.dirname(__FILE__), '../pacemaker_common.rb' + +Puppet::Type.type(:service).provide :pacemaker, :parent => Puppet::Provider::Pacemaker_common do + + has_feature :enableable + has_feature :refreshable + + commands :uname => 'uname' + commands :pcs => 'pcs' + commands :crm_resource => 'crm_resource' + commands :cibadmin => 'cibadmin' + + # hostname of the current node + # @return [String] + def hostname + return @hostname if @hostname + @hostname = (uname '-n').chomp.strip + end + + # original name passed from the type + # @return [String] + def title + @resource[:name] + end + + # primitive name with 'p_' added if needed + # @return [String] + def name + return @name if @name + primitive_name = title + if primitive_exists? primitive_name + Puppet.debug "Primitive with title '#{primitive_name}' was found in CIB" + @name = primitive_name + return @name + end + primitive_name = "p_#{primitive_name}" + if primitive_exists? primitive_name + Puppet.debug "Using '#{primitive_name}' name instead of '#{title}'" + @name = primitive_name + return @name + end + fail "Primitive '#{title}' was not found in CIB!" + end + + # full name of the primitive + # if resource is complex use group name + # @return [String] + def full_name + return @full_name if @full_name + if primitive_is_complex? name + full_name = primitives[name]['name'] + Puppet.debug "Using full name '#{full_name}' for complex primitive '#{name}'" + @full_name = full_name + else + @full_name = name + end + end + + # name of the basic service without 'p_' prefix + # used to disable the basic service + # @return [String] + def basic_service_name + return @basic_service_name if @basic_service_name + if name.start_with? 'p_' + basic_service_name = name.gsub /^p_/, '' + Puppet.debug "Using '#{basic_service_name}' as the basic service name for primitive '#{name}'" + @basic_service_name = basic_service_name + else + @basic_service_name = name + end + end + + # called by Puppet to determine if the service + # is running on the local node + # @return [:running,:stopped] + def status + wait_for_online + Puppet.debug "Call: 'status' for Pacemaker service '#{name}' on node '#{hostname}'" + cib_reset + out = get_primitive_puppet_status name, hostname + Puppet.debug get_cluster_debug_report + Puppet.debug "Return: '#{out}' (#{out.class})" + out + end + + # called by Puppet to start the service + def start + Puppet.debug "Call 'start' for Pacemaker service '#{name}' on node '#{hostname}'" + enable unless primitive_is_managed? name + disable_basic_service + constraint_location_add name, hostname + unban_primitive name, hostname + start_primitive name + cleanup_with_wait(name, hostname) if primitive_has_failures?(name, hostname) + + if primitive_is_multistate? name + Puppet.debug "Choose master start for Pacemaker service '#{name}'" + wait_for_master name + else + Puppet.debug "Choose global start for Pacemaker service '#{name}'" + wait_for_start name + end + end + + # called by Puppet to stop the service + def stop + Puppet.debug "Call 'stop' for Pacemaker service '#{name}' on node '#{hostname}'" + enable unless primitive_is_managed? name + cleanup_with_wait(name, hostname) if primitive_has_failures?(name, hostname) + + if primitive_is_complex? name + Puppet.debug "Choose local stop for Pacemaker service '#{name}' on node '#{hostname}'" + ban_primitive name, hostname + wait_for_stop name, hostname + else + Puppet.debug "Choose global stop for Pacemaker service '#{name}'" + stop_primitive name + wait_for_stop name + end + end + + # called by Puppet to restart the service + def restart + Puppet.debug "Call 'restart' for Pacemaker service '#{name}' on node '#{hostname}'" + unless primitive_is_running? name, hostname + Puppet.info "Pacemaker service '#{name}' is not running on node '#{hostname}'. Skipping restart!" + return + end + + begin + stop + rescue + nil + ensure + start + end + end + + # called by Puppet to enable the service + def enable + Puppet.debug "Call 'enable' for Pacemaker service '#{name}' on node '#{hostname}'" + manage_primitive name + end + + # called by Puppet to disable the service + def disable + Puppet.debug "Call 'disable' for Pacemaker service '#{name}' on node '#{hostname}'" + unmanage_primitive name + end + alias :manual_start :disable + + # called by Puppet to determine if the service is enabled + # @return [:true,:false] + def enabled? + Puppet.debug "Call 'enabled?' for Pacemaker service '#{name}' on node '#{hostname}'" + out = get_primitive_puppet_enable name + Puppet.debug "Return: '#{out}' (#{out.class})" + out + end + + # create an extra provider instance to deal with the basic service + # the provider will be chosen to match the current system + # @return [Puppet::Type::Service::Provider] + def extra_provider(provider_name = nil) + return @extra_provider if @extra_provider + begin + param_hash = {} + param_hash.store :name, basic_service_name + param_hash.store :provider, provider_name if provider_name + type = Puppet::Type::Service.new param_hash + @extra_provider = type.provider + rescue => e + Puppet.warning "Could not get extra provider for Pacemaker primitive '#{name}': #{e.message}" + @extra_provider = nil + end + end + + # disable and stop the basic service + def disable_basic_service + return unless extra_provider + begin + if extra_provider.enableable? and extra_provider.enabled? == :true + Puppet.info "Disable basic service '#{extra_provider.name}' using provider '#{extra_provider.class.name}'" + extra_provider.disable + else + Puppet.info "Basic service '#{extra_provider.name}' is disabled as reported by '#{extra_provider.class.name}' provider" + end + if extra_provider.status == :running + Puppet.info "Stop basic service '#{extra_provider.name}' using provider '#{extra_provider.class.name}'" + extra_provider.stop + else + Puppet.info "Basic service '#{extra_provider.name}' is stopped as reported by '#{extra_provider.class.name}' provider" + end + rescue => e + Puppet.warning "Could not disable basic service for Pacemaker primitive '#{name}' using '#{extra_provider.class.name}' provider: #{e.message}" + end + end + +end diff --git a/manifests/pacemaker/service.pp b/manifests/pacemaker/service.pp new file mode 100644 index 0000000..2805601 --- /dev/null +++ b/manifests/pacemaker/service.pp @@ -0,0 +1,192 @@ +# == Class: openstack_extras::pacemaker::service +# +# Configures Pacemaker resource for a specified service and +# overrides its service provider to Pacemaker. +# Assumes there is a service already exists in the Puppet catalog. +# For example, the one, such as nova-api, heat-engine, neutron-agent-l3 +# and so on, created by other core Puppet modules for Openstack. +# +# === Parameters +# +# [*ensure*] +# (optional) The state of the service provided by Pacemaker +# Defaults to present +# +# [*ocf_root_path*] +# (optional) The path for OCF scripts +# Defaults to /usr/lib/ocf +# +# [*primitive_class*] +# (optional) The class of Pacemaker resource (primitive) +# Defaults to ocf +# +# [*primitive_provider*] +# (optional) The provider of OCF scripts +# Defaults to pacemaker +# +# [*primitive_type*] +# (optional) The type of the primitive (OCF file name). +# Used with the other parameters as a full path to OCF script: +# primitive_class/primitive_provider/primitive_type +# resided at ocf_root_path/resource.d +# Defaults to false +# +# [*parameters*] +# (optional) The hash of parameters for a primitive +# Defaults to false +# +# [*operations*] +# (optional) The hash of operations for a primitive +# Defaults to false +# +# [*metadata*] +# (optional) The hash of metadata for a primitive +# Defaults to false +# +# [*ms_metadata*] +# (optional) The hash of ms_metadata for a primitive +# Defaults to false +# +# [*use_handler*] +# (optional) The handler (wrapper script) for OCF script +# Could be useful for debug and informational purposes. +# It sets some default values like OCF_ROOT in order to +# simplify debugging of OCF scripts +# Defaults to true +# +# [*handler_root_path*] +# (optional) The path for a handler script +# Defaults to /usr/local/bin +# +# [*ocf_script_template*] +# (optional) ERB template for OCF script for Pacemaker +# resource +# Defaults to false +# +# [*ocf_script_file*] +# (optional) OCF file for Pacemaker resource +# Defaults to false +# +# [*create_primitive*] +# (optional) Controls Pacemaker primitive creation +# Defaults to true +# +# === Examples +# +# Will create resource and ensure Pacemaker provider for +# 'some-api-service' with the given OCF scripte template and +# parameters: +# +# $metadata = { +# 'resource-stickiness' => '1' +# } +# $operations = { +# 'monitor' => { +# 'interval' => '20', +# 'timeout' => '30', +# }, +# 'start' => { +# 'timeout' => '60', +# }, +# 'stop' => { +# 'timeout' => '60', +# }, +# } +# $ms_metadata = { +# 'interleave' => true, +# } +# +# openstack_extras::pacemaker::service { 'some-api-service' : +# primitive_type => 'some-api-service', +# metadata => $metadata, +# ms_metadata => $ms_metadata, +# operations => $operations, +# ocf_script_template => 'some_module/some_api_service.ocf.erb', +# } +# +define openstack_extras::pacemaker::service ( + $ensure = 'present', + $ocf_root_path = '/usr/lib/ocf', + $primitive_class = 'ocf', + $primitive_provider = 'pacemaker', + $primitive_type = false, + $parameters = false, + $operations = false, + $metadata = false, + $ms_metadata = false, + $use_handler = true, + $handler_root_path = '/usr/local/bin', + $ocf_script_template = false, + $ocf_script_file = false, + $create_primitive = true, +) { + + $service_name = $title + $primitive_name = "p_${service_name}" + $ocf_script_name = "${service_name}-ocf-file" + $ocf_handler_name = "ocf_handler_${service_name}" + + $ocf_dir_path = "${ocf_root_path}/resource.d" + $ocf_script_path = "${ocf_dir_path}/${primitive_provider}/${$primitive_type}" + $ocf_handler_path = "${handler_root_path}/${ocf_handler_name}" + + Service<| title == $service_name |> { + provider => 'pacemaker', + } + + Service<| name == $service_name |> { + provider => 'pacemaker', + } + + if $create_primitive { + cs_primitive { $primitive_name : + ensure => $ensure, + primitive_class => $primitive_class, + primitive_type => $primitive_type, + provided_by => $primitive_provider, + parameters => $parameters, + operations => $operations, + metadata => $metadata, + ms_metadata => $ms_metadata, + } + } + + if $ocf_script_template or $ocf_script_file { + file { $ocf_script_name : + ensure => $ensure, + path => $ocf_script_path, + mode => '0755', + owner => 'root', + group => 'root', + } + + if $ocf_script_template { + File[$ocf_script_name] { + content => template($ocf_script_template), + } + } elsif $ocf_script_file { + File[$ocf_script_name] { + source => "puppet:///modules/${ocf_script_file}", + } + } + + } + + if ($primitive_class == 'ocf') and ($use_handler) { + file { $ocf_handler_name : + ensure => present, + path => $ocf_handler_path, + owner => 'root', + group => 'root', + mode => '0700', + content => template('openstack_extras/ocf_handler.erb'), + } + } + + File<| title == $ocf_script_name |> -> + Cs_primitive<| title == $primitive_name |> + File<| title == $ocf_script_name |> ~> Service[$service_name] + Cs_primitive<| title == $primitive_name |> -> Service[$service_name] + File<| title == $ocf_handler_name |> -> Service[$service_name] + +} diff --git a/metadata.json b/metadata.json index 9070e43..4c36ba5 100644 --- a/metadata.json +++ b/metadata.json @@ -31,5 +31,6 @@ ], "description": "Puppet module to add useful utilities for OpenStack deployments", "dependencies": [ + { "name": "puppetlabs/corosync", "version_requirement": ">=0.1.0" } ] } diff --git a/spec/defines/openstack_extras_pacemaker_service_spec.rb b/spec/defines/openstack_extras_pacemaker_service_spec.rb new file mode 100644 index 0000000..bb6717e --- /dev/null +++ b/spec/defines/openstack_extras_pacemaker_service_spec.rb @@ -0,0 +1,139 @@ +require 'spec_helper' + +describe 'openstack_extras::pacemaker::service', :type => :define do + + let :pre_condition do + "class { 'foo': }" + end + + let (:title) { 'foo-api' } + + let :default_params do + { + :ensure => 'present', + :ocf_root_path => '/usr/lib/ocf', + :primitive_class => 'ocf', + :primitive_provider => 'pacemaker', + :primitive_type => false, + :parameters => false, + :operations => false, + :metadata => false, + :ms_metadata => false, + :use_handler => true, + :handler_root_path => '/usr/local/bin', + :ocf_script_template => false, + :ocf_script_file => false, + :create_primitive => true + } + end + + context 'with defaults' do + it 'should contain openstack_extras::pacemaker::service definition' do + should contain_openstack_extras__pacemaker__service(title).with(default_params) + end + + it 'should override existing service provider' do + should contain_service('foo-api').with( + { + :provider => 'pacemaker' + }) + end + + it 'should create a pacemaker primitive' do + should contain_cs_primitive('p_foo-api').with( + { + 'ensure' => default_params[:ensure], + 'primitive_class' => default_params[:primitive_class], + 'primitive_type' => default_params[:primitive_type], + 'provided_by' => default_params[:primitive_provider], + 'parameters' => default_params[:parameters], + 'operations' => default_params[:operations], + 'metadata' => default_params[:metadata], + 'ms_metadata' => default_params[:ms_metadata], + }) + end + end + + context 'with custom OCF file' do + let :params do + default_params.merge( + { + :ocf_script_file => 'foo/scripts/foo.ocf' + } + ) + end + let (:ocf_dir_path) { "#{params[:ocf_root_path]}/resource.d" } + let (:ocf_script_path) { "#{ocf_dir_path}/#{params[:primitive_provider]}/#{params[:primitive_type]}" } + let (:ocf_handler_name) { "ocf_handler_#{title}" } + let (:ocf_handler_path) { "#{params[:handler_root_path]}/#{ocf_handler_name}" } + + it 'should create an OCF file' do + should contain_file("#{title}-ocf-file").with( + { + 'ensure' => 'present', + 'path' => ocf_script_path, + 'mode' => '0755', + 'owner' => 'root', + 'group' => 'root', + 'source' => "puppet:///modules/#{params[:ocf_script_file]}" + }) + end + + it 'should create a handler file' do + should contain_file("#{ocf_handler_name}").with( + { + 'ensure' => 'present', + 'path' => ocf_handler_path, + 'owner' => 'root', + 'group' => 'root', + 'mode' => '0700', + }).with_content(/OCF_ROOT/) + end + + end + + context 'with custom OCF path, provider, erb and w/o a wrapper' do + let(:params) do + default_params.merge( + { + :ocf_script_template => 'foo/foo.ocf.erb', + :use_handler => false, + :primitive_provider => 'some_provider', + :ocf_root_path => '/usr/lib/some_path', + }) + end + let (:ocf_dir_path) { "#{params[:ocf_root_path]}/resource.d" } + let (:ocf_script_path) { + "#{ocf_dir_path}/#{params[:primitive_provider]}/#{params[:primitive_type]}" + } + + it 'should create an OCF file from template' do + should contain_file("#{title}-ocf-file").with( + { + 'path' => ocf_script_path, + 'mode' => '0755', + 'owner' => 'root', + 'group' => 'root' + }).with_content(/erb/) + end + + it 'should not create a handler file' do + should_not contain_file("#{params[:ocf_handler_name]}") + end + + it 'should create a pacemaker primitive' do + should contain_cs_primitive('p_foo-api').with( + { + 'ensure' => params[:ensure], + 'primitive_class' => params[:primitive_class], + 'primitive_type' => params[:primitive_type], + 'provided_by' => params[:primitive_provider], + 'parameters' => params[:parameters], + 'operations' => params[:operations], + 'metadata' => params[:metadata], + 'ms_metadata' => params[:ms_metadata], + }) + end + end + +end diff --git a/spec/fixtures/manifests/site.pp b/spec/fixtures/manifests/site.pp new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/spec/fixtures/manifests/site.pp @@ -0,0 +1 @@ + diff --git a/spec/fixtures/modules/foo/files/scripts/foo.ocf b/spec/fixtures/modules/foo/files/scripts/foo.ocf new file mode 100644 index 0000000..e69de29 diff --git a/spec/fixtures/modules/foo/manifests/init.pp b/spec/fixtures/modules/foo/manifests/init.pp new file mode 100644 index 0000000..1e367d1 --- /dev/null +++ b/spec/fixtures/modules/foo/manifests/init.pp @@ -0,0 +1,3 @@ +class foo () { + service { 'foo-api': } +} diff --git a/spec/fixtures/modules/foo/templates/foo.ocf.erb b/spec/fixtures/modules/foo/templates/foo.ocf.erb new file mode 100644 index 0000000..c150680 --- /dev/null +++ b/spec/fixtures/modules/foo/templates/foo.ocf.erb @@ -0,0 +1 @@ +erb diff --git a/spec/init_spec.rb b/spec/init_spec.rb deleted file mode 100644 index f8ec369..0000000 --- a/spec/init_spec.rb +++ /dev/null @@ -1 +0,0 @@ -require 'spec_helper' diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 2c6f566..3d92005 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1 +1 @@ -require 'puppetlabs_spec_helper/module_spec_helper' +require 'puppetlabs_spec_helper/module_spec_helper' \ No newline at end of file diff --git a/spec/unit/puppet/provider/cib.xml b/spec/unit/puppet/provider/cib.xml new file mode 100644 index 0000000..4fdbf3d --- /dev/null +++ b/spec/unit/puppet/provider/cib.xml @@ -0,0 +1,483 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/spec/unit/puppet/provider/pacemaker_common_spec.rb b/spec/unit/puppet/provider/pacemaker_common_spec.rb new file mode 100644 index 0000000..902cc27 --- /dev/null +++ b/spec/unit/puppet/provider/pacemaker_common_spec.rb @@ -0,0 +1,222 @@ +require 'spec_helper' +require File.expand_path(File.join(File.dirname(__FILE__), '../../../../lib/puppet/provider/pacemaker_common.rb')) + +describe Puppet::Provider::Pacemaker_common do + + cib_xml_file = File.join File.dirname(__FILE__), 'cib.xml' + + let(:raw_cib) do + File.read cib_xml_file + end + + let(:resources_regexp) do + %r{nova|cinder|glance|keystone|neutron|sahara|murano|ceilometer|heat|swift} + end + + ########################### + + #-> Cloned primitive 'clone_p_neutron-plugin-openvswitch-agent' global status: start + #node-1: start | node-2: stop | node-3: stop + #-> Cloned primitive 'clone_ping_vip__public' global status: start + #node-1: start | node-2: start | node-3: start + #-> Cloned primitive 'clone_p_neutron-metadata-agent' global status: start + #node-1: start | node-2: stop | node-3: stop + #-> Simple primitive 'vip__management' global status: start + #node-1: start | node-2: stop | node-3: stop + #-> Cloned primitive 'clone_p_mysql' global status: start + #node-1: start | node-2: start | node-3: stop + #-> Multistate primitive 'master_p_rabbitmq-server' global status: master + #node-1: master | node-2: start | node-3: stop + #-> Cloned primitive 'clone_p_haproxy' global status: start + #node-1: start | node-2: start | node-3: stop + #-> Simple primitive 'p_ceilometer-alarm-evaluator' global status: stop + #node-1: stop | node-2: stop (FAIL) | node-3: stop (FAIL) + #-> Simple primitive 'p_ceilometer-agent-central' global status: stop + #node-1: stop | node-2: stop (FAIL) | node-3: stop (FAIL) + #-> Cloned primitive 'clone_p_neutron-l3-agent' global status: start + #node-1: start | node-2: stop | node-3: stop + #-> Simple primitive 'p_neutron-dhcp-agent' global status: start + #node-1: start | node-2: stop | node-3: stop + #-> Simple primitive 'vip__public' global status: start + #node-1: start | node-2: stop | node-3: stop + #-> Simple primitive 'p_heat-engine' global status: start + #node-1: start | node-2: stop | node-3: stop + + before(:each) do + @class = subject + @class.stubs(:raw_cib).returns raw_cib + @class.stubs(:pcs).returns true + end + + context 'configuration parser' do + it 'can obtain a CIB XML object' do + expect(@class.cib.to_s).to include '' + expect(@class.cib.to_s).to include '' + expect(@class.cib.to_s).to include '' + expect(@class.cib.to_s).to include '' + expect(@class.cib.to_s).to include '' + end + + it 'can get primitives section of CIB XML' do + expect(@class.cib_section_primitives).to be_a(Array) + expect(@class.cib_section_primitives.first.to_s).to start_with '' + end + + it 'can get primitives configuration' do + expect(@class.primitives).to be_a Hash + expect(@class.primitives['vip__public']).to be_a Hash + expect(@class.primitives['vip__public']['meta_attributes']).to be_a Hash + expect(@class.primitives['vip__public']['instance_attributes']).to be_a Hash + expect(@class.primitives['vip__public']['instance_attributes']['ip']).to be_a Hash + expect(@class.primitives['vip__public']['operations']).to be_a Hash + expect(@class.primitives['vip__public']['meta_attributes']['resource-stickiness']).to be_a Hash + expect(@class.primitives['vip__public']['operations']['vip__public-start-0']).to be_a Hash + end + + it 'can determine is primitive is simple or complex' do + expect(@class.primitive_is_complex? 'p_haproxy').to eq true + expect(@class.primitive_is_complex? 'vip__management').to eq false + end + end + + context 'node status parser' do + it 'can produce nodes structure' do + expect(@class.nodes).to be_a Hash + expect(@class.nodes['node-1']['primitives']['p_heat-engine']['status']).to eq('start') + #puts @class.get_cluster_debug_report + end + + it 'can determite a global primitive status' do + expect(@class.primitive_status 'p_heat-engine').to eq('start') + expect(@class.primitive_is_running? 'p_heat-engine').to eq true + expect(@class.primitive_status 'p_ceilometer-agent-central').to eq('stop') + expect(@class.primitive_is_running? 'p_ceilometer-agent-central').to eq false + expect(@class.primitive_is_running? 'UNKNOWN').to eq nil + expect(@class.primitive_status 'UNKNOWN').to eq nil + end + + it 'can determine a local primitive status on a node' do + expect(@class.primitive_status 'p_heat-engine', 'node-1').to eq('start') + expect(@class.primitive_is_running? 'p_heat-engine', 'node-1').to eq true + expect(@class.primitive_status 'p_heat-engine', 'node-2').to eq('stop') + expect(@class.primitive_is_running? 'p_heat-engine', 'node-2').to eq false + expect(@class.primitive_is_running? 'UNKNOWN', 'node-1').to eq nil + expect(@class.primitive_status 'UNKNOWN', 'node-1').to eq nil + end + + it 'can determine if primitive is managed or not' do + expect(@class.primitive_is_managed? 'p_heat-engine').to eq true + expect(@class.primitive_is_managed? 'p_haproxy').to eq true + expect(@class.primitive_is_managed? 'UNKNOWN').to eq nil + end + + it 'can determine if primitive is started or not' do + expect(@class.primitive_is_started? 'p_heat-engine').to eq true + expect(@class.primitive_is_started? 'p_haproxy').to eq true + expect(@class.primitive_is_started? 'UNKNOWN').to eq nil + end + + it 'can determine if primitive is failed or not globally' do + expect(@class.primitive_has_failures? 'p_ceilometer-agent-central').to eq true + expect(@class.primitive_has_failures? 'p_heat-engine').to eq false + expect(@class.primitive_has_failures? 'UNKNOWN').to eq nil + end + + it 'can determine if primitive is failed or not locally' do + expect(@class.primitive_has_failures? 'p_ceilometer-agent-central', 'node-1').to eq false + expect(@class.primitive_has_failures? 'p_ceilometer-agent-central', 'node-2').to eq true + expect(@class.primitive_has_failures? 'p_heat-engine', 'node-1').to eq false + expect(@class.primitive_has_failures? 'p_heat-engine', 'node-2').to eq false + expect(@class.primitive_has_failures? 'UNKNOWN', 'node-1').to eq nil + end + + it 'can determine that primitive is complex' do + expect(@class.primitive_is_complex? 'p_haproxy').to eq true + expect(@class.primitive_is_complex? 'p_heat-engine').to eq false + expect(@class.primitive_is_complex? 'p_rabbitmq-server').to eq true + expect(@class.primitive_is_complex? 'UNKNOWN').to eq nil + end + + it 'can determine that primitive is multistate' do + expect(@class.primitive_is_multistate? 'p_haproxy').to eq false + expect(@class.primitive_is_multistate? 'p_heat-engine').to eq false + expect(@class.primitive_is_multistate? 'p_rabbitmq-server').to eq true + expect(@class.primitive_is_multistate? 'UNKNOWN').to eq nil + end + + it 'can determine that primitive has master running' do + expect(@class.primitive_has_master_running? 'p_rabbitmq-server').to eq true + expect(@class.primitive_has_master_running? 'p_heat-engine').to eq false + expect(@class.primitive_has_master_running? 'UNKNOWN').to eq nil + end + + it 'can determine that primitive is clone' do + expect(@class.primitive_is_clone? 'p_haproxy').to eq true + expect(@class.primitive_is_clone? 'p_heat-engine').to eq false + expect(@class.primitive_is_clone? 'p_rabbitmq-server').to eq false + expect(@class.primitive_is_clone? 'UNKNOWN').to eq nil + end + + end + + context 'cluster control' do + it 'can enable maintenance mode' do + @class.expects(:pcs).with 'property', 'set', 'maintenance-mode=true' + @class.maintenance_mode 'true' + end + + it 'can disable maintenance mode' do + @class.expects(:pcs).with 'property', 'set', 'maintenance-mode=false' + @class.maintenance_mode 'false' + end + + it 'can set no-quorum policy' do + @class.expects(:pcs).with 'property', 'set', 'no-quorum-policy=ignore' + @class.no_quorum_policy 'ignore' + end + end + + context 'constraints control' do + it 'can add location constraint' do + @class.expects(:pcs).with 'constraint', 'location', 'add', 'myprimitive_on_mynode', 'myprimitive', 'mynode', '200' + @class.constraint_location_add 'myprimitive', 'mynode', '200' + end + + it 'can remove location constraint' do + @class.expects(:pcs).with 'constraint', 'location', 'remove', 'myprimitive_on_mynode' + @class.constraint_location_remove 'myprimitive', 'mynode' + end + end + + context 'wait functions' do + it 'retries block until it becomes true' do + @class.retry_block_until_true { true } + end + + it 'waits for Pacemaker to become ready' do + @class.stubs(:is_online?).returns true + @class.wait_for_online + end + + it 'cleanups primitive and waits for it to become online again' do + @class.stubs(:cleanup_primitive).with('myprimitive', 'mynode').returns true + @class.stubs(:cib_reset).returns true + @class.stubs(:primitive_status).returns 'stopped' + @class.cleanup_with_wait 'myprimitive', 'mynode' + end + + it 'waits for the service to start' do + @class.stubs(:cib_reset).returns true + @class.stubs(:primitive_is_running?).with('myprimitive', nil).returns true + @class.wait_for_start 'myprimitive' + end + + it 'waits for the service to stop' do + @class.stubs(:cib_reset).returns true + @class.stubs(:primitive_is_running?).with('myprimitive', nil).returns false + @class.wait_for_stop 'myprimitive' + end + end + +end diff --git a/spec/unit/puppet/provider/service/pacemaker_spec.rb b/spec/unit/puppet/provider/service/pacemaker_spec.rb new file mode 100644 index 0000000..ae7f9a5 --- /dev/null +++ b/spec/unit/puppet/provider/service/pacemaker_spec.rb @@ -0,0 +1,227 @@ +require 'spec_helper' + +describe Puppet::Type.type(:service).provider(:pacemaker) do + + let(:resource) { Puppet::Type.type(:service).new(:name => title, :provider=> :pacemaker) } + let(:provider) { resource.provider } + let(:title) { 'myservice' } + let(:full_name) { 'clone-p_myservice' } + let(:name) { 'p_myservice' } + let(:hostname) { 'mynode' } + + before :each do + @class = provider + + @class.stubs(:title).returns(title) + @class.stubs(:hostname).returns(hostname) + @class.stubs(:name).returns(name) + @class.stubs(:full_name).returns(full_name) + @class.stubs(:basic_service_name).returns(title) + + @class.stubs(:cib_reset).returns(true) + + @class.stubs(:wait_for_online).returns(true) + @class.stubs(:cleanup_with_wait).returns(true) + @class.stubs(:wait_for_start).returns(true) + @class.stubs(:wait_for_stop).returns(true) + + @class.stubs(:disable_basic_service).returns(true) + @class.stubs(:get_primitive_puppet_status).returns(:started) + @class.stubs(:get_primitive_puppet_enable).returns(:true) + + @class.stubs(:primitive_is_managed?).returns(true) + @class.stubs(:primitive_is_running?).returns(true) + @class.stubs(:primitive_has_failures?).returns(false) + @class.stubs(:primitive_is_complex?).returns(false) + @class.stubs(:primitive_is_multistate?).returns(false) + @class.stubs(:primitive_is_clone?).returns(false) + + @class.stubs(:unban_primitive).returns(true) + @class.stubs(:ban_primitive).returns(true) + @class.stubs(:start_primitive).returns(true) + @class.stubs(:stop_primitive).returns(true) + @class.stubs(:enable).returns(true) + @class.stubs(:disable).returns(true) + + @class.stubs(:constraint_location_add).returns(true) + @class.stubs(:constraint_location_remove).returns(true) + + @class.stubs(:get_cluster_debug_report).returns(true) + end + + context 'service name mangling' do + it 'uses title as the service name if it is found in CIB' do + @class.unstub(:name) + @class.stubs(:primitive_exists?).with(title).returns(true) + expect(@class.name).to eq(title) + end + + it 'uses "p_" prefix with name if found name with prefix' do + @class.unstub(:name) + @class.stubs(:primitive_exists?).with(title).returns(false) + @class.stubs(:primitive_exists?).with(name).returns(true) + expect(@class.name).to eq(name) + end + + it 'uses name without "p_" to disable basic service' do + @class.stubs(:name).returns(name) + expect(@class.basic_service_name).to eq(title) + end + end + + context '#status' do + it 'should wait for pacemaker to become online' do + @class.expects(:wait_for_online) + @class.status + end + + it 'should reset cib mnemoization on every call' do + @class.expects(:cib_reset) + @class.status + end + + it 'gets service status locally' do + @class.expects(:get_primitive_puppet_status).with name, hostname + @class.status + end + + end + + context '#start' do + it 'tries to enable service if it is not enabled to work with it' do + @class.stubs(:primitive_is_managed?).returns(false) + @class.expects(:enable).once + @class.start + @class.stubs(:primitive_is_managed?).returns(true) + @class.expects(:enable).never + @class.start + end + + it 'tries to disable a basic service with the same name' do + @class.expects(:disable_basic_service) + @class.start + end + + it 'should cleanup a primitive only if there are errors' do + @class.stubs(:primitive_has_failures?).returns(true) + @class.expects(:cleanup_with_wait).once + @class.start + @class.stubs(:primitive_has_failures?).returns(false) + @class.expects(:cleanup_with_wait).never + @class.start + end + + it 'tries to unban the service on the node by the name' do + @class.expects(:unban_primitive).with(name, hostname) + @class.start + end + + it 'tries to start the service by its name' do + @class.expects(:start_primitive).with(name) + @class.start + end + + it 'adds a location constraint for the service by its name' do + @class.expects(:constraint_location_add).with(name, hostname) + @class.start + end + + it 'waits for the service to start locally if primitive is clone' do + @class.stubs(:primitive_is_clone?).returns(true) + @class.stubs(:primitive_is_multistate?).returns(false) + @class.stubs(:primitive_is_complex?).returns(true) + @class.expects(:wait_for_start).with name + @class.start + end + + it 'waits for the service to start master anywhere if primitive is multistate' do + @class.stubs(:primitive_is_clone?).returns(false) + @class.stubs(:primitive_is_multistate?).returns(true) + @class.stubs(:primitive_is_complex?).returns(true) + @class.expects(:wait_for_master).with name + @class.start + end + + it 'waits for the service to start anywhere if primitive is simple' do + @class.stubs(:primitive_is_clone?).returns(false) + @class.stubs(:primitive_is_multistate?).returns(false) + @class.stubs(:primitive_is_complex?).returns(false) + @class.expects(:wait_for_start).with name + @class.start + end + end + + context '#stop' do + it 'tries to enable service if it is not enabled to work with it' do + @class.stubs(:primitive_is_managed?).returns(false) + @class.expects(:enable).once + @class.start + @class.stubs(:primitive_is_managed?).returns(true) + @class.expects(:enable).never + @class.start + end + + it 'should cleanup a primitive only if there are errors' do + @class.stubs(:primitive_has_failures?).returns(true) + @class.expects(:cleanup_with_wait).once + @class.start + @class.stubs(:primitive_has_failures?).returns(false) + @class.expects(:cleanup_with_wait).never + @class.start + end + + it 'uses Ban to stop the service and waits for it to stop locally if service is complex' do + @class.stubs(:primitive_is_complex?).returns(true) + @class.expects(:wait_for_stop).with name, hostname + @class.expects(:ban_primitive).with name, hostname + @class.stop + end + + it 'uses Stop to stop the service and waits for it to stop globally if service is simple' do + @class.stubs(:primitive_is_complex?).returns(false) + @class.expects(:wait_for_stop).with name + @class.expects(:stop_primitive).with name + @class.stop + end + end + + context '#restart' do + it 'does not stop or start the service if it is not locally running' do + @class.stubs(:primitive_is_running?).with(name, hostname).returns(false) + @class.expects(:stop).never + @class.expects(:start).never + @class.restart + end + + it 'stops and start the service if it is locally running' do + @class.stubs(:primitive_is_running?).with(name, hostname).returns(true) + restart_sequence = sequence('restart') + @class.expects(:stop).in_sequence(restart_sequence) + @class.expects(:start).in_sequence(restart_sequence) + @class.restart + end + end + + context 'basic service handling' do + before :each do + @class.unstub(:disable_basic_service) + @class.extra_provider.stubs(:enableable?).returns true + @class.extra_provider.stubs(:enabled?).returns :true + @class.extra_provider.stubs(:disable).returns true + @class.extra_provider.stubs(:stop).returns true + @class.extra_provider.stubs(:status).returns :running + end + + it 'tries to disable the basic service if it is enabled' do + @class.extra_provider.expects(:disable) + @class.disable_basic_service + end + + it 'tries to stop the service if it is running' do + @class.extra_provider.expects(:stop) + @class.disable_basic_service + end + end + +end + diff --git a/templates/ocf_handler.erb b/templates/ocf_handler.erb new file mode 100644 index 0000000..cd22f0d --- /dev/null +++ b/templates/ocf_handler.erb @@ -0,0 +1,118 @@ +#!/bin/sh +export PATH='/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin' +export OCF_ROOT='<%= @ocf_root_path %>' +export OCF_RA_VERSION_MAJOR='1' +export OCF_RA_VERSION_MINOR='0' +export OCF_RESOURCE_INSTANCE='<%= @primitive_name %>' + +# OCF Parameters +<% if @parameters.is_a? Hash -%> + <% @parameters.each do |k,v| -%> + <% v = v.to_s -%> + <% v = v + "'" unless v.end_with? "'" -%> + <% v = "'" + v unless v.start_with? "'" -%> + <%= "export OCF_RESKEY_#{k}=#{v}" %> + <% end -%> +<% end -%> + +help() { +cat< Pacemaker primitive + +Usage: <%= @ocf_handler_name %> [-dh] (action) + +Options: +-d - Use set -x to debug the shell script +-h - Show this help + +Main actions: +* start +* stop +* monitor +* meta-data +* validate-all + +Multistate: +* promote +* demote +* notify + +Migration: +* migrate_to +* migrate_from + +Optional and unused: +* usage +* help +* status +* reload +* restart +* recover +EOF +} + +red() { + echo -e "\033[31m${1}\033[0m" +} + +green() { + echo -e "\033[32m${1}\033[0m" +} + +blue() { + echo -e "\033[34m${1}\033[0m" +} + +ec2error() { + case "${1}" in + 0) green 'Success' ;; + 1) red 'Error: Generic' ;; + 2) red 'Error: Arguments' ;; + 3) red 'Error: Unimplemented' ;; + 4) red 'Error: Permissions' ;; + 5) red 'Error: Installation' ;; + 6) red 'Error: Configuration' ;; + 7) blue 'Not Running' ;; + 8) green 'Master Running' ;; + 9) red 'Master Failed' ;; + *) red "Unknown" ;; + esac +} + +DEBUG='0' +while getopts ':dh' opt; do + case $opt in + d) + DEBUG='1' + ;; + h) + help + exit 0 + ;; + \?) + echo "Invalid option: -${OPTARG}" >&2 + help + exit 1 + ;; + esac +done + +shift "$((OPTIND - 1))" + +ACTION="${1}" + +# set default action to monitor +if [ "${ACTION}" = '' ]; then + ACTION='monitor' +fi + +if [ "${DEBUG}" = '1' ]; then + bash -x <%= @ocf_script_path %> "${ACTION}" +else + <%= @ocf_script_path %> "${ACTION}" +fi +ec="${?}" + +message="$(ec2error ${ec})" +echo "Exit status: ${message} (${ec})" +exit "${ec}"