Ability to stop puppet deploy at every nodes
Change-Id: Ie8d0e82b04cc53f9cebe150415d00c1f3d9aa22e
This commit is contained in:
parent
f967bf7d64
commit
0843a91d79
@ -45,6 +45,9 @@ module Astute
|
||||
|
||||
# Sync puppet manifests and modules to every node (emulate puppet master)
|
||||
sync_puppet_manifests(part)
|
||||
|
||||
# Unlock puppet (can be lock if puppet was killed by user)
|
||||
enable_puppet_deploy(part.map{ |n| n['uid'] })
|
||||
end
|
||||
rescue => e
|
||||
Astute.logger.error("Unexpected error #{e.message} traceback #{e.format_backtrace}")
|
||||
@ -162,6 +165,11 @@ module Astute
|
||||
return status.exitstatus, stdout, stderr
|
||||
end
|
||||
|
||||
def enable_puppet_deploy(node_uids)
|
||||
puppetd = MClient.new(@ctx, "puppetd", node_uids)
|
||||
puppetd.enable
|
||||
end
|
||||
|
||||
def nodes_status(nodes, status, data_to_merge)
|
||||
{
|
||||
'nodes' => nodes.map do |n|
|
||||
|
@ -149,6 +149,12 @@ module Astute
|
||||
NodesRemover.new(Context.new(task_id, reporter), nodes, reboot).remove
|
||||
end
|
||||
|
||||
def stop_puppet_deploy(reporter, task_id, nodes)
|
||||
nodes_uids = nodes.map { |n| n['uid'] }.uniq
|
||||
puppetd = MClient.new(Context.new(task_id, reporter), "puppetd", nodes_uids)
|
||||
puppetd.stop_and_disable
|
||||
end
|
||||
|
||||
def dump_environment(reporter, task_id, lastdump)
|
||||
Dump.dump_environment(Context.new(task_id, reporter), lastdump)
|
||||
end
|
||||
|
@ -29,6 +29,12 @@ action "last_run_summary", :description => "Get a summary of the last puppet run
|
||||
:display_as => "Versions"
|
||||
end
|
||||
|
||||
action "stop_and_disable", :description => "Stop and disable puppet" do
|
||||
output :output,
|
||||
:description => "String indicating status",
|
||||
:display_as => "Status"
|
||||
end
|
||||
|
||||
action "enable", :description => "Enable puppet" do
|
||||
output :output,
|
||||
:description => "String indicating status",
|
||||
|
@ -12,6 +12,7 @@
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
require 'timeout'
|
||||
|
||||
module MCollective
|
||||
module Agent
|
||||
@ -43,6 +44,7 @@ module MCollective
|
||||
-p #{@lockfile} \
|
||||
/usr/bin/puppet apply /etc/puppet/manifests/site.pp"
|
||||
@last_summary = @config.pluginconf["puppet.summary"] || "/var/lib/puppet/state/last_run_summary.yaml"
|
||||
@lockmcofile = "/tmp/mcopuppetd.lock"
|
||||
end
|
||||
|
||||
action "last_run_summary" do
|
||||
@ -66,6 +68,10 @@ module MCollective
|
||||
set_status
|
||||
end
|
||||
|
||||
action "stop_and_disable" do
|
||||
stop_and_disable
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def last_run_summary
|
||||
@ -112,7 +118,7 @@ module MCollective
|
||||
|
||||
def puppet_daemon_status
|
||||
err_msg = ""
|
||||
alive = !!puppet_agent_pid
|
||||
alive = !!puppet_pid
|
||||
locked = File.exists?(@lockfile)
|
||||
disabled = locked && File::Stat.new(@lockfile).zero?
|
||||
|
||||
@ -137,30 +143,31 @@ module MCollective
|
||||
end
|
||||
|
||||
def runonce
|
||||
set_status
|
||||
case (reply[:status])
|
||||
when 'disabled' then # can't run
|
||||
reply.fail "Empty Lock file exists; puppet is disabled."
|
||||
lock_file(@lockmcofile) do
|
||||
set_status
|
||||
case (reply[:status])
|
||||
when 'disabled' then # can't run
|
||||
reply.fail "Empty Lock file exists; puppet is disabled."
|
||||
|
||||
when 'running' then # can't run two simultaniously
|
||||
reply.fail "Lock file and PID file exist; puppet is running."
|
||||
when 'running' then # can't run two simultaniously
|
||||
reply.fail "Lock file and PID file exist; puppet is running."
|
||||
|
||||
when 'idling' then # signal daemon
|
||||
pid = puppet_agent_pid
|
||||
begin
|
||||
::Process.kill('INT', pid)
|
||||
rescue Errno::ESRCH => e
|
||||
reply[:err_msg] = "Failed to signal the puppet apply daemon (process #{pid}): #{e}"
|
||||
ensure
|
||||
when 'idling' then # signal daemon
|
||||
pid = puppet_agent_pid
|
||||
begin
|
||||
::Process.kill('INT', pid)
|
||||
rescue Errno::ESRCH => e
|
||||
reply[:err_msg] = "Failed to signal the puppet apply daemon (process #{pid}): #{e}"
|
||||
ensure
|
||||
runonce_background
|
||||
reply[:output] = "Kill old idling puppet process #{pid})." + (reply[:output] || '')
|
||||
end
|
||||
|
||||
when 'stopped' then # just run
|
||||
runonce_background
|
||||
reply[:output] = "Kill old idling puppet process #{pid})." + (reply[:output] || '')
|
||||
else
|
||||
reply.fail "Unknown puppet status: #{reply[:status]}"
|
||||
end
|
||||
|
||||
when 'stopped' then # just run
|
||||
runonce_background
|
||||
|
||||
else
|
||||
reply.fail "Unknown puppet status: #{reply[:status]}"
|
||||
end
|
||||
end
|
||||
|
||||
@ -181,6 +188,22 @@ module MCollective
|
||||
reply[:output] = "Called #{cmd}, " + output + (reply[:output] || '')
|
||||
end
|
||||
|
||||
def stop_and_disable
|
||||
lock_file(@lockmcofile) do
|
||||
case puppet_daemon_status
|
||||
when 'stopped'
|
||||
disable
|
||||
when 'disabled'
|
||||
reply[:output] = "Puppet already stoped and disabled"
|
||||
return
|
||||
else
|
||||
kill_process
|
||||
disable
|
||||
end
|
||||
reply[:output] = "Puppet stoped and disabled"
|
||||
end
|
||||
end
|
||||
|
||||
def enable
|
||||
if File.exists?(@lockfile)
|
||||
stat = File::Stat.new(@lockfile)
|
||||
@ -211,10 +234,43 @@ module MCollective
|
||||
end
|
||||
end
|
||||
|
||||
def puppet_agent_pid
|
||||
private
|
||||
|
||||
def kill_process
|
||||
return if ['stopped', 'disabled'].include? puppet_daemon_status
|
||||
|
||||
begin
|
||||
Timeout.timeout(30) do
|
||||
Process.kill('TERM', puppet_pid)
|
||||
while puppet_pid do
|
||||
sleep 1
|
||||
end
|
||||
end
|
||||
rescue Timeout::Error
|
||||
Process.kill('KILL', puppet_pid)
|
||||
end
|
||||
#FIXME: Daemonized process do not update lock file when we send signal to kill him
|
||||
raise "Should never happen. Some process block lock file in critical section" unless rm_file(@lockfile)
|
||||
rescue => e
|
||||
reply.fail "Failed to kill the puppet daemon (process #{puppet_pid}): #{e}"
|
||||
end
|
||||
|
||||
def puppet_pid
|
||||
result = `ps -C puppet -o pid,comm --no-headers`.lines.first
|
||||
result && result.strip.split(' ')[0].to_i
|
||||
end
|
||||
|
||||
def lock_file(file_name, &block)
|
||||
File.open(file_name, 'w+') do |f|
|
||||
begin
|
||||
f.flock File::LOCK_EX
|
||||
yield
|
||||
ensure
|
||||
f.flock File::LOCK_UN
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
end
|
||||
|
@ -40,6 +40,7 @@ describe Astute::DeploymentEngine do
|
||||
deployer.stubs(:generate_ssh_keys)
|
||||
deployer.stubs(:upload_ssh_keys)
|
||||
deployer.stubs(:sync_puppet_manifests)
|
||||
deployer.stubs(:enable_puppet_deploy)
|
||||
end
|
||||
|
||||
it 'should generate and upload ssh keys' do
|
||||
@ -53,6 +54,15 @@ describe Astute::DeploymentEngine do
|
||||
deployer.deploy(nodes)
|
||||
end
|
||||
|
||||
it 'should enable puppet for all nodes' do
|
||||
nodes = [{'uid' => 1, 'deployment_id' => 1}, {'uid' => 2}, {'uid' => 1}]
|
||||
deployer.stubs(:deploy_piece)
|
||||
|
||||
deployer.expects(:enable_puppet_deploy).with([1,2]).returns()
|
||||
|
||||
deployer.deploy(nodes)
|
||||
end
|
||||
|
||||
it 'deploy nodes by order' do
|
||||
nodes = [{'uid' => 1, 'priority' => 10}, {'uid' => 2, 'priority' => 0}, {'uid' => 1, 'priority' => 15}]
|
||||
|
||||
@ -144,6 +154,7 @@ describe Astute::DeploymentEngine do
|
||||
deployer.stubs(:deploy_piece)
|
||||
deployer.stubs(:generate_ssh_keys)
|
||||
deployer.stubs(:upload_ssh_keys)
|
||||
deployer.stubs(:enable_puppet_deploy)
|
||||
end
|
||||
|
||||
let(:nodes) { [{'uid' => 1, 'deployment_id' => 1, 'master_ip' => '10.20.0.2'}, {'uid' => 2}] }
|
||||
@ -167,6 +178,7 @@ describe Astute::DeploymentEngine do
|
||||
Astute.config.PUPPET_SSH_KEYS = ['nova']
|
||||
deployer.stubs(:deploy_piece)
|
||||
deployer.stubs(:sync_puppet_manifests)
|
||||
deployer.stubs(:enable_puppet_deploy)
|
||||
end
|
||||
|
||||
let(:nodes) { [{'uid' => 1, 'deployment_id' => 1}, {'uid' => 2}] }
|
||||
|
@ -51,6 +51,7 @@ describe "NailyFact DeploymentEngine" do
|
||||
deploy_engine.stubs(:generate_ssh_keys).with(deploy_data.first['deployment_id'])
|
||||
deploy_engine.stubs(:upload_ssh_keys).with(uniq_nodes_uid, deploy_data.first['deployment_id'])
|
||||
deploy_engine.stubs(:sync_puppet_manifests).with(deploy_data.uniq { |n| n['uid'] })
|
||||
deploy_engine.stubs(:enable_puppet_deploy).with(uniq_nodes_uid)
|
||||
end
|
||||
|
||||
context 'log parsing' do
|
||||
|
Loading…
Reference in New Issue
Block a user