Ability to stop puppet deploy at every nodes
Change-Id: Ie8d0e82b04cc53f9cebe150415d00c1f3d9aa22e
This commit is contained in:
parent
f967bf7d64
commit
0843a91d79
@ -45,6 +45,9 @@ module Astute
|
|||||||
|
|
||||||
# Sync puppet manifests and modules to every node (emulate puppet master)
|
# Sync puppet manifests and modules to every node (emulate puppet master)
|
||||||
sync_puppet_manifests(part)
|
sync_puppet_manifests(part)
|
||||||
|
|
||||||
|
# Unlock puppet (can be lock if puppet was killed by user)
|
||||||
|
enable_puppet_deploy(part.map{ |n| n['uid'] })
|
||||||
end
|
end
|
||||||
rescue => e
|
rescue => e
|
||||||
Astute.logger.error("Unexpected error #{e.message} traceback #{e.format_backtrace}")
|
Astute.logger.error("Unexpected error #{e.message} traceback #{e.format_backtrace}")
|
||||||
@ -162,6 +165,11 @@ module Astute
|
|||||||
return status.exitstatus, stdout, stderr
|
return status.exitstatus, stdout, stderr
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def enable_puppet_deploy(node_uids)
|
||||||
|
puppetd = MClient.new(@ctx, "puppetd", node_uids)
|
||||||
|
puppetd.enable
|
||||||
|
end
|
||||||
|
|
||||||
def nodes_status(nodes, status, data_to_merge)
|
def nodes_status(nodes, status, data_to_merge)
|
||||||
{
|
{
|
||||||
'nodes' => nodes.map do |n|
|
'nodes' => nodes.map do |n|
|
||||||
|
@ -149,6 +149,12 @@ module Astute
|
|||||||
NodesRemover.new(Context.new(task_id, reporter), nodes, reboot).remove
|
NodesRemover.new(Context.new(task_id, reporter), nodes, reboot).remove
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def stop_puppet_deploy(reporter, task_id, nodes)
|
||||||
|
nodes_uids = nodes.map { |n| n['uid'] }.uniq
|
||||||
|
puppetd = MClient.new(Context.new(task_id, reporter), "puppetd", nodes_uids)
|
||||||
|
puppetd.stop_and_disable
|
||||||
|
end
|
||||||
|
|
||||||
def dump_environment(reporter, task_id, lastdump)
|
def dump_environment(reporter, task_id, lastdump)
|
||||||
Dump.dump_environment(Context.new(task_id, reporter), lastdump)
|
Dump.dump_environment(Context.new(task_id, reporter), lastdump)
|
||||||
end
|
end
|
||||||
|
@ -29,6 +29,12 @@ action "last_run_summary", :description => "Get a summary of the last puppet run
|
|||||||
:display_as => "Versions"
|
:display_as => "Versions"
|
||||||
end
|
end
|
||||||
|
|
||||||
|
action "stop_and_disable", :description => "Stop and disable puppet" do
|
||||||
|
output :output,
|
||||||
|
:description => "String indicating status",
|
||||||
|
:display_as => "Status"
|
||||||
|
end
|
||||||
|
|
||||||
action "enable", :description => "Enable puppet" do
|
action "enable", :description => "Enable puppet" do
|
||||||
output :output,
|
output :output,
|
||||||
:description => "String indicating status",
|
:description => "String indicating status",
|
||||||
|
@ -12,6 +12,7 @@
|
|||||||
# License for the specific language governing permissions and limitations
|
# License for the specific language governing permissions and limitations
|
||||||
# under the License.
|
# under the License.
|
||||||
|
|
||||||
|
require 'timeout'
|
||||||
|
|
||||||
module MCollective
|
module MCollective
|
||||||
module Agent
|
module Agent
|
||||||
@ -43,6 +44,7 @@ module MCollective
|
|||||||
-p #{@lockfile} \
|
-p #{@lockfile} \
|
||||||
/usr/bin/puppet apply /etc/puppet/manifests/site.pp"
|
/usr/bin/puppet apply /etc/puppet/manifests/site.pp"
|
||||||
@last_summary = @config.pluginconf["puppet.summary"] || "/var/lib/puppet/state/last_run_summary.yaml"
|
@last_summary = @config.pluginconf["puppet.summary"] || "/var/lib/puppet/state/last_run_summary.yaml"
|
||||||
|
@lockmcofile = "/tmp/mcopuppetd.lock"
|
||||||
end
|
end
|
||||||
|
|
||||||
action "last_run_summary" do
|
action "last_run_summary" do
|
||||||
@ -66,6 +68,10 @@ module MCollective
|
|||||||
set_status
|
set_status
|
||||||
end
|
end
|
||||||
|
|
||||||
|
action "stop_and_disable" do
|
||||||
|
stop_and_disable
|
||||||
|
end
|
||||||
|
|
||||||
private
|
private
|
||||||
|
|
||||||
def last_run_summary
|
def last_run_summary
|
||||||
@ -112,7 +118,7 @@ module MCollective
|
|||||||
|
|
||||||
def puppet_daemon_status
|
def puppet_daemon_status
|
||||||
err_msg = ""
|
err_msg = ""
|
||||||
alive = !!puppet_agent_pid
|
alive = !!puppet_pid
|
||||||
locked = File.exists?(@lockfile)
|
locked = File.exists?(@lockfile)
|
||||||
disabled = locked && File::Stat.new(@lockfile).zero?
|
disabled = locked && File::Stat.new(@lockfile).zero?
|
||||||
|
|
||||||
@ -137,30 +143,31 @@ module MCollective
|
|||||||
end
|
end
|
||||||
|
|
||||||
def runonce
|
def runonce
|
||||||
set_status
|
lock_file(@lockmcofile) do
|
||||||
case (reply[:status])
|
set_status
|
||||||
when 'disabled' then # can't run
|
case (reply[:status])
|
||||||
reply.fail "Empty Lock file exists; puppet is disabled."
|
when 'disabled' then # can't run
|
||||||
|
reply.fail "Empty Lock file exists; puppet is disabled."
|
||||||
|
|
||||||
when 'running' then # can't run two simultaniously
|
when 'running' then # can't run two simultaniously
|
||||||
reply.fail "Lock file and PID file exist; puppet is running."
|
reply.fail "Lock file and PID file exist; puppet is running."
|
||||||
|
|
||||||
when 'idling' then # signal daemon
|
when 'idling' then # signal daemon
|
||||||
pid = puppet_agent_pid
|
pid = puppet_agent_pid
|
||||||
begin
|
begin
|
||||||
::Process.kill('INT', pid)
|
::Process.kill('INT', pid)
|
||||||
rescue Errno::ESRCH => e
|
rescue Errno::ESRCH => e
|
||||||
reply[:err_msg] = "Failed to signal the puppet apply daemon (process #{pid}): #{e}"
|
reply[:err_msg] = "Failed to signal the puppet apply daemon (process #{pid}): #{e}"
|
||||||
ensure
|
ensure
|
||||||
|
runonce_background
|
||||||
|
reply[:output] = "Kill old idling puppet process #{pid})." + (reply[:output] || '')
|
||||||
|
end
|
||||||
|
|
||||||
|
when 'stopped' then # just run
|
||||||
runonce_background
|
runonce_background
|
||||||
reply[:output] = "Kill old idling puppet process #{pid})." + (reply[:output] || '')
|
else
|
||||||
|
reply.fail "Unknown puppet status: #{reply[:status]}"
|
||||||
end
|
end
|
||||||
|
|
||||||
when 'stopped' then # just run
|
|
||||||
runonce_background
|
|
||||||
|
|
||||||
else
|
|
||||||
reply.fail "Unknown puppet status: #{reply[:status]}"
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
@ -181,6 +188,22 @@ module MCollective
|
|||||||
reply[:output] = "Called #{cmd}, " + output + (reply[:output] || '')
|
reply[:output] = "Called #{cmd}, " + output + (reply[:output] || '')
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def stop_and_disable
|
||||||
|
lock_file(@lockmcofile) do
|
||||||
|
case puppet_daemon_status
|
||||||
|
when 'stopped'
|
||||||
|
disable
|
||||||
|
when 'disabled'
|
||||||
|
reply[:output] = "Puppet already stoped and disabled"
|
||||||
|
return
|
||||||
|
else
|
||||||
|
kill_process
|
||||||
|
disable
|
||||||
|
end
|
||||||
|
reply[:output] = "Puppet stoped and disabled"
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
def enable
|
def enable
|
||||||
if File.exists?(@lockfile)
|
if File.exists?(@lockfile)
|
||||||
stat = File::Stat.new(@lockfile)
|
stat = File::Stat.new(@lockfile)
|
||||||
@ -211,10 +234,43 @@ module MCollective
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
def puppet_agent_pid
|
private
|
||||||
|
|
||||||
|
def kill_process
|
||||||
|
return if ['stopped', 'disabled'].include? puppet_daemon_status
|
||||||
|
|
||||||
|
begin
|
||||||
|
Timeout.timeout(30) do
|
||||||
|
Process.kill('TERM', puppet_pid)
|
||||||
|
while puppet_pid do
|
||||||
|
sleep 1
|
||||||
|
end
|
||||||
|
end
|
||||||
|
rescue Timeout::Error
|
||||||
|
Process.kill('KILL', puppet_pid)
|
||||||
|
end
|
||||||
|
#FIXME: Daemonized process do not update lock file when we send signal to kill him
|
||||||
|
raise "Should never happen. Some process block lock file in critical section" unless rm_file(@lockfile)
|
||||||
|
rescue => e
|
||||||
|
reply.fail "Failed to kill the puppet daemon (process #{puppet_pid}): #{e}"
|
||||||
|
end
|
||||||
|
|
||||||
|
def puppet_pid
|
||||||
result = `ps -C puppet -o pid,comm --no-headers`.lines.first
|
result = `ps -C puppet -o pid,comm --no-headers`.lines.first
|
||||||
result && result.strip.split(' ')[0].to_i
|
result && result.strip.split(' ')[0].to_i
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def lock_file(file_name, &block)
|
||||||
|
File.open(file_name, 'w+') do |f|
|
||||||
|
begin
|
||||||
|
f.flock File::LOCK_EX
|
||||||
|
yield
|
||||||
|
ensure
|
||||||
|
f.flock File::LOCK_UN
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
@ -40,6 +40,7 @@ describe Astute::DeploymentEngine do
|
|||||||
deployer.stubs(:generate_ssh_keys)
|
deployer.stubs(:generate_ssh_keys)
|
||||||
deployer.stubs(:upload_ssh_keys)
|
deployer.stubs(:upload_ssh_keys)
|
||||||
deployer.stubs(:sync_puppet_manifests)
|
deployer.stubs(:sync_puppet_manifests)
|
||||||
|
deployer.stubs(:enable_puppet_deploy)
|
||||||
end
|
end
|
||||||
|
|
||||||
it 'should generate and upload ssh keys' do
|
it 'should generate and upload ssh keys' do
|
||||||
@ -53,6 +54,15 @@ describe Astute::DeploymentEngine do
|
|||||||
deployer.deploy(nodes)
|
deployer.deploy(nodes)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
it 'should enable puppet for all nodes' do
|
||||||
|
nodes = [{'uid' => 1, 'deployment_id' => 1}, {'uid' => 2}, {'uid' => 1}]
|
||||||
|
deployer.stubs(:deploy_piece)
|
||||||
|
|
||||||
|
deployer.expects(:enable_puppet_deploy).with([1,2]).returns()
|
||||||
|
|
||||||
|
deployer.deploy(nodes)
|
||||||
|
end
|
||||||
|
|
||||||
it 'deploy nodes by order' do
|
it 'deploy nodes by order' do
|
||||||
nodes = [{'uid' => 1, 'priority' => 10}, {'uid' => 2, 'priority' => 0}, {'uid' => 1, 'priority' => 15}]
|
nodes = [{'uid' => 1, 'priority' => 10}, {'uid' => 2, 'priority' => 0}, {'uid' => 1, 'priority' => 15}]
|
||||||
|
|
||||||
@ -144,6 +154,7 @@ describe Astute::DeploymentEngine do
|
|||||||
deployer.stubs(:deploy_piece)
|
deployer.stubs(:deploy_piece)
|
||||||
deployer.stubs(:generate_ssh_keys)
|
deployer.stubs(:generate_ssh_keys)
|
||||||
deployer.stubs(:upload_ssh_keys)
|
deployer.stubs(:upload_ssh_keys)
|
||||||
|
deployer.stubs(:enable_puppet_deploy)
|
||||||
end
|
end
|
||||||
|
|
||||||
let(:nodes) { [{'uid' => 1, 'deployment_id' => 1, 'master_ip' => '10.20.0.2'}, {'uid' => 2}] }
|
let(:nodes) { [{'uid' => 1, 'deployment_id' => 1, 'master_ip' => '10.20.0.2'}, {'uid' => 2}] }
|
||||||
@ -167,6 +178,7 @@ describe Astute::DeploymentEngine do
|
|||||||
Astute.config.PUPPET_SSH_KEYS = ['nova']
|
Astute.config.PUPPET_SSH_KEYS = ['nova']
|
||||||
deployer.stubs(:deploy_piece)
|
deployer.stubs(:deploy_piece)
|
||||||
deployer.stubs(:sync_puppet_manifests)
|
deployer.stubs(:sync_puppet_manifests)
|
||||||
|
deployer.stubs(:enable_puppet_deploy)
|
||||||
end
|
end
|
||||||
|
|
||||||
let(:nodes) { [{'uid' => 1, 'deployment_id' => 1}, {'uid' => 2}] }
|
let(:nodes) { [{'uid' => 1, 'deployment_id' => 1}, {'uid' => 2}] }
|
||||||
|
@ -51,6 +51,7 @@ describe "NailyFact DeploymentEngine" do
|
|||||||
deploy_engine.stubs(:generate_ssh_keys).with(deploy_data.first['deployment_id'])
|
deploy_engine.stubs(:generate_ssh_keys).with(deploy_data.first['deployment_id'])
|
||||||
deploy_engine.stubs(:upload_ssh_keys).with(uniq_nodes_uid, deploy_data.first['deployment_id'])
|
deploy_engine.stubs(:upload_ssh_keys).with(uniq_nodes_uid, deploy_data.first['deployment_id'])
|
||||||
deploy_engine.stubs(:sync_puppet_manifests).with(deploy_data.uniq { |n| n['uid'] })
|
deploy_engine.stubs(:sync_puppet_manifests).with(deploy_data.uniq { |n| n['uid'] })
|
||||||
|
deploy_engine.stubs(:enable_puppet_deploy).with(uniq_nodes_uid)
|
||||||
end
|
end
|
||||||
|
|
||||||
context 'log parsing' do
|
context 'log parsing' do
|
||||||
|
Loading…
Reference in New Issue
Block a user