Granular deployment: tests, fixes and refactoring
- fix problem with error nodes (endless loop); - add tests for puppet tasks. - update mcollective agent (should work with old version also). Closes-Bug: #1392779 (refactoring https://review.openstack.org/#/c/138790) Implements blueprint granular-deployment-based-on-tasks Change-Id: I9832e405143dbb6b2fbd22ed2d2bc7ad0bc85f01
This commit is contained in:
parent
18be5cd3b8
commit
85b1e638d8
|
@ -43,7 +43,7 @@ class Astute::DeploymentEngine::GranularDeployment < Astute::DeploymentEngine
|
||||||
report_nodes = nodes.uniq{ |n| n['uid'] }.map do |node|
|
report_nodes = nodes.uniq{ |n| n['uid'] }.map do |node|
|
||||||
{ 'uid' => node['uid'],
|
{ 'uid' => node['uid'],
|
||||||
'status' => 'error',
|
'status' => 'error',
|
||||||
'role' => 'hook',
|
'role' => node['role'],
|
||||||
'error_type' => 'deploy'
|
'error_type' => 'deploy'
|
||||||
}
|
}
|
||||||
end
|
end
|
||||||
|
@ -112,6 +112,7 @@ class Astute::DeploymentEngine::GranularDeployment < Astute::DeploymentEngine
|
||||||
else
|
else
|
||||||
Astute.logger.debug "No more tasks provided for node #{node_id}"
|
Astute.logger.debug "No more tasks provided for node #{node_id}"
|
||||||
end
|
end
|
||||||
|
end
|
||||||
|
|
||||||
@ctx.report_and_update_status('nodes' => nodes_to_report) if nodes_to_report.present?
|
@ctx.report_and_update_status('nodes' => nodes_to_report) if nodes_to_report.present?
|
||||||
|
|
||||||
|
@ -119,7 +120,6 @@ class Astute::DeploymentEngine::GranularDeployment < Astute::DeploymentEngine
|
||||||
sleep Astute.config.PUPPET_DEPLOY_INTERVAL
|
sleep Astute.config.PUPPET_DEPLOY_INTERVAL
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
|
||||||
|
|
||||||
def process_success_node(node_id, task)
|
def process_success_node(node_id, task)
|
||||||
Astute.logger.info "No more tasks provided for node #{node_id}. All node " \
|
Astute.logger.info "No more tasks provided for node #{node_id}. All node " \
|
||||||
|
@ -146,6 +146,7 @@ class Astute::DeploymentEngine::GranularDeployment < Astute::DeploymentEngine
|
||||||
end
|
end
|
||||||
|
|
||||||
def process_running_node(node_id, task, nodes)
|
def process_running_node(node_id, task, nodes)
|
||||||
|
Astute.logger.debug "Task '#{task}' on node uid=#{node_id} deploying"
|
||||||
begin
|
begin
|
||||||
# Pass nodes because logs calculation needs IP address of node, not just uid
|
# Pass nodes because logs calculation needs IP address of node, not just uid
|
||||||
nodes_progress = @ctx.deploy_log_parser.progress_calculate(Array(node_id), nodes)
|
nodes_progress = @ctx.deploy_log_parser.progress_calculate(Array(node_id), nodes)
|
||||||
|
|
|
@ -24,23 +24,22 @@ module Astute
|
||||||
@puppet_manifest = puppet_manifest || '/etc/puppet/manifests/site.pp'
|
@puppet_manifest = puppet_manifest || '/etc/puppet/manifests/site.pp'
|
||||||
@puppet_modules = puppet_modules || '/etc/puppet/modules'
|
@puppet_modules = puppet_modules || '/etc/puppet/modules'
|
||||||
@cwd = cwd || '/'
|
@cwd = cwd || '/'
|
||||||
@time_obsorver = TimeObserver.new(timeout || Astute.config.PUPPET_TIMEOUT)
|
@time_observer = TimeObserver.new(timeout || Astute.config.PUPPET_TIMEOUT)
|
||||||
@prev_summary = nil
|
@prev_summary = nil
|
||||||
@is_hung = false
|
@is_hung = false
|
||||||
end
|
end
|
||||||
|
|
||||||
def run
|
def run
|
||||||
Astute.logger.debug "Waiting for puppet to finish deployment on " \
|
Astute.logger.debug "Waiting for puppet to finish deployment on " \
|
||||||
"node #{@node['uid']} (timeout = #{@time_obsorver.time_limit} sec)..."
|
"node #{@node['uid']} (timeout = #{@time_observer.time_limit} sec)..."
|
||||||
@time_obsorver.start
|
@time_observer.start
|
||||||
@prev_summary ||= puppet_status
|
@prev_summary ||= puppet_status
|
||||||
puppetd_runonce
|
puppetd_runonce
|
||||||
end
|
end
|
||||||
|
|
||||||
# expect to run this method with respect of Astute.config.PUPPET_FADE_INTERVAL
|
# expect to run this method with respect of Astute.config.PUPPET_FADE_INTERVAL
|
||||||
def status
|
def status
|
||||||
# TODO(vsharshov): Should we raise error?
|
raise Timeout::Error unless @time_observer.enough_time?
|
||||||
raise Timeout::Error unless @time_obsorver.enough_time?
|
|
||||||
|
|
||||||
last_run = puppet_status
|
last_run = puppet_status
|
||||||
status = node_status(last_run)
|
status = node_status(last_run)
|
||||||
|
@ -146,21 +145,21 @@ module Astute
|
||||||
'error'
|
'error'
|
||||||
when succeed?(last_run) && !@is_hung
|
when succeed?(last_run) && !@is_hung
|
||||||
'succeed'
|
'succeed'
|
||||||
when (running?(last_run) || idling?(status)) && !@is_hung
|
when (running?(last_run) || idling?(last_run)) && !@is_hung
|
||||||
'running'
|
'running'
|
||||||
when stopped?(status) && !succeed?(last_run) && !@is_hung
|
when stopped?(last_run) && !succeed?(last_run) && !@is_hung
|
||||||
'error'
|
'error'
|
||||||
else
|
else
|
||||||
msg = "Unknow status: " \
|
msg = "Unknow status: " \
|
||||||
"is_hung #{@is_hung}, succeed? #{succeed?(last_run)}, " \
|
"is_hung #{@is_hung}, succeed? #{succeed?(last_run)}, " \
|
||||||
"running? #{running?(last_run)}, stopped? #{stopped?(status)}, " \
|
"running? #{running?(last_run)}, stopped? #{stopped?(last_run)}, " \
|
||||||
"idling? #{idling?(status)}"
|
"idling? #{idling?(last_run)}"
|
||||||
raise msg
|
raise msg
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
def processing_succeed_node
|
def processing_succeed_node
|
||||||
Astute.logger.debug "Puppet completed within #{@time_obsorver.stop} seconds"
|
Astute.logger.debug "Puppet completed within #{@time_observer.stop} seconds"
|
||||||
{ 'uid' => @node['uid'], 'status' => 'ready', 'role' => @node['role'] }
|
{ 'uid' => @node['uid'], 'status' => 'ready', 'role' => @node['role'] }
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
|
@ -109,8 +109,9 @@ module MCollective
|
||||||
end
|
end
|
||||||
|
|
||||||
def rm_file file
|
def rm_file file
|
||||||
|
return true unless File.exists?(file)
|
||||||
begin
|
begin
|
||||||
File.unlink(file)
|
File.unlink file
|
||||||
return true
|
return true
|
||||||
rescue
|
rescue
|
||||||
return false
|
return false
|
||||||
|
@ -119,9 +120,9 @@ module MCollective
|
||||||
|
|
||||||
def puppet_daemon_status
|
def puppet_daemon_status
|
||||||
err_msg = ""
|
err_msg = ""
|
||||||
alive = !!puppet_pid
|
alive = puppet_pid
|
||||||
locked = File.exists?(@lockfile)
|
locked = expected_puppet_pid == puppet_pid && !expected_puppet_pid.nil?
|
||||||
disabled = locked && File::Stat.new(@lockfile).zero?
|
disabled = File.exists?(@lockfile) && File::Stat.new(@lockfile).zero?
|
||||||
|
|
||||||
if locked && !disabled && !alive
|
if locked && !disabled && !alive
|
||||||
err_msg << "Process not running but not empty lockfile is present. Trying to remove lockfile..."
|
err_msg << "Process not running but not empty lockfile is present. Trying to remove lockfile..."
|
||||||
|
@ -154,16 +155,9 @@ module MCollective
|
||||||
reply.fail "Lock file and PID file exist; puppet is running."
|
reply.fail "Lock file and PID file exist; puppet is running."
|
||||||
|
|
||||||
when 'idling' then # signal daemon
|
when 'idling' then # signal daemon
|
||||||
pid = puppet_pid
|
kill_process
|
||||||
begin
|
set_status # recalculate state after puppet kill
|
||||||
::Process.kill('INT', pid)
|
reply[:err_msg] = "Looks like another puppet run at the same time. Try to kill it"
|
||||||
rescue Errno::ESRCH => e
|
|
||||||
reply[:err_msg] = "Failed to signal the puppet apply daemon (process #{pid}): #{e}"
|
|
||||||
ensure
|
|
||||||
runonce_background
|
|
||||||
reply[:output] = "Kill old idling puppet process #{pid})." + (reply[:output] || '')
|
|
||||||
end
|
|
||||||
|
|
||||||
when 'stopped' then # just run
|
when 'stopped' then # just run
|
||||||
runonce_background
|
runonce_background
|
||||||
else
|
else
|
||||||
|
@ -255,13 +249,13 @@ module MCollective
|
||||||
|
|
||||||
begin
|
begin
|
||||||
Timeout.timeout(30) do
|
Timeout.timeout(30) do
|
||||||
Process.kill('TERM', puppet_pid)
|
Process.kill('TERM', puppet_pid) if puppet_pid
|
||||||
while puppet_pid do
|
while puppet_pid do
|
||||||
sleep 1
|
sleep 1
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
rescue Timeout::Error
|
rescue Timeout::Error
|
||||||
Process.kill('KILL', puppet_pid)
|
Process.kill('KILL', puppet_pid) if puppet_pid
|
||||||
end
|
end
|
||||||
#FIXME: Daemonized process do not update lock file when we send signal to kill him
|
#FIXME: Daemonized process do not update lock file when we send signal to kill him
|
||||||
raise "Should never happen. Some process block lock file in critical section" unless rm_file(@lockfile)
|
raise "Should never happen. Some process block lock file in critical section" unless rm_file(@lockfile)
|
||||||
|
@ -269,9 +263,26 @@ module MCollective
|
||||||
reply.fail "Failed to kill the puppet daemon (process #{puppet_pid}): #{e}"
|
reply.fail "Failed to kill the puppet daemon (process #{puppet_pid}): #{e}"
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def expected_puppet_pid
|
||||||
|
File.read(@lockfile).to_i
|
||||||
|
rescue Errno::ENOENT
|
||||||
|
nil
|
||||||
|
end
|
||||||
|
|
||||||
def puppet_pid
|
def puppet_pid
|
||||||
result = `ps -C puppet -o pid,comm --no-headers`.lines.first
|
result = `ps -C puppet -o pid,comm --no-headers`.lines.first
|
||||||
result && result.strip.split(' ')[0].to_i
|
actual_pid = result && result.strip.split(' ')[0].to_i
|
||||||
|
expected_pid = expected_puppet_pid
|
||||||
|
|
||||||
|
case actual_pid
|
||||||
|
when expected_pid then expected_pid
|
||||||
|
else
|
||||||
|
reply[:err_msg] = "Potencial error. Looks like expecting puppet and actual are different. " \
|
||||||
|
"Expecting pid(lockfile): #{expected_pid}, actual(ps): #{actual_pid}"
|
||||||
|
actual_pid
|
||||||
|
end
|
||||||
|
rescue NoMethodError
|
||||||
|
nil
|
||||||
end
|
end
|
||||||
|
|
||||||
def lock_file(file_name, &block)
|
def lock_file(file_name, &block)
|
||||||
|
|
|
@ -0,0 +1,351 @@
|
||||||
|
# Copyright 2014 Mirantis, Inc.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
|
||||||
|
require File.join(File.dirname(__FILE__), '../spec_helper')
|
||||||
|
|
||||||
|
describe "Granular deployment engine" do
|
||||||
|
include SpecHelpers
|
||||||
|
|
||||||
|
let(:ctx) {
|
||||||
|
ctx = mock
|
||||||
|
ctx.stubs(:task_id)
|
||||||
|
ctx.stubs(:deploy_log_parser).returns(Astute::LogParser::NoParsing.new)
|
||||||
|
ctx.stubs(:status).returns({})
|
||||||
|
reporter = mock
|
||||||
|
reporter.stubs(:report)
|
||||||
|
up_reporter = Astute::ProxyReporter::DeploymentProxyReporter.new(reporter, nodes)
|
||||||
|
ctx.stubs(:reporter).returns(up_reporter)
|
||||||
|
ctx
|
||||||
|
}
|
||||||
|
|
||||||
|
let(:deploy_engine) do
|
||||||
|
Astute::DeploymentEngine::GranularDeployment.new(ctx)
|
||||||
|
end
|
||||||
|
|
||||||
|
let(:upload_file_hook) do
|
||||||
|
{
|
||||||
|
"priority" => 100,
|
||||||
|
"type" => "upload_file",
|
||||||
|
"fail_on_error" => false,
|
||||||
|
"diagnostic_name" => "upload-example-1.0",
|
||||||
|
"uids" => ['2', '3'],
|
||||||
|
"parameters" => {
|
||||||
|
"path" => "/etc/yum.repos.d/fuel_awesome_plugin-0.1.0.repo",
|
||||||
|
"data" => "[fuel_awesome_plugin-0.1.0]\\nname=Plugin fuel_awesome_plugin-0.1.0 repository\\nbaseurl=http => //10.20.0.2 => 8080/plugins/fuel_awesome_plugin-0.1.0/repositories/centos\\ngpgcheck=0"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
let(:sync_hook) do
|
||||||
|
{
|
||||||
|
"priority" => 200,
|
||||||
|
"type" => "sync",
|
||||||
|
"fail_on_error" => false,
|
||||||
|
"diagnostic_name" => "sync-example-1.0",
|
||||||
|
"uids" => ['1', '2'],
|
||||||
|
"parameters" => {
|
||||||
|
"src" => "rsync://10.20.0.2/plugins/fuel_awesome_plugin-0.1.0/deployment_scripts/",
|
||||||
|
"dst" => "/etc/fuel/plugins/fuel_awesome_plugin-0.1.0/"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
let(:shell_hook) do
|
||||||
|
{
|
||||||
|
"priority" => 100,
|
||||||
|
"type" => "shell",
|
||||||
|
"fail_on_error" => false,
|
||||||
|
"diagnostic_name" => "shell-example-1.0",
|
||||||
|
"uids" => ['1','2','3'],
|
||||||
|
"parameters" => {
|
||||||
|
"cmd" => "./deploy.sh",
|
||||||
|
"cwd" => "/etc/fuel/plugins/fuel_awesome_plugin-0.1.0/",
|
||||||
|
"timeout" => 60
|
||||||
|
}
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
let(:puppet_hook) do
|
||||||
|
{
|
||||||
|
"priority" => 300,
|
||||||
|
"type" => "puppet",
|
||||||
|
"fail_on_error" => false,
|
||||||
|
"diagnostic_name" => "puppet-example-1.0",
|
||||||
|
"uids" => ['1', '3'],
|
||||||
|
"parameters" => {
|
||||||
|
"puppet_manifest" => "cinder_glusterfs.pp",
|
||||||
|
"puppet_modules" => "modules",
|
||||||
|
"cwd" => "/etc/fuel/plugins/plugin_name-1.0",
|
||||||
|
"timeout" => 42
|
||||||
|
}
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
let(:nodes) do
|
||||||
|
[
|
||||||
|
{
|
||||||
|
'uid' => '45',
|
||||||
|
'priority' => 200,
|
||||||
|
'role' => 'ceph',
|
||||||
|
'tasks' => [
|
||||||
|
{
|
||||||
|
'priority' => 100,
|
||||||
|
'type' => 'puppet',
|
||||||
|
'uids' => ['45']
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'priority' => 300,
|
||||||
|
'type' => 'puppet',
|
||||||
|
'uids' => ['45']
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'uid' => '46',
|
||||||
|
'priority' => 200,
|
||||||
|
'role' => 'compute',
|
||||||
|
'tasks' => [
|
||||||
|
{
|
||||||
|
'priority' => 100,
|
||||||
|
'type' => 'puppet',
|
||||||
|
'uids' => ['46']
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'priority' => 200,
|
||||||
|
'type' => 'puppet',
|
||||||
|
'uids' => ['46']
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'priority' => 300,
|
||||||
|
'type' => 'puppet',
|
||||||
|
'uids' => ['46']
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
end
|
||||||
|
|
||||||
|
describe '#deploy_piace' do
|
||||||
|
it 'should run tasks using puppet task' do
|
||||||
|
ctx.stubs(:report_and_update_status)
|
||||||
|
deploy_engine.expects(:deploy_nodes).with(nodes)
|
||||||
|
|
||||||
|
deploy_engine.deploy_piece(nodes)
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'should report error status if error raised' do
|
||||||
|
error_report = {'nodes' =>
|
||||||
|
[
|
||||||
|
{
|
||||||
|
'uid' => '45',
|
||||||
|
'status' => 'error',
|
||||||
|
'role' => 'ceph',
|
||||||
|
'error_type' => 'deploy'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'uid' => '46',
|
||||||
|
'status' => 'error',
|
||||||
|
'role' => 'compute',
|
||||||
|
'error_type' => 'deploy'
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx.expects(:report_and_update_status).with(error_report)
|
||||||
|
deploy_engine.expects(:deploy_nodes).with(nodes).raises("Error simulation")
|
||||||
|
|
||||||
|
deploy_engine.deploy_piece(nodes) rescue nil
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'should not raise errir if no nodes was sent' do
|
||||||
|
expect{ deploy_engine.deploy_piece([])}.to_not raise_error
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'should prepare log for parsing' do
|
||||||
|
deploy_engine.stubs(:deploy_nodes).with(nodes)
|
||||||
|
|
||||||
|
ctx.deploy_log_parser.expects(:prepare).with(nodes).once
|
||||||
|
deploy_engine.deploy_piece(nodes)
|
||||||
|
end
|
||||||
|
end # 'deploy_piace'
|
||||||
|
|
||||||
|
describe '#deploy_nodes' do
|
||||||
|
it 'run deploy task on nodes' do
|
||||||
|
ctx.stubs(:report_and_update_status)
|
||||||
|
deploy_engine.expects(:run_task).times(5)
|
||||||
|
|
||||||
|
deploy_engine.stubs(:check_status).with('46')
|
||||||
|
.returns('deploying').then
|
||||||
|
.returns('ready').then
|
||||||
|
.returns('deploying').then
|
||||||
|
.returns('ready').then
|
||||||
|
.returns('deploying').then
|
||||||
|
.returns('ready')
|
||||||
|
|
||||||
|
deploy_engine.stubs(:check_status).with('45')
|
||||||
|
.returns('deploying').then
|
||||||
|
.returns('ready').then
|
||||||
|
.returns('deploying').then
|
||||||
|
.returns('ready')
|
||||||
|
|
||||||
|
deploy_engine.deploy_piece(nodes)
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'report status about nodes' do
|
||||||
|
deploy_engine.stubs(:run_task).times(5)
|
||||||
|
|
||||||
|
deploy_engine.stubs(:check_status).with('46')
|
||||||
|
.returns('deploying').then
|
||||||
|
.returns('ready').then
|
||||||
|
.returns('deploying').then
|
||||||
|
.returns('ready').then
|
||||||
|
.returns('deploying').then
|
||||||
|
.returns('ready')
|
||||||
|
|
||||||
|
deploy_engine.stubs(:check_status).with('45')
|
||||||
|
.returns('deploying').then
|
||||||
|
.returns('ready').then
|
||||||
|
.returns('deploying').then
|
||||||
|
.returns('ready')
|
||||||
|
|
||||||
|
succeed_report_node1 = {
|
||||||
|
'nodes' => [
|
||||||
|
{
|
||||||
|
'uid' => '45',
|
||||||
|
'status' => 'ready',
|
||||||
|
'role' => 'ceph',
|
||||||
|
'progress' => 100,
|
||||||
|
'task' => {'priority' => 300, 'type' => 'puppet', 'uids' => ['45']}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
succeed_report_node2 = {
|
||||||
|
'nodes' => [
|
||||||
|
{
|
||||||
|
'uid' => '46',
|
||||||
|
'status' => 'ready',
|
||||||
|
'role' => 'compute',
|
||||||
|
'progress' => 100,
|
||||||
|
'task' => {'priority' => 300, 'type' => 'puppet', 'uids' => ['46']}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx.expects(:report_and_update_status).with(succeed_report_node1).times(1)
|
||||||
|
ctx.expects(:report_and_update_status).with(succeed_report_node2).times(1)
|
||||||
|
|
||||||
|
deploy_engine.deploy_piece(nodes)
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'handle error nodes and do not process next tasks on problem node' do
|
||||||
|
deploy_engine.expects(:run_task).times(4)
|
||||||
|
|
||||||
|
deploy_engine.stubs(:check_status).with('46')
|
||||||
|
.returns('deploying').then
|
||||||
|
.returns('ready').then
|
||||||
|
.returns('deploying').then
|
||||||
|
.returns('error')
|
||||||
|
|
||||||
|
deploy_engine.stubs(:check_status).with('45')
|
||||||
|
.returns('deploying').then
|
||||||
|
.returns('ready').then
|
||||||
|
.returns('deploying').then
|
||||||
|
.returns('ready')
|
||||||
|
|
||||||
|
mixed_report = {'nodes' =>
|
||||||
|
[
|
||||||
|
{
|
||||||
|
'uid' => '45',
|
||||||
|
'status' => 'ready',
|
||||||
|
'role' => 'ceph',
|
||||||
|
'progress' => 100,
|
||||||
|
'task' => {'priority' => 300, 'type' => 'puppet', 'uids' => ['45']}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'uid' => '46',
|
||||||
|
'status' => 'error',
|
||||||
|
'error_type' => 'deploy',
|
||||||
|
'role' => 'compute',
|
||||||
|
'task' => {'priority' => 200, 'type' => 'puppet', 'uids' => ['46']}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
ctx.expects(:report_and_update_status).with(mixed_report)
|
||||||
|
|
||||||
|
deploy_engine.deploy_piece(nodes)
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'handle sutuation then all nodes failed' do
|
||||||
|
deploy_engine.expects(:run_task).times(2)
|
||||||
|
|
||||||
|
deploy_engine.stubs(:check_status).with('46')
|
||||||
|
.returns('deploying').then
|
||||||
|
.returns('error')
|
||||||
|
|
||||||
|
deploy_engine.stubs(:check_status).with('45')
|
||||||
|
.returns('deploying').then
|
||||||
|
.returns('error')
|
||||||
|
|
||||||
|
error_report = {'nodes' =>
|
||||||
|
[
|
||||||
|
{
|
||||||
|
'uid' => '45',
|
||||||
|
'status' => 'error',
|
||||||
|
'error_type' => 'deploy',
|
||||||
|
'role' => 'ceph',
|
||||||
|
'task' => {'priority' => 100, 'type' => 'puppet', 'uids' => ['45']}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'uid' => '46',
|
||||||
|
'status' => 'error',
|
||||||
|
'error_type' => 'deploy',
|
||||||
|
'role' => 'compute',
|
||||||
|
'task' => {'priority' => 100, 'type' => 'puppet', 'uids' => ['46']}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
ctx.expects(:report_and_update_status).with(error_report)
|
||||||
|
deploy_engine.deploy_piece(nodes)
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'should fail deployment if handler not provided' do
|
||||||
|
deploy_engine.expects(:run_task).times(2)
|
||||||
|
deploy_engine.stubs(:check_status).returns('unknown_handler')
|
||||||
|
|
||||||
|
error_report = {'nodes' =>
|
||||||
|
[
|
||||||
|
{
|
||||||
|
'uid' => '45',
|
||||||
|
'status' => 'error',
|
||||||
|
'error_type' => 'deploy',
|
||||||
|
'role' => 'ceph'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'uid' => '46',
|
||||||
|
'status' => 'error',
|
||||||
|
'error_type' => 'deploy',
|
||||||
|
'role' => 'compute'
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx.expects(:report_and_update_status).with(error_report)
|
||||||
|
expect {deploy_engine.deploy_piece(nodes) }
|
||||||
|
.to raise_error(/Internal error. Known status/)
|
||||||
|
end
|
||||||
|
|
||||||
|
end #deploy_nodes
|
||||||
|
|
||||||
|
end # 'describe'
|
|
@ -1,186 +0,0 @@
|
||||||
# Copyright 2014 Mirantis, Inc.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
||||||
# not use this file except in compliance with the License. You may obtain
|
|
||||||
# a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
||||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
||||||
# License for the specific language governing permissions and limitations
|
|
||||||
# under the License.
|
|
||||||
|
|
||||||
|
|
||||||
require File.join(File.dirname(__FILE__), '../spec_helper')
|
|
||||||
|
|
||||||
include Astute
|
|
||||||
|
|
||||||
describe "Granular deployment engine" do
|
|
||||||
include SpecHelpers
|
|
||||||
|
|
||||||
let(:ctx) {
|
|
||||||
ctx = mock
|
|
||||||
ctx.stubs(:task_id)
|
|
||||||
ctx.stubs(:deploy_log_parser).returns(Astute::LogParser::NoParsing.new)
|
|
||||||
ctx.stubs(:status).returns({})
|
|
||||||
reporter = mock
|
|
||||||
reporter.stubs(:report)
|
|
||||||
up_reporter = Astute::ProxyReporter::DeploymentProxyReporter.new(reporter, nodes)
|
|
||||||
ctx.stubs(:reporter).returns(up_reporter)
|
|
||||||
ctx
|
|
||||||
}
|
|
||||||
|
|
||||||
let(:deploy_engine) do
|
|
||||||
Astute::DeploymentEngine::GranularDeployment.new(ctx)
|
|
||||||
end
|
|
||||||
|
|
||||||
let(:upload_file_hook) do
|
|
||||||
{
|
|
||||||
"priority" => 100,
|
|
||||||
"type" => "upload_file",
|
|
||||||
"fail_on_error" => false,
|
|
||||||
"diagnostic_name" => "upload-example-1.0",
|
|
||||||
"uids" => ['2', '3'],
|
|
||||||
"parameters" => {
|
|
||||||
"path" => "/etc/yum.repos.d/fuel_awesome_plugin-0.1.0.repo",
|
|
||||||
"data" => "[fuel_awesome_plugin-0.1.0]\\nname=Plugin fuel_awesome_plugin-0.1.0 repository\\nbaseurl=http => //10.20.0.2 => 8080/plugins/fuel_awesome_plugin-0.1.0/repositories/centos\\ngpgcheck=0"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
end
|
|
||||||
|
|
||||||
let(:sync_hook) do
|
|
||||||
{
|
|
||||||
"priority" => 200,
|
|
||||||
"type" => "sync",
|
|
||||||
"fail_on_error" => false,
|
|
||||||
"diagnostic_name" => "sync-example-1.0",
|
|
||||||
"uids" => ['1', '2'],
|
|
||||||
"parameters" => {
|
|
||||||
"src" => "rsync://10.20.0.2/plugins/fuel_awesome_plugin-0.1.0/deployment_scripts/",
|
|
||||||
"dst" => "/etc/fuel/plugins/fuel_awesome_plugin-0.1.0/"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
end
|
|
||||||
|
|
||||||
let(:shell_hook) do
|
|
||||||
{
|
|
||||||
"priority" => 100,
|
|
||||||
"type" => "shell",
|
|
||||||
"fail_on_error" => false,
|
|
||||||
"diagnostic_name" => "shell-example-1.0",
|
|
||||||
"uids" => ['1','2','3'],
|
|
||||||
"parameters" => {
|
|
||||||
"cmd" => "./deploy.sh",
|
|
||||||
"cwd" => "/etc/fuel/plugins/fuel_awesome_plugin-0.1.0/",
|
|
||||||
"timeout" => 60
|
|
||||||
}
|
|
||||||
}
|
|
||||||
end
|
|
||||||
|
|
||||||
let(:puppet_hook) do
|
|
||||||
{
|
|
||||||
"priority" => 300,
|
|
||||||
"type" => "puppet",
|
|
||||||
"fail_on_error" => false,
|
|
||||||
"diagnostic_name" => "puppet-example-1.0",
|
|
||||||
"uids" => ['1', '3'],
|
|
||||||
"parameters" => {
|
|
||||||
"puppet_manifest" => "cinder_glusterfs.pp",
|
|
||||||
"puppet_modules" => "modules",
|
|
||||||
"cwd" => "/etc/fuel/plugins/plugin_name-1.0",
|
|
||||||
"timeout" => 42
|
|
||||||
}
|
|
||||||
}
|
|
||||||
end
|
|
||||||
|
|
||||||
let(:nodes) do
|
|
||||||
[
|
|
||||||
{
|
|
||||||
'uid' => '45',
|
|
||||||
'priority' => 200,
|
|
||||||
'role' => 'ceph',
|
|
||||||
'tasks' => [
|
|
||||||
{
|
|
||||||
'priority' => 100,
|
|
||||||
'type' => 'puppet',
|
|
||||||
'uids' => ['45']
|
|
||||||
},
|
|
||||||
{
|
|
||||||
'priority' => 300,
|
|
||||||
'type' => 'puppet',
|
|
||||||
'uids' => ['45']
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
'uid' => '46',
|
|
||||||
'priority' => 200,
|
|
||||||
'role' => 'compute',
|
|
||||||
'tasks' => [
|
|
||||||
{
|
|
||||||
'priority' => 100,
|
|
||||||
'type' => 'puppet',
|
|
||||||
'uids' => ['46']
|
|
||||||
},
|
|
||||||
{
|
|
||||||
'priority' => 200,
|
|
||||||
'type' => 'puppet',
|
|
||||||
'uids' => ['46']
|
|
||||||
},
|
|
||||||
{
|
|
||||||
'priority' => 300,
|
|
||||||
'type' => 'puppet',
|
|
||||||
'uids' => ['46']
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
end
|
|
||||||
|
|
||||||
describe '#deploy_piace' do
|
|
||||||
it 'should run tasks using puppet task' do
|
|
||||||
ctx.stubs(:report_and_update_status)
|
|
||||||
deploy_engine.expects(:deploy_nodes).with(nodes)
|
|
||||||
|
|
||||||
deploy_engine.deploy_piece(nodes)
|
|
||||||
end
|
|
||||||
|
|
||||||
it 'should report error status if error erased' do
|
|
||||||
error_report = {'nodes' =>
|
|
||||||
[
|
|
||||||
{
|
|
||||||
'uid' => '45',
|
|
||||||
'status' => 'error',
|
|
||||||
'role' => 'hook',
|
|
||||||
'error_type' => 'deploy'
|
|
||||||
},
|
|
||||||
{
|
|
||||||
'uid' => '46',
|
|
||||||
'status' => 'error',
|
|
||||||
'role' => 'hook',
|
|
||||||
'error_type' => 'deploy'
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx.expects(:report_and_update_status).with(error_report)
|
|
||||||
deploy_engine.expects(:deploy_nodes).with(nodes).raises("Error simulation")
|
|
||||||
|
|
||||||
deploy_engine.deploy_piece(nodes) rescue nil
|
|
||||||
end
|
|
||||||
|
|
||||||
it 'should not raise errir if no nodes was sent' do
|
|
||||||
expect{ deploy_engine.deploy_piece([])}.to_not raise_error
|
|
||||||
end
|
|
||||||
|
|
||||||
it 'should prepare log for parsing' do
|
|
||||||
deploy_engine.stubs(:deploy_nodes).with(nodes)
|
|
||||||
|
|
||||||
ctx.deploy_log_parser.expects(:prepare).with(nodes).once
|
|
||||||
deploy_engine.deploy_piece(nodes)
|
|
||||||
end
|
|
||||||
end # 'deploy_piace'
|
|
||||||
|
|
||||||
end # 'describe'
|
|
|
@ -0,0 +1,198 @@
|
||||||
|
# Copyright 2015 Mirantis, Inc.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
require File.join(File.dirname(__FILE__), '../spec_helper')
|
||||||
|
|
||||||
|
describe Astute::PuppetTask do
|
||||||
|
include SpecHelpers
|
||||||
|
|
||||||
|
let(:node) do
|
||||||
|
{
|
||||||
|
'uid' => '45',
|
||||||
|
'priority' => 200,
|
||||||
|
'role' => 'ceph',
|
||||||
|
'tasks' => [
|
||||||
|
{
|
||||||
|
'priority' => 100,
|
||||||
|
'type' => 'puppet',
|
||||||
|
'uids' => ['45']
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'priority' => 300,
|
||||||
|
'type' => 'puppet',
|
||||||
|
'uids' => ['45']
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
let(:ctx) {
|
||||||
|
ctx = mock
|
||||||
|
ctx.stubs(:task_id)
|
||||||
|
ctx.stubs(:deploy_log_parser).returns(Astute::LogParser::NoParsing.new)
|
||||||
|
ctx.stubs(:status).returns({})
|
||||||
|
reporter = mock
|
||||||
|
reporter.stubs(:report)
|
||||||
|
up_reporter = Astute::ProxyReporter::DeploymentProxyReporter.new(reporter, [node])
|
||||||
|
ctx.stubs(:reporter).returns(up_reporter)
|
||||||
|
ctx
|
||||||
|
}
|
||||||
|
|
||||||
|
let(:puppet_task) { Astute::PuppetTask.new(ctx, node)}
|
||||||
|
let(:puppet_task_wo_retries) { Astute::PuppetTask.new(ctx, node, retries=0)}
|
||||||
|
|
||||||
|
let(:mco_puppet_stopped) do
|
||||||
|
{
|
||||||
|
:changes => {"total" => 1},
|
||||||
|
:time => {"last_run" => 1358425701},
|
||||||
|
:resources => {"failed" => 0},
|
||||||
|
:status => "stopped",
|
||||||
|
:enabled => 1,
|
||||||
|
:stopped => 1,
|
||||||
|
:idling => 0,
|
||||||
|
:running => 0,
|
||||||
|
:runtime => 1358425701
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
let(:mco_puppet_running) do
|
||||||
|
mco_puppet_stopped.merge(
|
||||||
|
:status => 'running',
|
||||||
|
:running => 1,
|
||||||
|
:stopped => 0
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
let(:mco_puppet_fail) do
|
||||||
|
mco_puppet_running.merge(
|
||||||
|
:runtime => 1358426000,
|
||||||
|
:time => {"last_run" => 1358426000},
|
||||||
|
:resources => {"failed" => 1}
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
let(:mco_puppet_failed) do
|
||||||
|
mco_puppet_fail.merge(
|
||||||
|
:status => 'stopped',
|
||||||
|
:stopped => 1,
|
||||||
|
:running => 0
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
let(:mco_puppet_finished) do
|
||||||
|
mco_puppet_stopped.merge(
|
||||||
|
:time => {'last_run' => 1358428000},
|
||||||
|
:status => 'stopped'
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
let(:mco_puppet_idling) do
|
||||||
|
mco_puppet_stopped.merge(
|
||||||
|
:status => 'idling',
|
||||||
|
:running => 0,
|
||||||
|
:stopped => 0,
|
||||||
|
:idling => 1
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "#run" do
|
||||||
|
it 'run puppet using mcollective' do
|
||||||
|
puppet_task.expects(:puppet_status).returns(mco_puppet_stopped).times(2)
|
||||||
|
puppet_task.expects(:puppet_run)
|
||||||
|
puppet_task.run
|
||||||
|
end
|
||||||
|
end #run
|
||||||
|
|
||||||
|
describe "#status" do
|
||||||
|
before(:each) do
|
||||||
|
ctx.stubs(:report_and_update_status)
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'check puppet using mcollective' do
|
||||||
|
puppet_task.stubs(:puppet_status).returns(mco_puppet_stopped)
|
||||||
|
.then.returns(mco_puppet_stopped)
|
||||||
|
.then.returns(mco_puppet_running)
|
||||||
|
.then.returns(mco_puppet_finished)
|
||||||
|
|
||||||
|
puppet_task.expects(:puppet_run)
|
||||||
|
puppet_task.run
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'return error for node if puppet failed (a cycle w/o any transitional states)' do
|
||||||
|
puppet_task_wo_retries.stubs(:puppet_status).returns(mco_puppet_stopped)
|
||||||
|
.then.returns(mco_puppet_stopped)
|
||||||
|
.then.returns(mco_puppet_failed)
|
||||||
|
|
||||||
|
puppet_task_wo_retries.expects(:puppet_run)
|
||||||
|
puppet_task_wo_retries.run
|
||||||
|
expect(puppet_task_wo_retries.status).to eql('error')
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'retries to run puppet if it fails and return middle status' do
|
||||||
|
puppet_task.stubs(:puppet_status).returns(mco_puppet_stopped)
|
||||||
|
.then.returns(mco_puppet_stopped)
|
||||||
|
.then.returns(mco_puppet_failed)
|
||||||
|
.then.returns(mco_puppet_failed)
|
||||||
|
.then.returns(mco_puppet_finished)
|
||||||
|
|
||||||
|
puppet_task.expects(:puppet_run).times(2)
|
||||||
|
puppet_task.run
|
||||||
|
expect(puppet_task.status).to eql('deploying')
|
||||||
|
expect(puppet_task.status).to eql('ready')
|
||||||
|
end
|
||||||
|
|
||||||
|
it "return error for node if puppet failed (a cycle with one running state only)" do
|
||||||
|
puppet_task_wo_retries.stubs(:puppet_status).returns(mco_puppet_stopped)
|
||||||
|
.then.returns(mco_puppet_stopped)
|
||||||
|
.then.returns(mco_puppet_running)
|
||||||
|
.then.returns(mco_puppet_running)
|
||||||
|
.then.returns(mco_puppet_fail)
|
||||||
|
.then.returns(mco_puppet_failed)
|
||||||
|
|
||||||
|
puppet_task_wo_retries.expects(:puppet_run)
|
||||||
|
puppet_task_wo_retries.run
|
||||||
|
|
||||||
|
expect(puppet_task_wo_retries.status).to eql('deploying')
|
||||||
|
expect(puppet_task_wo_retries.status).to eql('deploying')
|
||||||
|
expect(puppet_task_wo_retries.status).to eql('deploying')
|
||||||
|
expect(puppet_task_wo_retries.status).to eql('error')
|
||||||
|
end
|
||||||
|
|
||||||
|
it "error status for node if puppet failed (a cycle w/o idle and finishing states)" do
|
||||||
|
puppet_task_wo_retries.stubs(:puppet_status).returns(mco_puppet_stopped)
|
||||||
|
.then.returns(mco_puppet_stopped)
|
||||||
|
.then.returns(mco_puppet_running)
|
||||||
|
.then.returns(mco_puppet_failed)
|
||||||
|
|
||||||
|
puppet_task_wo_retries.expects(:puppet_run)
|
||||||
|
puppet_task_wo_retries.run
|
||||||
|
expect(puppet_task_wo_retries.status).to eql('deploying')
|
||||||
|
expect(puppet_task_wo_retries.status).to eql('error')
|
||||||
|
end
|
||||||
|
|
||||||
|
it "retries to run puppet if it idling" do
|
||||||
|
puppet_task.stubs(:puppet_status).returns(mco_puppet_stopped)
|
||||||
|
.then.returns(mco_puppet_idling)
|
||||||
|
.then.returns(mco_puppet_stopped)
|
||||||
|
.then.returns(mco_puppet_running)
|
||||||
|
.then.returns(mco_puppet_finished)
|
||||||
|
|
||||||
|
puppet_task.expects(:puppet_run).times(2)
|
||||||
|
puppet_task.run
|
||||||
|
expect(puppet_task.status).to eql('deploying')
|
||||||
|
expect(puppet_task.status).to eql('ready')
|
||||||
|
end
|
||||||
|
end #status
|
||||||
|
|
||||||
|
end
|
Loading…
Reference in New Issue