openstack/fuel-astute
Code Issues Proposed changes

Granular deployment: tests, fixes and refactoring

Browse Source 
- fix problem with error nodes (endless loop);
- add tests for puppet tasks.
- update mcollective agent (should work with old version also).

Closes-Bug: #1392779 (refactoring https://review.openstack.org/#/c/138790)
Implements blueprint granular-deployment-based-on-tasks

Change-Id: I9832e405143dbb6b2fbd22ed2d2bc7ad0bc85f01
This commit is contained in:
Vladimir Sharshov (warpc) 2015-01-12 12:42:44 +03:00
parent 18be5cd3b8
commit 85b1e638d8
6 changed files with 593 additions and 219 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
lib/astute/deployment_engine/granular_deployment.rb
View File

@ -43,7 +43,7 @@ class Astute::DeploymentEngine::GranularDeployment < Astute::DeploymentEngine
report_nodes = nodes.uniq{ |n| n['uid'] }.map do |node|
{ 'uid' => node['uid'],
'status' => 'error',
'role' => 'hook',
'role' => node['role'],
'error_type' => 'deploy'
}
end
@ -112,12 +112,12 @@ class Astute::DeploymentEngine::GranularDeployment < Astute::DeploymentEngine
else
Astute.logger.debug "No more tasks provided for node #{node_id}"
end
@ctx.report_and_update_status('nodes' => nodes_to_report) if nodes_to_report.present?
break unless @task_manager.task_in_queue?
sleep Astute.config.PUPPET_DEPLOY_INTERVAL
end
@ctx.report_and_update_status('nodes' => nodes_to_report) if nodes_to_report.present?
break unless @task_manager.task_in_queue?
sleep Astute.config.PUPPET_DEPLOY_INTERVAL
end
end
@ -146,6 +146,7 @@ class Astute::DeploymentEngine::GranularDeployment < Astute::DeploymentEngine
end
def process_running_node(node_id, task, nodes)
Astute.logger.debug "Task '#{task}' on node uid=#{node_id} deploying"
begin
# Pass nodes because logs calculation needs IP address of node, not just uid
nodes_progress = @ctx.deploy_log_parser.progress_calculate(Array(node_id), nodes)

19
lib/astute/puppet_task.rb
View File

@ -24,23 +24,22 @@ module Astute
@puppet_manifest = puppet_manifest || '/etc/puppet/manifests/site.pp'
@puppet_modules = puppet_modules || '/etc/puppet/modules'
@cwd = cwd || '/'
@time_obsorver = TimeObserver.new(timeout || Astute.config.PUPPET_TIMEOUT)
@time_observer = TimeObserver.new(timeout || Astute.config.PUPPET_TIMEOUT)
@prev_summary = nil
@is_hung = false
end
def run
Astute.logger.debug "Waiting for puppet to finish deployment on " \
"node #{@node['uid']} (timeout = #{@time_obsorver.time_limit} sec)..."
@time_obsorver.start
"node #{@node['uid']} (timeout = #{@time_observer.time_limit} sec)..."
@time_observer.start
@prev_summary ||= puppet_status
puppetd_runonce
end
# expect to run this method with respect of Astute.config.PUPPET_FADE_INTERVAL
def status
# TODO(vsharshov): Should we raise error?
raise Timeout::Error unless @time_obsorver.enough_time?
raise Timeout::Error unless @time_observer.enough_time?
last_run = puppet_status
status = node_status(last_run)
@ -146,21 +145,21 @@ module Astute
'error'
when succeed?(last_run) && !@is_hung
'succeed'
when (running?(last_run) || idling?(status)) && !@is_hung
when (running?(last_run) || idling?(last_run)) && !@is_hung
'running'
when stopped?(status) && !succeed?(last_run) && !@is_hung
when stopped?(last_run) && !succeed?(last_run) && !@is_hung
'error'
else
msg = "Unknow status: " \
"is_hung #{@is_hung}, succeed? #{succeed?(last_run)}, " \
"running? #{running?(last_run)}, stopped? #{stopped?(status)}, " \
"idling? #{idling?(status)}"
"running? #{running?(last_run)}, stopped? #{stopped?(last_run)}, " \
"idling? #{idling?(last_run)}"
raise msg
end
end
def processing_succeed_node
Astute.logger.debug "Puppet completed within #{@time_obsorver.stop} seconds"
Astute.logger.debug "Puppet completed within #{@time_observer.stop} seconds"
{ 'uid' => @node['uid'], 'status' => 'ready', 'role' => @node['role'] }
end

45
mcagents/puppetd.rb
View File

@ -109,8 +109,9 @@ module MCollective
end
def rm_file file
return true unless File.exists?(file)
begin
File.unlink(file)
File.unlink file
return true
rescue
return false
@ -119,9 +120,9 @@ module MCollective
def puppet_daemon_status
err_msg = ""
alive = !!puppet_pid
locked = File.exists?(@lockfile)
disabled = locked && File::Stat.new(@lockfile).zero?
alive = puppet_pid
locked = expected_puppet_pid == puppet_pid && !expected_puppet_pid.nil?
disabled = File.exists?(@lockfile) && File::Stat.new(@lockfile).zero?
if locked && !disabled && !alive
err_msg << "Process not running but not empty lockfile is present. Trying to remove lockfile..."
@ -154,16 +155,9 @@ module MCollective
reply.fail "Lock file and PID file exist; puppet is running."
when 'idling' then # signal daemon
pid = puppet_pid
begin
::Process.kill('INT', pid)
rescue Errno::ESRCH => e
reply[:err_msg] = "Failed to signal the puppet apply daemon (process #{pid}): #{e}"
ensure
runonce_background
reply[:output] = "Kill old idling puppet process #{pid})." + (reply[:output] || '')
end
kill_process
set_status # recalculate state after puppet kill
reply[:err_msg] = "Looks like another puppet run at the same time. Try to kill it"
when 'stopped' then # just run
runonce_background
else
@ -255,13 +249,13 @@ module MCollective
begin
Timeout.timeout(30) do
Process.kill('TERM', puppet_pid)
Process.kill('TERM', puppet_pid) if puppet_pid
while puppet_pid do
sleep 1
end
end
rescue Timeout::Error
Process.kill('KILL', puppet_pid)
Process.kill('KILL', puppet_pid) if puppet_pid
end
#FIXME: Daemonized process do not update lock file when we send signal to kill him
raise "Should never happen. Some process block lock file in critical section" unless rm_file(@lockfile)
@ -269,9 +263,26 @@ module MCollective
reply.fail "Failed to kill the puppet daemon (process #{puppet_pid}): #{e}"
end
def expected_puppet_pid
File.read(@lockfile).to_i
rescue Errno::ENOENT
nil
end
def puppet_pid
result = `ps -C puppet -o pid,comm --no-headers`.lines.first
result && result.strip.split(' ')[0].to_i
actual_pid = result && result.strip.split(' ')[0].to_i
expected_pid = expected_puppet_pid
case actual_pid
when expected_pid then expected_pid
else
reply[:err_msg] = "Potencial error. Looks like expecting puppet and actual are different. " \
"Expecting pid(lockfile): #{expected_pid}, actual(ps): #{actual_pid}"
actual_pid
end
rescue NoMethodError
nil
end
def lock_file(file_name, &block)

351
spec/unit/granular_deployment_spec.rb Normal file
View File

@ -0,0 +1,351 @@
# Copyright 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
require File.join(File.dirname(__FILE__), '../spec_helper')
describe "Granular deployment engine" do
include SpecHelpers
let(:ctx) {
ctx = mock
ctx.stubs(:task_id)
ctx.stubs(:deploy_log_parser).returns(Astute::LogParser::NoParsing.new)
ctx.stubs(:status).returns({})
reporter = mock
reporter.stubs(:report)
up_reporter = Astute::ProxyReporter::DeploymentProxyReporter.new(reporter, nodes)
ctx.stubs(:reporter).returns(up_reporter)
ctx
}
let(:deploy_engine) do
Astute::DeploymentEngine::GranularDeployment.new(ctx)
end
let(:upload_file_hook) do
{
"priority" => 100,
"type" => "upload_file",
"fail_on_error" => false,
"diagnostic_name" => "upload-example-1.0",
"uids" => ['2', '3'],
"parameters" => {
"path" => "/etc/yum.repos.d/fuel_awesome_plugin-0.1.0.repo",
"data" => "[fuel_awesome_plugin-0.1.0]\\nname=Plugin fuel_awesome_plugin-0.1.0 repository\\nbaseurl=http => //10.20.0.2 => 8080/plugins/fuel_awesome_plugin-0.1.0/repositories/centos\\ngpgcheck=0"
}
}
end
let(:sync_hook) do
{
"priority" => 200,
"type" => "sync",
"fail_on_error" => false,
"diagnostic_name" => "sync-example-1.0",
"uids" => ['1', '2'],
"parameters" => {
"src" => "rsync://10.20.0.2/plugins/fuel_awesome_plugin-0.1.0/deployment_scripts/",
"dst" => "/etc/fuel/plugins/fuel_awesome_plugin-0.1.0/"
}
}
end
let(:shell_hook) do
{
"priority" => 100,
"type" => "shell",
"fail_on_error" => false,
"diagnostic_name" => "shell-example-1.0",
"uids" => ['1','2','3'],
"parameters" => {
"cmd" => "./deploy.sh",
"cwd" => "/etc/fuel/plugins/fuel_awesome_plugin-0.1.0/",
"timeout" => 60
}
}
end
let(:puppet_hook) do
{
"priority" => 300,
"type" => "puppet",
"fail_on_error" => false,
"diagnostic_name" => "puppet-example-1.0",
"uids" => ['1', '3'],
"parameters" => {
"puppet_manifest" => "cinder_glusterfs.pp",
"puppet_modules" => "modules",
"cwd" => "/etc/fuel/plugins/plugin_name-1.0",
"timeout" => 42
}
}
end
let(:nodes) do
[
{
'uid' => '45',
'priority' => 200,
'role' => 'ceph',
'tasks' => [
{
'priority' => 100,
'type' => 'puppet',
'uids' => ['45']
},
{
'priority' => 300,
'type' => 'puppet',
'uids' => ['45']
}
]
},
{
'uid' => '46',
'priority' => 200,
'role' => 'compute',
'tasks' => [
{
'priority' => 100,
'type' => 'puppet',
'uids' => ['46']
},
{
'priority' => 200,
'type' => 'puppet',
'uids' => ['46']
},
{
'priority' => 300,
'type' => 'puppet',
'uids' => ['46']
}
]
}
]
end
describe '#deploy_piace' do
it 'should run tasks using puppet task' do
ctx.stubs(:report_and_update_status)
deploy_engine.expects(:deploy_nodes).with(nodes)
deploy_engine.deploy_piece(nodes)
end
it 'should report error status if error raised' do
error_report = {'nodes' =>
[
{
'uid' => '45',
'status' => 'error',
'role' => 'ceph',
'error_type' => 'deploy'
},
{
'uid' => '46',
'status' => 'error',
'role' => 'compute',
'error_type' => 'deploy'
}
]
}
ctx.expects(:report_and_update_status).with(error_report)
deploy_engine.expects(:deploy_nodes).with(nodes).raises("Error simulation")
deploy_engine.deploy_piece(nodes) rescue nil
end
it 'should not raise errir if no nodes was sent' do
expect{ deploy_engine.deploy_piece([])}.to_not raise_error
end
it 'should prepare log for parsing' do
deploy_engine.stubs(:deploy_nodes).with(nodes)
ctx.deploy_log_parser.expects(:prepare).with(nodes).once
deploy_engine.deploy_piece(nodes)
end
end # 'deploy_piace'
describe '#deploy_nodes' do
it 'run deploy task on nodes' do
ctx.stubs(:report_and_update_status)
deploy_engine.expects(:run_task).times(5)
deploy_engine.stubs(:check_status).with('46')
.returns('deploying').then
.returns('ready').then
.returns('deploying').then
.returns('ready').then
.returns('deploying').then
.returns('ready')
deploy_engine.stubs(:check_status).with('45')
.returns('deploying').then
.returns('ready').then
.returns('deploying').then
.returns('ready')
deploy_engine.deploy_piece(nodes)
end
it 'report status about nodes' do
deploy_engine.stubs(:run_task).times(5)
deploy_engine.stubs(:check_status).with('46')
.returns('deploying').then
.returns('ready').then
.returns('deploying').then
.returns('ready').then
.returns('deploying').then
.returns('ready')
deploy_engine.stubs(:check_status).with('45')
.returns('deploying').then
.returns('ready').then
.returns('deploying').then
.returns('ready')
succeed_report_node1 = {
'nodes' => [
{
'uid' => '45',
'status' => 'ready',
'role' => 'ceph',
'progress' => 100,
'task' => {'priority' => 300, 'type' => 'puppet', 'uids' => ['45']}
}
]
}
succeed_report_node2 = {
'nodes' => [
{
'uid' => '46',
'status' => 'ready',
'role' => 'compute',
'progress' => 100,
'task' => {'priority' => 300, 'type' => 'puppet', 'uids' => ['46']}
}
]
}
ctx.expects(:report_and_update_status).with(succeed_report_node1).times(1)
ctx.expects(:report_and_update_status).with(succeed_report_node2).times(1)
deploy_engine.deploy_piece(nodes)
end
it 'handle error nodes and do not process next tasks on problem node' do
deploy_engine.expects(:run_task).times(4)
deploy_engine.stubs(:check_status).with('46')
.returns('deploying').then
.returns('ready').then
.returns('deploying').then
.returns('error')
deploy_engine.stubs(:check_status).with('45')
.returns('deploying').then
.returns('ready').then
.returns('deploying').then
.returns('ready')
mixed_report = {'nodes' =>
[
{
'uid' => '45',
'status' => 'ready',
'role' => 'ceph',
'progress' => 100,
'task' => {'priority' => 300, 'type' => 'puppet', 'uids' => ['45']}
},
{
'uid' => '46',
'status' => 'error',
'error_type' => 'deploy',
'role' => 'compute',
'task' => {'priority' => 200, 'type' => 'puppet', 'uids' => ['46']}
}
]
}
ctx.expects(:report_and_update_status).with(mixed_report)
deploy_engine.deploy_piece(nodes)
end
it 'handle sutuation then all nodes failed' do
deploy_engine.expects(:run_task).times(2)
deploy_engine.stubs(:check_status).with('46')
.returns('deploying').then
.returns('error')
deploy_engine.stubs(:check_status).with('45')
.returns('deploying').then
.returns('error')
error_report = {'nodes' =>
[
{
'uid' => '45',
'status' => 'error',
'error_type' => 'deploy',
'role' => 'ceph',
'task' => {'priority' => 100, 'type' => 'puppet', 'uids' => ['45']}
},
{
'uid' => '46',
'status' => 'error',
'error_type' => 'deploy',
'role' => 'compute',
'task' => {'priority' => 100, 'type' => 'puppet', 'uids' => ['46']}
}
]
}
ctx.expects(:report_and_update_status).with(error_report)
deploy_engine.deploy_piece(nodes)
end
it 'should fail deployment if handler not provided' do
deploy_engine.expects(:run_task).times(2)
deploy_engine.stubs(:check_status).returns('unknown_handler')
error_report = {'nodes' =>
[
{
'uid' => '45',
'status' => 'error',
'error_type' => 'deploy',
'role' => 'ceph'
},
{
'uid' => '46',
'status' => 'error',
'error_type' => 'deploy',
'role' => 'compute'
}
]
}
ctx.expects(:report_and_update_status).with(error_report)
expect {deploy_engine.deploy_piece(nodes) }
.to raise_error(/Internal error. Known status/)
end
end #deploy_nodes
end # 'describe'

186
spec/unit/nailgun_spec.rb
View File

@ -1,186 +0,0 @@
# Copyright 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
require File.join(File.dirname(__FILE__), '../spec_helper')
include Astute
describe "Granular deployment engine" do
include SpecHelpers
let(:ctx) {
ctx = mock
ctx.stubs(:task_id)
ctx.stubs(:deploy_log_parser).returns(Astute::LogParser::NoParsing.new)
ctx.stubs(:status).returns({})
reporter = mock
reporter.stubs(:report)
up_reporter = Astute::ProxyReporter::DeploymentProxyReporter.new(reporter, nodes)
ctx.stubs(:reporter).returns(up_reporter)
ctx
}
let(:deploy_engine) do
Astute::DeploymentEngine::GranularDeployment.new(ctx)
end
let(:upload_file_hook) do
{
"priority" => 100,
"type" => "upload_file",
"fail_on_error" => false,
"diagnostic_name" => "upload-example-1.0",
"uids" => ['2', '3'],
"parameters" => {
"path" => "/etc/yum.repos.d/fuel_awesome_plugin-0.1.0.repo",
"data" => "[fuel_awesome_plugin-0.1.0]\\nname=Plugin fuel_awesome_plugin-0.1.0 repository\\nbaseurl=http => //10.20.0.2 => 8080/plugins/fuel_awesome_plugin-0.1.0/repositories/centos\\ngpgcheck=0"
}
}
end
let(:sync_hook) do
{
"priority" => 200,
"type" => "sync",
"fail_on_error" => false,
"diagnostic_name" => "sync-example-1.0",
"uids" => ['1', '2'],
"parameters" => {
"src" => "rsync://10.20.0.2/plugins/fuel_awesome_plugin-0.1.0/deployment_scripts/",
"dst" => "/etc/fuel/plugins/fuel_awesome_plugin-0.1.0/"
}
}
end
let(:shell_hook) do
{
"priority" => 100,
"type" => "shell",
"fail_on_error" => false,
"diagnostic_name" => "shell-example-1.0",
"uids" => ['1','2','3'],
"parameters" => {
"cmd" => "./deploy.sh",
"cwd" => "/etc/fuel/plugins/fuel_awesome_plugin-0.1.0/",
"timeout" => 60
}
}
end
let(:puppet_hook) do
{
"priority" => 300,
"type" => "puppet",
"fail_on_error" => false,
"diagnostic_name" => "puppet-example-1.0",
"uids" => ['1', '3'],
"parameters" => {
"puppet_manifest" => "cinder_glusterfs.pp",
"puppet_modules" => "modules",
"cwd" => "/etc/fuel/plugins/plugin_name-1.0",
"timeout" => 42
}
}
end
let(:nodes) do
[
{
'uid' => '45',
'priority' => 200,
'role' => 'ceph',
'tasks' => [
{
'priority' => 100,
'type' => 'puppet',
'uids' => ['45']
},
{
'priority' => 300,
'type' => 'puppet',
'uids' => ['45']
}
]
},
{
'uid' => '46',
'priority' => 200,
'role' => 'compute',
'tasks' => [
{
'priority' => 100,
'type' => 'puppet',
'uids' => ['46']
},
{
'priority' => 200,
'type' => 'puppet',
'uids' => ['46']
},
{
'priority' => 300,
'type' => 'puppet',
'uids' => ['46']
}
]
}
]
end
describe '#deploy_piace' do
it 'should run tasks using puppet task' do
ctx.stubs(:report_and_update_status)
deploy_engine.expects(:deploy_nodes).with(nodes)
deploy_engine.deploy_piece(nodes)
end
it 'should report error status if error erased' do
error_report = {'nodes' =>
[
{
'uid' => '45',
'status' => 'error',
'role' => 'hook',
'error_type' => 'deploy'
},
{
'uid' => '46',
'status' => 'error',
'role' => 'hook',
'error_type' => 'deploy'
}
]
}
ctx.expects(:report_and_update_status).with(error_report)
deploy_engine.expects(:deploy_nodes).with(nodes).raises("Error simulation")
deploy_engine.deploy_piece(nodes) rescue nil
end
it 'should not raise errir if no nodes was sent' do
expect{ deploy_engine.deploy_piece([])}.to_not raise_error
end
it 'should prepare log for parsing' do
deploy_engine.stubs(:deploy_nodes).with(nodes)
ctx.deploy_log_parser.expects(:prepare).with(nodes).once
deploy_engine.deploy_piece(nodes)
end
end # 'deploy_piace'
end # 'describe'

198
spec/unit/puppet_task_spec.rb Normal file
View File

@ -0,0 +1,198 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
require File.join(File.dirname(__FILE__), '../spec_helper')
describe Astute::PuppetTask do
include SpecHelpers
let(:node) do
{
'uid' => '45',
'priority' => 200,
'role' => 'ceph',
'tasks' => [
{
'priority' => 100,
'type' => 'puppet',
'uids' => ['45']
},
{
'priority' => 300,
'type' => 'puppet',
'uids' => ['45']
}
]
}
end
let(:ctx) {
ctx = mock
ctx.stubs(:task_id)
ctx.stubs(:deploy_log_parser).returns(Astute::LogParser::NoParsing.new)
ctx.stubs(:status).returns({})
reporter = mock
reporter.stubs(:report)
up_reporter = Astute::ProxyReporter::DeploymentProxyReporter.new(reporter, [node])
ctx.stubs(:reporter).returns(up_reporter)
ctx
}
let(:puppet_task) { Astute::PuppetTask.new(ctx, node)}
let(:puppet_task_wo_retries) { Astute::PuppetTask.new(ctx, node, retries=0)}
let(:mco_puppet_stopped) do
{
:changes => {"total" => 1},
:time => {"last_run" => 1358425701},
:resources => {"failed" => 0},
:status => "stopped",
:enabled => 1,
:stopped => 1,
:idling => 0,
:running => 0,
:runtime => 1358425701
}
end
let(:mco_puppet_running) do
mco_puppet_stopped.merge(
:status => 'running',
:running => 1,
:stopped => 0
)
end
let(:mco_puppet_fail) do
mco_puppet_running.merge(
:runtime => 1358426000,
:time => {"last_run" => 1358426000},
:resources => {"failed" => 1}
)
end
let(:mco_puppet_failed) do
mco_puppet_fail.merge(
:status => 'stopped',
:stopped => 1,
:running => 0
)
end
let(:mco_puppet_finished) do
mco_puppet_stopped.merge(
:time => {'last_run' => 1358428000},
:status => 'stopped'
)
end
let(:mco_puppet_idling) do
mco_puppet_stopped.merge(
:status => 'idling',
:running => 0,
:stopped => 0,
:idling => 1
)
end
describe "#run" do
it 'run puppet using mcollective' do
puppet_task.expects(:puppet_status).returns(mco_puppet_stopped).times(2)
puppet_task.expects(:puppet_run)
puppet_task.run
end
end #run
describe "#status" do
before(:each) do
ctx.stubs(:report_and_update_status)
end
it 'check puppet using mcollective' do
puppet_task.stubs(:puppet_status).returns(mco_puppet_stopped)
.then.returns(mco_puppet_stopped)
.then.returns(mco_puppet_running)
.then.returns(mco_puppet_finished)
puppet_task.expects(:puppet_run)
puppet_task.run
end
it 'return error for node if puppet failed (a cycle w/o any transitional states)' do
puppet_task_wo_retries.stubs(:puppet_status).returns(mco_puppet_stopped)
.then.returns(mco_puppet_stopped)
.then.returns(mco_puppet_failed)
puppet_task_wo_retries.expects(:puppet_run)
puppet_task_wo_retries.run
expect(puppet_task_wo_retries.status).to eql('error')
end
it 'retries to run puppet if it fails and return middle status' do
puppet_task.stubs(:puppet_status).returns(mco_puppet_stopped)
.then.returns(mco_puppet_stopped)
.then.returns(mco_puppet_failed)
.then.returns(mco_puppet_failed)
.then.returns(mco_puppet_finished)
puppet_task.expects(:puppet_run).times(2)
puppet_task.run
expect(puppet_task.status).to eql('deploying')
expect(puppet_task.status).to eql('ready')
end
it "return error for node if puppet failed (a cycle with one running state only)" do
puppet_task_wo_retries.stubs(:puppet_status).returns(mco_puppet_stopped)
.then.returns(mco_puppet_stopped)
.then.returns(mco_puppet_running)
.then.returns(mco_puppet_running)
.then.returns(mco_puppet_fail)
.then.returns(mco_puppet_failed)
puppet_task_wo_retries.expects(:puppet_run)
puppet_task_wo_retries.run
expect(puppet_task_wo_retries.status).to eql('deploying')
expect(puppet_task_wo_retries.status).to eql('deploying')
expect(puppet_task_wo_retries.status).to eql('deploying')
expect(puppet_task_wo_retries.status).to eql('error')
end
it "error status for node if puppet failed (a cycle w/o idle and finishing states)" do
puppet_task_wo_retries.stubs(:puppet_status).returns(mco_puppet_stopped)
.then.returns(mco_puppet_stopped)
.then.returns(mco_puppet_running)
.then.returns(mco_puppet_failed)
puppet_task_wo_retries.expects(:puppet_run)
puppet_task_wo_retries.run
expect(puppet_task_wo_retries.status).to eql('deploying')
expect(puppet_task_wo_retries.status).to eql('error')
end
it "retries to run puppet if it idling" do
puppet_task.stubs(:puppet_status).returns(mco_puppet_stopped)
.then.returns(mco_puppet_idling)
.then.returns(mco_puppet_stopped)
.then.returns(mco_puppet_running)
.then.returns(mco_puppet_finished)
puppet_task.expects(:puppet_run).times(2)
puppet_task.run
expect(puppet_task.status).to eql('deploying')
expect(puppet_task.status).to eql('ready')
end
end #status
end