[FUEL-813] Refactoring and tests for mcollective provision method
This commit is contained in:
parent
05af849933
commit
862d96aed1
@ -11,7 +11,7 @@ Gem::Specification.new do |s|
|
||||
s.email = ['mscherbakov@mirantis.com']
|
||||
|
||||
s.add_dependency 'activesupport', '3.0.10'
|
||||
s.add_dependency 'mcollective-client', '~> 2.2.4'
|
||||
s.add_dependency 'mcollective-client', '2.3.1'
|
||||
s.add_dependency 'symboltable', '1.0.2'
|
||||
|
||||
s.add_development_dependency 'rspec', '2.13.0'
|
||||
|
@ -43,7 +43,8 @@ optparse = OptionParser.new do |o|
|
||||
|
||||
o.on("-h") { puts o; exit }
|
||||
|
||||
o.on("-c", "--command [COMMAND]", [:deploy, :provision], "Select operation (deploy, provision)") do |c|
|
||||
o.on("-c", "--command [COMMAND]", [:deploy, :fast_provision, :provision],
|
||||
"Select operation (deploy, fast_provision, provision)") do |c|
|
||||
opts[:command] = c || :deploy
|
||||
end
|
||||
|
||||
@ -74,7 +75,10 @@ case opts[:command]
|
||||
when :deploy
|
||||
orchestrator = Astute::Orchestrator.new(deploy_engine, log_parsing=false)
|
||||
orchestrator.deploy(reporter, environment['task_uuid'], environment['nodes'], environment['attributes'])
|
||||
when :fast_provision
|
||||
orchestrator = Astute::Orchestrator.new(deploy_engine, log_parsing=false)
|
||||
orchestrator.fast_provision(reporter, environment['engine'], environment['nodes'])
|
||||
when :provision
|
||||
orchestrator = Astute::Orchestrator.new(deploy_engine, log_parsing=false)
|
||||
orchestrator.provision(reporter, environment['engine'], environment['nodes'])
|
||||
orchestrator.provision(reporter, environment['task_uuid'], environment['nodes'])
|
||||
end
|
@ -16,15 +16,10 @@ module Astute
|
||||
class Orchestrator
|
||||
def initialize(deploy_engine=nil, log_parsing=false)
|
||||
@deploy_engine = deploy_engine || Astute::DeploymentEngine::NailyFact
|
||||
if log_parsing
|
||||
@log_parser = LogParser::ParseDeployLogs.new
|
||||
@provisionLogParser = LogParser::ParseProvisionLogs.new
|
||||
else
|
||||
@log_parser = LogParser::NoParsing.new
|
||||
@provisionLogParser = LogParser::NoParsing.new
|
||||
end
|
||||
@log_parser = log_parsing ? LogParser::ParseDeployLogs.new : LogParser::NoParsing.new
|
||||
end
|
||||
|
||||
# TODO(waprc): does this method should be private?
|
||||
def node_type(reporter, task_id, nodes, timeout=nil)
|
||||
context = Context.new(task_id, reporter)
|
||||
uids = nodes.map {|n| n['uid']}
|
||||
@ -54,7 +49,7 @@ module Astute
|
||||
deploy_engine_instance.deploy(nodes, attrs)
|
||||
end
|
||||
|
||||
def provision(reporter, engine_attrs, nodes)
|
||||
def fast_provision(reporter, engine_attrs, nodes)
|
||||
raise "Nodes to provision are not provided!" if nodes.empty?
|
||||
engine = create_engine(engine_attrs, reporter)
|
||||
|
||||
@ -88,57 +83,49 @@ module Astute
|
||||
return
|
||||
end
|
||||
|
||||
def naily_deploy(reporter, task_id, nodes, attrs)
|
||||
def provision(reporter, task_id, nodes)
|
||||
raise "Nodes to provision are not provided!" if nodes.empty?
|
||||
|
||||
# Following line fixes issues with uids: it should always be string
|
||||
nodes.map { |x| x['uid'] = x['uid'].to_s }
|
||||
nodes.map { |x| x['uid'] = x['uid'].to_s } # NOTE: perform that on environment['nodes'] initialization
|
||||
|
||||
nodes_uids = nodes.map { |n| n['uid'] }
|
||||
time = Time::now.to_f
|
||||
nodes_not_booted = nodes.map { |n| n['uid'] }
|
||||
Astute.logger.info "Starting OS provisioning for nodes: #{nodes_not_booted.join(',')}"
|
||||
|
||||
provisionLogParser = LogParser::ParseProvisionLogs.new
|
||||
sleep_not_greater_than(10) do # Wait while nodes going to reboot
|
||||
Astute.logger.info "Starting OS provisioning for nodes: #{nodes_uids.join(',')}"
|
||||
begin
|
||||
@provisionLogParser.prepare(nodes)
|
||||
rescue Exception => e
|
||||
provisionLogParser.prepare(nodes)
|
||||
rescue => e
|
||||
Astute.logger.warn "Some error occurred when prepare LogParser: #{e.message}, trace: #{e.backtrace.inspect}"
|
||||
end
|
||||
time = 10 + time - Time::now.to_f
|
||||
sleep (time) if time > 0 # Wait while nodes going to reboot. Sleep not greater than 10 sec.
|
||||
begin
|
||||
Timeout::timeout(Astute.config.PROVISIONING_TIMEOUT) do # Timeout for booting target OS
|
||||
while true
|
||||
time = Time::now.to_f
|
||||
types = node_type(reporter, task_id, nodes, 2)
|
||||
types.each do |t|
|
||||
Astute.logger.debug("Got node types: uid=#{t['uid']} type=#{t['node_type']}")
|
||||
end
|
||||
nodes_not_booted = nodes_uids.clone
|
||||
begin
|
||||
Timeout.timeout(Astute.config.PROVISIONING_TIMEOUT) do # Timeout for booting target OS
|
||||
catch :done do
|
||||
while true
|
||||
sleep_not_greater_than(5) do
|
||||
types = node_type(reporter, task_id, nodes, 2)
|
||||
types.each { |t| Astute.logger.debug("Got node types: uid=#{t['uid']} type=#{t['node_type']}") }
|
||||
|
||||
Astute.logger.debug("Not target nodes will be rejected")
|
||||
target_uids = types.reject{|n| n['node_type'] != 'target'}.map{|n| n['uid']}
|
||||
nodes_not_booted -= types.map { |n| n['uid'] }
|
||||
Astute.logger.debug "Not provisioned: #{nodes_not_booted.join(',')}, got target OSes: #{target_uids.join(',')}"
|
||||
|
||||
if nodes.length == target_uids.length
|
||||
Astute.logger.info "All nodes #{target_uids.join(',')} are provisioned."
|
||||
break
|
||||
throw :done
|
||||
else
|
||||
Astute.logger.debug("Nodes list length is not equal to target nodes list length: #{nodes.length} != #{target_uids.length}")
|
||||
end
|
||||
nodes_not_booted = nodes_uids - types.map { |n| n['uid'] }
|
||||
begin
|
||||
nodes_progress = @provisionLogParser.progress_calculate(nodes_uids, nodes)
|
||||
nodes_progress.each do |n|
|
||||
if target_uids.include?(n['uid'])
|
||||
n['progress'] = 100
|
||||
# TODO(mihgen): should we change status only once?
|
||||
n['status'] = 'provisioned'
|
||||
|
||||
report_about_progress(reporter, provisionLogParser, nodes_uids, target_uids, nodes)
|
||||
end
|
||||
end
|
||||
reporter.report({'nodes' => nodes_progress})
|
||||
rescue Exception => e
|
||||
Astute.logger.warn "Some error occurred when parse logs for nodes progress: #{e.message}, trace: #{e.backtrace.inspect}"
|
||||
end
|
||||
time = 5 + time - Time::now.to_f
|
||||
sleep (time) if time > 0 # Sleep not greater than 5 sec.
|
||||
end
|
||||
# We are here if jumped by break from while cycle
|
||||
# We are here if jumped by throw from while cycle
|
||||
end
|
||||
rescue Timeout::Error
|
||||
msg = "Timeout of provisioning is exceeded."
|
||||
@ -156,17 +143,6 @@ module Astute
|
||||
{'uid' => n['uid'], 'progress' => 100, 'status' => 'provisioned'}
|
||||
end
|
||||
reporter.report({'nodes' => nodes_progress})
|
||||
|
||||
begin
|
||||
result = deploy(reporter, task_id, nodes, attrs)
|
||||
rescue Timeout::Error
|
||||
msg = "Timeout of deployment is exceeded."
|
||||
Astute.logger.error msg
|
||||
reporter.report({'status' => 'error', 'error' => msg})
|
||||
return
|
||||
end
|
||||
|
||||
report_result(result, reporter)
|
||||
end
|
||||
|
||||
|
||||
@ -188,6 +164,13 @@ module Astute
|
||||
reporter.report(status)
|
||||
end
|
||||
|
||||
def sleep_not_greater_than(sleep_time, &block)
|
||||
time = Time.now.to_f
|
||||
block.call
|
||||
time = time + sleep_time - Time.now.to_f
|
||||
sleep (time) if time > 0
|
||||
end
|
||||
|
||||
def create_engine(engine_attrs, reporter)
|
||||
begin
|
||||
Astute.logger.info("Trying to instantiate cobbler engine: #{engine_attrs.inspect}")
|
||||
@ -249,5 +232,20 @@ module Astute
|
||||
failed_nodes
|
||||
end
|
||||
|
||||
def report_about_progress(reporter, provisionLogParser, nodes_uids, target_uids, nodes)
|
||||
begin
|
||||
nodes_progress = provisionLogParser.progress_calculate(nodes_uids, nodes)
|
||||
nodes_progress.each do |n|
|
||||
if target_uids.include?(n['uid']) && n['status'] != 'provisioned'
|
||||
n['progress'] = 100
|
||||
n['status'] = 'provisioned'
|
||||
end
|
||||
end
|
||||
reporter.report({'nodes' => nodes_progress})
|
||||
rescue => e
|
||||
Astute.logger.warn "Some error occurred when parse logs for nodes progress: #{e.message}, trace: #{e.backtrace.inspect}"
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
|
@ -28,6 +28,7 @@ require File.join(File.dirname(__FILE__), '../lib/astute')
|
||||
Astute.config.PUPPET_DEPLOY_INTERVAL = 0
|
||||
Astute.config.PUPPET_FADE_INTERVAL = 0
|
||||
Astute.config.MC_RETRY_INTERVAL = 0
|
||||
Astute.config.PROVISIONING_TIMEOUT = 0
|
||||
Astute.logger = Logger.new(STDERR)
|
||||
|
||||
RSpec.configure do |c|
|
||||
|
@ -218,7 +218,6 @@ describe Astute::Orchestrator do
|
||||
|
||||
it "remove_nodes do not fail if any of nodes failed"
|
||||
|
||||
describe '#provision' do
|
||||
|
||||
before(:all) do
|
||||
@data = {
|
||||
@ -230,6 +229,7 @@ describe Astute::Orchestrator do
|
||||
"task_uuid"=>"a5c44b9a-285a-4a0c-ae65-2ed6b3d250f4",
|
||||
"nodes" => [
|
||||
{
|
||||
'uid' => '1',
|
||||
'profile' => 'centos-x86_64',
|
||||
"name"=>"controller-1",
|
||||
'power_type' => 'ssh',
|
||||
@ -271,9 +271,11 @@ describe Astute::Orchestrator do
|
||||
}.freeze
|
||||
end
|
||||
|
||||
describe '#fast_provision' do
|
||||
|
||||
context 'cobler cases' do
|
||||
it "raise error if cobler settings empty" do
|
||||
expect {@orchestrator.provision(@reporter, {}, @data['nodes'])}.
|
||||
expect {@orchestrator.fast_provision(@reporter, {}, @data['nodes'])}.
|
||||
to raise_error(StopIteration)
|
||||
end
|
||||
end
|
||||
@ -292,7 +294,7 @@ describe Astute::Orchestrator do
|
||||
end
|
||||
|
||||
it "raises error if nodes list is empty" do
|
||||
expect {@orchestrator.provision(@reporter, @data['engine'], {})}.
|
||||
expect {@orchestrator.fast_provision(@reporter, @data['engine'], {})}.
|
||||
to raise_error(/Nodes to provision are not provided!/)
|
||||
end
|
||||
|
||||
@ -301,7 +303,7 @@ describe Astute::Orchestrator do
|
||||
expects(:power_reboot).with('controller-1')
|
||||
end
|
||||
@orchestrator.stubs(:check_reboot_nodes).returns([])
|
||||
@orchestrator.provision(@reporter, @data['engine'], @data['nodes'])
|
||||
@orchestrator.fast_provision(@reporter, @data['engine'], @data['nodes'])
|
||||
end
|
||||
|
||||
before(:each) { Astute::Provision::Cobbler.any_instance.stubs(:power_reboot).returns(333) }
|
||||
@ -315,19 +317,19 @@ describe Astute::Orchestrator do
|
||||
Astute::Provision::Cobbler.any_instance.stubs(:event_status).
|
||||
returns([Time.now.to_f, 'controller-1', 'complete'])
|
||||
|
||||
@orchestrator.provision(@reporter, @data['engine'], @data['nodes'])
|
||||
@orchestrator.fast_provision(@reporter, @data['engine'], @data['nodes'])
|
||||
end
|
||||
|
||||
it "report about success" do
|
||||
@reporter.expects(:report).with({'status' => 'ready', 'progress' => 100}).returns(true)
|
||||
@orchestrator.provision(@reporter, @data['engine'], @data['nodes'])
|
||||
@orchestrator.fast_provision(@reporter, @data['engine'], @data['nodes'])
|
||||
end
|
||||
|
||||
it "sync engine state" do
|
||||
Astute::Provision::Cobbler.any_instance do
|
||||
expects(:sync).once
|
||||
end
|
||||
@orchestrator.provision(@reporter, @data['engine'], @data['nodes'])
|
||||
@orchestrator.fast_provision(@reporter, @data['engine'], @data['nodes'])
|
||||
end
|
||||
|
||||
end
|
||||
@ -341,18 +343,77 @@ describe Astute::Orchestrator do
|
||||
expects(:sync).once
|
||||
end
|
||||
begin
|
||||
@orchestrator.provision(@reporter, @data['engine'], @data['nodes'])
|
||||
@orchestrator.fast_provision(@reporter, @data['engine'], @data['nodes'])
|
||||
rescue
|
||||
end
|
||||
end
|
||||
|
||||
it "raise error if failed node find" do
|
||||
expect {@orchestrator.provision(@reporter, @data['engine'], @data['nodes'])}.to raise_error(StopIteration)
|
||||
expect {@orchestrator.fast_provision(@reporter, @data['engine'], @data['nodes'])}.to raise_error(StopIteration)
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
|
||||
describe '#provision' do
|
||||
|
||||
it "raises error if nodes list is empty" do
|
||||
expect {@orchestrator.provision(@reporter, @data['task_uuid'], {})}.
|
||||
to raise_error(/Nodes to provision are not provided!/)
|
||||
end
|
||||
|
||||
it "prepare provision log for parsing" do
|
||||
Astute::LogParser::ParseProvisionLogs.any_instance do
|
||||
expects(:prepare).with(@data['nodes']).once
|
||||
end
|
||||
@orchestrator.stubs(:report_about_progress).returns()
|
||||
@orchestrator.stubs(:node_type).returns([{'uid' => 1, 'node_type' => 'target' }])
|
||||
|
||||
@orchestrator.provision(@reporter, @data['task_uuid'], @data['nodes'])
|
||||
end
|
||||
|
||||
it "ignore problem with parsing provision log" do
|
||||
Astute::LogParser::ParseProvisionLogs.any_instance do
|
||||
stubs(:prepare).with(@data['nodes']).raises
|
||||
end
|
||||
|
||||
@orchestrator.stubs(:report_about_progress).returns()
|
||||
@orchestrator.stubs(:node_type).returns([{'uid' => 1, 'node_type' => 'target' }])
|
||||
|
||||
@orchestrator.provision(@reporter, @data['task_uuid'], @data['nodes'])
|
||||
end
|
||||
|
||||
it 'provision nodes using mclient' do
|
||||
@orchestrator.stubs(:report_about_progress).returns()
|
||||
@orchestrator.expects(:node_type).returns([{'uid' => 1, 'node_type' => 'target' }])
|
||||
|
||||
@orchestrator.provision(@reporter, @data['task_uuid'], @data['nodes'])
|
||||
end
|
||||
|
||||
xit "fail if timeout of provisioning is exceeded" do
|
||||
Astute::LogParser::ParseProvisionLogs.any_instance do
|
||||
stubs(:prepare).returns()
|
||||
end
|
||||
|
||||
@orchestrator.stubs(:node_type).returns([{'uid' => 1, 'node_type' => 'target' }])
|
||||
|
||||
Timeout.stubs(:timeout).raises(Timeout::Error)
|
||||
|
||||
msg = 'Timeout of provisioning is exceeded'
|
||||
Astute.logger.expects(:error).with(msg)
|
||||
|
||||
# error_mgs = {'status' => 'error', 'error' => msg, 'nodes' => [{ 'uid' => 1,
|
||||
# 'status' => 'error',
|
||||
# 'error_msg' => msg,
|
||||
# 'progress' => 100,
|
||||
# 'error_type' => 'provision'}]}
|
||||
#
|
||||
#@reporter.expects(:report).with(error_mgs)
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user