openstack
/
fuel-astute
Code Issues Proposed changes

[FUEL-813] Refactoring and tests for mcollective provision method

This commit is contained in:
Vladmir Sharhsov(warpc) 2013-07-11 17:43:25 +04:00
parent 05af849933
commit 862d96aed1
5 changed files with 184 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
astute.gemspec
View File

@ -11,7 +11,7 @@ Gem::Specification.new do |s|
s.email = ['mscherbakov@mirantis.com']
s.add_dependency 'activesupport', '3.0.10'
s.add_dependency 'mcollective-client', '~> 2.2.4'
s.add_dependency 'mcollective-client', '2.3.1'
s.add_dependency 'symboltable', '1.0.2'
s.add_development_dependency 'rspec', '2.13.0'

8
bin/astute
View File

@ -43,7 +43,8 @@ optparse = OptionParser.new do |o|
o.on("-h") { puts o; exit }
o.on("-c", "--command [COMMAND]", [:deploy, :provision], "Select operation (deploy, provision)") do |c|
o.on("-c", "--command [COMMAND]", [:deploy, :fast_provision, :provision],
"Select operation (deploy, fast_provision, provision)") do |c|
opts[:command] = c || :deploy
end
@ -74,7 +75,10 @@ case opts[:command]
when :deploy
orchestrator = Astute::Orchestrator.new(deploy_engine, log_parsing=false)
orchestrator.deploy(reporter, environment['task_uuid'], environment['nodes'], environment['attributes'])
when :fast_provision
orchestrator = Astute::Orchestrator.new(deploy_engine, log_parsing=false)
orchestrator.fast_provision(reporter, environment['engine'], environment['nodes'])
when :provision
orchestrator = Astute::Orchestrator.new(deploy_engine, log_parsing=false)
orchestrator.provision(reporter, environment['engine'], environment['nodes'])
orchestrator.provision(reporter, environment['task_uuid'], environment['nodes'])
end

118
lib/astute/orchestrator.rb
View File

@ -16,15 +16,10 @@ module Astute
class Orchestrator
def initialize(deploy_engine=nil, log_parsing=false)
@deploy_engine = deploy_engine || Astute::DeploymentEngine::NailyFact
if log_parsing
@log_parser = LogParser::ParseDeployLogs.new
@provisionLogParser = LogParser::ParseProvisionLogs.new
else
@log_parser = LogParser::NoParsing.new
@provisionLogParser = LogParser::NoParsing.new
end
@log_parser = log_parsing ? LogParser::ParseDeployLogs.new : LogParser::NoParsing.new
end
# TODO(waprc): does this method should be private?
def node_type(reporter, task_id, nodes, timeout=nil)
context = Context.new(task_id, reporter)
uids = nodes.map {|n| n['uid']}
@ -54,7 +49,7 @@ module Astute
deploy_engine_instance.deploy(nodes, attrs)
end
def provision(reporter, engine_attrs, nodes)
def fast_provision(reporter, engine_attrs, nodes)
raise "Nodes to provision are not provided!" if nodes.empty?
engine = create_engine(engine_attrs, reporter)
@ -88,57 +83,49 @@ module Astute
return
end
def naily_deploy(reporter, task_id, nodes, attrs)
def provision(reporter, task_id, nodes)
raise "Nodes to provision are not provided!" if nodes.empty?
# Following line fixes issues with uids: it should always be string
nodes.map { |x| x['uid'] = x['uid'].to_s }
nodes.map { |x| x['uid'] = x['uid'].to_s } # NOTE: perform that on environment['nodes'] initialization
nodes_uids = nodes.map { |n| n['uid'] }
time = Time::now.to_f
nodes_not_booted = nodes.map { |n| n['uid'] }
Astute.logger.info "Starting OS provisioning for nodes: #{nodes_not_booted.join(',')}"
begin
@provisionLogParser.prepare(nodes)
rescue Exception => e
Astute.logger.warn "Some error occurred when prepare LogParser: #{e.message}, trace: #{e.backtrace.inspect}"
provisionLogParser = LogParser::ParseProvisionLogs.new
sleep_not_greater_than(10) do # Wait while nodes going to reboot
Astute.logger.info "Starting OS provisioning for nodes: #{nodes_uids.join(',')}"
begin
provisionLogParser.prepare(nodes)
rescue => e
Astute.logger.warn "Some error occurred when prepare LogParser: #{e.message}, trace: #{e.backtrace.inspect}"
end
end
time = 10 + time - Time::now.to_f
sleep (time) if time > 0 # Wait while nodes going to reboot. Sleep not greater than 10 sec.
nodes_not_booted = nodes_uids.clone
begin
Timeout::timeout(Astute.config.PROVISIONING_TIMEOUT) do # Timeout for booting target OS
while true
time = Time::now.to_f
types = node_type(reporter, task_id, nodes, 2)
types.each do |t|
Astute.logger.debug("Got node types: uid=#{t['uid']} type=#{t['node_type']}")
end
Astute.logger.debug("Not target nodes will be rejected")
target_uids = types.reject{|n| n['node_type'] != 'target'}.map{|n| n['uid']}
Astute.logger.debug "Not provisioned: #{nodes_not_booted.join(',')}, got target OSes: #{target_uids.join(',')}"
if nodes.length == target_uids.length
Astute.logger.info "All nodes #{target_uids.join(',')} are provisioned."
break
else
Astute.logger.debug("Nodes list length is not equal to target nodes list length: #{nodes.length} != #{target_uids.length}")
end
nodes_not_booted = nodes_uids - types.map { |n| n['uid'] }
begin
nodes_progress = @provisionLogParser.progress_calculate(nodes_uids, nodes)
nodes_progress.each do |n|
if target_uids.include?(n['uid'])
n['progress'] = 100
# TODO(mihgen): should we change status only once?
n['status'] = 'provisioned'
Timeout.timeout(Astute.config.PROVISIONING_TIMEOUT) do # Timeout for booting target OS
catch :done do
while true
sleep_not_greater_than(5) do
types = node_type(reporter, task_id, nodes, 2)
types.each { |t| Astute.logger.debug("Got node types: uid=#{t['uid']} type=#{t['node_type']}") }
Astute.logger.debug("Not target nodes will be rejected")
target_uids = types.reject{|n| n['node_type'] != 'target'}.map{|n| n['uid']}
nodes_not_booted -= types.map { |n| n['uid'] }
Astute.logger.debug "Not provisioned: #{nodes_not_booted.join(',')}, got target OSes: #{target_uids.join(',')}"
if nodes.length == target_uids.length
Astute.logger.info "All nodes #{target_uids.join(',')} are provisioned."
throw :done
else
Astute.logger.debug("Nodes list length is not equal to target nodes list length: #{nodes.length} != #{target_uids.length}")
end
report_about_progress(reporter, provisionLogParser, nodes_uids, target_uids, nodes)
end
reporter.report({'nodes' => nodes_progress})
rescue Exception => e
Astute.logger.warn "Some error occurred when parse logs for nodes progress: #{e.message}, trace: #{e.backtrace.inspect}"
end
time = 5 + time - Time::now.to_f
sleep (time) if time > 0 # Sleep not greater than 5 sec.
end
# We are here if jumped by break from while cycle
# We are here if jumped by throw from while cycle
end
rescue Timeout::Error
msg = "Timeout of provisioning is exceeded."
@ -156,17 +143,6 @@ module Astute
{'uid' => n['uid'], 'progress' => 100, 'status' => 'provisioned'}
end
reporter.report({'nodes' => nodes_progress})
begin
result = deploy(reporter, task_id, nodes, attrs)
rescue Timeout::Error
msg = "Timeout of deployment is exceeded."
Astute.logger.error msg
reporter.report({'status' => 'error', 'error' => msg})
return
end
report_result(result, reporter)
end
@ -188,6 +164,13 @@ module Astute
reporter.report(status)
end
def sleep_not_greater_than(sleep_time, &block)
time = Time.now.to_f
block.call
time = time + sleep_time - Time.now.to_f
sleep (time) if time > 0
end
def create_engine(engine_attrs, reporter)
begin
Astute.logger.info("Trying to instantiate cobbler engine: #{engine_attrs.inspect}")
@ -249,5 +232,20 @@ module Astute
failed_nodes
end
def report_about_progress(reporter, provisionLogParser, nodes_uids, target_uids, nodes)
begin
nodes_progress = provisionLogParser.progress_calculate(nodes_uids, nodes)
nodes_progress.each do |n|
if target_uids.include?(n['uid']) && n['status'] != 'provisioned'
n['progress'] = 100
n['status'] = 'provisioned'
end
end
reporter.report({'nodes' => nodes_progress})
rescue => e
Astute.logger.warn "Some error occurred when parse logs for nodes progress: #{e.message}, trace: #{e.backtrace.inspect}"
end
end
end
end

1
spec/spec_helper.rb
View File

@ -28,6 +28,7 @@ require File.join(File.dirname(__FILE__), '../lib/astute')
Astute.config.PUPPET_DEPLOY_INTERVAL = 0
Astute.config.PUPPET_FADE_INTERVAL = 0
Astute.config.MC_RETRY_INTERVAL = 0
Astute.config.PROVISIONING_TIMEOUT = 0
Astute.logger = Logger.new(STDERR)
RSpec.configure do |c|

175
spec/unit/orchestrator_spec.rb
View File

@ -218,62 +218,64 @@ describe Astute::Orchestrator do
it "remove_nodes do not fail if any of nodes failed"
describe '#provision' do
before(:all) do
@data = {
"engine"=>{
"url"=>"http://localhost/cobbler_api",
"username"=>"cobbler",
"password"=>"cobbler"
},
"task_uuid"=>"a5c44b9a-285a-4a0c-ae65-2ed6b3d250f4",
"nodes" => [
{
'profile' => 'centos-x86_64',
"name"=>"controller-1",
'power_type' => 'ssh',
'power_user' => 'root',
'power_pass' => '/root/.ssh/bootstrap.rsa',
'power-address' => '1.2.3.5',
'hostname' => 'name.domain.tld',
'name_servers' => '1.2.3.4 1.2.3.100',
'name_servers_search' => 'some.domain.tld domain.tld',
'netboot_enabled' => '1',
'ks_meta' => 'some_param=1 another_param=2',
'interfaces' => {
'eth0' => {
'mac_address' => '00:00:00:00:00:00',
'static' => '1',
'netmask' => '255.255.255.0',
'ip_address' => '1.2.3.5',
'dns_name' => 'node.mirantis.net',
},
'eth1' => {
'mac_address' => '00:00:00:00:00:01',
'static' => '0',
'netmask' => '255.255.255.0',
'ip_address' => '1.2.3.6',
}
before(:all) do
@data = {
"engine"=>{
"url"=>"http://localhost/cobbler_api",
"username"=>"cobbler",
"password"=>"cobbler"
},
"task_uuid"=>"a5c44b9a-285a-4a0c-ae65-2ed6b3d250f4",
"nodes" => [
{
'uid' => '1',
'profile' => 'centos-x86_64',
"name"=>"controller-1",
'power_type' => 'ssh',
'power_user' => 'root',
'power_pass' => '/root/.ssh/bootstrap.rsa',
'power-address' => '1.2.3.5',
'hostname' => 'name.domain.tld',
'name_servers' => '1.2.3.4 1.2.3.100',
'name_servers_search' => 'some.domain.tld domain.tld',
'netboot_enabled' => '1',
'ks_meta' => 'some_param=1 another_param=2',
'interfaces' => {
'eth0' => {
'mac_address' => '00:00:00:00:00:00',
'static' => '1',
'netmask' => '255.255.255.0',
'ip_address' => '1.2.3.5',
'dns_name' => 'node.mirantis.net',
},
'interfaces_extra' => {
'eth0' => {
'peerdns' => 'no',
'onboot' => 'yes',
},
'eth1' => {
'peerdns' => 'no',
'onboot' => 'yes',
}
'eth1' => {
'mac_address' => '00:00:00:00:00:01',
'static' => '0',
'netmask' => '255.255.255.0',
'ip_address' => '1.2.3.6',
}
},
'interfaces_extra' => {
'eth0' => {
'peerdns' => 'no',
'onboot' => 'yes',
},
'eth1' => {
'peerdns' => 'no',
'onboot' => 'yes',
}
}
]
}.freeze
end
}
]
}.freeze
end
describe '#fast_provision' do
context 'cobler cases' do
it "raise error if cobler settings empty" do
expect {@orchestrator.provision(@reporter, {}, @data['nodes'])}.
expect {@orchestrator.fast_provision(@reporter, {}, @data['nodes'])}.
to raise_error(StopIteration)
end
end
@ -292,7 +294,7 @@ describe Astute::Orchestrator do
end
it "raises error if nodes list is empty" do
expect {@orchestrator.provision(@reporter, @data['engine'], {})}.
expect {@orchestrator.fast_provision(@reporter, @data['engine'], {})}.
to raise_error(/Nodes to provision are not provided!/)
end
@ -301,7 +303,7 @@ describe Astute::Orchestrator do
expects(:power_reboot).with('controller-1')
end
@orchestrator.stubs(:check_reboot_nodes).returns([])
@orchestrator.provision(@reporter, @data['engine'], @data['nodes'])
@orchestrator.fast_provision(@reporter, @data['engine'], @data['nodes'])
end
before(:each) { Astute::Provision::Cobbler.any_instance.stubs(:power_reboot).returns(333) }
@ -315,19 +317,19 @@ describe Astute::Orchestrator do
Astute::Provision::Cobbler.any_instance.stubs(:event_status).
returns([Time.now.to_f, 'controller-1', 'complete'])
@orchestrator.provision(@reporter, @data['engine'], @data['nodes'])
@orchestrator.fast_provision(@reporter, @data['engine'], @data['nodes'])
end
it "report about success" do
@reporter.expects(:report).with({'status' => 'ready', 'progress' => 100}).returns(true)
@orchestrator.provision(@reporter, @data['engine'], @data['nodes'])
@orchestrator.fast_provision(@reporter, @data['engine'], @data['nodes'])
end
it "sync engine state" do
Astute::Provision::Cobbler.any_instance do
expects(:sync).once
end
@orchestrator.provision(@reporter, @data['engine'], @data['nodes'])
@orchestrator.fast_provision(@reporter, @data['engine'], @data['nodes'])
end
end
@ -341,18 +343,77 @@ describe Astute::Orchestrator do
expects(:sync).once
end
begin
@orchestrator.provision(@reporter, @data['engine'], @data['nodes'])
@orchestrator.fast_provision(@reporter, @data['engine'], @data['nodes'])
rescue
end
end
it "raise error if failed node find" do
expect {@orchestrator.provision(@reporter, @data['engine'], @data['nodes'])}.to raise_error(StopIteration)
expect {@orchestrator.fast_provision(@reporter, @data['engine'], @data['nodes'])}.to raise_error(StopIteration)
end
end
end
end
describe '#provision' do
it "raises error if nodes list is empty" do
expect {@orchestrator.provision(@reporter, @data['task_uuid'], {})}.
to raise_error(/Nodes to provision are not provided!/)
end
it "prepare provision log for parsing" do
Astute::LogParser::ParseProvisionLogs.any_instance do
expects(:prepare).with(@data['nodes']).once
end
@orchestrator.stubs(:report_about_progress).returns()
@orchestrator.stubs(:node_type).returns([{'uid' => 1, 'node_type' => 'target' }])
@orchestrator.provision(@reporter, @data['task_uuid'], @data['nodes'])
end
it "ignore problem with parsing provision log" do
Astute::LogParser::ParseProvisionLogs.any_instance do
stubs(:prepare).with(@data['nodes']).raises
end
@orchestrator.stubs(:report_about_progress).returns()
@orchestrator.stubs(:node_type).returns([{'uid' => 1, 'node_type' => 'target' }])
@orchestrator.provision(@reporter, @data['task_uuid'], @data['nodes'])
end
it 'provision nodes using mclient' do
@orchestrator.stubs(:report_about_progress).returns()
@orchestrator.expects(:node_type).returns([{'uid' => 1, 'node_type' => 'target' }])
@orchestrator.provision(@reporter, @data['task_uuid'], @data['nodes'])
end
xit "fail if timeout of provisioning is exceeded" do
Astute::LogParser::ParseProvisionLogs.any_instance do
stubs(:prepare).returns()
end
@orchestrator.stubs(:node_type).returns([{'uid' => 1, 'node_type' => 'target' }])
Timeout.stubs(:timeout).raises(Timeout::Error)
msg = 'Timeout of provisioning is exceeded'
Astute.logger.expects(:error).with(msg)
# error_mgs = {'status' => 'error', 'error' => msg, 'nodes' => [{ 'uid' => 1,
# 'status' => 'error',
# 'error_msg' => msg,
# 'progress' => 100,
# 'error_type' => 'provision'}]}
#
#@reporter.expects(:report).with(error_mgs)
end
end
end