From 862d96aed1543dc8d6c727bb33f7251ce319bfc1 Mon Sep 17 00:00:00 2001 From: "Vladmir Sharhsov(warpc)" Date: Thu, 11 Jul 2013 17:43:25 +0400 Subject: [PATCH] [FUEL-813] Refactoring and tests for mcollective provision method --- astute.gemspec | 2 +- bin/astute | 8 +- lib/astute/orchestrator.rb | 118 +++++++++++----------- spec/spec_helper.rb | 1 + spec/unit/orchestrator_spec.rb | 175 ++++++++++++++++++++++----------- 5 files changed, 184 insertions(+), 120 deletions(-) diff --git a/astute.gemspec b/astute.gemspec index f9d756a7..f65404be 100644 --- a/astute.gemspec +++ b/astute.gemspec @@ -11,7 +11,7 @@ Gem::Specification.new do |s| s.email = ['mscherbakov@mirantis.com'] s.add_dependency 'activesupport', '3.0.10' - s.add_dependency 'mcollective-client', '~> 2.2.4' + s.add_dependency 'mcollective-client', '2.3.1' s.add_dependency 'symboltable', '1.0.2' s.add_development_dependency 'rspec', '2.13.0' diff --git a/bin/astute b/bin/astute index 347bba87..c2d85c0b 100755 --- a/bin/astute +++ b/bin/astute @@ -43,7 +43,8 @@ optparse = OptionParser.new do |o| o.on("-h") { puts o; exit } - o.on("-c", "--command [COMMAND]", [:deploy, :provision], "Select operation (deploy, provision)") do |c| + o.on("-c", "--command [COMMAND]", [:deploy, :fast_provision, :provision], + "Select operation (deploy, fast_provision, provision)") do |c| opts[:command] = c || :deploy end @@ -74,7 +75,10 @@ case opts[:command] when :deploy orchestrator = Astute::Orchestrator.new(deploy_engine, log_parsing=false) orchestrator.deploy(reporter, environment['task_uuid'], environment['nodes'], environment['attributes']) + when :fast_provision + orchestrator = Astute::Orchestrator.new(deploy_engine, log_parsing=false) + orchestrator.fast_provision(reporter, environment['engine'], environment['nodes']) when :provision orchestrator = Astute::Orchestrator.new(deploy_engine, log_parsing=false) - orchestrator.provision(reporter, environment['engine'], environment['nodes']) + orchestrator.provision(reporter, environment['task_uuid'], environment['nodes']) end \ No newline at end of file diff --git a/lib/astute/orchestrator.rb b/lib/astute/orchestrator.rb index 8dbad9f6..73a51aed 100644 --- a/lib/astute/orchestrator.rb +++ b/lib/astute/orchestrator.rb @@ -16,15 +16,10 @@ module Astute class Orchestrator def initialize(deploy_engine=nil, log_parsing=false) @deploy_engine = deploy_engine || Astute::DeploymentEngine::NailyFact - if log_parsing - @log_parser = LogParser::ParseDeployLogs.new - @provisionLogParser = LogParser::ParseProvisionLogs.new - else - @log_parser = LogParser::NoParsing.new - @provisionLogParser = LogParser::NoParsing.new - end + @log_parser = log_parsing ? LogParser::ParseDeployLogs.new : LogParser::NoParsing.new end + # TODO(waprc): does this method should be private? def node_type(reporter, task_id, nodes, timeout=nil) context = Context.new(task_id, reporter) uids = nodes.map {|n| n['uid']} @@ -54,7 +49,7 @@ module Astute deploy_engine_instance.deploy(nodes, attrs) end - def provision(reporter, engine_attrs, nodes) + def fast_provision(reporter, engine_attrs, nodes) raise "Nodes to provision are not provided!" if nodes.empty? engine = create_engine(engine_attrs, reporter) @@ -88,57 +83,49 @@ module Astute return end - def naily_deploy(reporter, task_id, nodes, attrs) + def provision(reporter, task_id, nodes) + raise "Nodes to provision are not provided!" if nodes.empty? # Following line fixes issues with uids: it should always be string - nodes.map { |x| x['uid'] = x['uid'].to_s } + nodes.map { |x| x['uid'] = x['uid'].to_s } # NOTE: perform that on environment['nodes'] initialization nodes_uids = nodes.map { |n| n['uid'] } - time = Time::now.to_f - nodes_not_booted = nodes.map { |n| n['uid'] } - Astute.logger.info "Starting OS provisioning for nodes: #{nodes_not_booted.join(',')}" - begin - @provisionLogParser.prepare(nodes) - rescue Exception => e - Astute.logger.warn "Some error occurred when prepare LogParser: #{e.message}, trace: #{e.backtrace.inspect}" + + provisionLogParser = LogParser::ParseProvisionLogs.new + sleep_not_greater_than(10) do # Wait while nodes going to reboot + Astute.logger.info "Starting OS provisioning for nodes: #{nodes_uids.join(',')}" + begin + provisionLogParser.prepare(nodes) + rescue => e + Astute.logger.warn "Some error occurred when prepare LogParser: #{e.message}, trace: #{e.backtrace.inspect}" + end end - time = 10 + time - Time::now.to_f - sleep (time) if time > 0 # Wait while nodes going to reboot. Sleep not greater than 10 sec. + nodes_not_booted = nodes_uids.clone begin - Timeout::timeout(Astute.config.PROVISIONING_TIMEOUT) do # Timeout for booting target OS - while true - time = Time::now.to_f - types = node_type(reporter, task_id, nodes, 2) - types.each do |t| - Astute.logger.debug("Got node types: uid=#{t['uid']} type=#{t['node_type']}") - end - Astute.logger.debug("Not target nodes will be rejected") - target_uids = types.reject{|n| n['node_type'] != 'target'}.map{|n| n['uid']} - Astute.logger.debug "Not provisioned: #{nodes_not_booted.join(',')}, got target OSes: #{target_uids.join(',')}" - if nodes.length == target_uids.length - Astute.logger.info "All nodes #{target_uids.join(',')} are provisioned." - break - else - Astute.logger.debug("Nodes list length is not equal to target nodes list length: #{nodes.length} != #{target_uids.length}") - end - nodes_not_booted = nodes_uids - types.map { |n| n['uid'] } - begin - nodes_progress = @provisionLogParser.progress_calculate(nodes_uids, nodes) - nodes_progress.each do |n| - if target_uids.include?(n['uid']) - n['progress'] = 100 - # TODO(mihgen): should we change status only once? - n['status'] = 'provisioned' + Timeout.timeout(Astute.config.PROVISIONING_TIMEOUT) do # Timeout for booting target OS + catch :done do + while true + sleep_not_greater_than(5) do + types = node_type(reporter, task_id, nodes, 2) + types.each { |t| Astute.logger.debug("Got node types: uid=#{t['uid']} type=#{t['node_type']}") } + + Astute.logger.debug("Not target nodes will be rejected") + target_uids = types.reject{|n| n['node_type'] != 'target'}.map{|n| n['uid']} + nodes_not_booted -= types.map { |n| n['uid'] } + Astute.logger.debug "Not provisioned: #{nodes_not_booted.join(',')}, got target OSes: #{target_uids.join(',')}" + + if nodes.length == target_uids.length + Astute.logger.info "All nodes #{target_uids.join(',')} are provisioned." + throw :done + else + Astute.logger.debug("Nodes list length is not equal to target nodes list length: #{nodes.length} != #{target_uids.length}") end + + report_about_progress(reporter, provisionLogParser, nodes_uids, target_uids, nodes) end - reporter.report({'nodes' => nodes_progress}) - rescue Exception => e - Astute.logger.warn "Some error occurred when parse logs for nodes progress: #{e.message}, trace: #{e.backtrace.inspect}" end - time = 5 + time - Time::now.to_f - sleep (time) if time > 0 # Sleep not greater than 5 sec. end - # We are here if jumped by break from while cycle + # We are here if jumped by throw from while cycle end rescue Timeout::Error msg = "Timeout of provisioning is exceeded." @@ -156,17 +143,6 @@ module Astute {'uid' => n['uid'], 'progress' => 100, 'status' => 'provisioned'} end reporter.report({'nodes' => nodes_progress}) - - begin - result = deploy(reporter, task_id, nodes, attrs) - rescue Timeout::Error - msg = "Timeout of deployment is exceeded." - Astute.logger.error msg - reporter.report({'status' => 'error', 'error' => msg}) - return - end - - report_result(result, reporter) end @@ -188,6 +164,13 @@ module Astute reporter.report(status) end + def sleep_not_greater_than(sleep_time, &block) + time = Time.now.to_f + block.call + time = time + sleep_time - Time.now.to_f + sleep (time) if time > 0 + end + def create_engine(engine_attrs, reporter) begin Astute.logger.info("Trying to instantiate cobbler engine: #{engine_attrs.inspect}") @@ -249,5 +232,20 @@ module Astute failed_nodes end + def report_about_progress(reporter, provisionLogParser, nodes_uids, target_uids, nodes) + begin + nodes_progress = provisionLogParser.progress_calculate(nodes_uids, nodes) + nodes_progress.each do |n| + if target_uids.include?(n['uid']) && n['status'] != 'provisioned' + n['progress'] = 100 + n['status'] = 'provisioned' + end + end + reporter.report({'nodes' => nodes_progress}) + rescue => e + Astute.logger.warn "Some error occurred when parse logs for nodes progress: #{e.message}, trace: #{e.backtrace.inspect}" + end + end + end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 31d1aae1..e21c0418 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -28,6 +28,7 @@ require File.join(File.dirname(__FILE__), '../lib/astute') Astute.config.PUPPET_DEPLOY_INTERVAL = 0 Astute.config.PUPPET_FADE_INTERVAL = 0 Astute.config.MC_RETRY_INTERVAL = 0 +Astute.config.PROVISIONING_TIMEOUT = 0 Astute.logger = Logger.new(STDERR) RSpec.configure do |c| diff --git a/spec/unit/orchestrator_spec.rb b/spec/unit/orchestrator_spec.rb index 0282b02e..a7869826 100644 --- a/spec/unit/orchestrator_spec.rb +++ b/spec/unit/orchestrator_spec.rb @@ -218,62 +218,64 @@ describe Astute::Orchestrator do it "remove_nodes do not fail if any of nodes failed" - describe '#provision' do - - before(:all) do - @data = { - "engine"=>{ - "url"=>"http://localhost/cobbler_api", - "username"=>"cobbler", - "password"=>"cobbler" - }, - "task_uuid"=>"a5c44b9a-285a-4a0c-ae65-2ed6b3d250f4", - "nodes" => [ - { - 'profile' => 'centos-x86_64', - "name"=>"controller-1", - 'power_type' => 'ssh', - 'power_user' => 'root', - 'power_pass' => '/root/.ssh/bootstrap.rsa', - 'power-address' => '1.2.3.5', - 'hostname' => 'name.domain.tld', - 'name_servers' => '1.2.3.4 1.2.3.100', - 'name_servers_search' => 'some.domain.tld domain.tld', - 'netboot_enabled' => '1', - 'ks_meta' => 'some_param=1 another_param=2', - 'interfaces' => { - 'eth0' => { - 'mac_address' => '00:00:00:00:00:00', - 'static' => '1', - 'netmask' => '255.255.255.0', - 'ip_address' => '1.2.3.5', - 'dns_name' => 'node.mirantis.net', - }, - 'eth1' => { - 'mac_address' => '00:00:00:00:00:01', - 'static' => '0', - 'netmask' => '255.255.255.0', - 'ip_address' => '1.2.3.6', - } + + before(:all) do + @data = { + "engine"=>{ + "url"=>"http://localhost/cobbler_api", + "username"=>"cobbler", + "password"=>"cobbler" + }, + "task_uuid"=>"a5c44b9a-285a-4a0c-ae65-2ed6b3d250f4", + "nodes" => [ + { + 'uid' => '1', + 'profile' => 'centos-x86_64', + "name"=>"controller-1", + 'power_type' => 'ssh', + 'power_user' => 'root', + 'power_pass' => '/root/.ssh/bootstrap.rsa', + 'power-address' => '1.2.3.5', + 'hostname' => 'name.domain.tld', + 'name_servers' => '1.2.3.4 1.2.3.100', + 'name_servers_search' => 'some.domain.tld domain.tld', + 'netboot_enabled' => '1', + 'ks_meta' => 'some_param=1 another_param=2', + 'interfaces' => { + 'eth0' => { + 'mac_address' => '00:00:00:00:00:00', + 'static' => '1', + 'netmask' => '255.255.255.0', + 'ip_address' => '1.2.3.5', + 'dns_name' => 'node.mirantis.net', }, - 'interfaces_extra' => { - 'eth0' => { - 'peerdns' => 'no', - 'onboot' => 'yes', - }, - 'eth1' => { - 'peerdns' => 'no', - 'onboot' => 'yes', - } + 'eth1' => { + 'mac_address' => '00:00:00:00:00:01', + 'static' => '0', + 'netmask' => '255.255.255.0', + 'ip_address' => '1.2.3.6', + } + }, + 'interfaces_extra' => { + 'eth0' => { + 'peerdns' => 'no', + 'onboot' => 'yes', + }, + 'eth1' => { + 'peerdns' => 'no', + 'onboot' => 'yes', } } - ] - }.freeze - end + } + ] + }.freeze + end + + describe '#fast_provision' do context 'cobler cases' do it "raise error if cobler settings empty" do - expect {@orchestrator.provision(@reporter, {}, @data['nodes'])}. + expect {@orchestrator.fast_provision(@reporter, {}, @data['nodes'])}. to raise_error(StopIteration) end end @@ -292,7 +294,7 @@ describe Astute::Orchestrator do end it "raises error if nodes list is empty" do - expect {@orchestrator.provision(@reporter, @data['engine'], {})}. + expect {@orchestrator.fast_provision(@reporter, @data['engine'], {})}. to raise_error(/Nodes to provision are not provided!/) end @@ -301,7 +303,7 @@ describe Astute::Orchestrator do expects(:power_reboot).with('controller-1') end @orchestrator.stubs(:check_reboot_nodes).returns([]) - @orchestrator.provision(@reporter, @data['engine'], @data['nodes']) + @orchestrator.fast_provision(@reporter, @data['engine'], @data['nodes']) end before(:each) { Astute::Provision::Cobbler.any_instance.stubs(:power_reboot).returns(333) } @@ -315,19 +317,19 @@ describe Astute::Orchestrator do Astute::Provision::Cobbler.any_instance.stubs(:event_status). returns([Time.now.to_f, 'controller-1', 'complete']) - @orchestrator.provision(@reporter, @data['engine'], @data['nodes']) + @orchestrator.fast_provision(@reporter, @data['engine'], @data['nodes']) end it "report about success" do @reporter.expects(:report).with({'status' => 'ready', 'progress' => 100}).returns(true) - @orchestrator.provision(@reporter, @data['engine'], @data['nodes']) + @orchestrator.fast_provision(@reporter, @data['engine'], @data['nodes']) end it "sync engine state" do Astute::Provision::Cobbler.any_instance do expects(:sync).once end - @orchestrator.provision(@reporter, @data['engine'], @data['nodes']) + @orchestrator.fast_provision(@reporter, @data['engine'], @data['nodes']) end end @@ -341,18 +343,77 @@ describe Astute::Orchestrator do expects(:sync).once end begin - @orchestrator.provision(@reporter, @data['engine'], @data['nodes']) + @orchestrator.fast_provision(@reporter, @data['engine'], @data['nodes']) rescue end end it "raise error if failed node find" do - expect {@orchestrator.provision(@reporter, @data['engine'], @data['nodes'])}.to raise_error(StopIteration) + expect {@orchestrator.fast_provision(@reporter, @data['engine'], @data['nodes'])}.to raise_error(StopIteration) end end end end + + describe '#provision' do + + it "raises error if nodes list is empty" do + expect {@orchestrator.provision(@reporter, @data['task_uuid'], {})}. + to raise_error(/Nodes to provision are not provided!/) + end + + it "prepare provision log for parsing" do + Astute::LogParser::ParseProvisionLogs.any_instance do + expects(:prepare).with(@data['nodes']).once + end + @orchestrator.stubs(:report_about_progress).returns() + @orchestrator.stubs(:node_type).returns([{'uid' => 1, 'node_type' => 'target' }]) + + @orchestrator.provision(@reporter, @data['task_uuid'], @data['nodes']) + end + + it "ignore problem with parsing provision log" do + Astute::LogParser::ParseProvisionLogs.any_instance do + stubs(:prepare).with(@data['nodes']).raises + end + + @orchestrator.stubs(:report_about_progress).returns() + @orchestrator.stubs(:node_type).returns([{'uid' => 1, 'node_type' => 'target' }]) + + @orchestrator.provision(@reporter, @data['task_uuid'], @data['nodes']) + end + + it 'provision nodes using mclient' do + @orchestrator.stubs(:report_about_progress).returns() + @orchestrator.expects(:node_type).returns([{'uid' => 1, 'node_type' => 'target' }]) + + @orchestrator.provision(@reporter, @data['task_uuid'], @data['nodes']) + end + + xit "fail if timeout of provisioning is exceeded" do + Astute::LogParser::ParseProvisionLogs.any_instance do + stubs(:prepare).returns() + end + + @orchestrator.stubs(:node_type).returns([{'uid' => 1, 'node_type' => 'target' }]) + + Timeout.stubs(:timeout).raises(Timeout::Error) + + msg = 'Timeout of provisioning is exceeded' + Astute.logger.expects(:error).with(msg) + + # error_mgs = {'status' => 'error', 'error' => msg, 'nodes' => [{ 'uid' => 1, +# 'status' => 'error', +# 'error_msg' => msg, +# 'progress' => 100, +# 'error_type' => 'provision'}]} +# + #@reporter.expects(:report).with(error_mgs) + end + + end + end