openstack/fuel-library
Code Issues Proposed changes

Updated astute

Browse Source
This commit is contained in:
vic 2013-03-11 15:06:19 +04:00
parent 89922dc1ff
commit 415ed50b94
23 changed files with 614 additions and 775 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
deployment/mcollective/astute/astute.gemspec
View File

@ -16,7 +16,6 @@ Gem::Specification.new do |s|
s.files = Dir.glob("{bin,lib,spec,samples,templates}/**/*")
s.executables = ['astute', 'astute_run', 'openstack_system']
s.extra_rdoc_files = %w< README >
s.require_path = 'lib'
end

3
deployment/mcollective/astute/lib/astute.rb
View File

@ -17,6 +17,9 @@ module Astute
autoload 'MClient', 'astute/mclient'
autoload 'ProxyReporter', 'astute/reporter'
autoload 'NodeRemoval', 'astute/node_removal'
LogParser.autoload :ParseDeployLogs, 'astute/logparser/deployment'
LogParser.autoload :ParseProvisionLogs, 'astute/logparser/provision'
LogParser.autoload :Patterns, 'astute/logparser/parser_patterns'
def self.logger
@logger ||= Logger.new('/var/log/astute.log')

33
deployment/mcollective/astute/lib/astute/deployment_engine.rb
View File

@ -13,6 +13,7 @@ module Astute
def deploy(nodes, attrs)
# See implementation in subclasses, this may be overriden
attrs['deployment_mode'] ||= 'multinode_compute' # simple multinode deployment is the default
@ctx.deploy_log_parser.deploy_type = attrs['deployment_mode']
Astute.logger.info "Deployment mode #{attrs['deployment_mode']}"
result = self.send("deploy_#{attrs['deployment_mode']}", nodes, attrs)
end
@ -61,12 +62,10 @@ module Astute
Astute.logger.info "Starting deployment of controllers"
deploy_piece(ctrl_nodes, attrs)
@ctx.deploy_log_parser.pattern_spec['expected_line_number'] = 380
compute_nodes = nodes.select {|n| n['role'] == 'compute'}
Astute.logger.info "Starting deployment of computes"
deploy_piece(compute_nodes, attrs)
@ctx.deploy_log_parser.pattern_spec['expected_line_number'] = 300
other_nodes = nodes - ctrl_nodes - compute_nodes
Astute.logger.info "Starting deployment of other nodes"
deploy_piece(other_nodes, attrs)
@ -97,26 +96,23 @@ module Astute
def deploy_ha_compute(nodes, attrs)
ctrl_nodes = nodes.select {|n| n['role'] == 'controller'}
Astute.logger.info "Starting deployment of all controllers one by one, ignoring failure"
ctrl_nodes.each {|n| deploy_piece([n], attrs, retries=0, ignore_failure=true)}
ctrl_nodes.each {|n| deploy_piece([n], attrs, retries=0, change_node_status=false)}
Astute.logger.info "Starting deployment of all controllers, ignoring failure"
deploy_piece(ctrl_nodes, attrs, retries=0, ignore_failure=true)
deploy_piece(ctrl_nodes, attrs, retries=0, change_node_status=false)
Astute.logger.info "Starting deployment of 1st controller again, ignoring failure"
deploy_piece([ctrl_nodes[0]], attrs, retries=0, ignore_failure=true)
deploy_piece([ctrl_nodes[0]], attrs, retries=0, change_node_status=false)
retries = 1
Astute.logger.info "Starting deployment of all controllers until it completes, "\
"allowed retries: #{retries}"
deploy_piece(ctrl_nodes, attrs, retries=retries)
# FIXME(mihgen): put right numbers for logs
@ctx.deploy_log_parser.pattern_spec['expected_line_number'] = 380
compute_nodes = nodes.select {|n| n['role'] == 'compute'}
Astute.logger.info "Starting deployment of computes"
deploy_piece(compute_nodes, attrs)
@ctx.deploy_log_parser.pattern_spec['expected_line_number'] = 300
other_nodes = nodes - ctrl_nodes - compute_nodes
Astute.logger.info "Starting deployment of other nodes"
deploy_piece(other_nodes, attrs)
@ -129,13 +125,13 @@ module Astute
other_nodes = nodes - ctrl_nodes - compute_nodes
Astute.logger.info "Starting deployment of all controllers one by one, ignoring failure"
ctrl_nodes.each {|n| deploy_piece([n], attrs, retries=0, ignore_failure=true)}
ctrl_nodes.each {|n| deploy_piece([n], attrs, retries=0, change_node_status=false)}
Astute.logger.info "Starting deployment of 1st controller again, ignoring failure"
deploy_piece(ctrl_nodes[0..0], attrs, retries=0, ignore_failure=true)
deploy_piece(ctrl_nodes[0..0], attrs, retries=0, change_node_status=false)
Astute.logger.info "Starting deployment of controllers exclude first, ignoring failure"
deploy_piece(ctrl_nodes[1..-1], attrs, retries=0, ignore_failure=true)
deploy_piece(ctrl_nodes[1..-1], attrs, retries=0, change_node_status=false)
Astute.logger.info "Starting deployment of 1st controller again, ignoring failure"
deploy_piece(ctrl_nodes[0..0], attrs, retries=0)
@ -174,13 +170,13 @@ module Astute
deploy_piece(compute_nodes, attrs)
Astute.logger.info "Starting deployment of storages, ignoring failure"
deploy_piece(storage_nodes, attrs, ignore_failure=true)
deploy_piece(storage_nodes, attrs, change_node_status=false)
Astute.logger.info "Starting deployment of storages, ignoring failure"
deploy_piece(storage_nodes, attrs, ignore_failure=true)
deploy_piece(storage_nodes, attrs, change_node_status=false)
Astute.logger.info "Starting deployment of all proxies one by one, ignoring failure"
proxy_nodes.each {|n| deploy_piece([n], attrs, retries=0, ignore_failure=true)}
proxy_nodes.each {|n| deploy_piece([n], attrs, retries=0, change_node_status=false)}
Astute.logger.info "Starting deployment of storages"
deploy_piece(storage_nodes, attrs)
@ -194,8 +190,8 @@ module Astute
end
private
def nodes_status(nodes, status)
{'nodes' => nodes.map { |n| {'uid' => n['uid'], 'status' => status} }}
def nodes_status(nodes, status, data_to_merge)
{'nodes' => nodes.map { |n| {'uid' => n['uid'], 'status' => status}.merge(data_to_merge) }}
end
def validate_nodes(nodes)
@ -229,6 +225,9 @@ module Astute
interfaces[name]['broadcast'] = iface['brd']
end
end
if iface['gateway'] and iface['name'] =~ /^public$/i
interfaces[name]['gateway'] = iface['gateway']
end
interfaces[name]['ensure'] = 'present'
Astute.logger.debug "Calculated network for interface: #{name}, data: #{interfaces[name].inspect}"
end
@ -243,4 +242,4 @@ module Astute
end
end
end

6
deployment/mcollective/astute/lib/astute/deployment_engine/nailyfact.rb
View File

@ -33,9 +33,9 @@ class Astute::DeploymentEngine::NailyFact < Astute::DeploymentEngine
metapublisher.call(@ctx, node['uid'], metadata)
end
def deploy_piece(nodes, attrs, retries=2, ignore_failure=false)
def deploy_piece(nodes, attrs, retries=2, change_node_status=true)
return false unless validate_nodes(nodes)
@ctx.reporter.report nodes_status(nodes, 'deploying')
@ctx.reporter.report nodes_status(nodes, 'deploying', {'progress' => 0})
Astute.logger.info "#{@ctx.task_id}: Calculation of required attributes to pass, include netw.settings"
nodes.each do |node|
@ -43,7 +43,7 @@ class Astute::DeploymentEngine::NailyFact < Astute::DeploymentEngine
end
Astute.logger.info "#{@ctx.task_id}: All required attrs/metadata passed via facts extension. Starting deployment."
Astute::PuppetdDeployer.deploy(@ctx, nodes, retries, ignore_failure)
Astute::PuppetdDeployer.deploy(@ctx, nodes, retries, change_node_status)
nodes_roles = nodes.map { |n| { n['uid'] => n['role'] } }
Astute.logger.info "#{@ctx.task_id}: Finished deployment of nodes => roles: #{nodes_roles.inspect}"
end

12
deployment/mcollective/astute/lib/astute/deployment_engine/puppet_kernel.rb
View File

@ -1,12 +0,0 @@
class Astute::DeploymentEngine::PuppetKernel < Astute::DeploymentEngine
# NOTE(mihgen): Not completed
def deploy_piece(nodes, attrs)
return false unless validate_nodes(nodes)
case nodes[0]['role']
when "controller"
classes = {"nailytest::test_rpuppet" => {"rpuppet" => ["controller", "privet"]}}
Astute::RpuppetDeployer.rpuppet_deploy(@ctx, nodes, attrs, classes)
# network_data = calculate_networks(node['network_data'])
end
end
end

2
deployment/mcollective/astute/lib/astute/deployment_engine/simple_puppet.rb
View File

@ -3,7 +3,7 @@ class Astute::DeploymentEngine::SimplePuppet < Astute::DeploymentEngine
# with all required parameters for modules
def deploy_piece(nodes, *args)
return false unless validate_nodes(nodes)
@ctx.reporter.report nodes_status(nodes, 'deploying')
@ctx.reporter.report nodes_status(nodes, 'deploying', {'progress' => 0})
Astute::PuppetdDeployer.deploy(@ctx, nodes)
nodes_roles = nodes.map { |n| { n['uid'] => n['role'] } }
Astute.logger.info "#{@ctx.task_id}: Finished deployment of nodes => roles: #{nodes_roles.inspect}"

234
deployment/mcollective/astute/lib/astute/logparser.rb
View File

@ -1,13 +1,13 @@
module Astute
module LogParser
@separator = "SEPARATOR\n"
@log_portion = 10000
LOG_PORTION = 10000
# Default values. Can be overrided by pattern_spec.
# E.g. pattern_spec = {'separator' => 'new_separator', ...}
PATH_PREFIX = '/var/log/remote/'
SEPARATOR = "SEPARATOR\n"
class NoParsing
attr_accessor :pattern_spec
def initialize(*args)
@pattern_spec = {}
end
def method_missing(*args)
@ -20,181 +20,113 @@ module Astute
end
class ParseNodeLogs
attr_accessor :pattern_spec
attr_reader :pattern_spec
def initialize(filename, pattern_spec=nil)
@filename = filename
if pattern_spec.nil?
@pattern_spec = {'type' => 'count-lines',
'endlog_patterns' => [{'pattern' => /Finished catalog run in [0-9]+\.[0-9]* seconds\n/, 'progress' => 1.0}],
'expected_line_number' => 500}
else
@pattern_spec = pattern_spec
end
def initialize(pattern_spec)
@nodes_states = {}
@pattern_spec = pattern_spec
@pattern_spec['path_prefix'] ||= PATH_PREFIX.to_s
@pattern_spec['separator'] ||= SEPARATOR.to_s
end
def progress_calculate(uids_to_calc, nodes)
nodes_progress = []
uids_to_calc.each do |uid|
node = nodes.select {|n| n['uid'] == uid}[0]
path = "/var/log/remote/#{node['ip']}/#{@filename}"
node_pattern_spec = @nodes_states[uid]
unless node_pattern_spec
node_pattern_spec = Marshal.load(Marshal.dump(@pattern_spec))
@nodes_states[uid] = node_pattern_spec
end
path = "#{@pattern_spec['path_prefix']}#{node['ip']}/#{@pattern_spec['filename']}"
begin
progress = (get_log_progress(path, node_pattern_spec)*100).to_i # Return percent of progress
rescue Exception => e
Astute.logger.warn "Some error occurred when calculate progress for node '#{uid}': #{e.message}, trace: #{e.backtrace.inspect}"
progress = 0
end
nodes_progress << {
'uid' => uid,
'progress' => (LogParser::get_log_progress(path, @pattern_spec)*100).to_i # Return percent of progress
'progress' => progress
}
end
return nodes_progress
end
def add_separator(nodes)
def prepare(nodes)
@nodes_states = {}
nodes.each do |node|
path = "/var/log/remote/#{node['ip']}/#{@filename}"
LogParser::add_log_separator(path)
path = "#{@pattern_spec['path_prefix']}#{node['ip']}/#{@pattern_spec['filename']}"
File.open(path, 'a') {|fo| fo.write @pattern_spec['separator'] } if File.writable?(path)
end
end
end
def pattern_spec= (pattern_spec)
initialise(pattern_spec)
end
public
def self.add_log_separator(path, separator=@separator)
File.open(path, 'a') {|fo| fo.write separator } if File.readable?(path)
end
private
def get_log_progress(path, node_pattern_spec)
unless File.readable?(path)
Astute.logger.debug "Can't read file with logs: #{path}"
return 0
end
if node_pattern_spec.nil?
Astute.logger.warn "Can't parse logs. Pattern_spec is empty."
return 0
end
progress = nil
File.open(path) do |fo|
# Try to find well-known ends of log.
endlog = find_endlog_patterns(fo, node_pattern_spec)
return endlog if endlog
# Start reading from end of file.
fo.pos = fo.stat.size
def self.get_log_progress(path, pattern_spec)
# Pattern specification example:
# pattern_spec = {'type' => 'pattern-list', 'separator' => "custom separator\n",
# 'chunk_size' => 10000,
# 'pattern_list' => [
# {'pattern' => 'to step installpackages', 'progress' => 0.16},
# {'pattern' => 'Installing',
# 'number' => 210, # Now it install 205 packets. Add 5 packets for growth in future.
# 'p_min' => 0.16, # min percent
# 'p_max' => 0.87 # max percent
# }
# ]
# }
# Method 'calculate' should be defined at child classes.
progress = calculate(fo, node_pattern_spec)
node_pattern_spec['file_pos'] = fo.pos
end
unless progress
Astute.logger.warn("Wrong pattern #{node_pattern_spec.inspect} defined for calculating progress via logs.")
return 0
end
return progress
end
return 0 unless File.readable?(path)
progress = nil
File.open(path) do |fo|
# Try to find well-known ends of log.
endlog = find_endlog_patterns(fo, pattern_spec)
return endlog if endlog
# Start reading from end of file.
def find_endlog_patterns(fo, pattern_spec)
# Pattern example:
# pattern_spec = {...,
# 'endlog_patterns' => [{'pattern' => /Finished catalog run in [0-9]+\.[0-9]* seconds\n/, 'progress' => 1.0}],
# }
endlog_patterns = pattern_spec['endlog_patterns']
return nil unless endlog_patterns
fo.pos = fo.stat.size
if pattern_spec['type'] == 'count-lines'
progress = simple_line_counter(fo, pattern_spec)
elsif pattern_spec['type'] = 'pattern-list'
progress = simple_pattern_finder(fo, pattern_spec)
chunk = get_chunk(fo, 100)
return nil unless chunk
endlog_patterns.each do |pattern|
return pattern['progress'] if chunk.end_with?(pattern['pattern'])
end
end
unless progress
Naily.logger.warn("Wrong pattern #{pattern_spec.inspect} defined for calculating progress via logs.")
return 0
end
return progress
end
private
def self.simple_pattern_finder(fo, pattern_spec)
# Use custom separator if defined.
separator = pattern_spec['separator']
separator = @separator unless separator
log_patterns = pattern_spec['pattern_list']
unless log_patterns
Naily.logger.warn("Wrong pattern #{pattern_spec.inspect} defined for calculating progress via logs.")
return 0
return nil
end
chunk = get_chunk(fo, pattern_spec['chunk_size'])
# NOTE(mihgen): Following line fixes "undefined method `rindex' for nil:NilClass" for empty log file
return 0 unless chunk
pos = chunk.rindex(separator)
chunk = chunk.slice((pos + separator.size)..-1) if pos
block = chunk.split("\n")
return 0 unless block
while true
string = block.pop
return 0 unless string # If we found nothing
log_patterns.each do |pattern|
if string.include?(pattern['pattern'])
return pattern['progress'] if pattern['progress']
if pattern['number']
string = block.pop
counter = 1
while string
counter += 1 if string.include?(pattern['pattern'])
string = block.pop
end
progress = counter.to_f / pattern['number']
progress = 1 if progress > 1
progress = pattern['p_min'] + progress * (pattern['p_max'] - pattern['p_min'])
return progress
end
Naily.logger.warn("Wrong pattern #{pattern_spec.inspect} defined for calculating progress via log.")
end
end
end
end
def self.find_endlog_patterns(fo, pattern_spec)
endlog_patterns = pattern_spec['endlog_patterns']
return nil unless endlog_patterns
fo.pos = fo.stat.size
chunk = get_chunk(fo, 100)
endlog_patterns.each do |pattern|
return pattern['progress'] if chunk.end_with?(pattern['pattern'])
end
return nil
end
def self.simple_line_counter(fo, pattern_spec)
# Use custom separator if defined.
separator = pattern_spec['separator']
separator = @separator unless separator
counter = 0
end_of_scope = false
previous_subchunk = ''
until end_of_scope
chunk = get_chunk(fo, pattern_spec['chunk_size'])
break unless chunk
# Trying to find separator on border between chunks.
subchunk = chunk.slice((1-separator.size)..-1)
# End of file reached. Exit from cycle.
end_of_scope = true unless subchunk
if subchunk and (subchunk + previous_subchunk).include?(separator)
# Separator found on border between chunks. Exit from cycle.
end_of_scope = true
continue
end
pos = chunk.rindex(separator)
def get_chunk(fo, size=nil, pos=nil)
if pos
end_of_scope = true
chunk = chunk.slice((pos + separator.size)..-1)
fo.pos = pos
return fo.read
end
counter += chunk.count("\n")
size = LOG_PORTION unless size
return nil if fo.pos == 0
size = fo.pos if fo.pos < size
next_pos = fo.pos - size
fo.pos = next_pos
block = fo.read(size)
fo.pos = next_pos
return block
end
number = pattern_spec['expected_line_number']
unless number
Naily.logger.warn("Wrong pattern #{pattern_spec.inspect} defined for calculating progress via log.")
return 0
end
progress = counter.to_f / number
progress = 1 if progress > 1
return progress
end
def self.get_chunk(fo, size=nil)
size = @log_portion unless size
return nil if fo.pos == 0
size = fo.pos if fo.pos < size
next_pos = fo.pos - size
fo.pos = next_pos
block = fo.read(size)
fo.pos = next_pos
return block
end
end
end

39
deployment/mcollective/astute/lib/astute/mclient.rb
View File

@ -11,7 +11,15 @@ module Astute
@agent = agent
@nodes = nodes.map { |n| n.to_s }
@check_result = check_result
@mc = rpcclient(agent, :exit_on_failure => false)
unless Thread.current['semaphore'].nil?
Thread.current['semaphore'].synchronize do
Thread.current['mclient'] = rpcclient(agent, :exit_on_failure => false)
end
else
Thread.current['mclient'] = rpcclient(agent, :exit_on_failure => false)
end
@mc = Thread.current['mclient']
@mc.timeout = timeout if timeout
@mc.progress = false
@retries = Astute.config.MC_RETRIES
@ -21,7 +29,15 @@ module Astute
end
def method_missing(method, *args)
res = @mc.send(method, *args)
unless Thread.current['semaphore'].nil?
Thread.current['semaphore'].synchronize do
Thread.current['mc_res'] = @mc.send(method, *args)
end
else
Thread.current['mc_res'] = @mc.send(method, *args)
end
res = Thread.current['mc_res']
if method == :discover
@nodes = args[0][:nodes]
return res
@ -42,7 +58,16 @@ module Astute
not_responded = @nodes - nodes_responded
Astute.logger.debug "Retry ##{retry_index} to run mcollective agent on nodes: '#{not_responded.join(',')}'"
@mc.discover(:nodes => not_responded)
new_res = @mc.send(method, *args)
unless Thread.current['semaphore'].nil?
Thread.current['semaphore'].synchronize do
Thread.current['mc_new_res'] = @mc.send(method, *args)
end
else
Thread.current['mc_new_res'] = @mc.send(method, *args)
end
new_res = Thread.current['mc_new_res']
log_result(new_res, method)
# new_res can have some nodes which finally responded
res += new_res
@ -52,15 +77,15 @@ module Astute
if res.length < @nodes.length
nodes_responded = res.map { |n| n.results[:sender] }
not_responded = @nodes - nodes_responded
err_msg += "#{@task_id}: MCollective agents '#{not_responded.join(',')}' didn't respond.\n"
err_msg += "MCollective agents '#{not_responded.join(',')}' didn't respond. \n"
end
end
failed = res.select { |x| x.results[:statuscode] != 0 }
if failed.any?
err_msg += "#{@task_id}: MCollective call failed in agent '#{@agent}', "\
"method '#{method}', failed nodes: #{failed.map{|x| x.results[:sender]}.join(',')}"
err_msg += "MCollective call failed in agent '#{@agent}', "\
"method '#{method}', failed nodes: #{failed.map{|x| x.results[:sender]}.join(',')} \n"
end
raise err_msg unless err_msg.empty?
raise "#{@task_id}: #{err_msg}" unless err_msg.empty?
return res
end

9
deployment/mcollective/astute/lib/astute/network.rb
View File

@ -16,13 +16,14 @@ module Astute
# TODO Everything breakes if agent not found. We have to handle that
net_probe = MClient.new(ctx, "net_probe", uids)
net_probe.start_frame_listeners(:iflist => ['eth0'].to_json)
ctx.reporter.report({'progress' => 30, 'status' => 'verification'})
data_to_send = {'eth0' => networks.map {|n| n['vlan_id']}.join(',')}
net_probe.start_frame_listeners(:interfaces => data_to_send.to_json)
ctx.reporter.report({'progress' => 30})
# Interface name is hardcoded for now. Later we expect it to be passed from Nailgun backend
data_to_send = {'eth0' => networks.map {|n| n['vlan_id']}.join(',')}
net_probe.send_probing_frames(:interfaces => data_to_send.to_json)
ctx.reporter.report({'progress' => 60, 'status' => 'verification'})
ctx.reporter.report({'progress' => 60})
stats = net_probe.get_probing_info
result = stats.map {|node| {'uid' => node.results[:sender],

7
deployment/mcollective/astute/lib/astute/orchestrator.rb
View File

@ -3,7 +3,7 @@ module Astute
def initialize(deploy_engine=nil, log_parsing=false)
@deploy_engine = deploy_engine ||= Astute::DeploymentEngine::NailyFact
if log_parsing
@log_parser = LogParser::ParseNodeLogs.new('puppet-agent.log')
@log_parser = LogParser::ParseDeployLogs.new
else
@log_parser = LogParser::NoParsing.new
end
@ -26,6 +26,11 @@ module Astute
context = Context.new(task_id, proxy_reporter, @log_parser)
deploy_engine_instance = @deploy_engine.new(context)
Astute.logger.info "Using #{deploy_engine_instance.class} for deployment."
begin
@log_parser.prepare(nodes)
rescue Exception => e
Astute.logger.warn "Some error occurred when prepare LogParser: #{e.message}, trace: #{e.backtrace.inspect}"
end
deploy_engine_instance.deploy(nodes, attrs)
end

20
deployment/mcollective/astute/lib/astute/puppetd.rb
View File

@ -49,14 +49,14 @@ module Astute
nodes_to_check = running_nodes + succeed_nodes + error_nodes
unless nodes_to_check.size == last_run.size
raise "Should never happen. Internal error in nodes statuses calculation. Statuses calculated for: #{nodes_to_check.inspect},"
raise "Shoud never happen. Internal error in nodes statuses calculation. Statuses calculated for: #{nodes_to_check.inspect},"
"nodes passed to check statuses of: #{last_run.map {|n| n.results[:sender]}}"
end
return {'succeed' => succeed_nodes, 'error' => error_nodes, 'running' => running_nodes}
end
public
def self.deploy(ctx, nodes, retries=2, ignore_failure=false)
def self.deploy(ctx, nodes, retries=2, change_node_status=true)
# TODO: can we hide retries, ignore_failure into @ctx ?
uids = nodes.map {|n| n['uid']}
# TODO(mihgen): handle exceptions from mclient, raised if agent does not respond or responded with error
@ -67,12 +67,6 @@ module Astute
node_retries = {}
uids.each {|x| node_retries.merge!({x => retries}) }
begin
ctx.deploy_log_parser.add_separator(nodes)
rescue Exception => e
Astute.logger.warn "Some error occurred when add separator to logs: #{e.message}, trace: #{e.backtrace.inspect}"
end
Astute.logger.debug "Waiting for puppet to finish deployment on all nodes (timeout = #{Astute.config.PUPPET_TIMEOUT} sec)..."
time_before = Time.now
Timeout::timeout(Astute.config.PUPPET_TIMEOUT) do
@ -84,12 +78,8 @@ module Astute
Astute.logger.debug "Nodes statuses: #{calc_nodes.inspect}"
# At least we will report about successfully deployed nodes
nodes_to_report = calc_nodes['succeed'].map { |n| {'uid' => n, 'status' => 'ready'} }
if last_run[0].results[:data][:resources]["failed"]
puts "Puppet error while installing " + nodes_to_report.inspect.to_str
exit!
end
nodes_to_report = []
nodes_to_report.concat(calc_nodes['succeed'].map { |n| {'uid' => n, 'status' => 'ready'} }) if change_node_status
# Process retries
nodes_to_retry = []
@ -101,7 +91,7 @@ module Astute
nodes_to_retry << uid
else
Astute.logger.debug "Node #{uid.inspect} has failed to deploy. There is no more retries for puppet run."
nodes_to_report << {'uid' => uid, 'status' => 'error', 'error_type' => 'deploy'} unless ignore_failure
nodes_to_report << {'uid' => uid, 'status' => 'error', 'error_type' => 'deploy'} if change_node_status
end
end
if nodes_to_retry.any?

108
deployment/mcollective/astute/lib/astute/reporter.rb
View File

@ -2,7 +2,6 @@ require 'set'
STATES = {'offline' => 0,
'discover' => 10,
'verification' => 20,
'provisioning' => 30,
'provisioned' => 40,
'deploying' => 50,
@ -20,74 +19,87 @@ module Astute
nodes_to_report = []
nodes = (data['nodes'] or [])
nodes.each do |node|
node = validate(node)
node_here = @nodes.select {|x| x['uid'] == node['uid']}
if node_here.empty?
nodes_to_report << node
next
end
node_here = node_here[0]
# We need to update node here only if progress is greater, or status changed
if node_here.eql?(node)
next
end
unless node['status'].nil?
node_here_state = (STATES[node_here['status']] or 0)
if STATES[node['status']] < node_here_state
Astute.logger.error("Attempt to assign lower status detected: "\
"Status was: #{node_here['status']}, attempted to "\
"assign: #{node['status']}. Skipping this node (id=#{node['uid']})")
next
end
end
nodes_to_report << node
node = node_validate(node)
nodes_to_report << node if node
end
# Let's report only if nodes updated
if nodes_to_report.any?
data['nodes'] = nodes_to_report
@up_reporter.report(data)
# Replacing current list of nodes with the updated one, keeping not updated elements
uids = nodes_to_report.map {|x| x['uid']}
@nodes.delete_if {|x| uids.include?(x['uid'])}
@nodes.concat(nodes_to_report)
# Update nodes attributes in @nodes.
nodes_to_report.each do |node|
saved_node = @nodes.select {|x| x['uid'] == node['uid']}.first
if saved_node
node.each {|k, v| saved_node[k] = v}
else
@nodes << node
end
end
end
end
private
def validate(node)
err = ''
unless node['status'].nil?
err += "Status provided #{node['status']} is not supported." if STATES[node['status']].nil?
def node_validate(node)
# Validate basic correctness of attributes.
err = []
if node['status'].nil?
err << "progress value provided, but no status" unless node['progress'].nil?
else
err << "Status provided #{node['status']} is not supported" if STATES[node['status']].nil?
end
unless node['uid']
err += "node uid is not provided."
err << "Node uid is not provided"
end
unless node['progress'].nil?
err = "progress value provided, but no status." if node['status'].nil?
if err.any?
msg = "Validation of node: #{node.inspect} for report failed: #{err.join('; ')}."
Astute.logger.error(msg)
raise msg
end
raise "Validation of node: #{node.inspect} for report failed: #{err}" if err.any?
# Validate progress field.
unless node['progress'].nil?
if node['progress'] > 100
Astute.logger.error("Passed report for node with progress > 100: "\
Astute.logger.warn("Passed report for node with progress > 100: "\
"#{node.inspect}. Adjusting progress to 100.")
node['progress'] = 100
end
unless node['status'].nil?
if node['status'] == 'ready' and node['progress'] != 100
Astute.logger.error("In ready state node should have progress 100, "\
"but node passed: #{node.inspect}. Setting it to 100")
node['progress'] = 100
end
if node['status'] == 'verification'
# FIXME(mihgen): Currently our backend doesn't support such status. So let's just remove it...
node.delete('status')
end
if node['progress'] < 0
Astute.logger.warn("Passed report for node with progress < 0: "\
"#{node.inspect}. Adjusting progress to 0.")
node['progress'] = 0
end
end
if not node['status'].nil? and ['provisioned', 'ready'].include?(node['status']) and node['progress'] != 100
Astute.logger.warn("In #{node['status']} state node should have progress 100, "\
"but node passed: #{node.inspect}. Setting it to 100")
node['progress'] = 100
end
# Comparison with previous state.
saved_node = @nodes.select {|x| x['uid'] == node['uid']}.first
unless saved_node.nil?
saved_status = (STATES[saved_node['status']] or 0)
node_status = (STATES[node['status']] or saved_status)
saved_progress = (saved_node['progress'] or 0)
node_progress = (node['progress'] or saved_progress)
if node_status < saved_status
Astute.logger.warn("Attempt to assign lower status detected: "\
"Status was: #{saved_status}, attempted to "\
"assign: #{node_status}. Skipping this node (id=#{node['uid']})")
return
end
if node_progress < saved_progress and node_status == saved_status
Astute.logger.warn("Attempt to assign lesser progress detected: "\
"Progress was: #{saved_progress}, attempted to "\
"assign: #{node_progress}. Skipping this node (id=#{node['uid']})")
return
end
# We need to update node here only if progress is greater, or status changed
return if node.select{|k, v| not saved_node[k].eql?(v)}.empty?
end
return node
end
end

12
deployment/mcollective/astute/samples/compact.yaml
View File

@ -1,12 +0,0 @@
use_case: compact
fuel-controller-01.your-domain-name.com:
role: controller
fuel-controller-02.your-domain-name.com:
role: controller
fuel-controller-03.your-domain-name.com:
role: controller
fuel-compute-01.your-domain-name.com:
role: compute
fuel-compute-02.your-domain-name.com:
role: compute

19
deployment/mcollective/astute/samples/full.yaml
View File

@ -1,19 +0,0 @@
use_case: full
fuel-controller-01.your-domain-name.com:
role: controller
fuel-controller-02.your-domain-name.com:
role: controller
fuel-compute-01.your-domain-name.com:
role: compute
fuel-compute-02.your-domain-name.com:
role: compute
fuel-swift-01.your-domain-name.com:
role: storage
fuel-swift-02.your-domain-name.com:
role: storage
fuel-swift-03.your-domain-name.com:
role: storage
fuel-swiftproxy-01.your-domain-name.com:
role: proxy
fuel-swiftproxy-02.your-domain-name.com:
role: proxy

9
deployment/mcollective/astute/samples/minimal.yaml
View File

@ -1,9 +0,0 @@
use_case: minimal
fuel-controller-01.your-domain-name.com:
role: controller
fuel-controller-02.your-domain-name.com:
role: controller
fuel-compute-01.your-domain-name.com:
role: compute
fuel-compute-02.your-domain-name.com:
role: compute

9
deployment/mcollective/astute/samples/simple.yaml
View File

@ -1,9 +0,0 @@
use_case: simple
fuel-controller-01.your-domain-name.com:
role: controller
fuel-compute-01.your-domain-name.com:
role: compute
fuel-compute-02.your-domain-name.com:
role: compute
fuel-compute-03.your-domain-name.com:
role: compute

318
deployment/mcollective/astute/spec/unit/logparser_spec.rb
View File

@ -1,105 +1,267 @@
#!/usr/bin/env rspec
require File.join(File.dirname(__FILE__), "..", "spec_helper")
require 'tempfile'
require 'tmpdir'
require 'date'
include Astute
describe LogParser do
context "Pattern-based progress bar calculation (anaconda.log)" do
before :each do
@pattern_spec = {'type' => 'pattern-list', 'chunk_size' => 40000, # Size of block which reads for pattern searching.
'pattern_list' => [
{'pattern' => 'Running kickstart %%pre script', 'progress' => 0.08},
{'pattern' => 'to step enablefilesystems', 'progress' => 0.09},
{'pattern' => 'to step reposetup', 'progress' => 0.13},
{'pattern' => 'to step installpackages', 'progress' => 0.16},
{'pattern' => 'Installing',
'number' => 210, # Now it install 205 packets. Add 5 packets for growth in future.
'p_min' => 0.16, # min percent
'p_max' => 0.87 # max percent
},
{'pattern' => 'to step postinstallconfig', 'progress' => 0.87},
{'pattern' => 'to step dopostaction', 'progress' => 0.92},
].reverse
}
def get_statistics_variables(progress_table)
# Calculate some statistics variables: expectancy, standart deviation and
# correlation coefficient between real and ideal progress calculation.
total_time = 0
real_expectancy = 0
real_sqr_expectancy = 0
prev_event_date = nil
progress_table.each do |el|
date = el[:date]
prev_event_date = date unless prev_event_date
progress = el[:progress].to_f
period = date - prev_event_date
hours, mins, secs, frac = Date::day_fraction_to_time(period)
period_in_sec = hours * 60 * 60 + mins * 60 + secs
total_time += period_in_sec
real_expectancy += period_in_sec * progress
real_sqr_expectancy += period_in_sec * progress ** 2
el[:time_delta] = period_in_sec
prev_event_date = date
end
def test_supposed_time_parser(pattern_spec)
# Calculate standart deviation for real progress distibution.
real_expectancy = real_expectancy.to_f / total_time
real_sqr_expectancy = real_sqr_expectancy.to_f / total_time
real_standart_deviation = Math.sqrt(real_sqr_expectancy - real_expectancy ** 2)
# Calculate PCC (correlation coefficient).
ideal_sqr_expectancy = 0
ideal_expectancy = 0
t = 0
ideal_delta = 100.0 / total_time
mixed_expectancy = 0
progress_table.each do |el|
t += el[:time_delta]
ideal_progress = t * ideal_delta
ideal_expectancy += ideal_progress * el[:time_delta]
ideal_sqr_expectancy += ideal_progress ** 2 * el[:time_delta]
el[:ideal_progress] = ideal_progress
mixed_expectancy += el[:progress] * ideal_progress * el[:time_delta]
end
ideal_expectancy = ideal_expectancy / total_time
ideal_sqr_expectancy = ideal_sqr_expectancy / total_time
mixed_expectancy = mixed_expectancy / total_time
ideal_standart_deviation = Math.sqrt(ideal_sqr_expectancy - ideal_expectancy ** 2)
covariance = mixed_expectancy - ideal_expectancy * real_expectancy
pcc = covariance / (ideal_standart_deviation * real_standart_deviation)
statistics = {
'real_expectancy' => real_expectancy,
'real_sqr_expectancy' => real_sqr_expectancy,
'real_standart_deviation' => real_standart_deviation,
'ideal_expectancy' => ideal_expectancy,
'ideal_sqr_expectancy' => ideal_sqr_expectancy,
'ideal_standart_deviation' => ideal_standart_deviation,
'mixed_expectancy' => mixed_expectancy,
'covariance' => covariance,
'pcc' => pcc,
'total_time' => total_time,
}
return statistics
end
def get_next_line(fo, date_regexp, date_format)
until fo.eof?
line = fo.readline
date_string = line.match(date_regexp)
if date_string
date = DateTime.strptime(date_string[0], date_format)
return line, date
end
end
end
def get_next_lines_by_date(fo, now, date_regexp, date_format)
lines = ''
until fo.eof?
pos = fo.pos
line, date = get_next_line(fo, date_regexp, date_format)
if date <= now
lines += line
else
fo.pos = pos
return lines
end
end
return lines
end
context "Correlation coeff. (PCC) of Provisioning progress bar calculation" do
def provision_parser_wrapper(node)
uids = [node['uid']]
nodes = [node]
time_delta = 5.0/24/60/60
log_delay = 6*time_delta
deploy_parser = Astute::LogParser::ParseProvisionLogs.new
pattern_spec = deploy_parser.pattern_spec
date_regexp = '^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}'
date_format = '%Y-%m-%dT%H:%M:%S'
fo = Tempfile.new('logparse')
logfile = File.join(File.dirname(__FILE__), "..", "example-logs", "anaconda.log_")
path = fo.path
initial_progress = Astute::LogParser.get_log_progress(path, pattern_spec)
initial_progress.should eql(0)
progress_table = []
File.open(logfile).each do |line|
fo.write(line)
fo.flush
date_string = line.match(date_regexp)
if date_string
date = DateTime.strptime(date_string[0], date_format)
progress = Astute::LogParser.get_log_progress(path, pattern_spec)
progress_table << {'date' => date, 'progress' => progress}
Dir.mktmpdir do |dir|
# Create temp log files and structures.
pattern_spec['path_prefix'] = "#{dir}/"
path = "#{pattern_spec['path_prefix']}#{node['ip']}/#{pattern_spec['filename']}"
Dir.mkdir(File.dirname(File.dirname(path)))
Dir.mkdir(File.dirname(path))
node['file'] = File.open(path, 'w')
src_filename = File.join(File.dirname(__FILE__), "..", "example-logs", node['src_filename'])
node['src'] = File.open(src_filename)
line, date = get_next_line(node['src'], date_regexp, date_format)
node['src'].pos = 0
node['now'] = date - log_delay
node['progress_table'] ||= []
# End 'while' cycle if reach EOF at all src files.
until node['src'].eof?
# Copy logs line by line from example logfile to tempfile and collect progress for each step.
lines, date = get_next_lines_by_date(node['src'], node['now'], date_regexp, date_format)
node['file'].write(lines)
node['file'].flush
node['last_lines'] = lines
DateTime.stubs(:now).returns(node['now'])
node_progress = deploy_parser.progress_calculate(uids, nodes)[0]
node['progress_table'] << {:date => node['now'], :progress => node_progress['progress']}
node['now'] += time_delta
end
nodes.each do |node|
node['statistics'] = get_statistics_variables(node['progress_table'])
end
# Clear temp files.
node['file'].close
File.unlink(node['file'].path)
Dir.unlink(File.dirname(node['file'].path))
end
fo.close!
first_event_date, first_progress = progress_table[0]['date'], progress_table[0]['progress']
last_event_date, last_progress = progress_table[-1]['date'], progress_table[-1]['progress']
period = (last_event_date - first_event_date) / (last_progress - first_progress)
hours, mins, secs, frac = Date::day_fraction_to_time(period)
# FIXME(mihgen): I hope this calculation can be much simplified: needs refactoring
# Assuming server was in reboot for reboot_time
reboot_time = 30
# period will be useful for other test cases
period_in_sec = hours * 60 * 60 + mins * 60 + secs + reboot_time
# Let's normalize the time in table
progress_table.each do |el|
delta = el['date'] - first_event_date
hours, mins, secs, frac = Date::day_fraction_to_time(delta)
delta_in_sec = hours * 60 * 60 + mins * 60 + secs
el['time'] = delta_in_sec + reboot_time
end
return progress_table, period_in_sec
return node
end
it "new progress must be equal or greater than previous" do
progress_table, period_in_sec = test_supposed_time_parser(@pattern_spec)
progress_table.each_cons(2) do |el|
el[1]['progress'].should be >= el[0]['progress']
el[0]['progress'].should be >= 0
el[1]['progress'].should be <= 1
end
it "should be greather than 0.96" do
node = {'uid' => '1', 'ip' => '1.0.0.1', 'role' => 'controller', 'src_filename' => 'anaconda.log_',
'meta' => { 'disks' =>
[
{'name' => 'flash drive', 'removable' => true, 'size' => 1000},
{'name' => 'sda', 'removable'=> false, 'size' => 32*1000*1000*1000},
]
}
}
calculated_node = provision_parser_wrapper(node)
calculated_node['statistics']['pcc'].should > 0.96
end
it "it should move smoothly"
it "it must be updated at least 5 times" do
# Otherwise progress bar has no meaning I guess...
pending('Not yet implemented')
end
end
end
context "Correlation coeff. (PCC) of Deploying progress bar calculation" do
def deployment_parser_wrapper(cluster_type, nodes)
uids = nodes.map{|n| n['uid']}
#pattern_spec = {'type' => 'supposed_time',
#'chunk_size' => 10000,
#'date_format' => '%Y-%m-%dT%H:%M:%S',
#'date_regexp' => '^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}',
#'pattern_list' => [
#{'pattern' => 'Running anaconda script', 'supposed_time' => 60},
#{'pattern' => 'moving (1) to step enablefilesystems', 'supposed_time' => 3},
#{'pattern' => "notifying kernel of 'change' event on device", 'supposed_time' => 97},
#{'pattern' => 'Preparing to install packages', 'supposed_time' => 8},
#{'pattern' => 'Installing glibc-common-2.12', 'supposed_time' => 9},
#{'pattern' => 'Installing bash-4.1.2', 'supposed_time' => 10},
#{'pattern' => 'Installing coreutils-8.4-19', 'supposed_time' => 20},
#{'pattern' => 'Installing centos-release-6-3', 'supposed_time' => 20},
#{'pattern' => 'Installing attr-2.4.44', 'supposed_time' => 19},
#{'pattern' => 'leaving (1) step installpackages', 'supposed_time' => 51},
#{'pattern' => 'moving (1) to step postscripts', 'supposed_time' => 3},
#{'pattern' => 'leaving (1) step postscripts', 'supposed_time' => 132},
#].reverse,
#}
deploy_parser = Astute::LogParser::ParseDeployLogs.new(cluster_type)
pattern_spec = deploy_parser.pattern_spec
date_regexp = '^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}'
date_format = '%Y-%m-%dT%H:%M:%S'
Dir.mktmpdir do |dir|
# Create temp log files and structures.
pattern_spec['path_prefix'] = "#{dir}/"
nodes.each do |node|
path = "#{pattern_spec['path_prefix']}#{node['ip']}/#{pattern_spec['filename']}"
Dir.mkdir(File.dirname(path))
node['file'] = File.open(path, 'w')
src_filename = File.join(File.dirname(__FILE__), "..", "example-logs", node['src_filename'])
node['src'] = File.open(src_filename)
node['progress_table'] ||= []
end
# End 'while' cycle if reach EOF at all src files.
while nodes.index{|n| not n['src'].eof?}
# Copy logs line by line from example logfile to tempfile and collect progress for each step.
nodes.each do |node|
unless node['src'].eof?
line = node['src'].readline
node['file'].write(line)
node['file'].flush
node['last_line'] = line
else
node['last_line'] = ''
end
end
nodes_progress = deploy_parser.progress_calculate(uids, nodes)
nodes_progress.each do |progress|
node = nodes.at(nodes.index{|n| n['uid'] == progress['uid']})
date_string = node['last_line'].match(date_regexp)
if date_string
date = DateTime.strptime(date_string[0], date_format)
node['progress_table'] << {:date => date, :progress => progress['progress']}
end
end
end
nodes.each do |node|
node['statistics'] = get_statistics_variables(node['progress_table'])
end
# Clear temp files.
nodes.each do |n|
n['file'].close
File.unlink(n['file'].path)
Dir.unlink(File.dirname(n['file'].path))
end
end
return nodes
end
it "should be greather than 0.85 for HA deployment" do
nodes = [
{'uid' => '1', 'ip' => '1.0.0.1', 'role' => 'controller', 'src_filename' => 'puppet-agent.log.ha.contr.2'},
{'uid' => '2', 'ip' => '1.0.0.2', 'role' => 'compute', 'src_filename' => 'puppet-agent.log.ha.compute'},
]
calculated_nodes = deployment_parser_wrapper('ha_compute', nodes)
calculated_nodes.each {|node| node['statistics']['pcc'].should > 0.85}
# For debug purposes.
# print "\n"
# calculated_nodes.each do |node|
# print node['statistics'].inspect, "\n", node['statistics']['pcc'], "\n", node['progress_table'][-1][:progress], "\n"
# end
end
it "should be greather than 0.97 for singlenode deployment" do
nodes = [
{'uid' => '1', 'ip' => '1.0.0.1', 'role' => 'controller', 'src_filename' => 'puppet-agent.log.singlenode'},
]
calculated_nodes = deployment_parser_wrapper('singlenode_compute', nodes)
calculated_nodes.each {|node| node['statistics']['pcc'].should > 0.97}
end
it "should be greather than 0.94 for multinode deployment" do
nodes = [
{'uid' => '1', 'ip' => '1.0.0.1', 'role' => 'controller', 'src_filename' => 'puppet-agent.log.multi.contr'},
{'uid' => '2', 'ip' => '1.0.0.2', 'role' => 'compute', 'src_filename' => 'puppet-agent.log.multi.compute'},
]
calculated_nodes = deployment_parser_wrapper('multinode_compute', nodes)
calculated_nodes.each {|node| node['statistics']['pcc'].should > 0.94}
end
end
end

2
deployment/mcollective/astute/spec/unit/mclient_spec.rb
View File

@ -74,6 +74,6 @@ describe MClient do
mclient = MClient.new(@ctx, "faketest", nodes.map {|x| x['uid']})
mclient.retries = 1
expect { mclient.echo(:msg => 'hello world') }.to \
raise_error(/MCollective agents '3' didn't respond.\n.* failed nodes: 2/)
raise_error(/MCollective agents '3' didn't respond. \n.* failed nodes: 2/)
end
end

20
deployment/mcollective/astute/spec/unit/nailyfact_deploy_spec.rb
View File

@ -142,19 +142,31 @@ describe "NailyFact DeploymentEngine" do
@data['args']['attributes']['deployment_mode'] = "multinode_compute"
Astute::Metadata.expects(:publish_facts).times(@data['args']['nodes'].size)
# we got two calls, one for controller, and another for all computes
Astute::PuppetdDeployer.expects(:deploy).twice
controller_nodes = @data['args']['nodes'].select{|n| n['role'] == 'controller'}
compute_nodes = @data['args']['nodes'].select{|n| n['role'] == 'compute'}
Astute::PuppetdDeployer.expects(:deploy).with(@ctx, controller_nodes, instance_of(Fixnum), true).once
Astute::PuppetdDeployer.expects(:deploy).with(@ctx, compute_nodes, instance_of(Fixnum), true).once
@deploy_engine.deploy(@data['args']['nodes'], @data['args']['attributes'])
end
it "ha_compute deploy should not raise any exception" do
Astute::Metadata.expects(:publish_facts).at_least_once
Astute::PuppetdDeployer.expects(:deploy).times(7)
controller_nodes = @data_ha['args']['nodes'].select{|n| n['role'] == 'controller'}
compute_nodes = @data_ha['args']['nodes'].select{|n| n['role'] == 'compute'}
controller_nodes.each do |n|
Astute::PuppetdDeployer.expects(:deploy).with(@ctx, [n], 0, false).once
end
Astute::PuppetdDeployer.expects(:deploy).with(@ctx, controller_nodes, 0, false).once
Astute::PuppetdDeployer.expects(:deploy).with(@ctx, [controller_nodes.first], 0, false).once
Astute::PuppetdDeployer.expects(:deploy).with(@ctx, controller_nodes, 0, false).once
Astute::PuppetdDeployer.expects(:deploy).with(@ctx, controller_nodes, 3, true).once
Astute::PuppetdDeployer.expects(:deploy).with(@ctx, compute_nodes, instance_of(Fixnum), true).once
@deploy_engine.deploy(@data_ha['args']['nodes'], @data_ha['args']['attributes'])
end
it "ha_compute deploy should not raise any exception if there are only one controller" do
Astute::Metadata.expects(:publish_facts).at_least_once
Astute::PuppetdDeployer.expects(:deploy).times(4)
Astute::PuppetdDeployer.expects(:deploy).times(5)
ctrl = @data_ha['args']['nodes'].select {|n| n['role'] == 'controller'}[0]
@deploy_engine.deploy([ctrl], @data_ha['args']['attributes'])
end
@ -163,7 +175,7 @@ describe "NailyFact DeploymentEngine" do
@data['args']['attributes']['deployment_mode'] = "singlenode_compute"
@data['args']['nodes'] = [@data['args']['nodes'][0]] # We have only one node in singlenode
Astute::Metadata.expects(:publish_facts).times(@data['args']['nodes'].size)
Astute::PuppetdDeployer.expects(:deploy).once # one call for one node
Astute::PuppetdDeployer.expects(:deploy).with(@ctx, @data['args']['nodes'], instance_of(Fixnum), true).once
@deploy_engine.deploy(@data['args']['nodes'], @data['args']['attributes'])
end
end

74
deployment/mcollective/astute/spec/unit/puppetd_spec.rb
View File

@ -14,7 +14,7 @@ describe "Puppetd" do
end
it "reports ready status for node if puppet deploy finished successfully" do
@reporter.expects(:report).with('nodes' => [{'uid' => '1', 'status' => 'ready'}])
@reporter.expects(:report).with('nodes' => [{'uid' => '1', 'status' => 'ready', 'progress' => 100}])
last_run_result = {:data=>
{:time=>{"last_run"=>1358425701},
:status => "running", :resources => {'failed' => 0},
@ -45,8 +45,41 @@ describe "Puppetd" do
Astute::PuppetdDeployer.deploy(@ctx, nodes, retries=0)
end
it "doesn't report ready status for node if change_node_status disabled" do
@reporter.expects(:report).never
last_run_result = {:data=>
{:time=>{"last_run"=>1358425701},
:status => "running", :resources => {'failed' => 0},
:running => 1, :idling => 0},
:sender=>"1"}
last_run_result_new = Marshal.load(Marshal.dump(last_run_result))
last_run_result_new[:data][:time]['last_run'] = 1358426000
last_run_result_finished = Marshal.load(Marshal.dump(last_run_result))
last_run_result_finished[:data][:status] = 'stopped'
last_run_result_finished[:data][:time]['last_run'] = 1358427000
nodes = [{'uid' => '1'}]
rpcclient = mock_rpcclient(nodes)
rpcclient_valid_result = mock_mc_result(last_run_result)
rpcclient_new_res = mock_mc_result(last_run_result_new)
rpcclient_finished_res = mock_mc_result(last_run_result_finished)
rpcclient.stubs(:last_run_summary).returns([rpcclient_valid_result]).then.
returns([rpcclient_valid_result]).then.
returns([rpcclient_new_res]).then.
returns([rpcclient_finished_res])
rpcclient.expects(:runonce).at_least_once.returns([rpcclient_valid_result])
Astute::PuppetdDeployer.deploy(@ctx, nodes, retries=0, change_node_status=false)
end
it "publishes error status for node if puppet failed" do
@reporter.expects(:report).with('nodes' => [{'status' => 'error', 'error_type' => 'deploy', 'uid' => '1'}])
@reporter.expects(:report).with('nodes' => [{'status' => 'error',
'error_type' => 'deploy', 'uid' => '1'}])
last_run_result = {:statuscode=>0, :data=>
{:changes=>{"total"=>1}, :time=>{"last_run"=>1358425701},
@ -80,8 +113,43 @@ describe "Puppetd" do
Astute::PuppetdDeployer.deploy(@ctx, nodes, retries=0)
end
it "doesn't publish error status for node if change_node_status disabled" do
@reporter.expects(:report).never
last_run_result = {:statuscode=>0, :data=>
{:changes=>{"total"=>1}, :time=>{"last_run"=>1358425701},
:resources=>{"failed"=>0}, :status => "running",
:running => 1, :idling => 0, :runtime => 100},
:sender=>"1"}
last_run_result_new = Marshal.load(Marshal.dump(last_run_result))
last_run_result_new[:data][:time]['last_run'] = 1358426000
last_run_result_new[:data][:resources]['failed'] = 1
nodes = [{'uid' => '1'}]
last_run_result_finished = Marshal.load(Marshal.dump(last_run_result))
last_run_result_finished[:data][:status] = 'stopped'
last_run_result_finished[:data][:time]['last_run'] = 1358427000
last_run_result_finished[:data][:resources]['failed'] = 1
rpcclient = mock_rpcclient(nodes)
rpcclient_valid_result = mock_mc_result(last_run_result)
rpcclient_new_res = mock_mc_result(last_run_result_new)
rpcclient_finished_res = mock_mc_result(last_run_result_finished)
rpcclient.stubs(:last_run_summary).returns([rpcclient_valid_result]).then.
returns([rpcclient_valid_result]).then.
returns([rpcclient_new_res]).then.
returns([rpcclient_finished_res])
rpcclient.expects(:runonce).at_least_once.returns([rpcclient_valid_result])
MClient.any_instance.stubs(:rpcclient).returns(rpcclient)
Astute::PuppetdDeployer.deploy(@ctx, nodes, retries=0, change_node_status=false)
end
it "retries to run puppet if it fails" do
@reporter.expects(:report).with('nodes' => [{'uid' => '1', 'status' => 'ready'}])
@reporter.expects(:report).with('nodes' => [{'uid' => '1', 'status' => 'ready', 'progress' => 100}])
last_run_result = {:statuscode=>0, :data=>
{:changes=>{"total"=>1}, :time=>{"last_run"=>1358425701},

81
deployment/mcollective/astute/spec/unit/reporter_spec.rb
View File

@ -53,20 +53,45 @@ describe "ProxyReporter" do
end
it "adjusts progress to 100 if passed greater" do
expected_msg = Marshal.load(Marshal.dump(@msg_pr))
expected_msg['nodes'][1]['progress'] = 100
@msg_pr['nodes'][1]['progress'] = 120
input_msg = {'nodes' => [{'uid' => 1, 'status' => 'deploying', 'progress' => 120}]}
expected_msg = {'nodes' => [{'uid' => 1, 'status' => 'deploying', 'progress' => 100}]}
@up_reporter.expects(:report).with(expected_msg)
@reporter.report(@msg_pr)
@reporter.report(input_msg)
end
it "adjusts progress to 100 if status ready" do
expected_msg = Marshal.load(Marshal.dump(@msg_pr))
expected_msg['nodes'][1]['progress'] = 100
expected_msg['nodes'][1]['status'] = 'ready'
@msg_pr['nodes'][1]['status'] = 'ready'
it "adjusts progress to 0 if passed less" do
input_msg = {'nodes' => [{'uid' => 1, 'status' => 'deploying', 'progress' => -20}]}
expected_msg = {'nodes' => [{'uid' => 1, 'status' => 'deploying', 'progress' => 0}]}
@up_reporter.expects(:report).with(expected_msg)
@reporter.report(@msg_pr)
@reporter.report(input_msg)
end
it "adjusts progress to 100 if status provisioned and no progress given" do
input_msg = {'nodes' => [{'uid' => 1, 'status' => 'provisioned'}]}
expected_msg = {'nodes' => [{'uid' => 1, 'status' => 'provisioned', 'progress' => 100}]}
@up_reporter.expects(:report).with(expected_msg)
@reporter.report(input_msg)
end
it "adjusts progress to 100 if status ready and no progress given" do
input_msg = {'nodes' => [{'uid' => 1, 'status' => 'ready'}]}
expected_msg = {'nodes' => [{'uid' => 1, 'status' => 'ready', 'progress' => 100}]}
@up_reporter.expects(:report).with(expected_msg)
@reporter.report(input_msg)
end
it "adjusts progress to 100 if status provisioned with progress" do
input_msg = {'nodes' => [{'uid' => 1, 'status' => 'provisioned', 'progress' => 50}]}
expected_msg = {'nodes' => [{'uid' => 1, 'status' => 'provisioned', 'progress' => 100}]}
@up_reporter.expects(:report).with(expected_msg)
@reporter.report(input_msg)
end
it "adjusts progress to 100 if status ready with progress" do
input_msg = {'nodes' => [{'uid' => 1, 'status' => 'ready', 'progress' => 50}]}
expected_msg = {'nodes' => [{'uid' => 1, 'status' => 'ready', 'progress' => 100}]}
@up_reporter.expects(:report).with(expected_msg)
@reporter.report(input_msg)
end
it "does not report if node was in ready, and trying to set is deploying" do
@ -125,5 +150,41 @@ describe "ProxyReporter" do
@up_reporter.expects(:report).with(msgs[3])
msgs.each {|msg| @reporter.report(msg)}
end
it "doesn't update progress if it less than previous progress with same status" do
msgs = [{'nodes' => [{'uid' => 1, 'status' => 'provisioning', 'progress' => 50}]},
{'nodes' => [{'uid' => 1, 'status' => 'provisioning', 'progress' => 10}]},
{'nodes' => [{'uid' => 1, 'status' => 'deploying', 'progress' => 50}]},
{'nodes' => [{'uid' => 1, 'status' => 'deploying', 'progress' => 10}]}]
@up_reporter.expects(:report).with(msgs[0])
@up_reporter.expects(:report).with(msgs[2])
@up_reporter.expects(:report).never
msgs.each {|msg| @reporter.report(msg)}
end
it "updates progress if it less than previous progress when changing status" do
msgs = [{'nodes' => [{'uid' => 1, 'status' => 'provisioning', 'progress' => 50}]},
{'nodes' => [{'uid' => 1, 'status' => 'provisioned'}]},
{'nodes' => [{'uid' => 1, 'status' => 'provisioned', 'progress' => 100}]},
{'nodes' => [{'uid' => 1, 'status' => 'deploying', 'progress' => 0}]}]
@up_reporter.expects(:report).with(msgs[0])
@up_reporter.expects(:report).with(msgs[2])
@up_reporter.expects(:report).with(msgs[3])
@up_reporter.expects(:report).never
msgs.each {|msg| @reporter.report(msg)}
end
it "doesn't forget previously reported attributes" do
msgs = [{'nodes' => [{'uid' => 1, 'status' => 'provisioning', 'progress' => 50}]},
{'nodes' => [{'uid' => 1, 'status' => 'provisioning'}]},
{'nodes' => [{'uid' => 1, 'status' => 'provisioning', 'key' => 'value'}]},
{'nodes' => [{'uid' => 1, 'status' => 'provisioning', 'progress' => 0}]},
]
@up_reporter.expects(:report).with(msgs[0])
@up_reporter.expects(:report).with(msgs[2])
@up_reporter.expects(:report).never
msgs.each {|msg| @reporter.report(msg)}
end
end
end

2
deployment/mcollective/astute/spec/unit/simplepuppet_deploy_spec.rb
View File

@ -42,7 +42,7 @@ describe "SimplePuppet DeploymentEngine" do
it "ha_compute deploy should not raise any exception" do
@env['attributes']['deployment_mode'] = "ha_compute"
Astute::Metadata.expects(:publish_facts).never
Astute::PuppetdDeployer.expects(:deploy).times(5)
Astute::PuppetdDeployer.expects(:deploy).times(6)
@deploy_engine.deploy(@env['nodes'], @env['attributes'])
end

369
deployment/mcollective/astute/templates/site_openstack_swift_standalone.erb
View File

@ -1,369 +0,0 @@
#
# Parameter values in this file should be changed, taking into consideration your
# networking setup and desired OpenStack settings.
#
# Please consult with the latest Fuel User Guide before making edits.
#
# This is a name of public interface. Public network provides address space for Floating IPs, as well as public IP accessibility to the API endpoints.
$public_interface = 'eth1'
# This is a name of internal interface. It will be hooked to the management network, where data exchange between components of the OpenStack cluster will happen.
$internal_interface = 'eth0'
# This is a name of private interface. All traffic within OpenStack tenants' networks will go through this interface.
$private_interface = 'eth2'
# Public and Internal VIPs. These virtual addresses are required by HA topology and will be managed by keepalived.
$internal_virtual_ip = '<%= internal_virtual_ip %>'
$public_virtual_ip = '<%= public_virtual_ip %>'
# Map of controller IP addresses on internal interfaces. Must have an entry for every controller node.
$controller_internal_addresses = {'fuel-controller-01' => '10.0.0.101','fuel-controller-02' => '10.0.0.102','fuel-controller-03' => '10.0.0.103'}
# Specify pools for Floating IP and Fixed IP.
# Floating IP addresses are used for communication of VM instances with the outside world (e.g. Internet).
# Fixed IP addresses are typically used for communication between VM instances.
$create_networks = <%= create_networks %>
$floating_range = '<%= floating_range %>'
$fixed_range = '<%= fixed_range %>'
$num_networks = 1
$network_size = 31
$vlan_start = <%= startvlan %>
# If $external_ipinfo option is not defined the addresses will be calculated automatically from $floating_range:
# the first address will be defined as an external default router
# second address will be set to an uplink bridge interface (br-ex)
# remaining addresses are utilized for ip floating pool
$external_ipinfo = {}
## $external_ipinfo = {
## 'public_net_router' => '<%= public_net_router %>',
## 'ext_bridge' => '<%= ext_bridge %>',
## 'pool_start' => '<%= pool_start %>',
## 'pool_end' => '<%= pool_end %>',
## }
# For VLAN networks: valid VLAN VIDs are 1 through 4094.
# For GRE networks: Valid tunnel IDs are any 32 bit unsigned integer.
$segment_range = '<%= segment_range %>'
# it should be set to an integer value (valid range is 0..254)
$deployment_id = '59'
# Here you can enable or disable different services, based on the chosen deployment topology.
$cinder = true
$cinder_on_computes = false
$multi_host = true
$manage_volumes = true
$quantum = true
$auto_assign_floating_ip = false
$glance_backend = 'swift'
# Use loopback device for swift
# set 'loopback' or false
$swift_loopback = 'loopback'
# Set master hostname for the HA cluster of controller nodes, as well as hostnames for every controller in the cluster.
# Set master hostname as fully qualified domain name if FQDNs are used in $controller_internal_addresses
$master_hostname = '<%= master_hostname %>'
# Set short hostnames only to $controller_hostnames. RabbitMQ will not work if Fully Qualified domain names set here!
$controller_hostnames = ['fuel-controller-01', 'fuel-controller-02', 'fuel-controller-03']
$swift_master = 'fuel-swiftproxy-01'
# Set up OpenStack network manager
$network_manager = 'nova.network.manager.FlatDHCPManager'
# Setup network interface, which Cinder used for export iSCSI targets.
$cinder_iscsi_bind_iface = $internal_interface
# Here you can add physical volumes to cinder. Please replace values with the actual names of devices.
$nv_physical_volume = ['/dev/sdz', '/dev/sdy', '/dev/sdx']
# Specify credentials for different services
$mysql_root_password = 'nova'
$admin_email = 'openstack@openstack.org'
$admin_password = 'nova'
$keystone_db_password = 'nova'
$keystone_admin_token = 'nova'
$glance_db_password = 'nova'
$glance_user_password = 'nova'
$nova_db_password = 'nova'
$nova_user_password = 'nova'
$rabbit_password = 'nova'
$rabbit_user = 'nova'
$swift_user_password = 'swift_pass'
$swift_shared_secret = 'changeme'
$quantum_user_password = 'quantum_pass'
$quantum_db_password = 'quantum_pass'
$quantum_db_user = 'quantum'
$quantum_db_dbname = 'quantum'
$tenant_network_type = 'gre'
$quantum_host = $internal_virtual_ip
$use_syslog = <%= use_syslog %>
if $use_syslog {
class { "::rsyslog::client":
log_local => true,
log_auth_local => true,
server => '<%= syslog_server %>',
port => '514'
}
}
case $::osfamily {
"Debian": {
$rabbitmq_version_string = '2.7.1-0ubuntu4'
}
"RedHat": {
$rabbitmq_version_string = '2.8.7-2.el6'
}
}
# OpenStack packages to be installed
$openstack_version = {
'keystone' => 'latest',
'glance' => 'latest',
'horizon' => 'latest',
'nova' => 'latest',
'novncproxy' => 'latest',
'cinder' => 'latest',
'rabbitmq_version' => $rabbitmq_version_string,
}
$mirror_type = 'external'
$internal_address = getvar("::ipaddress_${internal_interface}")
$quantum_sql_connection = "mysql://${quantum_db_user}:${quantum_db_password}@${quantum_host}/${quantum_db_dbname}"
$controller_node_public = $internal_virtual_ip
$swift_local_net_ip = $internal_address
$swift_proxies = {'fuel-swiftproxy-01' => '10.0.0.108','fuel-swiftproxy-02' => '10.0.0.109'}
$verbose = true
Exec { logoutput => true }
# Globally apply an environment-based tag to all resources on each node.
tag("${::deployment_id}::${::environment}")
stage { 'openstack-custom-repo': before => Stage['main'] }
class { 'openstack::mirantis_repos': stage => 'openstack-custom-repo', type => $mirror_type }
if $::operatingsystem == 'Ubuntu' {
class { 'openstack::apparmor::disable': stage => 'openstack-custom-repo' }
}
#Rate Limits for cinder and Nova
#Cinder and Nova can rate-limit your requests to API services
#These limits can be small for your installation or usage scenario
#Change the following variables if you want. The unit is requests per minute.
$nova_rate_limits = { 'POST' => 1000,
'POST_SERVERS' => 1000,
'PUT' => 1000, 'GET' => 1000,
'DELETE' => 1000 }
$cinder_rate_limits = { 'POST' => 1000,
'POST_SERVERS' => 1000,
'PUT' => 1000, 'GET' => 1000,
'DELETE' => 1000 }
sysctl::value { 'net.ipv4.conf.all.rp_filter': value => '0' }
# Definition of OpenStack controller nodes.
node /fuel-controller-[\d+]/ {
class { 'openstack::controller_ha':
controller_public_addresses => $controller_public_addresses,
controller_internal_addresses => $controller_internal_addresses,
internal_address => $internal_address,
public_interface => $public_interface,
internal_interface => $internal_interface,
private_interface => $private_interface,
internal_virtual_ip => $internal_virtual_ip,
public_virtual_ip => $public_virtual_ip,
master_hostname => $master_hostname,
floating_range => $floating_range,
fixed_range => $fixed_range,
multi_host => $multi_host,
network_manager => $network_manager,
num_networks => $num_networks,
network_size => $network_size,
network_config => { 'vlan_start' => $vlan_start },
verbose => $verbose,
auto_assign_floating_ip => $auto_assign_floating_ip,
mysql_root_password => $mysql_root_password,
admin_email => $admin_email,
admin_password => $admin_password,
keystone_db_password => $keystone_db_password,
keystone_admin_token => $keystone_admin_token,
glance_db_password => $glance_db_password,
glance_user_password => $glance_user_password,
nova_db_password => $nova_db_password,
nova_user_password => $nova_user_password,
rabbit_password => $rabbit_password,
rabbit_user => $rabbit_user,
rabbit_nodes => $controller_hostnames,
memcached_servers => $controller_hostnames,
export_resources => false,
glance_backend => $glance_backend,
swift_proxies => $swift_proxies,
quantum => $quantum,
quantum_user_password => $quantum_user_password,
quantum_db_password => $quantum_db_password,
quantum_db_user => $quantum_db_user,
quantum_db_dbname => $quantum_db_dbname,
tenant_network_type => $tenant_network_type,
segment_range => $segment_range,
cinder => $cinder,
cinder_iscsi_bind_iface => $cinder_iscsi_bind_iface,
manage_volumes => $manage_volumes,
galera_nodes => $controller_hostnames,
nv_physical_volume => $nv_physical_volume,
use_syslog => $use_syslog,
nova_rate_limits => $nova_rate_limits,
cinder_rate_limits => $cinder_rate_limits
}
class { 'swift::keystone::auth':
password => $swift_user_password,
public_address => $public_virtual_ip,
internal_address => $internal_virtual_ip,
admin_address => $internal_virtual_ip,
}
}
# Definition of OpenStack compute nodes.
node /fuel-compute-[\d+]/ {
class { 'openstack::compute':
public_interface => $public_interface,
private_interface => $private_interface,
internal_address => $internal_address,
libvirt_type => 'qemu',
fixed_range => $fixed_range,
network_manager => $network_manager,
network_config => { 'vlan_start' => $vlan_start },
multi_host => $multi_host,
sql_connection => "mysql://nova:${nova_db_password}@${internal_virtual_ip}/nova",
rabbit_nodes => $controller_hostnames,
rabbit_password => $rabbit_password,
rabbit_user => $rabbit_user,
glance_api_servers => "${internal_virtual_ip}:9292",
vncproxy_host => $public_virtual_ip,
verbose => $verbose,
vnc_enabled => true,
manage_volumes => $manage_volumes,
nova_user_password => $nova_user_password,
cache_server_ip => $controller_hostnames,
service_endpoint => $internal_virtual_ip,
quantum => $quantum,
quantum_host => $quantum_host,
quantum_sql_connection => $quantum_sql_connection,
quantum_user_password => $quantum_user_password,
tenant_network_type => $tenant_network_type,
segment_range => $segment_range,
cinder => $cinder_on_computes,
cinder_iscsi_bind_iface=> $cinder_iscsi_bind_iface,
nv_physical_volume => $nv_physical_volume,
db_host => $internal_virtual_ip,
ssh_private_key => 'puppet:///ssh_keys/openstack',
ssh_public_key => 'puppet:///ssh_keys/openstack.pub',
use_syslog => $use_syslog,
nova_rate_limits => $nova_rate_limits,
cinder_rate_limits => $cinder_rate_limits
}
}
# Definition of the first OpenStack Swift node.
node /fuel-swift-01/ {
$swift_zone = 1
class { 'openstack::swift::storage_node':
storage_type => $swift_loopback,
swift_zone => $swift_zone,
swift_local_net_ip => $internal_address,
}
}
# Definition of the second OpenStack Swift node.
node /fuel-swift-02/ {
$swift_zone = 2
class { 'openstack::swift::storage_node':
storage_type => $swift_loopback,
swift_zone => $swift_zone,
swift_local_net_ip => $internal_address,
}
}
# Definition of the third OpenStack Swift node.
node /fuel-swift-03/ {
$swift_zone = 3
class { 'openstack::swift::storage_node':
storage_type => $swift_loopback,
swift_zone => $swift_zone,
swift_local_net_ip => $internal_address,
}
}
# Definition of OpenStack Swift proxy nodes.
node /fuel-swiftproxy-[\d+]/ {
class { 'openstack::swift::proxy':
swift_proxies => $swift_proxies,
swift_master => $swift_master,
controller_node_address => $internal_virtual_ip,
swift_local_net_ip => $internal_address,
}
}
# Definition of OpenStack Quantum node.
node /fuel-quantum/ {
class { 'openstack::quantum_router':
db_host => $internal_virtual_ip,
service_endpoint => $internal_virtual_ip,
auth_host => $internal_virtual_ip,
internal_address => $internal_address,
public_interface => $public_interface,
private_interface => $private_interface,
floating_range => $floating_range,
fixed_range => $fixed_range,
create_networks => $create_networks,
verbose => $verbose,
rabbit_password => $rabbit_password,
rabbit_user => $rabbit_user,
rabbit_nodes => $controller_hostnames,
quantum => $quantum,
quantum_user_password => $quantum_user_password,
quantum_db_password => $quantum_db_password,
quantum_db_user => $quantum_db_user,
quantum_db_dbname => $quantum_db_dbname,
tenant_network_type => $tenant_network_type,
segment_range => $segment_range,
external_ipinfo => $external_ipinfo,
api_bind_address => $internal_address,
use_syslog => $use_syslog,
}
class { 'openstack::auth_file':
admin_password => $admin_password,
keystone_admin_token => $keystone_admin_token,
controller_node => $internal_virtual_ip,
before => Class['openstack::quantum_router'],
}
}
# This configuration option is deprecated and will be removed in future releases. It's currently kept for backward compatibility.
$controller_public_addresses = {'fuel-controller-01' => '10.0.2.15','fuel-controller-02' => '10.0.2.16','fuel-controller-03' => '10.0.2.17'}