Move Naily to Astute
Change-Id: I9151f65d63211859cdc6756bc68f84ac99926afc Implements: blueprint move-naily-to-astute-repo
This commit is contained in:
parent
d7c6c4d00f
commit
cb1392ee81
|
@ -39,14 +39,14 @@ orchestrator = Astute::Orchestrator.new(deploy_engine, log_parsing=false)
|
|||
# Add systems to cobbler, reboot and start installation process.
|
||||
orchestrator.provision(reporter, environment['engine'], environment['nodes'])
|
||||
|
||||
# Observation OS installation
|
||||
# Observation OS installation
|
||||
orchestrator.watch_provision_progress(reporter, environment['task_uuid'], environment['nodes'])
|
||||
|
||||
# Deploy OpenStack
|
||||
orchestrator.deploy(reporter, environment['task_uuid'], environment['nodes'])
|
||||
|
||||
```
|
||||
Example of using Astute as library: https://github.com/stackforge/fuel-web/blob/master/naily/lib/naily/dispatcher.rb
|
||||
Example of using Astute as library: lib/astute/server/dispatcher.rb
|
||||
|
||||
|
||||
Using as CLI
|
||||
|
|
|
@ -16,6 +16,10 @@ Gem::Specification.new do |s|
|
|||
s.add_dependency 'rest-client', '~> 1.6.7'
|
||||
s.add_dependency 'popen4', '~> 0.1.2'
|
||||
|
||||
# Astute as service
|
||||
s.add_dependency 'amqp', '0.9.10'
|
||||
s.add_dependency 'raemon', '0.3.0'
|
||||
|
||||
s.add_development_dependency 'rake', '10.0.4'
|
||||
s.add_development_dependency 'rspec', '2.13.0'
|
||||
s.add_development_dependency 'mocha', '0.13.3'
|
||||
|
@ -23,7 +27,7 @@ Gem::Specification.new do |s|
|
|||
s.add_development_dependency 'simplecov-rcov', '~> 0.2.3'
|
||||
|
||||
s.files = Dir.glob("{bin,lib,spec,examples}/**/*")
|
||||
s.executables = ['astute']
|
||||
s.executables = ['astuted']
|
||||
s.require_path = 'lib'
|
||||
end
|
||||
|
||||
|
|
|
@ -0,0 +1,89 @@
|
|||
#!/usr/bin/env ruby
|
||||
|
||||
# Copyright 2013 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
require 'astute'
|
||||
require 'logger'
|
||||
require 'ostruct'
|
||||
require 'optparse'
|
||||
require 'yaml'
|
||||
require 'amqp'
|
||||
require 'raemon'
|
||||
|
||||
options = OpenStruct.new
|
||||
options.daemonize = false
|
||||
options.pidfile = '/var/run/astuted.pid'
|
||||
options.config_path = '/etc/astute/astuted.conf'
|
||||
options.log_path = nil
|
||||
options.log_level = 'debug'
|
||||
options.workers = 1
|
||||
|
||||
OptionParser.new do |opts|
|
||||
opts.banner = 'Usage: astuted [options]'
|
||||
opts.separator "\nOptions:"
|
||||
opts.on('-d', '--[no-]daemonize', 'Daemonize server') do |flag|
|
||||
options.daemonize = flag
|
||||
end
|
||||
opts.on('-P', '--pidfile PATH', 'Path to pidfile') do |path|
|
||||
options.pidfile = path
|
||||
end
|
||||
opts.on('-w', '--workers NUMBER', 'Number of worker processes') do |number|
|
||||
options.workers = number.to_i
|
||||
end
|
||||
opts.on('-c', '--config PATH', 'Use custom config file') do |path|
|
||||
unless File.exists?(path)
|
||||
puts "Error: config file #{path} was not found"
|
||||
exit
|
||||
end
|
||||
options.config_path = path
|
||||
end
|
||||
opts.on('-l', '--logfile PATH' 'Log file path') do |path|
|
||||
options.log_path = path
|
||||
end
|
||||
levels = %w{fatal error warn info debug}
|
||||
opts.on('--loglevel LEVEL', levels, "Logging level (#{levels.join(', ')})") do |level|
|
||||
options.log_level = level
|
||||
end
|
||||
opts.on_tail('-h', '--help', 'Show this message') do
|
||||
puts opts
|
||||
exit
|
||||
end
|
||||
opts.on_tail('-v', '--version', 'Show version') do
|
||||
puts Astute::VERSION
|
||||
exit
|
||||
end
|
||||
end.parse!
|
||||
|
||||
if options.daemonize
|
||||
# After daemonize we can't log to STDOUT, pick a default log file
|
||||
options.log_path ||= "#{Dir.pwd}/astute.log"
|
||||
end
|
||||
|
||||
Astute.config.update(YAML.load(File.read(options.config_path))) if File.exists?(options.config_path)
|
||||
Astute.logger = options.log_path ? Logger.new(options.log_path) : Logger.new(STDOUT)
|
||||
Astute.logger.level = Logger.const_get(options.log_level.upcase)
|
||||
Astute.logger.formatter = proc do |severity, datetime, progname, msg|
|
||||
severity_map = {'DEBUG' => 'debug', 'INFO' => 'info', 'WARN' => 'warning', 'ERROR' => 'err', 'FATAL' => 'crit'}
|
||||
"#{datetime.strftime("%Y-%m-%dT%H:%M:%S")} #{severity_map[severity]}: [#{Process.pid}] #{msg}\n"
|
||||
end
|
||||
|
||||
Astute.logger.info "Starting..."
|
||||
|
||||
Raemon::Master.start(options.workers, Astute::Server::Worker,
|
||||
:detach => options.daemonize,
|
||||
:name => 'astute',
|
||||
:pid_file => options.pidfile,
|
||||
:logger => Astute.logger
|
||||
)
|
|
@ -37,7 +37,15 @@ require 'astute/post_deploy_actions/restart_radosgw'
|
|||
require 'astute/post_deploy_actions/update_cluster_hosts_info'
|
||||
require 'astute/post_deploy_actions/upload_cirros_image'
|
||||
|
||||
# Server
|
||||
require 'astute/server/worker'
|
||||
require 'astute/server/server'
|
||||
require 'astute/server/producer'
|
||||
require 'astute/server/dispatcher'
|
||||
require 'astute/server/reporter'
|
||||
|
||||
module Astute
|
||||
# Library
|
||||
autoload 'Context', 'astute/context'
|
||||
autoload 'MClient', 'astute/mclient'
|
||||
autoload 'ProxyReporter', 'astute/reporter'
|
||||
|
|
|
@ -53,6 +53,8 @@ module Astute
|
|||
|
||||
def self.default_config
|
||||
conf = {}
|
||||
|
||||
# Library settings
|
||||
conf[:PUPPET_TIMEOUT] = 90 * 60 # maximum time it waits for the whole deployment
|
||||
conf[:PUPPET_DEPLOY_INTERVAL] = 2 # sleep for ## sec, then check puppet status again
|
||||
conf[:PUPPET_FADE_TIMEOUT] = 120 # how long it can take for puppet to exit after dumping to last_run_summary
|
||||
|
@ -71,6 +73,17 @@ module Astute
|
|||
#and uploaded to all nodes before deploy
|
||||
conf[:MAX_NODES_PER_CALL] = 50 # how many nodes to deploy in one puppet call
|
||||
|
||||
# Server settings
|
||||
conf[:broker_host] = 'localhost'
|
||||
conf[:broker_port] = 5672
|
||||
conf[:broker_username] = 'mcollective'
|
||||
conf[:broker_password] = 'mcollective'
|
||||
|
||||
conf[:broker_service_queue] = 'naily_service'
|
||||
conf[:broker_queue] = 'naily'
|
||||
conf[:broker_publisher_queue] = 'nailgun'
|
||||
conf[:broker_exchange] = 'nailgun'
|
||||
|
||||
conf
|
||||
end
|
||||
end
|
||||
|
|
|
@ -17,14 +17,14 @@ module Astute
|
|||
class Context
|
||||
attr_accessor :reporter, :deploy_log_parser
|
||||
attr_reader :task_id, :status
|
||||
|
||||
|
||||
def initialize(task_id, reporter, deploy_log_parser=nil)
|
||||
@task_id = task_id
|
||||
@reporter = reporter
|
||||
@status = {}
|
||||
@deploy_log_parser = deploy_log_parser
|
||||
end
|
||||
|
||||
|
||||
def report_and_update_status(data)
|
||||
if data['nodes']
|
||||
data['nodes'].each do |node|
|
||||
|
@ -33,6 +33,6 @@ module Astute
|
|||
end
|
||||
reporter.report(data)
|
||||
end
|
||||
|
||||
|
||||
end
|
||||
end
|
||||
|
|
|
@ -0,0 +1,209 @@
|
|||
# Copyright 2013 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
require 'astute/server/reporter'
|
||||
|
||||
module Astute
|
||||
module Server
|
||||
|
||||
class Dispatcher
|
||||
def initialize(producer)
|
||||
@orchestrator = Astute::Orchestrator.new(nil, log_parsing=true)
|
||||
@producer = producer
|
||||
@provisionLogParser = Astute::LogParser::ParseProvisionLogs.new
|
||||
end
|
||||
|
||||
def echo(args)
|
||||
Astute.logger.info 'Running echo command'
|
||||
args
|
||||
end
|
||||
|
||||
#
|
||||
# Main worker actions
|
||||
#
|
||||
|
||||
def download_release(data)
|
||||
# Example of message = {
|
||||
# {'method': 'download_release',
|
||||
# 'respond_to': 'download_release_resp',
|
||||
# 'args':{
|
||||
# 'task_uuid': 'task UUID',
|
||||
# 'release_info':{
|
||||
# 'release_id': 'release ID',
|
||||
# 'redhat':{
|
||||
# 'license_type' :"rhn" or "rhsm",
|
||||
# 'username': 'username',
|
||||
# 'password': 'password',
|
||||
# 'satellite': 'satellite host (for RHN license)'
|
||||
# 'activation_key': 'activation key (for RHN license)'
|
||||
# }
|
||||
# }
|
||||
# }}
|
||||
Astute.logger.info("'download_release' method called with data: #{data.inspect}")
|
||||
reporter = Astute::Server::Reporter.new(@producer, data['respond_to'], data['args']['task_uuid'])
|
||||
release_info = data['args']['release_info']['redhat']
|
||||
begin
|
||||
result = @orchestrator.download_release(reporter, data['args']['task_uuid'], release_info)
|
||||
rescue Timeout::Error
|
||||
msg = "Timeout of release download is exceeded."
|
||||
Astute.logger.error msg
|
||||
reporter.report({'status' => 'error', 'error' => msg})
|
||||
return
|
||||
end
|
||||
end
|
||||
|
||||
def check_redhat_credentials(data)
|
||||
release = data['args']['release_info']
|
||||
task_id = data['args']['task_uuid']
|
||||
reporter = Astute::Server::Reporter.new(@producer, data['respond_to'], task_id)
|
||||
@orchestrator.check_redhat_credentials(reporter, task_id, release)
|
||||
end
|
||||
|
||||
def check_redhat_licenses(data)
|
||||
release = data['args']['release_info']
|
||||
nodes = data['args']['nodes']
|
||||
task_id = data['args']['task_uuid']
|
||||
reporter = Astute::Server::Reporter.new(@producer, data['respond_to'], task_id)
|
||||
@orchestrator.check_redhat_licenses(reporter, task_id, release, nodes)
|
||||
end
|
||||
|
||||
def provision(data)
|
||||
Astute.logger.info("'provision' method called with data: #{data.inspect}")
|
||||
|
||||
reporter = Astute::Server::Reporter.new(@producer, data['respond_to'], data['args']['task_uuid'])
|
||||
begin
|
||||
@orchestrator.provision(reporter,
|
||||
data['args']['provisioning_info']['engine'],
|
||||
data['args']['provisioning_info']['nodes'])
|
||||
rescue => e
|
||||
Astute.logger.error "Error running provisioning: #{e.message}, trace: #{e.backtrace.inspect}"
|
||||
raise StopIteration
|
||||
end
|
||||
|
||||
@orchestrator.watch_provision_progress(
|
||||
reporter, data['args']['task_uuid'], data['args']['provisioning_info']['nodes'])
|
||||
end
|
||||
|
||||
def deploy(data)
|
||||
Astute.logger.info("'deploy' method called with data: #{data.inspect}")
|
||||
|
||||
reporter = Astute::Server::Reporter.new(@producer, data['respond_to'], data['args']['task_uuid'])
|
||||
begin
|
||||
@orchestrator.deploy(reporter, data['args']['task_uuid'], data['args']['deployment_info'])
|
||||
reporter.report('status' => 'ready', 'progress' => 100)
|
||||
rescue Timeout::Error
|
||||
msg = "Timeout of deployment is exceeded."
|
||||
Astute.logger.error msg
|
||||
reporter.report('status' => 'error', 'error' => msg)
|
||||
end
|
||||
end
|
||||
|
||||
def verify_networks(data)
|
||||
reporter = Astute::Server::SubtaskReporter.new(@producer, data['respond_to'], data['args']['task_uuid'], data['subtasks'])
|
||||
result = @orchestrator.verify_networks(reporter, data['args']['task_uuid'], data['args']['nodes'])
|
||||
report_result(result, reporter)
|
||||
end
|
||||
|
||||
def dump_environment(data)
|
||||
task_id = data['args']['task_uuid']
|
||||
reporter = Astute::Server::Reporter.new(@producer, data['respond_to'], task_id)
|
||||
@orchestrator.dump_environment(reporter, task_id, data['args']['lastdump'])
|
||||
end
|
||||
|
||||
def remove_nodes(data)
|
||||
task_uuid = data['args']['task_uuid']
|
||||
reporter = Astute::Server::Reporter.new(@producer, data['respond_to'], task_uuid)
|
||||
nodes = data['args']['nodes']
|
||||
engine = data['args']['engine']
|
||||
|
||||
result = if nodes.empty?
|
||||
Astute.logger.debug("#{task_uuid} Node list is empty")
|
||||
nil
|
||||
else
|
||||
@orchestrator.remove_nodes(reporter, task_uuid, engine, nodes)
|
||||
end
|
||||
|
||||
report_result(result, reporter)
|
||||
end
|
||||
|
||||
def reset_environment(data)
|
||||
remove_nodes(data)
|
||||
end
|
||||
|
||||
#
|
||||
# Service worker actions
|
||||
#
|
||||
|
||||
def stop_deploy_task(data, service_data)
|
||||
Astute.logger.debug("'stop_deploy_task' service method called with data: #{data.inspect}")
|
||||
target_task_uuid = data['args']['stop_task_uuid']
|
||||
task_uuid = data['args']['task_uuid']
|
||||
|
||||
return unless task_in_queue?(target_task_uuid, service_data[:tasks_queue])
|
||||
|
||||
Astute.logger.debug("Cancel task #{target_task_uuid}. Start")
|
||||
if target_task_uuid == service_data[:tasks_queue].current_task_id
|
||||
reporter = Astute::Server::Reporter.new(@producer, data['respond_to'], task_uuid)
|
||||
result = stop_current_task(data, service_data, reporter)
|
||||
report_result(result, reporter)
|
||||
else
|
||||
replace_future_task(data, service_data)
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def task_in_queue?(task_uuid, tasks_queue)
|
||||
tasks_queue.task_in_queue?(task_uuid)
|
||||
end
|
||||
|
||||
def replace_future_task(data, service_data)
|
||||
target_task_uuid = data['args']['stop_task_uuid']
|
||||
task_uuid = data['args']['task_uuid']
|
||||
|
||||
new_task_data = data_for_rm_nodes(data)
|
||||
Astute.logger.info("Replace running task #{target_task_uuid} to new #{task_uuid} with data: #{new_task_data.inspect}")
|
||||
service_data[:tasks_queue].replace_task(target_task_uuid, new_task_data)
|
||||
end
|
||||
|
||||
def stop_current_task(data, service_data, reporter)
|
||||
target_task_uuid = data['args']['stop_task_uuid']
|
||||
task_uuid = data['args']['task_uuid']
|
||||
nodes = data['args']['nodes']
|
||||
|
||||
Astute.logger.info "Try to kill running task #{target_task_uuid}"
|
||||
service_data[:main_work_thread].kill
|
||||
|
||||
result = if service_data[:tasks_queue].current_task_method == 'deploy'
|
||||
@orchestrator.stop_puppet_deploy(reporter, task_uuid, nodes)
|
||||
@orchestrator.remove_nodes(reporter, task_uuid, data['args']['engine'], nodes)
|
||||
else
|
||||
@orchestrator.stop_provision(reporter, task_uuid, data['args']['engine'], nodes)
|
||||
end
|
||||
end
|
||||
|
||||
def data_for_rm_nodes(data)
|
||||
data['method'] = 'remove_nodes'
|
||||
data
|
||||
end
|
||||
|
||||
def report_result(result, reporter)
|
||||
result = {} unless result.instance_of?(Hash)
|
||||
status = {'status' => 'ready', 'progress' => 100}.merge(result)
|
||||
reporter.report(status)
|
||||
end
|
||||
end
|
||||
|
||||
end #Server
|
||||
end #Astute
|
|
@ -1,5 +1,3 @@
|
|||
#!/usr/bin/env ruby
|
||||
|
||||
# Copyright 2013 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
|
@ -14,9 +12,26 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
puts <<-EOF
|
||||
CLI interface in Astute no longer supported. Please use new Fuel-CLI.
|
||||
More details you can get by link: https://github.com/Mirantis/fuel-docs/blob/master/pages/user-guide/cli.rst
|
||||
EOF
|
||||
module Astute
|
||||
module Server
|
||||
|
||||
exit 1
|
||||
class Producer
|
||||
def initialize(exchange)
|
||||
@exchange = exchange
|
||||
end
|
||||
|
||||
def publish(message, options={})
|
||||
default_options = {:routing_key => Astute.config.broker_publisher_queue,
|
||||
:content_type => 'application/json'}
|
||||
options = default_options.merge(options)
|
||||
|
||||
begin
|
||||
@exchange.publish(message.to_json, options)
|
||||
rescue
|
||||
Astute.logger.error "Error publishing message: #{$!}"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
end #Server
|
||||
end #Astute
|
|
@ -0,0 +1,53 @@
|
|||
# Copyright 2013 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
module Astute
|
||||
module Server
|
||||
|
||||
class Reporter
|
||||
def initialize(producer, method, task_uuid)
|
||||
@producer = producer
|
||||
@method = method
|
||||
@task_uuid = task_uuid
|
||||
end
|
||||
|
||||
def report(msg)
|
||||
msg_with_task = {'task_uuid' => @task_uuid}.merge(msg)
|
||||
message = {'method' => @method, 'args' => msg_with_task}
|
||||
Astute.logger.info "Casting message to fuel: #{message.inspect}"
|
||||
@producer.publish(message)
|
||||
end
|
||||
end
|
||||
|
||||
class SubtaskReporter < Reporter
|
||||
def initialize(producer, method, task_uuid, subtasks)
|
||||
super(producer, method, task_uuid)
|
||||
@subtasks = subtasks
|
||||
end
|
||||
|
||||
def report_to_subtask(subtask_name, msg)
|
||||
if @subtasks[subtask_name] and @subtasks[subtask_name].any?
|
||||
subtask_msg = {'task_uuid'=>@subtasks[subtask_name]['task_uuid']}.merge(msg)
|
||||
message = {'method' => @subtasks[subtask_name]['respond_to'],
|
||||
'args' => subtask_msg}
|
||||
Astute.logger.info "Casting message to fuel: #{message.inspect}"
|
||||
@producer.publish(message)
|
||||
else
|
||||
Astute.logger.info "No subtask #{subtask_name} for : #{@method}"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
end #Server
|
||||
end #Astute
|
|
@ -0,0 +1,178 @@
|
|||
# Copyright 2013 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
require 'json'
|
||||
require 'astute/server/task_queue'
|
||||
|
||||
module Astute
|
||||
module Server
|
||||
|
||||
class Server
|
||||
def initialize(channel, exchange, delegate, producer, service_channel, service_exchange)
|
||||
@channel = channel
|
||||
@exchange = exchange
|
||||
@delegate = delegate
|
||||
@producer = producer
|
||||
@service_channel = service_channel
|
||||
@service_exchange = service_exchange
|
||||
end
|
||||
|
||||
def run
|
||||
@queue = @channel.queue(Astute.config.broker_queue, :durable => true).bind(@exchange)
|
||||
@service_queue = @service_channel.queue("", :exclusive => true, :auto_delete => true).bind(@service_exchange)
|
||||
|
||||
@main_work_thread = nil
|
||||
@tasks_queue = TaskQueue.new
|
||||
|
||||
Thread.new(&method(:register_callbacks))
|
||||
self
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def register_callbacks
|
||||
main_worker
|
||||
service_worker
|
||||
end
|
||||
|
||||
def main_worker
|
||||
@consumer = AMQP::Consumer.new(@channel, @queue)
|
||||
@consumer.on_delivery do |metadata, payload|
|
||||
Astute.logger.debug "Process message from worker queue: #{payload.inspect}"
|
||||
perform_main_job(metadata, payload)
|
||||
end
|
||||
@consumer.consume
|
||||
end
|
||||
|
||||
def service_worker
|
||||
@service_queue.subscribe do |_, payload|
|
||||
Astute.logger.debug "Process message from service queue: #{payload.inspect}"
|
||||
perform_service_job(nil, payload)
|
||||
end
|
||||
end
|
||||
|
||||
def perform_main_job(metadata, payload)
|
||||
@main_work_thread = Thread.new do
|
||||
begin
|
||||
data = parse_data(payload)
|
||||
@tasks_queue = Astute::Server::TaskQueue.new
|
||||
|
||||
@tasks_queue.add_task(data)
|
||||
dispatch(@tasks_queue)
|
||||
ensure
|
||||
metadata.ack
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def perform_service_job(metadata, payload)
|
||||
Thread.new do
|
||||
service_data = {:main_work_thread => @main_work_thread, :tasks_queue => @tasks_queue}
|
||||
dispatch(parse_data(payload), service_data)
|
||||
end
|
||||
end
|
||||
|
||||
def dispatch(data, service_data=nil)
|
||||
data.each_with_index do |message, i|
|
||||
begin
|
||||
dispatch_message message, service_data
|
||||
rescue StopIteration
|
||||
Astute.logger.debug "Dispatching aborted by #{message['method']}"
|
||||
abort_messages messages[(i + 1)..-1]
|
||||
break
|
||||
rescue => ex
|
||||
Astute.logger.error "Error running RPC method #{message['method']}: #{ex.message}, trace: #{ex.backtrace.inspect}"
|
||||
return_results message, {
|
||||
'status' => 'error',
|
||||
'error' => "Error occurred while running method '#{message['method']}'. Inspect Orchestrator logs for the details."
|
||||
}
|
||||
break
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def dispatch_message(data, service_data=nil)
|
||||
Astute.logger.debug "Dispatching message: #{data.inspect}"
|
||||
|
||||
if Astute.config.fake_dispatch
|
||||
Astute.logger.debug "Fake dispatch"
|
||||
return
|
||||
end
|
||||
|
||||
unless @delegate.respond_to?(data['method'])
|
||||
Astute.logger.error "Unsupported RPC call '#{data['method']}'"
|
||||
return_results data, {
|
||||
'status' => 'error',
|
||||
'error' => "Unsupported method '#{data['method']}' called."
|
||||
}
|
||||
return
|
||||
end
|
||||
|
||||
Astute.logger.debug "Main worker task id is #{@tasks_queue.current_task_id}" if service_data.nil?
|
||||
|
||||
Astute.logger.info "Processing RPC call '#{data['method']}'"
|
||||
if !service_data
|
||||
@delegate.send(data['method'], data)
|
||||
else
|
||||
@delegate.send(data['method'], data, service_data)
|
||||
end
|
||||
end
|
||||
|
||||
def return_results(message, results)
|
||||
if results.is_a?(Hash) && message['respond_to']
|
||||
reporter = Astute::Server::Reporter.new(@producer, message['respond_to'], message['args']['task_uuid'])
|
||||
reporter.report results
|
||||
end
|
||||
end
|
||||
|
||||
def parse_data(data)
|
||||
Astute.logger.debug "Got message with payload #{data.inspect}"
|
||||
messages = nil
|
||||
begin
|
||||
messages = JSON.load(data)
|
||||
rescue => e
|
||||
Astute.logger.error "Error deserializing payload: #{e.message}, trace: #{e.backtrace.inspect}"
|
||||
end
|
||||
messages.is_a?(Array) ? messages : [messages]
|
||||
end
|
||||
|
||||
def abort_messages(messages)
|
||||
return unless messages && messages.size > 0
|
||||
messages.each do |message|
|
||||
begin
|
||||
Astute.logger.debug "Aborting '#{message['method']}'"
|
||||
err_msg = {
|
||||
'status' => 'error',
|
||||
'error' => 'Task aborted',
|
||||
'progress' => 100
|
||||
}
|
||||
|
||||
if message['args']['nodes'].instance_of?(Array)
|
||||
err_nodes = message['args']['nodes'].map do |node|
|
||||
{'uid' => node['uid'], 'status' => 'error', 'error_type' => 'provision', 'progress' => 0}
|
||||
end
|
||||
|
||||
err_msg.merge!('nodes' => err_nodes)
|
||||
end
|
||||
|
||||
return_results(message, err_msg)
|
||||
rescue => ex
|
||||
Astute.logger.debug "Failed to abort '#{message['method']}': #{ex.inspect}"
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
end #Server
|
||||
end #Astute
|
|
@ -0,0 +1,89 @@
|
|||
# Copyright 2013 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
require 'thread'
|
||||
|
||||
module Astute
|
||||
module Server
|
||||
|
||||
class TaskQueue
|
||||
include Enumerable
|
||||
|
||||
attr_reader :current_task_id
|
||||
attr_reader :current_task_method
|
||||
|
||||
def initialize
|
||||
@queue = []
|
||||
@semaphore = Mutex.new
|
||||
@current_task_id = nil
|
||||
@current_task_method = nil
|
||||
end
|
||||
|
||||
def add_task(data)
|
||||
@semaphore.synchronize { data.compact.each { |t| @queue << t } }
|
||||
end
|
||||
|
||||
def replace_task(replacing_task_id, new_task_data)
|
||||
@semaphore.synchronize do
|
||||
@queue.map! { |x| find_task_id(x) == replacing_task_id ? new_task_data : x }.flatten!
|
||||
end
|
||||
end
|
||||
|
||||
def remove_task(replacing_task_id)
|
||||
replace_task(replacing_task_id, nil)
|
||||
end
|
||||
|
||||
def clear_queue
|
||||
@semaphore.synchronize { @queue.map! { |x| nil } }
|
||||
end
|
||||
|
||||
def task_in_queue?(task_id)
|
||||
@semaphore.synchronize { @queue.find { |t| find_task_id(t) == task_id } }
|
||||
end
|
||||
|
||||
def each(&block)
|
||||
@queue.each do |task|
|
||||
@semaphore.synchronize do
|
||||
next if task.nil?
|
||||
@current_task_id = find_task_id(task)
|
||||
@current_task_method = find_task_method(task)
|
||||
end
|
||||
|
||||
if block_given?
|
||||
block.call task
|
||||
else
|
||||
yield task
|
||||
end
|
||||
end
|
||||
ensure
|
||||
@semaphore.synchronize do
|
||||
@current_task_id = nil
|
||||
@current_task_method = nil
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def find_task_id(data)
|
||||
data && data['args'] && data['args']['task_uuid'] ? data['args']['task_uuid'] : nil
|
||||
end
|
||||
|
||||
def find_task_method(data)
|
||||
data && data['method']
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
end #Server
|
||||
end #Astute
|
|
@ -0,0 +1,108 @@
|
|||
# Copyright 2013 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
require 'raemon'
|
||||
|
||||
module Astute
|
||||
module Server
|
||||
|
||||
class Worker
|
||||
include Raemon::Worker
|
||||
|
||||
def start
|
||||
super
|
||||
start_heartbeat
|
||||
end
|
||||
|
||||
def stop
|
||||
super
|
||||
begin
|
||||
@connection.close{ stop_event_machine }
|
||||
ensure
|
||||
stop_event_machine
|
||||
end
|
||||
end
|
||||
|
||||
def run
|
||||
EM.run do
|
||||
run_server
|
||||
end
|
||||
rescue => e
|
||||
Astute.logger.error "Exception during worker initialization: #{e.inspect}, trace: #{e.backtrace.inspect}"
|
||||
sleep 5
|
||||
retry
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def start_heartbeat
|
||||
@heartbeat ||= Thread.new do
|
||||
sleep 30
|
||||
heartbeat!
|
||||
end
|
||||
end
|
||||
|
||||
def run_server
|
||||
AMQP.logging = true
|
||||
AMQP.connect(connection_options) do |connection|
|
||||
@connection = configure_connection(connection)
|
||||
|
||||
@channel = create_channel(@connection)
|
||||
@exchange = @channel.topic(Astute.config.broker_exchange, :durable => true)
|
||||
@service_channel = create_channel(@connection, prefetch=false)
|
||||
@service_exchange = @service_channel.fanout(Astute.config.broker_service_queue, :auto_delete => true)
|
||||
|
||||
@producer = Astute::Server::Producer.new(@exchange)
|
||||
@delegate = Astute.config.delegate || Astute::Server::Dispatcher.new(@producer)
|
||||
@server = Astute::Server::Server.new(@channel, @exchange, @delegate, @producer, @service_channel, @service_exchange)
|
||||
|
||||
@server.run
|
||||
end
|
||||
end
|
||||
|
||||
def configure_connection(connection)
|
||||
connection.on_tcp_connection_loss do |conn, settings|
|
||||
Astute.logger.warn "Trying to reconnect to message broker..."
|
||||
conn.reconnect
|
||||
end
|
||||
connection
|
||||
end
|
||||
|
||||
def create_channel(connection, prefetch=true)
|
||||
prefetch_opts = ( prefetch ? {:prefetch => 1} : {} )
|
||||
channel = AMQP::Channel.new(connection, AMQP::Channel.next_channel_id, prefetch_opts)
|
||||
channel.auto_recovery = true
|
||||
channel.on_error do |ch, error|
|
||||
Astute.logger.fatal "Channel error #{error.inspect}"
|
||||
stop
|
||||
end
|
||||
channel
|
||||
end
|
||||
|
||||
def connection_options
|
||||
{
|
||||
:host => Astute.config.broker_host,
|
||||
:port => Astute.config.broker_port,
|
||||
:username => Astute.config.broker_username,
|
||||
:password => Astute.config.broker_password,
|
||||
}.reject{|k, v| v.nil? }
|
||||
end
|
||||
|
||||
def stop_event_machine
|
||||
EM.stop_event_loop if EM.reactor_running?
|
||||
end
|
||||
end
|
||||
|
||||
end #Server
|
||||
end #Astute
|
Loading…
Reference in New Issue