Move from amqp-gem to bunny
Differents: - separate independent chanel for outgoing report; - solid way to redeclare already existed queues; - auto recovery mode in case of network problem by default; - more solid, modern and simple library for AMQP. Also: - implement asynchronous logger for event callbacks. Short words from both gems authors: amqp gem brings in a fair share of EventMachine complexity which cannot be fully eliminated. Event loop blocking, writes that happen at the end of loop tick, uncaught exceptions in event loop silently killing it: it's not worth the pain unless you've already deeply invested in EventMachine and understand how it works. Closes-Bug: #1498847 Closes-Bug: #1487397 Closes-Bug: #1461562 Related-Bug: #1485895 Related-Bug: #1483182 Change-Id: I52d005498ccb978ada158bfa64b1c7de1a24e9b0
This commit is contained in:
Vladimir Sharshov (warpc)
parent
d2c1b40181
commit
b60624ee2c
@ -17,7 +17,7 @@ Gem::Specification.new do |s|
|
||||
s.add_dependency 'net-ssh-multi', '~> 1.1'
|
||||
|
||||
# Astute as service
|
||||
s.add_dependency 'amqp', '1.4.1'
|
||||
s.add_dependency 'bunny', ">= 2.0"
|
||||
s.add_dependency 'raemon', '0.3.0'
|
||||
|
||||
s.add_development_dependency 'facter'
|
||||
|
||||
@ -19,7 +19,6 @@ require 'logger'
|
||||
require 'ostruct'
|
||||
require 'optparse'
|
||||
require 'yaml'
|
||||
require 'amqp'
|
||||
require 'raemon'
|
||||
|
||||
options = OpenStruct.new
|
||||
|
||||
@ -20,6 +20,7 @@ require 'logger'
|
||||
require 'shellwords'
|
||||
require 'active_support/all'
|
||||
require 'pp'
|
||||
require 'bunny'
|
||||
|
||||
require 'astute/ext/exception'
|
||||
require 'astute/ext/deep_copy'
|
||||
@ -43,6 +44,8 @@ require 'astute/puppet_task'
|
||||
require 'astute/task_manager'
|
||||
require 'astute/pre_delete'
|
||||
require 'astute/version'
|
||||
require 'astute/server/async_logger'
|
||||
|
||||
|
||||
['/astute/pre_deployment_actions/*.rb',
|
||||
'/astute/pre_deploy_actions/*.rb',
|
||||
|
||||
79
lib/astute/server/async_logger.rb
Normal file
79
lib/astute/server/async_logger.rb
Normal file
@ -0,0 +1,79 @@
|
||||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
require 'thread'
|
||||
|
||||
module Astute
|
||||
module Server
|
||||
|
||||
# Asynchronous singleton logger, which should be used
|
||||
# in event callbacks of event machine, it doesn't block
|
||||
# callbacks because writing a message to log takes some time.
|
||||
# Also synchronous logger, potentially could lead to deadlocks.
|
||||
# See:
|
||||
# https://bugs.launchpad.net/fuel/+bug/1453573
|
||||
# https://bugs.launchpad.net/fuel/+bug/1487397
|
||||
module AsyncLogger
|
||||
def self.start_up(logger=Logger.new(STDOUT))
|
||||
@queue ||= Queue.new
|
||||
@log = logger
|
||||
@thread = Thread.new { flush_messages }
|
||||
end
|
||||
|
||||
def self.shutdown
|
||||
@thread.kill
|
||||
end
|
||||
|
||||
def self.add(severity, msg=nil)
|
||||
return if @shutdown
|
||||
|
||||
@queue.push([severity, msg])
|
||||
end
|
||||
|
||||
def self.debug(msg=nil)
|
||||
add(Logger::Severity::DEBUG, msg)
|
||||
end
|
||||
|
||||
def self.info(msg=nil)
|
||||
add(Logger::Severity::INFO, msg)
|
||||
end
|
||||
|
||||
def self.warn(msg=nil)
|
||||
add(Logger::Severity::WARN, msg)
|
||||
end
|
||||
|
||||
def self.error(msg=nil)
|
||||
add(Logger::Severity::ERROR, msg)
|
||||
end
|
||||
|
||||
def self.fatal(msg=nil)
|
||||
add(Logger::Severity::FATAL, msg)
|
||||
end
|
||||
|
||||
def self.unknown(msg=nil)
|
||||
add(Logger::Severity::UNKNOWN, msg)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def self.flush_messages
|
||||
loop do
|
||||
severity, msg = @queue.pop
|
||||
@log.add(severity, msg)
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -25,7 +25,7 @@ module Astute
|
||||
end
|
||||
|
||||
def echo(args)
|
||||
Astute.logger.info 'Running echo command'
|
||||
Astute.logger.info('Running echo command')
|
||||
args
|
||||
end
|
||||
|
||||
@ -43,9 +43,10 @@ module Astute
|
||||
|
||||
def provision(data, provision_method)
|
||||
|
||||
Astute.logger.info("'provision' method called with data:\n#{data.pretty_inspect}")
|
||||
Astute.logger.info("'provision' method called with data:\n"\
|
||||
"#{data.pretty_inspect}")
|
||||
|
||||
reporter = Astute::Server::Reporter.new(@producer, data['respond_to'], data['args']['task_uuid'])
|
||||
reporter = create_reporter(data)
|
||||
begin
|
||||
result = @orchestrator.provision(
|
||||
reporter,
|
||||
@ -53,20 +54,19 @@ module Astute
|
||||
data['args']['provisioning_info'],
|
||||
provision_method
|
||||
)
|
||||
|
||||
#TODO(vsharshov): Refactoring the deployment aborting messages (StopIteration)
|
||||
rescue => e
|
||||
Astute.logger.error "Error running provisioning: #{e.message}, trace: #{e.format_backtrace}"
|
||||
Astute.logger.error("Error running provisioning: #{e.message}, "\
|
||||
"trace: #{e.format_backtrace}")
|
||||
raise StopIteration
|
||||
end
|
||||
raise StopIteration if result && result['status'] == 'error'
|
||||
end
|
||||
|
||||
def deploy(data)
|
||||
Astute.logger.info("'deploy' method called with data:\n#{data.pretty_inspect}")
|
||||
|
||||
reporter = Astute::Server::Reporter.new(@producer, data['respond_to'], data['args']['task_uuid'])
|
||||
Astute.logger.info("'deploy' method called with data:\n"\
|
||||
"#{data.pretty_inspect}")
|
||||
|
||||
reporter = create_reporter(data)
|
||||
begin
|
||||
@orchestrator.deploy(
|
||||
reporter,
|
||||
@ -78,15 +78,16 @@ module Astute
|
||||
reporter.report('status' => 'ready', 'progress' => 100)
|
||||
rescue Timeout::Error
|
||||
msg = "Timeout of deployment is exceeded."
|
||||
Astute.logger.error msg
|
||||
Astute.logger.error(msg)
|
||||
reporter.report('status' => 'error', 'error' => msg)
|
||||
end
|
||||
end
|
||||
|
||||
def granular_deploy(data)
|
||||
Astute.logger.info("'granular_deploy' method called with data:\n#{data.pretty_inspect}")
|
||||
Astute.logger.info("'granular_deploy' method called with data:\n"\
|
||||
"#{data.pretty_inspect}")
|
||||
|
||||
reporter = Astute::Server::Reporter.new(@producer, data['respond_to'], data['args']['task_uuid'])
|
||||
reporter = create_reporter(data)
|
||||
begin
|
||||
@orchestrator.granular_deploy(
|
||||
reporter,
|
||||
@ -98,7 +99,7 @@ module Astute
|
||||
reporter.report('status' => 'ready', 'progress' => 100)
|
||||
rescue Timeout::Error
|
||||
msg = "Timeout of deployment is exceeded."
|
||||
Astute.logger.error msg
|
||||
Astute.logger.error(msg)
|
||||
reporter.report('status' => 'error', 'error' => msg)
|
||||
end
|
||||
end
|
||||
@ -111,60 +112,80 @@ module Astute
|
||||
Astute.logger.warn("No method for #{subtask}")
|
||||
end
|
||||
end
|
||||
reporter = Astute::Server::Reporter.new(@producer, data['respond_to'], data['args']['task_uuid'])
|
||||
result = @orchestrator.verify_networks(reporter, data['args']['task_uuid'], data['args']['nodes'])
|
||||
reporter = create_reporter(data)
|
||||
result = @orchestrator.verify_networks(
|
||||
reporter,
|
||||
data['args']['task_uuid'],
|
||||
data['args']['nodes']
|
||||
)
|
||||
report_result(result, reporter)
|
||||
end
|
||||
|
||||
def check_dhcp(data)
|
||||
reporter = Astute::Server::Reporter.new(@producer, data['respond_to'], data['args']['task_uuid'])
|
||||
result = @orchestrator.check_dhcp(reporter, data['args']['task_uuid'], data['args']['nodes'])
|
||||
reporter = create_reporter(data)
|
||||
result = @orchestrator.check_dhcp(
|
||||
reporter,
|
||||
data['args']['task_uuid'],
|
||||
data['args']['nodes']
|
||||
)
|
||||
report_result(result, reporter)
|
||||
end
|
||||
|
||||
def multicast_verification(data)
|
||||
reporter = Astute::Server::Reporter.new(@producer, data['respond_to'], data['args']['task_uuid'])
|
||||
result = @orchestrator.multicast_verification(reporter, data['args']['task_uuid'], data['args']['nodes'])
|
||||
reporter = create_reporter(data)
|
||||
result = @orchestrator.multicast_verification(
|
||||
reporter,
|
||||
data['args']['task_uuid'],
|
||||
data['args']['nodes']
|
||||
)
|
||||
report_result(result, reporter)
|
||||
end
|
||||
|
||||
def check_repositories(data)
|
||||
reporter = Astute::Server::Reporter.new(@producer, data['respond_to'], data['args']['task_uuid'])
|
||||
result = @orchestrator.check_repositories(reporter, data['args']['task_uuid'], data['args']['nodes'], data['args']['urls'])
|
||||
reporter = create_reporter(data)
|
||||
result = @orchestrator.check_repositories(
|
||||
reporter,
|
||||
data['args']['task_uuid'],
|
||||
data['args']['nodes'],
|
||||
data['args']['urls']
|
||||
)
|
||||
report_result(result, reporter)
|
||||
end
|
||||
|
||||
def check_repositories_with_setup(data)
|
||||
reporter = Astute::Server::Reporter.new(@producer, data['respond_to'], data['args']['task_uuid'])
|
||||
result = @orchestrator.check_repositories_with_setup(reporter, data['args']['task_uuid'], data['args']['nodes'])
|
||||
reporter = create_reporter(data)
|
||||
result = @orchestrator.check_repositories_with_setup(
|
||||
reporter,
|
||||
data['args']['task_uuid'],
|
||||
data['args']['nodes']
|
||||
)
|
||||
report_result(result, reporter)
|
||||
end
|
||||
|
||||
def dump_environment(data)
|
||||
task_id = data['args']['task_uuid']
|
||||
reporter = Astute::Server::Reporter.new(@producer, data['respond_to'], task_id)
|
||||
@orchestrator.dump_environment(reporter, task_id, data['args']['settings'])
|
||||
@orchestrator.dump_environment(
|
||||
create_reporter(data),
|
||||
data['args']['task_uuid'],
|
||||
data['args']['settings']
|
||||
)
|
||||
end
|
||||
|
||||
def remove_nodes(data, reset=false)
|
||||
task_uuid = data['args']['task_uuid']
|
||||
reporter = Astute::Server::Reporter.new(@producer, data['respond_to'], task_uuid)
|
||||
nodes = data['args']['nodes']
|
||||
engine = data['args']['engine']
|
||||
check_ceph = data['args']['check_ceph']
|
||||
reporter = create_reporter(data)
|
||||
|
||||
result = if nodes.empty?
|
||||
result = if data['args']['nodes'].empty?
|
||||
Astute.logger.debug("#{task_uuid} Node list is empty")
|
||||
nil
|
||||
else
|
||||
@orchestrator.remove_nodes(
|
||||
reporter,
|
||||
task_uuid,
|
||||
engine,
|
||||
nodes,
|
||||
data['args']['engine'],
|
||||
data['args']['nodes'],
|
||||
{
|
||||
:reboot => true,
|
||||
:check_ceph => check_ceph,
|
||||
:check_ceph => data['args']['check_ceph'],
|
||||
:reset => reset
|
||||
}
|
||||
)
|
||||
@ -178,16 +199,9 @@ module Astute
|
||||
end
|
||||
|
||||
def execute_tasks(data)
|
||||
task_uuid = data['args']['task_uuid']
|
||||
reporter = Astute::Server::Reporter.new(
|
||||
@producer,
|
||||
data['respond_to'],
|
||||
task_uuid
|
||||
)
|
||||
|
||||
@orchestrator.execute_tasks(
|
||||
reporter,
|
||||
task_uuid,
|
||||
create_reporter(data),
|
||||
data['args']['task_uuid'],
|
||||
data['args']['tasks']
|
||||
)
|
||||
end
|
||||
@ -197,15 +211,17 @@ module Astute
|
||||
#
|
||||
|
||||
def stop_deploy_task(data, service_data)
|
||||
Astute.logger.debug("'stop_deploy_task' service method called with data:\n#{data.pretty_inspect}")
|
||||
Astute.logger.debug("'stop_deploy_task' service method called with"\
|
||||
"data:\n#{data.pretty_inspect}")
|
||||
target_task_uuid = data['args']['stop_task_uuid']
|
||||
task_uuid = data['args']['task_uuid']
|
||||
|
||||
return unless task_in_queue?(target_task_uuid, service_data[:tasks_queue])
|
||||
return unless task_in_queue?(target_task_uuid,
|
||||
service_data[:tasks_queue])
|
||||
|
||||
Astute.logger.debug("Cancel task #{target_task_uuid}. Start")
|
||||
if target_task_uuid == service_data[:tasks_queue].current_task_id
|
||||
reporter = Astute::Server::Reporter.new(@producer, data['respond_to'], task_uuid)
|
||||
reporter = create_reporter(data)
|
||||
result = stop_current_task(data, service_data, reporter)
|
||||
report_result(result, reporter)
|
||||
else
|
||||
@ -215,6 +231,14 @@ module Astute
|
||||
|
||||
private
|
||||
|
||||
def create_reporter(data)
|
||||
Astute::Server::Reporter.new(
|
||||
@producer,
|
||||
data['respond_to'],
|
||||
data['args']['task_uuid']
|
||||
)
|
||||
end
|
||||
|
||||
def task_in_queue?(task_uuid, tasks_queue)
|
||||
tasks_queue.task_in_queue?(task_uuid)
|
||||
end
|
||||
@ -224,8 +248,13 @@ module Astute
|
||||
task_uuid = data['args']['task_uuid']
|
||||
|
||||
new_task_data = data_for_rm_nodes(data)
|
||||
Astute.logger.info("Replace running task #{target_task_uuid} to new #{task_uuid} with data:\n#{new_task_data.pretty_inspect}")
|
||||
service_data[:tasks_queue].replace_task(target_task_uuid, new_task_data)
|
||||
Astute.logger.info("Replace running task #{target_task_uuid} to "\
|
||||
"new #{task_uuid} with data:\n"\
|
||||
"#{new_task_data.pretty_inspect}")
|
||||
service_data[:tasks_queue].replace_task(
|
||||
target_task_uuid,
|
||||
new_task_data
|
||||
)
|
||||
end
|
||||
|
||||
def stop_current_task(data, service_data, reporter)
|
||||
@ -233,15 +262,25 @@ module Astute
|
||||
task_uuid = data['args']['task_uuid']
|
||||
nodes = data['args']['nodes']
|
||||
|
||||
Astute.logger.info "Try to kill running task #{target_task_uuid}"
|
||||
Astute.logger.info("Try to kill running task #{target_task_uuid}")
|
||||
service_data[:main_work_thread].kill
|
||||
|
||||
result = if ['deploy', 'task_deployment', 'granular_deploy'].include? (
|
||||
service_data[:tasks_queue].current_task_method)
|
||||
@orchestrator.stop_puppet_deploy(reporter, task_uuid, nodes)
|
||||
@orchestrator.remove_nodes(reporter, task_uuid, data['args']['engine'], nodes)
|
||||
@orchestrator.remove_nodes(
|
||||
reporter,
|
||||
task_uuid,
|
||||
data['args']['engine'],
|
||||
nodes
|
||||
)
|
||||
else
|
||||
@orchestrator.stop_provision(reporter, task_uuid, data['args']['engine'], nodes)
|
||||
@orchestrator.stop_provision(
|
||||
reporter,
|
||||
task_uuid,
|
||||
data['args']['engine'],
|
||||
nodes
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
@ -14,27 +14,48 @@
|
||||
|
||||
module Astute
|
||||
module Server
|
||||
|
||||
class Producer
|
||||
def initialize(exchange)
|
||||
@exchange = exchange
|
||||
@publish_queue = Queue.new
|
||||
@publish_consumer = Thread.new do
|
||||
loop do
|
||||
msg = @publish_queue.pop
|
||||
publish_from_queue msg
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def publish_from_queue(message)
|
||||
Astute.logger.info "Casting message to Nailgun:\n"\
|
||||
"#{message[:message].pretty_inspect}"
|
||||
@exchange.publish(message[:message].to_json, message[:options])
|
||||
rescue => e
|
||||
Astute.logger.error "Error publishing message: #{e.message}"
|
||||
end
|
||||
|
||||
def publish(message, options={})
|
||||
default_options = {:routing_key => Astute.config.broker_publisher_queue,
|
||||
:content_type => 'application/json'}
|
||||
options = default_options.merge(options)
|
||||
|
||||
EM.next_tick {
|
||||
begin
|
||||
Astute.logger.info "Casting message to Nailgun:\n#{message.pretty_inspect}"
|
||||
@exchange.publish(message.to_json, options)
|
||||
rescue
|
||||
Astute.logger.error "Error publishing message: #{$!}"
|
||||
end
|
||||
default_options = {
|
||||
:routing_key => Astute.config.broker_publisher_queue,
|
||||
:content_type => 'application/json'
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
# Status message manage task status in Nailgun. If we miss some of them,
|
||||
# user need manually delete them or change it status using DB.
|
||||
# Persistent option tell RabbitMQ to save message in case of
|
||||
# unexpected/expected restart.
|
||||
if message.respond_to?(:keys) && message.keys.map(&:to_s).include?('status')
|
||||
default_options.merge!({:persistent => true})
|
||||
end
|
||||
|
||||
options = default_options.merge(options)
|
||||
@publish_queue << {:message => message, :options => options}
|
||||
end
|
||||
|
||||
def stop
|
||||
@publish_consumer.kill
|
||||
end
|
||||
|
||||
end # Producer
|
||||
end #Server
|
||||
end #Astute
|
||||
|
||||
@ -20,63 +20,82 @@ module Astute
|
||||
module Server
|
||||
|
||||
class Server
|
||||
def initialize(channel, exchange, delegate, producer, service_channel, service_exchange)
|
||||
@channel = channel
|
||||
@exchange = exchange
|
||||
def initialize(channels_and_exchanges, delegate, producer)
|
||||
@channel = channels_and_exchanges[:channel]
|
||||
@exchange = channels_and_exchanges[:exchange]
|
||||
@delegate = delegate
|
||||
@producer = producer
|
||||
@service_channel = service_channel
|
||||
@service_exchange = service_exchange
|
||||
@service_channel = channels_and_exchanges[:service_channel]
|
||||
@service_exchange = channels_and_exchanges[:service_exchange]
|
||||
# NOTE(eli): Generate unique name for service queue
|
||||
# See bug: https://bugs.launchpad.net/fuel/+bug/1485895
|
||||
@service_queue_name = "naily_service_#{SecureRandom.uuid}"
|
||||
@watch_thread = nil
|
||||
end
|
||||
|
||||
def run
|
||||
@queue = @channel.queue(Astute.config.broker_queue, :durable => true).bind(@exchange)
|
||||
@service_queue = @service_channel.queue(@service_queue_name, :exclusive => true, :auto_delete => true).bind(@service_exchange)
|
||||
@queue = @channel.queue(
|
||||
Astute.config.broker_queue,
|
||||
:durable => true
|
||||
)
|
||||
@queue.bind(@exchange)
|
||||
|
||||
@service_queue = @service_channel.queue(
|
||||
@service_queue_name,
|
||||
:exclusive => true,
|
||||
:auto_delete => true
|
||||
)
|
||||
@service_queue.bind(@service_exchange)
|
||||
|
||||
@main_work_thread = nil
|
||||
@tasks_queue = TaskQueue.new
|
||||
|
||||
Thread.new(&method(:register_callbacks))
|
||||
self
|
||||
register_callbacks
|
||||
|
||||
run_infinite_loop
|
||||
@watch_thread.join
|
||||
end
|
||||
|
||||
def stop
|
||||
@watch_thread.wakeup
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def run_infinite_loop
|
||||
@watch_thread = Thread.new do
|
||||
Thread.stop
|
||||
Astute.logger.debug "Stop main thread"
|
||||
end
|
||||
end
|
||||
|
||||
def register_callbacks
|
||||
main_worker
|
||||
service_worker
|
||||
end
|
||||
|
||||
def main_worker
|
||||
@consumer = AMQP::Consumer.new(@channel, @queue, consumer_tag=nil, exclusive=false)
|
||||
@consumer.on_cancel do |basic_cancel|
|
||||
Astute.logger.debug("Received cancel notification from in main worker.")
|
||||
@exchange.auto_recover
|
||||
@service_exchange.auto_recover
|
||||
@queue.auto_recover
|
||||
@service_queue.auto_recover
|
||||
end
|
||||
@consumer.on_delivery do |metadata, payload|
|
||||
@queue.subscribe(:manual_ack => true) do |delivery_info, _, payload|
|
||||
if @main_work_thread.nil? || !@main_work_thread.alive?
|
||||
Astute.logger.debug "Process message from worker queue:\n#{payload.pretty_inspect}"
|
||||
metadata.ack
|
||||
perform_main_job(metadata, payload)
|
||||
Astute.logger.debug "Process message from worker queue:\n"\
|
||||
"#{payload.pretty_inspect}"
|
||||
@channel.acknowledge(delivery_info.delivery_tag, false)
|
||||
perform_main_job(payload)
|
||||
else
|
||||
Astute.logger.debug "Requeue message because worker is busy:\n#{payload.pretty_inspect}"
|
||||
# Avoid throttle by consume/reject cycle if only one worker is running
|
||||
EM.add_timer(2) { metadata.reject(:requeue => true) }
|
||||
Astute.logger.debug "Requeue message because worker is busy:"\
|
||||
"\n#{payload.pretty_inspect}"
|
||||
# Avoid throttle by consume/reject cycle
|
||||
# if only one worker is running
|
||||
@channel.reject(delivery_info.delivery_tag, true)
|
||||
end
|
||||
end
|
||||
@consumer.consume
|
||||
end
|
||||
|
||||
def service_worker
|
||||
@service_queue.subscribe do |_, payload|
|
||||
Astute.logger.debug "Process message from service queue:\n#{payload.pretty_inspect}"
|
||||
perform_service_job(nil, payload)
|
||||
@service_queue.subscribe do |_delivery_info, _properties, payload|
|
||||
Astute.logger.debug "Process message from service queue:\n"\
|
||||
"#{payload.pretty_inspect}"
|
||||
perform_service_job(payload)
|
||||
end
|
||||
end
|
||||
|
||||
@ -94,7 +113,7 @@ module Astute
|
||||
end
|
||||
end
|
||||
|
||||
def perform_main_job(metadata, payload)
|
||||
def perform_main_job(payload)
|
||||
@main_work_thread = Thread.new do
|
||||
data = parse_data(payload)
|
||||
@tasks_queue = Astute::Server::TaskQueue.new
|
||||
@ -109,9 +128,12 @@ module Astute
|
||||
end
|
||||
end
|
||||
|
||||
def perform_service_job(metadata, payload)
|
||||
def perform_service_job(payload)
|
||||
Thread.new do
|
||||
service_data = {:main_work_thread => @main_work_thread, :tasks_queue => @tasks_queue}
|
||||
service_data = {
|
||||
:main_work_thread => @main_work_thread,
|
||||
:tasks_queue => @tasks_queue
|
||||
}
|
||||
data = parse_data(payload)
|
||||
send_message_task_in_orchestrator(data)
|
||||
dispatch(data, service_data)
|
||||
@ -127,8 +149,9 @@ module Astute
|
||||
abort_messages data[(i + 1)..-1]
|
||||
break
|
||||
rescue => ex
|
||||
Astute.logger.error "Error running RPC method #{message['method']}: #{ex.message}, " \
|
||||
"trace: #{ex.format_backtrace}"
|
||||
Astute.logger.error "Error running RPC method "\
|
||||
"#{message['method']}: #{ex.message}, "\
|
||||
"trace: #{ex.format_backtrace}"
|
||||
return_results message, {
|
||||
'status' => 'error',
|
||||
'error' => "Method #{message['method']}. #{ex.message}.\n" \
|
||||
@ -156,7 +179,10 @@ module Astute
|
||||
return
|
||||
end
|
||||
|
||||
Astute.logger.debug "Main worker task id is #{@tasks_queue.current_task_id}" if service_data.nil?
|
||||
if service_data.nil?
|
||||
Astute.logger.debug "Main worker task id is "\
|
||||
"#{@tasks_queue.current_task_id}"
|
||||
end
|
||||
|
||||
Astute.logger.info "Processing RPC call '#{data['method']}'"
|
||||
if !service_data
|
||||
@ -168,7 +194,11 @@ module Astute
|
||||
|
||||
def return_results(message, results={})
|
||||
if results.is_a?(Hash) && message['respond_to']
|
||||
reporter = Astute::Server::Reporter.new(@producer, message['respond_to'], message['args']['task_uuid'])
|
||||
reporter = Astute::Server::Reporter.new(
|
||||
@producer,
|
||||
message['respond_to'],
|
||||
message['args']['task_uuid']
|
||||
)
|
||||
reporter.report results
|
||||
end
|
||||
end
|
||||
@ -179,7 +209,8 @@ module Astute
|
||||
begin
|
||||
messages = JSON.load(data)
|
||||
rescue => e
|
||||
Astute.logger.error "Error deserializing payload: #{e.message}, trace:\n#{e.backtrace.pretty_inspect}"
|
||||
Astute.logger.error "Error deserializing payload: #{e.message},"\
|
||||
" trace:\n#{e.backtrace.pretty_inspect}"
|
||||
end
|
||||
messages.is_a?(Array) ? messages : [messages]
|
||||
end
|
||||
@ -197,7 +228,12 @@ module Astute
|
||||
|
||||
if message['args']['nodes'].instance_of?(Array)
|
||||
err_nodes = message['args']['nodes'].map do |node|
|
||||
{'uid' => node['uid'], 'status' => 'error', 'error_type' => 'provision', 'progress' => 0}
|
||||
{
|
||||
'uid' => node['uid'],
|
||||
'status' => 'error',
|
||||
'error_type' => 'provision',
|
||||
'progress' => 0
|
||||
}
|
||||
end
|
||||
|
||||
err_msg.merge!('nodes' => err_nodes)
|
||||
@ -205,7 +241,8 @@ module Astute
|
||||
|
||||
return_results(message, err_msg)
|
||||
rescue => ex
|
||||
Astute.logger.debug "Failed to abort '#{message['method']}':\n#{ex.pretty_inspect}"
|
||||
Astute.logger.debug "Failed to abort '#{message['method']}':\n"\
|
||||
"#{ex.pretty_inspect}"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@ -14,6 +14,7 @@
|
||||
|
||||
require 'raemon'
|
||||
require 'net/http'
|
||||
require 'bunny'
|
||||
|
||||
module Astute
|
||||
module Server
|
||||
@ -26,39 +27,42 @@ module Astute
|
||||
def start
|
||||
super
|
||||
start_heartbeat
|
||||
Astute::Server::AsyncLogger.start_up(Astute.logger)
|
||||
Astute.logger = Astute::Server::AsyncLogger
|
||||
end
|
||||
|
||||
def stop
|
||||
super
|
||||
begin
|
||||
@connection.close{ stop_event_machine } if @connection
|
||||
ensure
|
||||
stop_event_machine
|
||||
end
|
||||
@connection.stop if defined?(@connection) && @connection.present?
|
||||
@producer.stop if defined?(@producer) && @producer.present?
|
||||
@server.stop if defined?(@server) && @server.present?
|
||||
Astute::Server::AsyncLogger.shutdown
|
||||
end
|
||||
|
||||
def run
|
||||
Astute.logger.info "Worker initialization"
|
||||
EM.run do
|
||||
run_server
|
||||
end
|
||||
rescue AMQP::TCPConnectionFailed => e
|
||||
Astute.logger.warn "TCP connection to AMQP failed: #{e.message}. Retry #{DELAY_SEC} sec later..."
|
||||
run_server
|
||||
rescue Bunny::TCPConnectionFailed => e
|
||||
Astute.logger.warn "TCP connection to AMQP failed: #{e.message}. "\
|
||||
"Retry #{DELAY_SEC} sec later..."
|
||||
sleep DELAY_SEC
|
||||
retry
|
||||
rescue AMQP::PossibleAuthenticationFailureError => e
|
||||
Astute.logger.warn "If problem repeated more than 5 minutes, please check " \
|
||||
"authentication parameters. #{e.message}. Retry #{DELAY_SEC} sec later..."
|
||||
rescue Bunny::PossibleAuthenticationFailureError => e
|
||||
Astute.logger.warn "If problem repeated more than 5 minutes, "\
|
||||
"please check "\
|
||||
"authentication parameters. #{e.message}. "\
|
||||
"Retry #{DELAY_SEC} sec later..."
|
||||
sleep DELAY_SEC
|
||||
retry
|
||||
rescue => e
|
||||
Astute.logger.error "Exception during worker initialization: #{e.message}, trace: #{e.format_backtrace}"
|
||||
Astute.logger.error "Exception during worker initialization:"\
|
||||
" #{e.message}, trace: #{e.format_backtrace}"
|
||||
Astute.logger.warn "Retry #{DELAY_SEC} sec later..."
|
||||
sleep DELAY_SEC
|
||||
retry
|
||||
end
|
||||
|
||||
private
|
||||
private
|
||||
|
||||
def start_heartbeat
|
||||
@heartbeat ||= Thread.new do
|
||||
@ -68,76 +72,97 @@ module Astute
|
||||
end
|
||||
|
||||
def run_server
|
||||
AMQP.logging = true
|
||||
AMQP.connect(connection_options) do |connection|
|
||||
@connection = configure_connection(connection)
|
||||
@connection = Bunny.new(connection_options)
|
||||
@connection.start
|
||||
channels_and_exchanges = declare_channels_and_exchanges(@connection)
|
||||
|
||||
@channel = create_channel(@connection)
|
||||
@exchange = @channel.topic(Astute.config.broker_exchange, :durable => true)
|
||||
@service_channel = create_channel(@connection, prefetch=false)
|
||||
@service_exchange = @service_channel.fanout(Astute.config.broker_service_exchange, :auto_delete => true)
|
||||
@producer = Astute::Server::Producer.new(
|
||||
channels_and_exchanges[:report_exchange]
|
||||
)
|
||||
delegate = Astute::Server::Dispatcher.new(@producer)
|
||||
@server = Astute::Server::Server.new(
|
||||
channels_and_exchanges,
|
||||
delegate,
|
||||
@producer
|
||||
)
|
||||
|
||||
@producer = Astute::Server::Producer.new(@exchange)
|
||||
@delegate = Astute.config.delegate || Astute::Server::Dispatcher.new(@producer)
|
||||
@server = Astute::Server::Server.new(@channel, @exchange, @delegate, @producer, @service_channel, @service_exchange)
|
||||
|
||||
@server.run
|
||||
end
|
||||
@server.run
|
||||
end
|
||||
|
||||
def configure_connection(connection)
|
||||
connection.on_tcp_connection_loss do |conn, settings|
|
||||
Astute.logger.warn "Trying to reconnect to message broker. Retry #{DELAY_SEC} sec later..."
|
||||
EM.add_timer(DELAY_SEC) { conn.reconnect }
|
||||
end
|
||||
connection
|
||||
end
|
||||
def declare_channels_and_exchanges(connection)
|
||||
# WARN: Bunny::Channel are designed to assume they are
|
||||
# not shared between threads.
|
||||
channel = @connection.create_channel
|
||||
exchange = channel.topic(
|
||||
Astute.config.broker_exchange,
|
||||
:durable => true
|
||||
)
|
||||
|
||||
def create_channel(connection, prefetch=true)
|
||||
prefetch_opts = ( prefetch ? {:prefetch => 1} : {} )
|
||||
channel = AMQP::Channel.new(connection, connection.next_channel_id, prefetch_opts)
|
||||
channel.auto_recovery = true
|
||||
channel.on_error do |ch, error|
|
||||
if error.reply_code == 406 #PRECONDITION_FAILED
|
||||
cleanup_rabbitmq_stuff
|
||||
else
|
||||
Astute.logger.fatal "Channel error\n#{error.pretty_inspect}"
|
||||
end
|
||||
sleep DELAY_SEC # avoid race condition
|
||||
stop
|
||||
report_channel = @connection.create_channel
|
||||
report_exchange = report_channel.topic(
|
||||
Astute.config.broker_exchange,
|
||||
:durable => true
|
||||
)
|
||||
|
||||
service_channel = @connection.create_channel
|
||||
service_channel.prefetch(0)
|
||||
|
||||
service_exchange = service_channel.fanout(
|
||||
Astute.config.broker_service_exchange,
|
||||
:auto_delete => true
|
||||
)
|
||||
|
||||
return {
|
||||
:exchange => exchange,
|
||||
:service_exchange => service_exchange,
|
||||
:channel => channel,
|
||||
:service_channel => service_channel,
|
||||
:report_channel => report_channel,
|
||||
:report_exchange => report_exchange
|
||||
}
|
||||
rescue Bunny::PreconditionFailed => e
|
||||
Astute.logger.warn "Try to remove problem exchanges and queues"
|
||||
if connection.queue_exists? Astute.config.broker_queue
|
||||
channel.queue_delete Astute.config.broker_queue
|
||||
end
|
||||
channel
|
||||
if connection.queue_exists? Astute.config.broker_publisher_queue
|
||||
channel.queue_delete Astute.config.broker_publisher_queue
|
||||
end
|
||||
|
||||
cleanup_rabbitmq_stuff
|
||||
raise e
|
||||
end
|
||||
|
||||
def connection_options
|
||||
{
|
||||
:host => Astute.config.broker_host,
|
||||
:port => Astute.config.broker_port,
|
||||
:username => Astute.config.broker_username,
|
||||
:password => Astute.config.broker_password,
|
||||
:user => Astute.config.broker_username,
|
||||
:pass => Astute.config.broker_password,
|
||||
:heartbeat => :server
|
||||
}.reject{|k, v| v.nil? }
|
||||
end
|
||||
|
||||
def stop_event_machine
|
||||
EM.stop_event_loop if EM.reactor_running?
|
||||
end
|
||||
|
||||
def cleanup_rabbitmq_stuff
|
||||
Astute.logger.warn "Try to remove problem exchanges and queues"
|
||||
|
||||
[Astute.config.broker_exchange, Astute.config.broker_service_exchange].each do |exchange|
|
||||
rest_delete("/api/exchanges/%2F/#{exchange}")
|
||||
end
|
||||
|
||||
[Astute.config.broker_queue, Astute.config.broker_publisher_queue].each do |queue|
|
||||
rest_delete("/api/queues/%2F/#{queue}")
|
||||
[Astute.config.broker_exchange,
|
||||
Astute.config.broker_service_exchange].each do |exchange|
|
||||
rest_delete("/api/exchanges/%2F/#{exchange}")
|
||||
end
|
||||
end
|
||||
|
||||
def rest_delete(url)
|
||||
http = Net::HTTP.new(Astute.config.broker_host, Astute.config.broker_rest_api_port)
|
||||
http = Net::HTTP.new(
|
||||
Astute.config.broker_host,
|
||||
Astute.config.broker_rest_api_port
|
||||
)
|
||||
request = Net::HTTP::Delete.new(url)
|
||||
request.basic_auth(Astute.config.broker_username, Astute.config.broker_password)
|
||||
request.basic_auth(
|
||||
Astute.config.broker_username,
|
||||
Astute.config.broker_password
|
||||
)
|
||||
|
||||
response = http.request(request)
|
||||
|
||||
@ -145,13 +170,13 @@ module Astute
|
||||
when 204 then Astute.logger.debug "Successfully delete object at #{url}"
|
||||
when 404 then
|
||||
else
|
||||
Astute.logger.error "Failed to perform delete request. Debug information: "\
|
||||
"http code: #{response.code}, message: #{response.message},"\
|
||||
"body #{response.body}"
|
||||
Astute.logger.error "Failed to perform delete request. Debug"\
|
||||
" information: http code: #{response.code},"\
|
||||
" message: #{response.message},"\
|
||||
" body #{response.body}"
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
end # Worker
|
||||
end #Server
|
||||
end #Astute
|
||||
|
||||
@ -20,7 +20,7 @@ Requires: ruby21-rubygem-activesupport = 3.0.10
|
||||
Requires: ruby21-rubygem-mcollective-client = 2.4.1
|
||||
Requires: ruby21-rubygem-symboltable = 1.0.2
|
||||
Requires: ruby21-rubygem-rest-client = 1.6.7
|
||||
Requires: ruby21-rubygem-amqp = 1.4.1
|
||||
Requires: ruby21-rubygem-bunny
|
||||
Requires: ruby21-rubygem-raemon = 0.3.0
|
||||
Requires: ruby21-rubygem-net-ssh = 2.8.0
|
||||
Requires: ruby21-rubygem-net-ssh-gateway = 1.2.0
|
||||
|
||||
Loading…
Reference in New Issue
Block a user