Upload file task timeout support

Astute will not retry and will not wait around 10 minutes for
every node which connection was missed in case of
upload file task. For now it will wait only default upload
timeout.

Default timeout for upload now can be setup in config. For now
it is 60 seconds. Also upload file task now support timeout
parameter which will overide default.

Change-Id: Ice8207f539566a50d4eb30c04ab563c3ee1278ec
Closes-Bug: #1629031
This commit is contained in:
Vladimir Sharshov (warpc) 2016-12-27 17:50:14 +03:00
parent 1dc4d754b7
commit f475c45dfc
7 changed files with 123 additions and 46 deletions

View File

@ -74,6 +74,7 @@ module Astute
conf[:shell_retries] = 2 # default retries for shell task
conf[:shell_interval] = 2 # default interval for shell task
conf[:shell_timeout] = 300 # default timeout for shell task
conf[:upload_timeout] = 60 # default timeout for upload task
conf[:shell_cwd] = '/' # default cwd for shell task
conf[:stop_timeout] = 600 # how long it can take for stop
conf[:rsync_options] = '-c -r --delete -l' # default rsync options

View File

@ -66,7 +66,8 @@ module Astute
"data" => node.to_json,
"user_owner" => 'root',
"group_owner" => 'root',
"overwrite" => true
"overwrite" => true,
"timeout" => Astute.config.upload_timeout
}
}
end

View File

@ -0,0 +1,95 @@
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
module Astute
class UploadFileMClient
def initialize(ctx, node_id)
@ctx = ctx
@node_id = node_id
end
# Run upload without check using mcollective agent
# @param [Hash] mco_params Upload file options
# @param [Integer] timeout Timeout for upload command
# @return [true, false] upload result
def upload_without_check(mco_params)
mco_params = setup_default(mco_params)
results = upload_file(_check_result=false, mco_params['timeout'])
.upload(
:path => mco_params['path'],
:content => mco_params['content'],
:overwrite => mco_params['overwrite'],
:parents => mco_params['parents'],
:permissions => mco_params['permissions'],
:user_owner => mco_params['user_owner'],
:group_owner => mco_params['group_owner'],
:dir_permissions => mco_params['dir_permissions']
)
if results.present? && results.first[:statuscode] == 0
Astute.logger.debug("#{@ctx.task_id}: file was uploaded "\
"#{details_for_log(mco_params)} successfully")
true
else
Astute.logger.error("#{@ctx.task_id}: file was not uploaded "\
"#{details_for_log(mco_params)}: "\
"#{results.first[:msg] if results.present? }")
false
end
rescue MClientTimeout, MClientError => e
Astute.logger.error("#{@ctx.task_id}: file was not uploaded "\
"#{details_for_log(mco_params)}: #{e.message}")
false
end
private
# Create configured shell mcollective agent
# @return [Astute::MClient]
def upload_file(check_result=false, timeout=2)
MClient.new(
@ctx,
"uploadfile",
[@node_id],
check_result,
timeout
)
end
# Setup default value for upload mcollective agent
# @param [Hash] mco_params Upload file options
# @return [Hash] mco_params
def setup_default(mco_params)
mco_params['timeout'] ||= Astute.config.upload_timeout
mco_params['overwrite'] = true if mco_params['overwrite'].nil?
mco_params['parents'] = true if mco_params['parents'].nil?
mco_params['permissions'] ||= '0644'
mco_params['user_owner'] ||= 'root'
mco_params['group_owner'] ||= 'root'
mco_params['dir_permissions'] ||= '0755'
mco_params
end
# Return short useful info about node and shell task
# @return [String] detail info about upload task
def details_for_log(mco_params)
"#{mco_params['path']} on node #{@node_id} "\
"with timeout #{mco_params['timeout']}"
end
end
end

View File

@ -145,37 +145,8 @@ module Astute
# In other case please use separate thread or
# use upload file task.
# Synchronous (blocking) call
def upload_file(node_uid, mco_params={})
upload_mclient = MClient.new(
@ctx,
"uploadfile",
Array(node_uid)
)
mco_params['overwrite'] = true if mco_params['overwrite'].nil?
mco_params['parents'] = true if mco_params['parents'].nil?
mco_params['permissions'] ||= '0644'
mco_params['user_owner'] ||= 'root'
mco_params['group_owner'] ||= 'root'
mco_params['dir_permissions'] ||= '0755'
upload_mclient.upload(
:path => mco_params['path'],
:content => mco_params['content'],
:overwrite => mco_params['overwrite'],
:parents => mco_params['parents'],
:permissions => mco_params['permissions'],
:user_owner => mco_params['user_owner'],
:group_owner => mco_params['group_owner'],
:dir_permissions => mco_params['dir_permissions']
)
Astute.logger.debug("#{@ctx.task_id}: file was uploaded " \
"#{mco_params['path']} on node #{node_uid} successfully")
true
rescue MClientTimeout, MClientError => e
Astute.logger.error("#{@ctx.task_id}: file was not uploaded "\
"#{mco_params['path']} on node #{node_uid}: #{e.message}")
false
def upload_file(node_uid, mco_params)
UploadFileMClient.new(@ctx, node_uid).upload_without_check(mco_params)
end
def failed!

View File

@ -41,7 +41,8 @@ module Astute
def setup_default
@task['parameters']['content'] = @task['parameters']['data']
@task['parameters']['timeout'] ||= Astute.config.upload_timeout
end
end
end
end

View File

@ -157,7 +157,8 @@ describe Astute::ImageProvision do
"data" => node.to_json,
"user_owner" => 'root',
"group_owner" => 'root',
"overwrite" => true
"overwrite" => true,
"timeout" => Astute.config.upload_timeout
}
}
end
@ -212,4 +213,3 @@ describe Astute::ImageProvision do
end
end

View File

@ -37,7 +37,6 @@ describe Astute::Shell do
describe '#run' do
it 'should create puppet wrapper' do
mclient = mock_rpcclient
Astute::Shell.any_instance.stubs(:run_shell_without_check)
Astute::Puppet.any_instance.stubs(:run)
@ -53,18 +52,27 @@ describe Astute::Shell do
}
eos
mclient.expects(:upload).with({
:path => '/etc/puppet/shell_manifests/shell_task_id_manifest.pp',
:content => content,
:overwrite => true,
:parents => true,
:permissions => '0755',
:user_owner => 'root',
:group_owner => 'root',
:dir_permissions => '0755'})
manifest_content = <<-eos
#!/bin/bash
# Puppet shell wrapper for task: shell_task_id
# Manifest: /etc/puppet/shell_manifests/shell_task_id_manifest.pp
cd / && sh some_command
eos
Astute::UploadFileMClient.any_instance.expects(:upload_without_check).with({
'path' => '/etc/puppet/shell_manifests/shell_task_id_manifest.pp',
'content' => content,
'permissions' => '0755'
})
Astute::UploadFileMClient.any_instance.expects(:upload_without_check).with({
'path' => '/etc/puppet/shell_manifests/shell_task_id_command.sh',
'content' => manifest_content,
'permissions' => '0755'
})
subject.run
end
end
end