Extend command timeout

In some deployments with huge number of resources, some commands
(especially list commands) may take long time. Extend the timeout to
avoid giving up commands too early.

Also extend the whole timeout and sleep between command executions,
to avoid too many requests within timeouts.

Change-Id: I3610340067d416dc7be94696f2fba921fa015eb9
This commit is contained in:
Takashi Kajinami 2024-10-01 21:19:58 +09:00
parent 2238cf2347
commit 533c346467
2 changed files with 13 additions and 14 deletions

View File

@ -14,11 +14,9 @@ class Puppet::Provider::Openstack < Puppet::Provider
commands :openstack_command => 'openstack' commands :openstack_command => 'openstack'
@@no_retry_actions = %w(create remove delete) @@no_retry_actions = %w(create remove delete)
@@command_timeout = 40 @@command_timeout = 90
# Fails on the 5th retry for a max of 212s (~3.5min) before total @@request_timeout = 300
# failure. @@retry_sleep = 10
@@request_timeout = 170
@@retry_sleep = 3
class << self class << self
[:no_retry_actions, :request_timeout, :retry_sleep].each do |m| [:no_retry_actions, :request_timeout, :retry_sleep].each do |m|
define_method m do define_method m do
@ -89,8 +87,8 @@ class Puppet::Provider::Openstack < Puppet::Provider
Puppet::Util.withenv(env) do Puppet::Util.withenv(env) do
rv = nil rv = nil
end_time = current_time + request_timeout
start_time = current_time start_time = current_time
end_time = start_time + request_timeout
retry_count = 0 retry_count = 0
loop do loop do
begin begin
@ -129,9 +127,10 @@ class Puppet::Provider::Openstack < Puppet::Provider
rescue Puppet::ExecutionFailure => exception rescue Puppet::ExecutionFailure => exception
raise Puppet::Error::OpenstackUnauthorizedError, 'Could not authenticate' if exception.message =~ /HTTP 40[13]/ raise Puppet::Error::OpenstackUnauthorizedError, 'Could not authenticate' if exception.message =~ /HTTP 40[13]/
raise Puppet::Error::OpenstackUnauthorizedError, 'Could not authenticate' if exception.message.match(/Missing value \S* required for auth plugin/) raise Puppet::Error::OpenstackUnauthorizedError, 'Could not authenticate' if exception.message.match(/Missing value \S* required for auth plugin/)
if current_time > end_time remaining_time = end_time - current_time
if remaining_time < 0
error_message = exception.message error_message = exception.message
error_message += " (tried #{retry_count}, for a total of #{end_time - start_time } seconds)" error_message += " (tried #{retry_count}, for a total of #{end_time - start_time} seconds)"
raise(Puppet::ExecutionFailure, error_message) raise(Puppet::ExecutionFailure, error_message)
end end
@ -142,7 +141,7 @@ class Puppet::Provider::Openstack < Puppet::Provider
raise exception if exception.message.match(nr) raise exception if exception.message.match(nr)
end end
end end
debug "Non-fatal error: '#{exception.message}'. Retrying for #{end_time - current_time} more seconds" debug "Non-fatal error: '#{exception.message}'. Retrying for #{remaining_time} more seconds"
sleep retry_sleep sleep retry_sleep
retry_count += 1 retry_count += 1
retry retry

View File

@ -111,7 +111,7 @@ name="test"
lambda { |*args| raise Puppet::ExecutionFailure, 'Unable to establish connection' }, lambda { |*args| raise Puppet::ExecutionFailure, 'Unable to establish connection' },
lambda { |*args| return list_data } lambda { |*args| return list_data }
) )
expect(provider.class).to receive(:sleep).with(3).and_return(nil) expect(provider.class).to receive(:sleep).with(10).and_return(nil)
response = Puppet::Provider::Openstack.request('project', 'list', ['--long']) response = Puppet::Provider::Openstack.request('project', 'list', ['--long'])
expect(response.first[:description]).to eq 'Test tenant' expect(response.first[:description]).to eq 'Test tenant'
end end
@ -119,10 +119,10 @@ name="test"
it 'fails after the timeout and redacts' do it 'fails after the timeout and redacts' do
expect(provider.class).to receive(:execute) expect(provider.class).to receive(:execute)
.and_raise(Puppet::ExecutionFailure, "Execution of 'openstack user create foo --password secret' returned 1: command failed") .and_raise(Puppet::ExecutionFailure, "Execution of 'openstack user create foo --password secret' returned 1: command failed")
.exactly(3).times .exactly(6).times
allow(provider.class).to receive(:sleep) allow(provider.class).to receive(:sleep)
allow(provider.class).to receive(:current_time) allow(provider.class).to receive(:current_time)
.and_return(0, 10, 10, 20, 20, 200, 200) .and_return(0, 10, 20, 100, 200, 300, 400)
expect do expect do
Puppet::Provider::Openstack.request('project', 'list', ['--long']) Puppet::Provider::Openstack.request('project', 'list', ['--long'])
end.to raise_error Puppet::ExecutionFailure, /Execution of \'openstack user create foo --password \[redacted secret\]\' returned 1/ end.to raise_error Puppet::ExecutionFailure, /Execution of \'openstack user create foo --password \[redacted secret\]\' returned 1/
@ -132,10 +132,10 @@ name="test"
expect(provider.class).to receive(:openstack) expect(provider.class).to receive(:openstack)
.with('project', 'list', '--quiet', '--format', 'csv', ['--long']) .with('project', 'list', '--quiet', '--format', 'csv', ['--long'])
.and_raise(Puppet::ExecutionFailure, 'Unable to establish connection') .and_raise(Puppet::ExecutionFailure, 'Unable to establish connection')
.exactly(3).times .exactly(6).times
allow(provider.class).to receive(:sleep) allow(provider.class).to receive(:sleep)
allow(provider.class).to receive(:current_time) allow(provider.class).to receive(:current_time)
.and_return(0, 10, 10, 20, 20, 200, 200) .and_return(0, 10, 20, 100, 200, 300, 400)
expect do expect do
Puppet::Provider::Openstack.request('project', 'list', ['--long']) Puppet::Provider::Openstack.request('project', 'list', ['--long'])
end.to raise_error Puppet::ExecutionFailure, /Unable to establish connection/ end.to raise_error Puppet::ExecutionFailure, /Unable to establish connection/