openstack/puppet-openstacklib
Code Issues Proposed changes

Merge "Extend command timeout"

Browse Source
This commit is contained in:
Zuul 2024-10-03 15:41:19 +00:00 committed by Gerrit Code Review
commit 8a51244df9
2 changed files with 13 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
lib/puppet/provider/openstack.rb
View File

@ -14,11 +14,9 @@ class Puppet::Provider::Openstack < Puppet::Provider
commands :openstack_command => 'openstack' commands :openstack_command => 'openstack'
@@no_retry_actions = %w(create remove delete) @@no_retry_actions = %w(create remove delete)
@@command_timeout = 40 @@command_timeout = 90
# Fails on the 5th retry for a max of 212s (~3.5min) before total @@request_timeout = 300
# failure. @@retry_sleep = 10
@@request_timeout = 170
@@retry_sleep = 3
class << self class << self
[:no_retry_actions, :request_timeout, :retry_sleep].each do |m| [:no_retry_actions, :request_timeout, :retry_sleep].each do |m|
define_method m do define_method m do
@ -89,8 +87,8 @@ class Puppet::Provider::Openstack < Puppet::Provider
Puppet::Util.withenv(env) do Puppet::Util.withenv(env) do
rv = nil rv = nil
end_time = current_time + request_timeout
start_time = current_time start_time = current_time
end_time = start_time + request_timeout
retry_count = 0 retry_count = 0
loop do loop do
begin begin
@ -129,9 +127,10 @@ class Puppet::Provider::Openstack < Puppet::Provider
rescue Puppet::ExecutionFailure => exception rescue Puppet::ExecutionFailure => exception
raise Puppet::Error::OpenstackUnauthorizedError, 'Could not authenticate' if exception.message =~ /HTTP 40[13]/ raise Puppet::Error::OpenstackUnauthorizedError, 'Could not authenticate' if exception.message =~ /HTTP 40[13]/
raise Puppet::Error::OpenstackUnauthorizedError, 'Could not authenticate' if exception.message.match(/Missing value \S* required for auth plugin/) raise Puppet::Error::OpenstackUnauthorizedError, 'Could not authenticate' if exception.message.match(/Missing value \S* required for auth plugin/)
if current_time > end_time remaining_time = end_time - current_time
if remaining_time < 0
error_message = exception.message error_message = exception.message
error_message += " (tried #{retry_count}, for a total of #{end_time - start_time } seconds)" error_message += " (tried #{retry_count}, for a total of #{end_time - start_time} seconds)"
raise(Puppet::ExecutionFailure, error_message) raise(Puppet::ExecutionFailure, error_message)
end end
@ -142,7 +141,7 @@ class Puppet::Provider::Openstack < Puppet::Provider
raise exception if exception.message.match(nr) raise exception if exception.message.match(nr)
end end
end end
debug "Non-fatal error: '#{exception.message}'. Retrying for #{end_time - current_time} more seconds" debug "Non-fatal error: '#{exception.message}'. Retrying for #{remaining_time} more seconds"
sleep retry_sleep sleep retry_sleep
retry_count += 1 retry_count += 1
retry retry

10
spec/unit/provider/openstack_spec.rb
View File

@ -111,7 +111,7 @@ name="test"
lambda { |*args| raise Puppet::ExecutionFailure, 'Unable to establish connection' }, lambda { |*args| raise Puppet::ExecutionFailure, 'Unable to establish connection' },
lambda { |*args| return list_data } lambda { |*args| return list_data }
) )
expect(provider.class).to receive(:sleep).with(3).and_return(nil) expect(provider.class).to receive(:sleep).with(10).and_return(nil)
response = Puppet::Provider::Openstack.request('project', 'list', ['--long']) response = Puppet::Provider::Openstack.request('project', 'list', ['--long'])
expect(response.first[:description]).to eq 'Test tenant' expect(response.first[:description]).to eq 'Test tenant'
end end
@ -119,10 +119,10 @@ name="test"
it 'fails after the timeout and redacts' do it 'fails after the timeout and redacts' do
expect(provider.class).to receive(:execute) expect(provider.class).to receive(:execute)
.and_raise(Puppet::ExecutionFailure, "Execution of 'openstack user create foo --password secret' returned 1: command failed") .and_raise(Puppet::ExecutionFailure, "Execution of 'openstack user create foo --password secret' returned 1: command failed")
.exactly(3).times .exactly(6).times
allow(provider.class).to receive(:sleep) allow(provider.class).to receive(:sleep)
allow(provider.class).to receive(:current_time) allow(provider.class).to receive(:current_time)
.and_return(0, 10, 10, 20, 20, 200, 200) .and_return(0, 10, 20, 100, 200, 300, 400)
expect do expect do
Puppet::Provider::Openstack.request('project', 'list', ['--long']) Puppet::Provider::Openstack.request('project', 'list', ['--long'])
end.to raise_error Puppet::ExecutionFailure, /Execution of \'openstack user create foo --password \[redacted secret\]\' returned 1/ end.to raise_error Puppet::ExecutionFailure, /Execution of \'openstack user create foo --password \[redacted secret\]\' returned 1/
@ -132,10 +132,10 @@ name="test"
expect(provider.class).to receive(:openstack) expect(provider.class).to receive(:openstack)
.with('project', 'list', '--quiet', '--format', 'csv', ['--long']) .with('project', 'list', '--quiet', '--format', 'csv', ['--long'])
.and_raise(Puppet::ExecutionFailure, 'Unable to establish connection') .and_raise(Puppet::ExecutionFailure, 'Unable to establish connection')
.exactly(3).times .exactly(6).times
allow(provider.class).to receive(:sleep) allow(provider.class).to receive(:sleep)
allow(provider.class).to receive(:current_time) allow(provider.class).to receive(:current_time)
.and_return(0, 10, 10, 20, 20, 200, 200) .and_return(0, 10, 20, 100, 200, 300, 400)
expect do expect do
Puppet::Provider::Openstack.request('project', 'list', ['--long']) Puppet::Provider::Openstack.request('project', 'list', ['--long'])
end.to raise_error Puppet::ExecutionFailure, /Unable to establish connection/ end.to raise_error Puppet::ExecutionFailure, /Unable to establish connection/