openstack/ironic
Code Issues Proposed changes

Merge "Follow-up Retries and timeout for IPA command"

Browse Source
This commit is contained in:
Zuul 2018-12-07 12:51:40 +00:00 committed by Gerrit Code Review
commit 36a81170dc
3 changed files with 80 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ironic/conf/agent.py
View File

@ -104,12 +104,12 @@ opts = [
'service.')),
cfg.IntOpt('command_timeout',
default=60,
help=_('Timeout (in seconds) for IPA commands')),
help=_('Timeout (in seconds) for IPA commands.')),
cfg.IntOpt('max_command_attempts',
default=3,
help=_('This is the maximum number of attempts that will be '
'done for IPA commands that fails due to network '
'problems')),
'problems.')),
]

139
ironic/tests/unit/drivers/modules/test_agent_client.py
View File

@ -16,6 +16,7 @@ import json
import mock
import requests
import retrying
import six
from six.moves import http_client
@ -153,68 +154,6 @@ class TestAgentClient(base.TestCase):
{'method': method, 'node': self.node.uuid,
'error': error}, str(e))
def test__command_fail_all_attempts(self):
error = 'Connection Timeout'
method = 'standby.run_image'
image_info = {'image_id': 'test_image'}
params = {'image_info': image_info}
self.client.session.post.side_effect = [requests.Timeout(error),
requests.Timeout(error),
requests.Timeout(error),
requests.Timeout(error)]
self.client._get_command_url(self.node)
self.client._get_command_body(method, params)
e = self.assertRaises(exception.AgentConnectionFailed,
self.client._command,
self.node, method, params)
self.assertEqual('Connection to agent failed: Failed to connect to '
'the agent running on node %(node)s for invoking '
'command %(method)s. Error: %(error)s' %
{'method': method, 'node': self.node.uuid,
'error': error}, str(e))
self.assertEqual(3, self.client.session.post.call_count)
def test__command_succeed_after_two_timeouts(self):
error = 'Connection Timeout'
response_data = {'status': 'ok'}
response_text = json.dumps(response_data)
method = 'standby.run_image'
image_info = {'image_id': 'test_image'}
params = {'image_info': image_info}
self.client.session.post.side_effect = [requests.Timeout(error),
requests.Timeout(error),
MockResponse(response_text)]
response = self.client._command(self.node, method, params)
self.assertEqual(3, self.client.session.post.call_count)
self.assertEqual(response, response_data)
self.client.session.post.assert_called_with(
self.client._get_command_url(self.node),
data=self.client._get_command_body(method, params),
params={'wait': 'false'},
timeout=60)
def test__command_succeed_after_one_timeout(self):
error = 'Connection Timeout'
response_data = {'status': 'ok'}
response_text = json.dumps(response_data)
method = 'standby.run_image'
image_info = {'image_id': 'test_image'}
params = {'image_info': image_info}
self.client.session.post.side_effect = [requests.Timeout(error),
MockResponse(response_text),
requests.Timeout(error)]
response = self.client._command(self.node, method, params)
self.assertEqual(2, self.client.session.post.call_count)
self.assertEqual(response, response_data)
self.client.session.post.assert_called_with(
self.client._get_command_url(self.node),
data=self.client._get_command_body(method, params),
params={'wait': 'false'},
timeout=60)
def test__command_error_code(self):
response_text = '{"faultstring": "you dun goofd"}'
self.client.session.post.return_value = MockResponse(
@ -397,3 +336,79 @@ class TestAgentClient(base.TestCase):
self.client.finalize_rescue,
self.node)
self.assertFalse(self.client._command.called)
class TestAgentClientAttempts(base.TestCase):
def setUp(self):
super(TestAgentClientAttempts, self).setUp()
self.client = agent_client.AgentClient()
self.client.session = mock.MagicMock(autospec=requests.Session)
self.node = MockNode()
@mock.patch.object(retrying.time, 'sleep', autospec=True)
def test__command_fail_all_attempts(self, mock_sleep):
mock_sleep.return_value = None
error = 'Connection Timeout'
method = 'standby.run_image'
image_info = {'image_id': 'test_image'}
params = {'image_info': image_info}
self.client.session.post.side_effect = [requests.Timeout(error),
requests.Timeout(error),
requests.Timeout(error),
requests.Timeout(error)]
self.client._get_command_url(self.node)
self.client._get_command_body(method, params)
e = self.assertRaises(exception.AgentConnectionFailed,
self.client._command,
self.node, method, params)
self.assertEqual('Connection to agent failed: Failed to connect to '
'the agent running on node %(node)s for invoking '
'command %(method)s. Error: %(error)s' %
{'method': method, 'node': self.node.uuid,
'error': error}, str(e))
self.assertEqual(3, self.client.session.post.call_count)
@mock.patch.object(retrying.time, 'sleep', autospec=True)
def test__command_succeed_after_two_timeouts(self, mock_sleep):
mock_sleep.return_value = None
error = 'Connection Timeout'
response_data = {'status': 'ok'}
response_text = json.dumps(response_data)
method = 'standby.run_image'
image_info = {'image_id': 'test_image'}
params = {'image_info': image_info}
self.client.session.post.side_effect = [requests.Timeout(error),
requests.Timeout(error),
MockResponse(response_text)]
response = self.client._command(self.node, method, params)
self.assertEqual(3, self.client.session.post.call_count)
self.assertEqual(response, response_data)
self.client.session.post.assert_called_with(
self.client._get_command_url(self.node),
data=self.client._get_command_body(method, params),
params={'wait': 'false'},
timeout=60)
@mock.patch.object(retrying.time, 'sleep', autospec=True)
def test__command_succeed_after_one_timeout(self, mock_sleep):
mock_sleep.return_value = None
error = 'Connection Timeout'
response_data = {'status': 'ok'}
response_text = json.dumps(response_data)
method = 'standby.run_image'
image_info = {'image_id': 'test_image'}
params = {'image_info': image_info}
self.client.session.post.side_effect = [requests.Timeout(error),
MockResponse(response_text),
requests.Timeout(error)]
response = self.client._command(self.node, method, params)
self.assertEqual(2, self.client.session.post.call_count)
self.assertEqual(response, response_data)
self.client.session.post.assert_called_with(
self.client._get_command_url(self.node),
data=self.client._get_command_body(method, params),
params={'wait': 'false'},
timeout=60)

1
releasenotes/notes/ipa-command-retries-and-timeout-29b0be3f2c21328c.yaml
View File

@ -3,3 +3,4 @@ fixes:
- |
Adds ``command_timeout`` and ``max_command_attempts`` configuration options
to IPA, so when connection errors occur the command will be executed again.
The options are located in the ``[agent]`` section.