diff --git a/ironic/conf/agent.py b/ironic/conf/agent.py
index d92eec9b8a..7b2ea9ac57 100644
--- a/ironic/conf/agent.py
+++ b/ironic/conf/agent.py
@@ -104,12 +104,12 @@ opts = [
'service.')),
cfg.IntOpt('command_timeout',
default=60,
- help=_('Timeout (in seconds) for IPA commands')),
+ help=_('Timeout (in seconds) for IPA commands.')),
cfg.IntOpt('max_command_attempts',
default=3,
help=_('This is the maximum number of attempts that will be '
'done for IPA commands that fails due to network '
- 'problems')),
+ 'problems.')),
]
diff --git a/ironic/tests/unit/drivers/modules/test_agent_client.py b/ironic/tests/unit/drivers/modules/test_agent_client.py
index 17d210f14c..1c563f7fd3 100644
--- a/ironic/tests/unit/drivers/modules/test_agent_client.py
+++ b/ironic/tests/unit/drivers/modules/test_agent_client.py
@@ -16,6 +16,7 @@ import json
import mock
import requests
+import retrying
import six
from six.moves import http_client
@@ -153,68 +154,6 @@ class TestAgentClient(base.TestCase):
{'method': method, 'node': self.node.uuid,
'error': error}, str(e))
- def test__command_fail_all_attempts(self):
- error = 'Connection Timeout'
- method = 'standby.run_image'
- image_info = {'image_id': 'test_image'}
- params = {'image_info': image_info}
- self.client.session.post.side_effect = [requests.Timeout(error),
- requests.Timeout(error),
- requests.Timeout(error),
- requests.Timeout(error)]
- self.client._get_command_url(self.node)
- self.client._get_command_body(method, params)
-
- e = self.assertRaises(exception.AgentConnectionFailed,
- self.client._command,
- self.node, method, params)
- self.assertEqual('Connection to agent failed: Failed to connect to '
- 'the agent running on node %(node)s for invoking '
- 'command %(method)s. Error: %(error)s' %
- {'method': method, 'node': self.node.uuid,
- 'error': error}, str(e))
- self.assertEqual(3, self.client.session.post.call_count)
-
- def test__command_succeed_after_two_timeouts(self):
- error = 'Connection Timeout'
- response_data = {'status': 'ok'}
- response_text = json.dumps(response_data)
- method = 'standby.run_image'
- image_info = {'image_id': 'test_image'}
- params = {'image_info': image_info}
- self.client.session.post.side_effect = [requests.Timeout(error),
- requests.Timeout(error),
- MockResponse(response_text)]
-
- response = self.client._command(self.node, method, params)
- self.assertEqual(3, self.client.session.post.call_count)
- self.assertEqual(response, response_data)
- self.client.session.post.assert_called_with(
- self.client._get_command_url(self.node),
- data=self.client._get_command_body(method, params),
- params={'wait': 'false'},
- timeout=60)
-
- def test__command_succeed_after_one_timeout(self):
- error = 'Connection Timeout'
- response_data = {'status': 'ok'}
- response_text = json.dumps(response_data)
- method = 'standby.run_image'
- image_info = {'image_id': 'test_image'}
- params = {'image_info': image_info}
- self.client.session.post.side_effect = [requests.Timeout(error),
- MockResponse(response_text),
- requests.Timeout(error)]
-
- response = self.client._command(self.node, method, params)
- self.assertEqual(2, self.client.session.post.call_count)
- self.assertEqual(response, response_data)
- self.client.session.post.assert_called_with(
- self.client._get_command_url(self.node),
- data=self.client._get_command_body(method, params),
- params={'wait': 'false'},
- timeout=60)
-
def test__command_error_code(self):
response_text = '{"faultstring": "you dun goofd"}'
self.client.session.post.return_value = MockResponse(
@@ -397,3 +336,79 @@ class TestAgentClient(base.TestCase):
self.client.finalize_rescue,
self.node)
self.assertFalse(self.client._command.called)
+
+
+class TestAgentClientAttempts(base.TestCase):
+ def setUp(self):
+ super(TestAgentClientAttempts, self).setUp()
+ self.client = agent_client.AgentClient()
+ self.client.session = mock.MagicMock(autospec=requests.Session)
+ self.node = MockNode()
+
+ @mock.patch.object(retrying.time, 'sleep', autospec=True)
+ def test__command_fail_all_attempts(self, mock_sleep):
+ mock_sleep.return_value = None
+ error = 'Connection Timeout'
+ method = 'standby.run_image'
+ image_info = {'image_id': 'test_image'}
+ params = {'image_info': image_info}
+ self.client.session.post.side_effect = [requests.Timeout(error),
+ requests.Timeout(error),
+ requests.Timeout(error),
+ requests.Timeout(error)]
+ self.client._get_command_url(self.node)
+ self.client._get_command_body(method, params)
+
+ e = self.assertRaises(exception.AgentConnectionFailed,
+ self.client._command,
+ self.node, method, params)
+ self.assertEqual('Connection to agent failed: Failed to connect to '
+ 'the agent running on node %(node)s for invoking '
+ 'command %(method)s. Error: %(error)s' %
+ {'method': method, 'node': self.node.uuid,
+ 'error': error}, str(e))
+ self.assertEqual(3, self.client.session.post.call_count)
+
+ @mock.patch.object(retrying.time, 'sleep', autospec=True)
+ def test__command_succeed_after_two_timeouts(self, mock_sleep):
+ mock_sleep.return_value = None
+ error = 'Connection Timeout'
+ response_data = {'status': 'ok'}
+ response_text = json.dumps(response_data)
+ method = 'standby.run_image'
+ image_info = {'image_id': 'test_image'}
+ params = {'image_info': image_info}
+ self.client.session.post.side_effect = [requests.Timeout(error),
+ requests.Timeout(error),
+ MockResponse(response_text)]
+
+ response = self.client._command(self.node, method, params)
+ self.assertEqual(3, self.client.session.post.call_count)
+ self.assertEqual(response, response_data)
+ self.client.session.post.assert_called_with(
+ self.client._get_command_url(self.node),
+ data=self.client._get_command_body(method, params),
+ params={'wait': 'false'},
+ timeout=60)
+
+ @mock.patch.object(retrying.time, 'sleep', autospec=True)
+ def test__command_succeed_after_one_timeout(self, mock_sleep):
+ mock_sleep.return_value = None
+ error = 'Connection Timeout'
+ response_data = {'status': 'ok'}
+ response_text = json.dumps(response_data)
+ method = 'standby.run_image'
+ image_info = {'image_id': 'test_image'}
+ params = {'image_info': image_info}
+ self.client.session.post.side_effect = [requests.Timeout(error),
+ MockResponse(response_text),
+ requests.Timeout(error)]
+
+ response = self.client._command(self.node, method, params)
+ self.assertEqual(2, self.client.session.post.call_count)
+ self.assertEqual(response, response_data)
+ self.client.session.post.assert_called_with(
+ self.client._get_command_url(self.node),
+ data=self.client._get_command_body(method, params),
+ params={'wait': 'false'},
+ timeout=60)
diff --git a/releasenotes/notes/ipa-command-retries-and-timeout-29b0be3f2c21328c.yaml b/releasenotes/notes/ipa-command-retries-and-timeout-29b0be3f2c21328c.yaml
index f9e87c3770..f1e1e75e3b 100644
--- a/releasenotes/notes/ipa-command-retries-and-timeout-29b0be3f2c21328c.yaml
+++ b/releasenotes/notes/ipa-command-retries-and-timeout-29b0be3f2c21328c.yaml
@@ -3,3 +3,4 @@ fixes:
- |
Adds ``command_timeout`` and ``max_command_attempts`` configuration options
to IPA, so when connection errors occur the command will be executed again.
+ The options are located in the ``[agent]`` section.