diff --git a/heat-config/bin/heat-config-notify b/heat-config/bin/heat-config-notify index 3f115b0..30eb8c4 100755 --- a/heat-config/bin/heat-config-notify +++ b/heat-config/bin/heat-config-notify @@ -19,6 +19,9 @@ import sys import requests +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry + try: from heatclient import client as heatclient except ImportError: @@ -105,12 +108,25 @@ def main(argv=sys.argv, stdin=sys.stdin): # we need to trim log content because Heat response size is limited # by max_json_body_size = 1048576 str_signal_data = trim_response(signal_data) + session = requests.Session() + # Retry if connection issues occur or the service is returning a 5xx + retry = Retry( + total=10, + read=10, + connect=10, + backoff_factor=0.5, + status_forcelist=(500, 502, 503, 504) + ) + adapter = HTTPAdapter(max_retries=retry) + session.mount('http://', adapter) + session.mount('https://', adapter) + if sigverb == 'PUT': - r = requests.put(sigurl, data=str_signal_data, - headers={'content-type': 'application/json'}) + r = session.put(sigurl, data=str_signal_data, + headers={'content-type': 'application/json'}) else: - r = requests.post(sigurl, data=str_signal_data, - headers={'content-type': 'application/json'}) + r = session.post(sigurl, data=str_signal_data, + headers={'content-type': 'application/json'}) log.debug('Response %s ' % r) if 'deploy_queue_id' in iv: diff --git a/releasenotes/notes/heat-config-notify-retry-e01b995d239bb1fd.yaml b/releasenotes/notes/heat-config-notify-retry-e01b995d239bb1fd.yaml new file mode 100644 index 0000000..5551bda --- /dev/null +++ b/releasenotes/notes/heat-config-notify-retry-e01b995d239bb1fd.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Add retry logic if 500, 502, 503 or 504 responses are returned or network + connection issues occur during heat signals using http or https. diff --git a/tests/test_heat_config_notify.py b/tests/test_heat_config_notify.py index 15ca558..2b2dd9a 100644 --- a/tests/test_heat_config_notify.py +++ b/tests/test_heat_config_notify.py @@ -111,9 +111,14 @@ class HeatConfigNotifyTest(common.RunScriptTest): def test_notify_signal_id(self): requests = mock.MagicMock() - hcn.requests = requests + session = mock.MagicMock() + requests.Session.return_value = session + retry = mock.MagicMock() + httpadapter = mock.MagicMock() - requests.post.return_value = '[200]' + hcn.requests = requests + hcn.Retry = retry + hcn.HTTPAdapter = httpadapter signal_data = json.dumps({'foo': 'bar'}) self.stdin.write(signal_data) @@ -124,16 +129,23 @@ class HeatConfigNotifyTest(common.RunScriptTest): 0, hcn.main(['heat-config-notify', config_file.name], self.stdin)) - requests.post.assert_called_once_with( + session.post.assert_called_once_with( 'mock://192.0.2.3/foo', data=signal_data, headers={'content-type': 'application/json'}) def test_notify_signal_id_put(self): requests = mock.MagicMock() - hcn.requests = requests + session = mock.MagicMock() + requests.Session.return_value = session + retry = mock.MagicMock() + httpadapter = mock.MagicMock() - requests.post.return_value = '[200]' + hcn.requests = requests + hcn.Retry = retry + hcn.HTTPAdapter = httpadapter + + session.post.return_value = '[200]' signal_data = json.dumps({'foo': 'bar'}) self.stdin.write(signal_data) @@ -144,32 +156,46 @@ class HeatConfigNotifyTest(common.RunScriptTest): 0, hcn.main(['heat-config-notify', config_file.name], self.stdin)) - requests.put.assert_called_once_with( + session.put.assert_called_once_with( 'mock://192.0.2.3/foo', data=signal_data, headers={'content-type': 'application/json'}) def test_notify_signal_id_empty_data(self): requests = mock.MagicMock() - hcn.requests = requests + session = mock.MagicMock() + requests.Session.return_value = session + retry = mock.MagicMock() + httpadapter = mock.MagicMock() - requests.post.return_value = '[200]' + hcn.requests = requests + hcn.Retry = retry + hcn.HTTPAdapter = httpadapter + + session.post.return_value = '[200]' with self.write_config_file(self.data_signal_id) as config_file: self.assertEqual( 0, hcn.main(['heat-config-notify', config_file.name], self.stdin)) - requests.post.assert_called_once_with( + session.post.assert_called_once_with( 'mock://192.0.2.3/foo', data='{}', headers={'content-type': 'application/json'}) def test_notify_signal_id_invalid_json_data(self): requests = mock.MagicMock() - hcn.requests = requests + session = mock.MagicMock() + requests.Session.return_value = session + retry = mock.MagicMock() + httpadapter = mock.MagicMock() - requests.post.return_value = '[200]' + hcn.requests = requests + hcn.Retry = retry + hcn.HTTPAdapter = httpadapter + + session.post.return_value = '[200]' signal_data = json.dumps({'foo': 'bar'}) self.stdin.write(signal_data) @@ -181,7 +207,7 @@ class HeatConfigNotifyTest(common.RunScriptTest): 0, hcn.main(['heat-config-notify', config_file.name], self.stdin)) - requests.post.assert_called_once_with( + session.post.assert_called_once_with( 'mock://192.0.2.3/foo', data='{}', headers={'content-type': 'application/json'})