Retry logic for url request in heat-config-notify

Adds retry logic for software deployments using the url signals
to ensure that requests are retried if network connection issues
occur or a 500, 502, 503, or 504 is returned by the http or https
endpoint.

Note: this does not add retry logic to heatclient or zaqarclient
if they are used for signaling.

Change-Id: I82dff4a4b9fac05c5ec649db3eb379bdec71e208
Related-Bug: #1731540
This commit is contained in:
Alex Schultz 2017-11-13 09:33:41 -07:00
parent 9dad9eefd7
commit 756fcafdf0
3 changed files with 63 additions and 16 deletions

View File

@ -19,6 +19,9 @@ import sys
import requests import requests
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
try: try:
from heatclient import client as heatclient from heatclient import client as heatclient
except ImportError: except ImportError:
@ -105,12 +108,25 @@ def main(argv=sys.argv, stdin=sys.stdin):
# we need to trim log content because Heat response size is limited # we need to trim log content because Heat response size is limited
# by max_json_body_size = 1048576 # by max_json_body_size = 1048576
str_signal_data = trim_response(signal_data) str_signal_data = trim_response(signal_data)
session = requests.Session()
# Retry if connection issues occur or the service is returning a 5xx
retry = Retry(
total=10,
read=10,
connect=10,
backoff_factor=0.5,
status_forcelist=(500, 502, 503, 504)
)
adapter = HTTPAdapter(max_retries=retry)
session.mount('http://', adapter)
session.mount('https://', adapter)
if sigverb == 'PUT': if sigverb == 'PUT':
r = requests.put(sigurl, data=str_signal_data, r = session.put(sigurl, data=str_signal_data,
headers={'content-type': 'application/json'}) headers={'content-type': 'application/json'})
else: else:
r = requests.post(sigurl, data=str_signal_data, r = session.post(sigurl, data=str_signal_data,
headers={'content-type': 'application/json'}) headers={'content-type': 'application/json'})
log.debug('Response %s ' % r) log.debug('Response %s ' % r)
if 'deploy_queue_id' in iv: if 'deploy_queue_id' in iv:

View File

@ -0,0 +1,5 @@
---
features:
- |
Add retry logic if 500, 502, 503 or 504 responses are returned or network
connection issues occur during heat signals using http or https.

View File

@ -111,9 +111,14 @@ class HeatConfigNotifyTest(common.RunScriptTest):
def test_notify_signal_id(self): def test_notify_signal_id(self):
requests = mock.MagicMock() requests = mock.MagicMock()
hcn.requests = requests session = mock.MagicMock()
requests.Session.return_value = session
retry = mock.MagicMock()
httpadapter = mock.MagicMock()
requests.post.return_value = '[200]' hcn.requests = requests
hcn.Retry = retry
hcn.HTTPAdapter = httpadapter
signal_data = json.dumps({'foo': 'bar'}) signal_data = json.dumps({'foo': 'bar'})
self.stdin.write(signal_data) self.stdin.write(signal_data)
@ -124,16 +129,23 @@ class HeatConfigNotifyTest(common.RunScriptTest):
0, 0,
hcn.main(['heat-config-notify', config_file.name], self.stdin)) hcn.main(['heat-config-notify', config_file.name], self.stdin))
requests.post.assert_called_once_with( session.post.assert_called_once_with(
'mock://192.0.2.3/foo', 'mock://192.0.2.3/foo',
data=signal_data, data=signal_data,
headers={'content-type': 'application/json'}) headers={'content-type': 'application/json'})
def test_notify_signal_id_put(self): def test_notify_signal_id_put(self):
requests = mock.MagicMock() requests = mock.MagicMock()
hcn.requests = requests session = mock.MagicMock()
requests.Session.return_value = session
retry = mock.MagicMock()
httpadapter = mock.MagicMock()
requests.post.return_value = '[200]' hcn.requests = requests
hcn.Retry = retry
hcn.HTTPAdapter = httpadapter
session.post.return_value = '[200]'
signal_data = json.dumps({'foo': 'bar'}) signal_data = json.dumps({'foo': 'bar'})
self.stdin.write(signal_data) self.stdin.write(signal_data)
@ -144,32 +156,46 @@ class HeatConfigNotifyTest(common.RunScriptTest):
0, 0,
hcn.main(['heat-config-notify', config_file.name], self.stdin)) hcn.main(['heat-config-notify', config_file.name], self.stdin))
requests.put.assert_called_once_with( session.put.assert_called_once_with(
'mock://192.0.2.3/foo', 'mock://192.0.2.3/foo',
data=signal_data, data=signal_data,
headers={'content-type': 'application/json'}) headers={'content-type': 'application/json'})
def test_notify_signal_id_empty_data(self): def test_notify_signal_id_empty_data(self):
requests = mock.MagicMock() requests = mock.MagicMock()
hcn.requests = requests session = mock.MagicMock()
requests.Session.return_value = session
retry = mock.MagicMock()
httpadapter = mock.MagicMock()
requests.post.return_value = '[200]' hcn.requests = requests
hcn.Retry = retry
hcn.HTTPAdapter = httpadapter
session.post.return_value = '[200]'
with self.write_config_file(self.data_signal_id) as config_file: with self.write_config_file(self.data_signal_id) as config_file:
self.assertEqual( self.assertEqual(
0, 0,
hcn.main(['heat-config-notify', config_file.name], self.stdin)) hcn.main(['heat-config-notify', config_file.name], self.stdin))
requests.post.assert_called_once_with( session.post.assert_called_once_with(
'mock://192.0.2.3/foo', 'mock://192.0.2.3/foo',
data='{}', data='{}',
headers={'content-type': 'application/json'}) headers={'content-type': 'application/json'})
def test_notify_signal_id_invalid_json_data(self): def test_notify_signal_id_invalid_json_data(self):
requests = mock.MagicMock() requests = mock.MagicMock()
hcn.requests = requests session = mock.MagicMock()
requests.Session.return_value = session
retry = mock.MagicMock()
httpadapter = mock.MagicMock()
requests.post.return_value = '[200]' hcn.requests = requests
hcn.Retry = retry
hcn.HTTPAdapter = httpadapter
session.post.return_value = '[200]'
signal_data = json.dumps({'foo': 'bar'}) signal_data = json.dumps({'foo': 'bar'})
self.stdin.write(signal_data) self.stdin.write(signal_data)
@ -181,7 +207,7 @@ class HeatConfigNotifyTest(common.RunScriptTest):
0, 0,
hcn.main(['heat-config-notify', config_file.name], self.stdin)) hcn.main(['heat-config-notify', config_file.name], self.stdin))
requests.post.assert_called_once_with( session.post.assert_called_once_with(
'mock://192.0.2.3/foo', 'mock://192.0.2.3/foo',
data='{}', data='{}',
headers={'content-type': 'application/json'}) headers={'content-type': 'application/json'})