Retry logic for url request in heat-config-notify

Adds retry logic for software deployments using the url signals
to ensure that requests are retried if network connection issues
occur or a 500, 502, 503, or 504 is returned by the http or https
endpoint.

Note: this does not add retry logic to heatclient or zaqarclient
if they are used for signaling.

Change-Id: I82dff4a4b9fac05c5ec649db3eb379bdec71e208
Related-Bug: #1731540

heat-config/bin/heat-config-notify

@@ -19,6 +19,9 @@ import sys
 
 import requests
 
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3.util.retry import Retry
+
 try:
     from heatclient import client as heatclient
 except ImportError:
@@ -105,12 +108,25 @@ def main(argv=sys.argv, stdin=sys.stdin):
         # we need to trim log content because Heat response size is limited
         # by max_json_body_size = 1048576
         str_signal_data = trim_response(signal_data)
+        session = requests.Session()
+        # Retry if connection issues occur or the service is returning a 5xx
+        retry = Retry(
+            total=10,
+            read=10,
+            connect=10,
+            backoff_factor=0.5,
+            status_forcelist=(500, 502, 503, 504)
+        )
+        adapter = HTTPAdapter(max_retries=retry)
+        session.mount('http://', adapter)
+        session.mount('https://', adapter)
+
         if sigverb == 'PUT':
-            r = requests.put(sigurl, data=str_signal_data,
+            r = session.put(sigurl, data=str_signal_data,
-                             headers={'content-type': 'application/json'})
+                            headers={'content-type': 'application/json'})
         else:
-            r = requests.post(sigurl, data=str_signal_data,
+            r = session.post(sigurl, data=str_signal_data,
-                              headers={'content-type': 'application/json'})
+                             headers={'content-type': 'application/json'})
         log.debug('Response %s ' % r)
 
     if 'deploy_queue_id' in iv:

releasenotes/notes/heat-config-notify-retry-e01b995d239bb1fd.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Add retry logic if 500, 502, 503 or 504 responses are returned or network
+    connection issues occur during heat signals using http or https.

tests/test_heat_config_notify.py

@@ -111,9 +111,14 @@ class HeatConfigNotifyTest(common.RunScriptTest):
 
     def test_notify_signal_id(self):
         requests = mock.MagicMock()
-        hcn.requests = requests
+        session = mock.MagicMock()
+        requests.Session.return_value = session
+        retry = mock.MagicMock()
+        httpadapter = mock.MagicMock()
 
-        requests.post.return_value = '[200]'
+        hcn.requests = requests
+        hcn.Retry = retry
+        hcn.HTTPAdapter = httpadapter
 
         signal_data = json.dumps({'foo': 'bar'})
         self.stdin.write(signal_data)
@@ -124,16 +129,23 @@ class HeatConfigNotifyTest(common.RunScriptTest):
                 0,
                 hcn.main(['heat-config-notify', config_file.name], self.stdin))
 
-        requests.post.assert_called_once_with(
+        session.post.assert_called_once_with(
             'mock://192.0.2.3/foo',
             data=signal_data,
             headers={'content-type': 'application/json'})
 
     def test_notify_signal_id_put(self):
         requests = mock.MagicMock()
-        hcn.requests = requests
+        session = mock.MagicMock()
+        requests.Session.return_value = session
+        retry = mock.MagicMock()
+        httpadapter = mock.MagicMock()
 
-        requests.post.return_value = '[200]'
+        hcn.requests = requests
+        hcn.Retry = retry
+        hcn.HTTPAdapter = httpadapter
+
+        session.post.return_value = '[200]'
 
         signal_data = json.dumps({'foo': 'bar'})
         self.stdin.write(signal_data)
@@ -144,32 +156,46 @@ class HeatConfigNotifyTest(common.RunScriptTest):
                 0,
                 hcn.main(['heat-config-notify', config_file.name], self.stdin))
 
-        requests.put.assert_called_once_with(
+        session.put.assert_called_once_with(
             'mock://192.0.2.3/foo',
             data=signal_data,
             headers={'content-type': 'application/json'})
 
     def test_notify_signal_id_empty_data(self):
         requests = mock.MagicMock()
-        hcn.requests = requests
+        session = mock.MagicMock()
+        requests.Session.return_value = session
+        retry = mock.MagicMock()
+        httpadapter = mock.MagicMock()
 
-        requests.post.return_value = '[200]'
+        hcn.requests = requests
+        hcn.Retry = retry
+        hcn.HTTPAdapter = httpadapter
+
+        session.post.return_value = '[200]'
 
         with self.write_config_file(self.data_signal_id) as config_file:
             self.assertEqual(
                 0,
                 hcn.main(['heat-config-notify', config_file.name], self.stdin))
 
-        requests.post.assert_called_once_with(
+        session.post.assert_called_once_with(
             'mock://192.0.2.3/foo',
             data='{}',
             headers={'content-type': 'application/json'})
 
     def test_notify_signal_id_invalid_json_data(self):
         requests = mock.MagicMock()
-        hcn.requests = requests
+        session = mock.MagicMock()
+        requests.Session.return_value = session
+        retry = mock.MagicMock()
+        httpadapter = mock.MagicMock()
 
-        requests.post.return_value = '[200]'
+        hcn.requests = requests
+        hcn.Retry = retry
+        hcn.HTTPAdapter = httpadapter
+
+        session.post.return_value = '[200]'
 
         signal_data = json.dumps({'foo': 'bar'})
         self.stdin.write(signal_data)
@@ -181,7 +207,7 @@ class HeatConfigNotifyTest(common.RunScriptTest):
                 0,
                 hcn.main(['heat-config-notify', config_file.name], self.stdin))
 
-        requests.post.assert_called_once_with(
+        session.post.assert_called_once_with(
             'mock://192.0.2.3/foo',
             data='{}',
             headers={'content-type': 'application/json'})