Browse Source

Retry logic for url request in heat-config-notify

Adds retry logic for software deployments using the url signals
to ensure that requests are retried if network connection issues
occur or a 500, 502, 503, or 504 is returned by the http or https
endpoint.

Note: this does not add retry logic to heatclient or zaqarclient
if they are used for signaling.

Change-Id: I82dff4a4b9fac05c5ec649db3eb379bdec71e208
Related-Bug: #1731540
Alex Schultz 1 year ago
parent
commit
756fcafdf0

+ 20
- 4
heat-config/bin/heat-config-notify View File

@@ -19,6 +19,9 @@ import sys
19 19
 
20 20
 import requests
21 21
 
22
+from requests.adapters import HTTPAdapter
23
+from requests.packages.urllib3.util.retry import Retry
24
+
22 25
 try:
23 26
     from heatclient import client as heatclient
24 27
 except ImportError:
@@ -105,12 +108,25 @@ def main(argv=sys.argv, stdin=sys.stdin):
105 108
         # we need to trim log content because Heat response size is limited
106 109
         # by max_json_body_size = 1048576
107 110
         str_signal_data = trim_response(signal_data)
111
+        session = requests.Session()
112
+        # Retry if connection issues occur or the service is returning a 5xx
113
+        retry = Retry(
114
+            total=10,
115
+            read=10,
116
+            connect=10,
117
+            backoff_factor=0.5,
118
+            status_forcelist=(500, 502, 503, 504)
119
+        )
120
+        adapter = HTTPAdapter(max_retries=retry)
121
+        session.mount('http://', adapter)
122
+        session.mount('https://', adapter)
123
+
108 124
         if sigverb == 'PUT':
109
-            r = requests.put(sigurl, data=str_signal_data,
110
-                             headers={'content-type': 'application/json'})
125
+            r = session.put(sigurl, data=str_signal_data,
126
+                            headers={'content-type': 'application/json'})
111 127
         else:
112
-            r = requests.post(sigurl, data=str_signal_data,
113
-                              headers={'content-type': 'application/json'})
128
+            r = session.post(sigurl, data=str_signal_data,
129
+                             headers={'content-type': 'application/json'})
114 130
         log.debug('Response %s ' % r)
115 131
 
116 132
     if 'deploy_queue_id' in iv:

+ 5
- 0
releasenotes/notes/heat-config-notify-retry-e01b995d239bb1fd.yaml View File

@@ -0,0 +1,5 @@
1
+---
2
+features:
3
+  - |
4
+    Add retry logic if 500, 502, 503 or 504 responses are returned or network
5
+    connection issues occur during heat signals using http or https.

+ 35
- 9
tests/test_heat_config_notify.py View File

@@ -111,9 +111,14 @@ class HeatConfigNotifyTest(common.RunScriptTest):
111 111
 
112 112
     def test_notify_signal_id(self):
113 113
         requests = mock.MagicMock()
114
-        hcn.requests = requests
114
+        session = mock.MagicMock()
115
+        requests.Session.return_value = session
116
+        retry = mock.MagicMock()
117
+        httpadapter = mock.MagicMock()
115 118
 
116
-        requests.post.return_value = '[200]'
119
+        hcn.requests = requests
120
+        hcn.Retry = retry
121
+        hcn.HTTPAdapter = httpadapter
117 122
 
118 123
         signal_data = json.dumps({'foo': 'bar'})
119 124
         self.stdin.write(signal_data)
@@ -124,16 +129,23 @@ class HeatConfigNotifyTest(common.RunScriptTest):
124 129
                 0,
125 130
                 hcn.main(['heat-config-notify', config_file.name], self.stdin))
126 131
 
127
-        requests.post.assert_called_once_with(
132
+        session.post.assert_called_once_with(
128 133
             'mock://192.0.2.3/foo',
129 134
             data=signal_data,
130 135
             headers={'content-type': 'application/json'})
131 136
 
132 137
     def test_notify_signal_id_put(self):
133 138
         requests = mock.MagicMock()
139
+        session = mock.MagicMock()
140
+        requests.Session.return_value = session
141
+        retry = mock.MagicMock()
142
+        httpadapter = mock.MagicMock()
143
+
134 144
         hcn.requests = requests
145
+        hcn.Retry = retry
146
+        hcn.HTTPAdapter = httpadapter
135 147
 
136
-        requests.post.return_value = '[200]'
148
+        session.post.return_value = '[200]'
137 149
 
138 150
         signal_data = json.dumps({'foo': 'bar'})
139 151
         self.stdin.write(signal_data)
@@ -144,32 +156,46 @@ class HeatConfigNotifyTest(common.RunScriptTest):
144 156
                 0,
145 157
                 hcn.main(['heat-config-notify', config_file.name], self.stdin))
146 158
 
147
-        requests.put.assert_called_once_with(
159
+        session.put.assert_called_once_with(
148 160
             'mock://192.0.2.3/foo',
149 161
             data=signal_data,
150 162
             headers={'content-type': 'application/json'})
151 163
 
152 164
     def test_notify_signal_id_empty_data(self):
153 165
         requests = mock.MagicMock()
166
+        session = mock.MagicMock()
167
+        requests.Session.return_value = session
168
+        retry = mock.MagicMock()
169
+        httpadapter = mock.MagicMock()
170
+
154 171
         hcn.requests = requests
172
+        hcn.Retry = retry
173
+        hcn.HTTPAdapter = httpadapter
155 174
 
156
-        requests.post.return_value = '[200]'
175
+        session.post.return_value = '[200]'
157 176
 
158 177
         with self.write_config_file(self.data_signal_id) as config_file:
159 178
             self.assertEqual(
160 179
                 0,
161 180
                 hcn.main(['heat-config-notify', config_file.name], self.stdin))
162 181
 
163
-        requests.post.assert_called_once_with(
182
+        session.post.assert_called_once_with(
164 183
             'mock://192.0.2.3/foo',
165 184
             data='{}',
166 185
             headers={'content-type': 'application/json'})
167 186
 
168 187
     def test_notify_signal_id_invalid_json_data(self):
169 188
         requests = mock.MagicMock()
189
+        session = mock.MagicMock()
190
+        requests.Session.return_value = session
191
+        retry = mock.MagicMock()
192
+        httpadapter = mock.MagicMock()
193
+
170 194
         hcn.requests = requests
195
+        hcn.Retry = retry
196
+        hcn.HTTPAdapter = httpadapter
171 197
 
172
-        requests.post.return_value = '[200]'
198
+        session.post.return_value = '[200]'
173 199
 
174 200
         signal_data = json.dumps({'foo': 'bar'})
175 201
         self.stdin.write(signal_data)
@@ -181,7 +207,7 @@ class HeatConfigNotifyTest(common.RunScriptTest):
181 207
                 0,
182 208
                 hcn.main(['heat-config-notify', config_file.name], self.stdin))
183 209
 
184
-        requests.post.assert_called_once_with(
210
+        session.post.assert_called_once_with(
185 211
             'mock://192.0.2.3/foo',
186 212
             data='{}',
187 213
             headers={'content-type': 'application/json'})

Loading…
Cancel
Save