From 023dd35a5ab3b0278b7c13cbc8b5ff427257fc83 Mon Sep 17 00:00:00 2001
From: Alex Kavanagh <alex@ajkavanagh.co.uk>
Date: Tue, 16 Apr 2019 15:50:38 +0100
Subject: [PATCH] Ensure that the manager.py re-connects if it gets a 500 error

This patch works around a bug during upgrades where (it is believed)
that the manager.py script can hold a 'stale' connection to the apache2
hosted keystone, which then returns a 500 HTTP error when next used.

Change-Id: Id9c9b967742467f30270f52c2a8b6634f6480324
Related-Bug: #1823743
---
 hooks/keystone_utils.py | 11 +++++++++++
 hooks/manager.py        |  6 ++++++
 2 files changed, 17 insertions(+)

diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py
index 0104f0a1..c2e602c3 100644
--- a/hooks/keystone_utils.py
+++ b/hooks/keystone_utils.py
@@ -1061,6 +1061,11 @@ JSON_ENCODE_OPTIONS = dict(
 )
 
 
+class RetryProxyManagerCall(Exception):
+    pass
+
+
+@retry_on_exception(5, base_delay=3, exc_type=RetryProxyManagerCall)
 def _proxy_manager_call(path, api_version, args, kwargs):
     package = dict(path=path,
                    api_version=api_version,
@@ -1080,8 +1085,14 @@ def _proxy_manager_call(path, api_version, args, kwargs):
                  "The call was: path={}, args={}, kwargs={}, api_version={}"
                  .format(result['error'], path, args, kwargs, api_version))
             log(s, level=ERROR)
+            if result.get('retry'):
+                stop_manager_instance()
+                raise RetryProxyManagerCall()
             raise RuntimeError(s)
         return json.loads(result_str)['result']
+    except RetryProxyManagerCall:
+        # cause a retry
+        raise
     except RuntimeError as e:
         raise e
     except Exception as e:
diff --git a/hooks/manager.py b/hooks/manager.py
index 05abaaa6..6a215968 100755
--- a/hooks/manager.py
+++ b/hooks/manager.py
@@ -560,6 +560,12 @@ if __name__ == '__main__':
                 _callable = getattr(_callable, attr)
             # now make the call and return the arguments
             result = {'result': _callable(*spec['args'], **spec['kwargs'])}
+        except exceptions.InternalServerError as e:
+            # we've hit a 500 error, which is bad, and really we want the
+            # parent process to restart us to try again.
+            print(str(e))
+            result = {'error': str(e),
+                      'retry': True}
         except uds.UDSException as e:
             print(str(e))
             import traceback