Ensure that the manager.py re-connects if it gets a 500 error

This patch works around a bug during upgrades where (it is believed)
that the manager.py script can hold a 'stale' connection to the apache2
hosted keystone, which then returns a 500 HTTP error when next used.

Change-Id: Id9c9b967742467f30270f52c2a8b6634f6480324
Related-Bug: #1823743

hooks/keystone_utils.py

@@ -1061,6 +1061,11 @@ JSON_ENCODE_OPTIONS = dict(
 )
 
 
+class RetryProxyManagerCall(Exception):
+    pass
+
+
+@retry_on_exception(5, base_delay=3, exc_type=RetryProxyManagerCall)
 def _proxy_manager_call(path, api_version, args, kwargs):
     package = dict(path=path,
                    api_version=api_version,
@@ -1080,8 +1085,14 @@ def _proxy_manager_call(path, api_version, args, kwargs):
                  "The call was: path={}, args={}, kwargs={}, api_version={}"
                  .format(result['error'], path, args, kwargs, api_version))
             log(s, level=ERROR)
+            if result.get('retry'):
+                stop_manager_instance()
+                raise RetryProxyManagerCall()
             raise RuntimeError(s)
         return json.loads(result_str)['result']
+    except RetryProxyManagerCall:
+        # cause a retry
+        raise
     except RuntimeError as e:
         raise e
     except Exception as e:

hooks/manager.py

@@ -560,6 +560,12 @@ if __name__ == '__main__':
                 _callable = getattr(_callable, attr)
             # now make the call and return the arguments
             result = {'result': _callable(*spec['args'], **spec['kwargs'])}
+        except exceptions.InternalServerError as e:
+            # we've hit a 500 error, which is bad, and really we want the
+            # parent process to restart us to try again.
+            print(str(e))
+            result = {'error': str(e),
+                      'retry': True}
         except uds.UDSException as e:
             print(str(e))
             import traceback