Merge "Check for connectivity to the tiller postgres backend."

sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py

@@ -3258,8 +3258,12 @@ class ArmadaHelper(object):
 
         # Wait for armada to be ready for cmd execution.
         # NOTE: make_armada_requests() also has retry mechanism
-        timeout = 30
-        while True:
+        TIMEOUT_DELTA = 5
+        TIMEOUT_SLEEP = 5
+        TIMEOUT_START_VALUE = 30
+
+        timeout = TIMEOUT_START_VALUE
+        while timeout > 0:
             try:
                 pods = self._kube.kube_get_pods_by_selector(
                     ARMADA_NAMESPACE,
@@ -3287,20 +3291,42 @@ class ArmadaHelper(object):
                         LOG.error("Failed to copy %s to %s, error: %s",
                                   src, dest_dir, stderr)
                         raise RuntimeError('armada pod not ready')
-                    else:
-                        return True
-                    return True
+                    break
 
             except Exception as e:
                 LOG.info("Could not get Armada service : %s " % e)
 
-            if timeout <= 0:
-                break
-            time.sleep(5)
-            timeout -= 5
+            time.sleep(TIMEOUT_SLEEP)
+            timeout -= TIMEOUT_DELTA
 
-        LOG.error("Failed to get Armada service after 30 seconds.")
-        return False
+        if timeout <= 0:
+            LOG.error("Failed to get Armada service after {seconds} seconds.".
+                      format(seconds=TIMEOUT_START_VALUE))
+            return False
+
+        # We don't need to loop through the code that checks the pod's status
+        # again. Once the previous loop exits with pod 'Running' we can test
+        # the connectivity to the tiller postgres backend:
+        timeout = TIMEOUT_START_VALUE
+        while timeout > 0:
+            try:
+                _ = helm_utils.retrieve_helm_releases()
+                break
+            except exception.HelmTillerFailure:
+                LOG.warn("Could not query Helm/Tiller releases")
+                time.sleep(TIMEOUT_SLEEP)
+                timeout -= TIMEOUT_DELTA
+                continue
+            except Exception as ex:
+                LOG.error("Unhandled exception : {error}".format(error=str(ex)))
+                return False
+
+        if timeout <= 0:
+            LOG.error("Failed to query Helm/Tiller for {seconds} seconds.".
+                      format(seconds=TIMEOUT_START_VALUE))
+            return False
+
+        return True
 
     def stop_armada_request(self):
         """A simple way to cancel an on-going manifest apply/rollback/delete

sysinv/sysinv/sysinv/sysinv/helm/utils.py

@@ -1,6 +1,6 @@
 # sim: tabstop=4 shiftwidth=4 softtabstop=4
 #
-# Copyright (c) 2019 Wind River Systems, Inc.
+# Copyright (c) 2019-2021 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #