From 222f7b68775b6f6063bbf0f8148f8a1ffb853a25 Mon Sep 17 00:00:00 2001
From: "Gupta, Sangeet (sg774j)" <sg774j@att.com>
Date: Tue, 24 Aug 2021 15:00:13 +0000
Subject: [PATCH] cert-rotation: Correct and enhance the rotation script.

Corrected the counter increment and enhanced the script to handle
situation if the certificate is stuck in issuing state.

Change-Id: Ib8a84831a605bb3e5a1fc5b5a909c827ec864797
---
 cert-rotation/Chart.yaml                      |  2 +-
 .../templates/bin/_rotate-certs.sh.tpl        | 25 ++++++++++++++++---
 releasenotes/notes/cert-rotation.yaml         |  1 +
 3 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/cert-rotation/Chart.yaml b/cert-rotation/Chart.yaml
index 91e7743b5..9725c2b44 100644
--- a/cert-rotation/Chart.yaml
+++ b/cert-rotation/Chart.yaml
@@ -16,5 +16,5 @@ appVersion: "1.0"
 description: Rotate the certificates generated by cert-manager
 home: https://cert-manager.io/
 name: cert-rotation
-version: 0.1.1
+version: 0.1.2
 ...
diff --git a/cert-rotation/templates/bin/_rotate-certs.sh.tpl b/cert-rotation/templates/bin/_rotate-certs.sh.tpl
index 6504679ef..e71ba60ca 100644
--- a/cert-rotation/templates/bin/_rotate-certs.sh.tpl
+++ b/cert-rotation/templates/bin/_rotate-certs.sh.tpl
@@ -55,17 +55,34 @@ function rotate_and_get_certs_list(){
         for cert in ${certRotated[@]}
         do
             counter=0
+            retried=false
             while [ "$(kubectl get certificate -n ${namespace} ${cert} -o json | jq -r '.status.conditions[].status')" != "True" ]
             do
                 # Wait for secret to become ready. Wait for 300 seconds maximum. Sleep for 10 seconds
                 if [ ${counter} -ge 30 ]
                 then
-                    echo "ERROR: Rotated certificate  ${cert} in ${namespace} is not ready."
-                    # Continue so that the certificates that are rotated successfully are deployed.
-                    break
+                    # Seems certificate is not in ready state yet, may be there is an issue be renewing the certificate.
+                    # Try one more time before failing it. The name of the secret would be different at this time (when in
+                    # process of issuing)
+                    priSeckeyName=$(kubectl get certificate -n ${namespace} ${cert} -o json | jq -r '.status["nextPrivateKeySecretName"]')
+
+                    if [ ${retried} = false ] && [ ! -z ${priSeckeyName} ]
+                    then
+                        echo "Deleting interim failed secret ${priSeckeyName} in namespace ${namespace}"
+                        kubectl delete secret -n ${namespace} ${priSeckeyName}
+                        retried=true
+                        counter=0
+                    else
+                        # Tried 2 times to renew the certificate, something is not right. Log error and
+                        # continue to check the status of next certificate. Once the status of all the
+                        # certificates has been checked, the pods need to be restarted so that the successfully
+                        # renewed certificates can be deployed.
+                        echo "ERROR: Rotated certificate  ${cert} in ${namespace} is not ready."
+                        break
+                    fi
                 fi
                 echo "Rotated certificate ${cert} in ${namespace} is not ready yet ... waiting"
-                counter+=(${counter+=1})
+                counter=$((counter+1))
                 sleep 10
             done
 
diff --git a/releasenotes/notes/cert-rotation.yaml b/releasenotes/notes/cert-rotation.yaml
index 390466543..2328b8e59 100644
--- a/releasenotes/notes/cert-rotation.yaml
+++ b/releasenotes/notes/cert-rotation.yaml
@@ -2,4 +2,5 @@
 cert-rotation:
   - 0.1.0 Initial Chart
   - 0.1.1 Return true if grep finds no match
+  - 0.1.2 Correct and enhance the rotation script
 ...