Use upgraded kubelet on unlock after kubelet upgrade

When unlocking after kubelet upgrade, there is a problem here:
c784e1ca9a/sysinv/sysinv/sysinv/sysinv/puppet/kubernetes.py (L383)
the kube_host_upgrade status is no longer equal to
KUBE_HOST_UPGRADING_KUBELET and the kubelet version is taken from
the DB which is the old one.
The kubelet value from the DB is updated when the k8s upgrade is
complete. (i.e after doing: system kube-upgrade-complete)

To fix this we added a new state to kube_host_upgrade.status
defined as kubernetes.KUBE_HOST_UPGRADED_KUBELET. This state will
indicate that the kubelet was upgraded for a certain host.

When we complete the upgrade (i.e system kube-upgrade-complete)
the per-host kube_host_upgrade.status values will be set to None
so that they're blank in prep for the next k8s upgrade.

Closes-Bug: 1943690
Change-Id: Ifdb5c51649831deed76ab9a5370cd293f8a5784a
Signed-off-by: Mihnea Saracin <Mihnea.Saracin@windriver.com>

sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_upgrade.py

@@ -393,7 +393,8 @@ class KubeUpgradeController(rest.RestController):
             kube_host_upgrades = \
                 pecan.request.dbapi.kube_host_upgrade_get_list()
             for kube_host_upgrade in kube_host_upgrades:
-                if kube_host_upgrade.status is not None:
+                if kube_host_upgrade.status != \
+                        kubernetes.KUBE_HOST_UPGRADED_KUBELET:
                     raise wsme.exc.ClientSideError(_(
                         "At least one host has not completed the kubernetes "
                         "upgrade"))
@@ -411,6 +412,11 @@ class KubeUpgradeController(rest.RestController):
             kube_cmd_versions.kubelet_version = kube_upgrade_obj.to_version.lstrip('v')
             kube_cmd_versions.save()
 
+            # The global kubelet version is set, clear the per-host status.
+            for kube_host_upgrade in kube_host_upgrades:
+                pecan.request.dbapi.kube_host_upgrade_update(
+                    kube_host_upgrade.id, {'status': None})
+
             # All is well, mark the upgrade as complete
             kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_COMPLETE
             kube_upgrade_obj.save()

sysinv/sysinv/sysinv/sysinv/common/kubernetes.py

@@ -88,6 +88,7 @@ KUBE_HOST_UPGRADING_CONTROL_PLANE = 'upgrading-control-plane'
 KUBE_HOST_UPGRADING_CONTROL_PLANE_FAILED = 'upgrading-control-plane-failed'
 KUBE_HOST_UPGRADING_KUBELET = 'upgrading-kubelet'
 KUBE_HOST_UPGRADING_KUBELET_FAILED = 'upgrading-kubelet-failed'
+KUBE_HOST_UPGRADED_KUBELET = 'upgraded-kubelet'
 
 # Kubernetes rootca update states
 KUBE_ROOTCA_UPDATE_STARTED = 'update-started'

sysinv/sysinv/sysinv/sysinv/conductor/manager.py

@@ -13665,7 +13665,7 @@ class ConductorManager(service.PeriodicService):
         # The kubelet update was successful
         kube_host_upgrade_obj = objects.kube_host_upgrade.get_by_host_id(
             context, host_obj.id)
-        kube_host_upgrade_obj.status = None
+        kube_host_upgrade_obj.status = kubernetes.KUBE_HOST_UPGRADED_KUBELET
         kube_host_upgrade_obj.save()
 
     def kube_upgrade_networking(self, context, kube_version):

sysinv/sysinv/sysinv/sysinv/puppet/kubernetes.py

@@ -380,7 +380,9 @@ class KubernetesPuppet(base.BasePuppet):
             if kube_upgrade_state:
                 kube_host_upgrade = objects.kube_host_upgrade.get_by_host_id(
                     self.context, host.id)
-                if kube_host_upgrade.status == kubernetes.KUBE_HOST_UPGRADING_KUBELET:
+                if kube_host_upgrade.status in [
+                        kubernetes.KUBE_HOST_UPGRADING_KUBELET,
+                        kubernetes.KUBE_HOST_UPGRADED_KUBELET]:
                     kubelet_version = kube_host_upgrade.target_version.lstrip('v')
 
             config.update({'platform::kubernetes::params::kubeadm_version': kubeadm_version})

sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_upgrade.py

@@ -728,6 +728,12 @@ class TestPatch(TestKubeUpgrade,
             state=kubernetes.KUBE_UPGRADING_KUBELETS)
         uuid = kube_upgrade.uuid
 
+        # Mark the kube host upgrade as upgraded-kubelet
+        values = {
+            'status': kubernetes.KUBE_HOST_UPGRADED_KUBELET
+        }
+        self.dbapi.kube_host_upgrade_update(1, values)
+
         # Update state
         new_state = kubernetes.KUBE_UPGRADE_COMPLETE
         response = self.patch_json('/kube_upgrade',

sysinv/sysinv/sysinv/sysinv/tests/conductor/test_manager.py

@@ -1222,9 +1222,10 @@ class ManagerTestCase(base.DbTestCase):
         self.assertEqual(updated_upgrade.state,
                          kubernetes.KUBE_UPGRADED_SECOND_MASTER)
 
-        # Verify that the host upgrade status was cleared
+        # Verify that the host upgrade status is upgraded-kubelet
         updated_host_upgrade = self.dbapi.kube_host_upgrade_get(1)
-        self.assertEqual(updated_host_upgrade.status, None)
+        self.assertEqual(updated_host_upgrade.status,
+                         kubernetes.KUBE_HOST_UPGRADED_KUBELET)
 
     def test_kube_upgrade_kubelet_second_master(self):
         # Create an upgrade