Fix rolling upgrade

There are two changes in this patch:

1) Fix the kube_image_digest for both master and worker nodes
2) source the bashrc to get the kubectl command worked

story: 2008628

Change-Id: Iccde333b8f2d219de9c5923d4adb822c7d3f19f4

magnum/drivers/common/templates/kubernetes/fragments/upgrade-kubernetes.sh

@@ -1,6 +1,11 @@
+echo "START: upgrade k8s versions and operating system"
+
+set +x
 . /etc/sysconfig/heat-params
 set -x
 
+set -eu -o pipefail
+
 ssh_cmd="ssh -F /srv/magnum/.ssh/config root@localhost"
 KUBECONFIG="/etc/kubernetes/kubelet-config.yaml"
 if [ "$(echo $USE_PODMAN | tr '[:upper:]' '[:lower:]')" == "true" ]; then
@@ -155,3 +160,10 @@ elif [ "$new_ostree_remote" != "" ] && [ "$current_ostree_remote" != "$new_ostre
     ${ssh_cmd} rpm-ostree rebase $new_ostree_remote
     shutdown --reboot --no-wall -t 1
 fi
+
+# We need to double check if this is a k8s version upgrade only action, if so, the command should exit to avoid run the later
+# cert rotate scripts.
+if [ "${new_kube_tag}" != "${KUBE_TAG}" ]; then
+    echo "END: Upgrade k8s versions and operating system"
+    exit 0
+fi

magnum/drivers/heat/driver.py

@@ -320,11 +320,13 @@ class FedoraKubernetesDriver(KubernetesDriver):
         heat_params = {}
         try:
             kube_tag = cluster_template.labels["kube_tag"]
+            image_digest = cluster_template.labels.get("kube_image_digest")
             kube_tag_params = {
                 "kube_tag": kube_tag,
                 "kube_version": kube_tag,
                 "master_kube_tag": kube_tag,
                 "minion_kube_tag": kube_tag,
+                "kube_image_digest": image_digest,
             }
             heat_params.update(kube_tag_params)
         except KeyError:

magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml

@@ -1515,6 +1515,7 @@ resources:
           ostree_commit: {get_param: ostree_commit}
           use_podman: {get_param: use_podman}
           selinux_mode: {get_param: selinux_mode}
+          kube_image_digest: {get_param: kube_image_digest}
           container_runtime: {get_param: container_runtime}
           containerd_version: {get_param: containerd_version}
           containerd_tarball_url: {get_param: containerd_tarball_url}

magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml

@@ -1060,6 +1060,7 @@ resources:
           - "\n"
           -
             - "#!/bin/bash"
+            - "source /etc/bashrc"
             - get_file: ../../common/templates/kubernetes/fragments/upgrade-kubernetes.sh
             - get_file: ../../common/templates/kubernetes/fragments/make-cert.sh
             - get_file: ../../common/templates/kubernetes/fragments/rotate-kubernetes-ca-certs-master.sh

magnum/drivers/k8s_fedora_coreos_v1/templates/kubeminion.yaml

@@ -334,6 +334,12 @@ parameters:
     description: >
       Choose SELinux mode
 
+  kube_image_digest:
+    type: string
+    description: >
+      The digest of the image which should match the given kube_tag
+    default: ''
+
   container_runtime:
     type: string
     description: The container runtime to install
@@ -601,6 +607,7 @@ resources:
       group: script
       inputs:
       - name: kube_tag_input
+      - name: kube_image_digest_input
       - name: ostree_remote_input
       - name: ostree_commit_input
       - name: kube_service_account_key_input
@@ -610,6 +617,7 @@ resources:
           - "\n"
           -
             - "#!/bin/bash"
+            - "source /etc/bashrc"
             - get_file: ../../common/templates/kubernetes/fragments/upgrade-kubernetes.sh
             - get_file: ../../common/templates/kubernetes/fragments/make-cert-client.sh
             - get_file: ../../common/templates/kubernetes/fragments/rotate-kubernetes-ca-certs-worker.sh
@@ -623,6 +631,7 @@ resources:
       actions: ['UPDATE']
       input_values:
         kube_tag_input: {get_param: kube_tag}
+        kube_image_digest_input: {get_param: kube_image_digest}
         ostree_remote_input: {get_param: ostree_remote}
         ostree_commit_input: {get_param: ostree_commit}
         kube_service_account_key_input: {get_param: kube_service_account_key}