From d6076ad0c652456dc5ba2cc9992b4ddcddc77c9b Mon Sep 17 00:00:00 2001 From: Feilong Wang Date: Wed, 17 Feb 2021 09:01:40 +1300 Subject: [PATCH] Fix rolling upgrade There are two changes in this patch: 1) Fix the kube_image_digest for both master and worker nodes 2) source the bashrc to get the kubectl command worked story: 2008628 Change-Id: Iccde333b8f2d219de9c5923d4adb822c7d3f19f4 --- .../kubernetes/fragments/upgrade-kubernetes.sh | 12 ++++++++++++ magnum/drivers/heat/driver.py | 2 ++ .../k8s_fedora_coreos_v1/templates/kubecluster.yaml | 1 + .../k8s_fedora_coreos_v1/templates/kubemaster.yaml | 1 + .../k8s_fedora_coreos_v1/templates/kubeminion.yaml | 9 +++++++++ 5 files changed, 25 insertions(+) diff --git a/magnum/drivers/common/templates/kubernetes/fragments/upgrade-kubernetes.sh b/magnum/drivers/common/templates/kubernetes/fragments/upgrade-kubernetes.sh index 3c03af691e..2b1c2bf55b 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/upgrade-kubernetes.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/upgrade-kubernetes.sh @@ -1,6 +1,11 @@ +echo "START: upgrade k8s versions and operating system" + +set +x . /etc/sysconfig/heat-params set -x +set -eu -o pipefail + ssh_cmd="ssh -F /srv/magnum/.ssh/config root@localhost" KUBECONFIG="/etc/kubernetes/kubelet-config.yaml" if [ "$(echo $USE_PODMAN | tr '[:upper:]' '[:lower:]')" == "true" ]; then @@ -155,3 +160,10 @@ elif [ "$new_ostree_remote" != "" ] && [ "$current_ostree_remote" != "$new_ostre ${ssh_cmd} rpm-ostree rebase $new_ostree_remote shutdown --reboot --no-wall -t 1 fi + +# We need to double check if this is a k8s version upgrade only action, if so, the command should exit to avoid run the later +# cert rotate scripts. +if [ "${new_kube_tag}" != "${KUBE_TAG}" ]; then + echo "END: Upgrade k8s versions and operating system" + exit 0 +fi diff --git a/magnum/drivers/heat/driver.py b/magnum/drivers/heat/driver.py index 5e67bd5e9a..03fd5c1632 100755 --- a/magnum/drivers/heat/driver.py +++ b/magnum/drivers/heat/driver.py @@ -320,11 +320,13 @@ class FedoraKubernetesDriver(KubernetesDriver): heat_params = {} try: kube_tag = cluster_template.labels["kube_tag"] + image_digest = cluster_template.labels.get("kube_image_digest") kube_tag_params = { "kube_tag": kube_tag, "kube_version": kube_tag, "master_kube_tag": kube_tag, "minion_kube_tag": kube_tag, + "kube_image_digest": image_digest, } heat_params.update(kube_tag_params) except KeyError: diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml index 6b6876f495..65531e310e 100644 --- a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml @@ -1515,6 +1515,7 @@ resources: ostree_commit: {get_param: ostree_commit} use_podman: {get_param: use_podman} selinux_mode: {get_param: selinux_mode} + kube_image_digest: {get_param: kube_image_digest} container_runtime: {get_param: container_runtime} containerd_version: {get_param: containerd_version} containerd_tarball_url: {get_param: containerd_tarball_url} diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml index fc2f3f4432..744a10d258 100644 --- a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml @@ -1060,6 +1060,7 @@ resources: - "\n" - - "#!/bin/bash" + - "source /etc/bashrc" - get_file: ../../common/templates/kubernetes/fragments/upgrade-kubernetes.sh - get_file: ../../common/templates/kubernetes/fragments/make-cert.sh - get_file: ../../common/templates/kubernetes/fragments/rotate-kubernetes-ca-certs-master.sh diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubeminion.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubeminion.yaml index f058b87daf..bd7c0e51d2 100644 --- a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubeminion.yaml +++ b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubeminion.yaml @@ -334,6 +334,12 @@ parameters: description: > Choose SELinux mode + kube_image_digest: + type: string + description: > + The digest of the image which should match the given kube_tag + default: '' + container_runtime: type: string description: The container runtime to install @@ -601,6 +607,7 @@ resources: group: script inputs: - name: kube_tag_input + - name: kube_image_digest_input - name: ostree_remote_input - name: ostree_commit_input - name: kube_service_account_key_input @@ -610,6 +617,7 @@ resources: - "\n" - - "#!/bin/bash" + - "source /etc/bashrc" - get_file: ../../common/templates/kubernetes/fragments/upgrade-kubernetes.sh - get_file: ../../common/templates/kubernetes/fragments/make-cert-client.sh - get_file: ../../common/templates/kubernetes/fragments/rotate-kubernetes-ca-certs-worker.sh @@ -623,6 +631,7 @@ resources: actions: ['UPDATE'] input_values: kube_tag_input: {get_param: kube_tag} + kube_image_digest_input: {get_param: kube_image_digest} ostree_remote_input: {get_param: ostree_remote} ostree_commit_input: {get_param: ostree_commit} kube_service_account_key_input: {get_param: kube_service_account_key}