Fix rolling upgrade
There are two changes in this patch: 1) Fix the kube_image_digest for both master and worker nodes 2) Source the bashrc to get the kubectl command worked story: 2008628 Change-Id: Iccde333b8f2d219de9c5923d4adb822c7d3f19f4
This commit is contained in:
Feilong Wang
parent
439548e3de
commit
b7ae361e3b
|
|
@ -1,6 +1,11 @@
|
|||
echo "START: upgrade k8s versions and operating system"
|
||||
|
||||
set +x
|
||||
. /etc/sysconfig/heat-params
|
||||
set -x
|
||||
|
||||
set -eu -o pipefail
|
||||
|
||||
ssh_cmd="ssh -F /srv/magnum/.ssh/config root@localhost"
|
||||
KUBECONFIG="/etc/kubernetes/kubelet-config.yaml"
|
||||
if [ "$(echo $USE_PODMAN | tr '[:upper:]' '[:lower:]')" == "true" ]; then
|
||||
|
|
@ -126,20 +131,40 @@ current_ostree_remote=`${ssh_cmd} rpm-ostree status | awk '/* ostree/{print $0}'
|
|||
remote_list=`${ssh_cmd} ostree remote list`
|
||||
|
||||
# NOTE(flwang): This part is only applicable for fedora atomic
|
||||
if [[ $current_ostree_remote == *"fedora-atomic"* ]]; then
|
||||
# Fedora Atomic 29 will be the last release before migrating to Fedora CoreOS, so we're OK to add 28 and 29 remotes directly
|
||||
if [[ ! " ${remote_list[@]} " =~ "fedora-atomic-28" ]]; then
|
||||
${ssh_cmd} ostree remote add --set=gpgkeypath=/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-28-primary --contenturl=mirrorlist=https://ostree.fedoraproject.org/mirrorlist fedora-atomic-28 https://kojipkgs.fedoraproject.org/atomic/repo/
|
||||
fi
|
||||
if [[ ! " ${remote_list[@]} " =~ "fedora-atomic-29" ]]; then
|
||||
${ssh_cmd} ostree remote add --set=gpgkeypath=/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-29-primary --contenturl=mirrorlist=https://ostree.fedoraproject.org/mirrorlist fedora-atomic-29 https://kojipkgs.fedoraproject.org/atomic/repo/
|
||||
fi
|
||||
# The uri of existing Fedora Atomic 27 remote is not accessible now, so replace it with correct uri
|
||||
if [[ " ${remote_list[@]} " =~ "fedora-atomic" ]]; then
|
||||
sed -i '
|
||||
/^url=/ s|=.*|=https://kojipkgs.fedoraproject.org/atomic/repo/|
|
||||
' /etc/ostree/remotes.d/fedora-atomic.conf
|
||||
fi
|
||||
if [[ $current_ostree_remote == *"fedora:fedora/x86_64/coreos/stable"* ]]; then
|
||||
# By default there is no RPM PGP key for FC33 if the server built on
|
||||
# FC31, so add the GPG key to make sure server can upgrade from FC31 to FC33.
|
||||
$ssh_cmd cat > /etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-33-primary << EOF
|
||||
-----BEGIN PGP PUBLIC KEY BLOCK-----
|
||||
|
||||
mQINBF4wBvsBEADQmcGbVUbDRUoXADReRmOOEMeydHghtKC9uRs9YNpGYZIB+bie
|
||||
bGYZmflQayfh/wEpO2W/IZfGpHPL42V7SbyvqMjwNls/fnXsCtf4LRofNK8Qd9fN
|
||||
kYargc9R7BEz/mwXKMiRQVx+DzkmqGWy2gq4iD0/mCyf5FdJCE40fOWoIGJXaOI1
|
||||
Tz1vWqKwLS5T0dfmi9U4Tp/XsKOZGvN8oi5h0KmqFk7LEZr1MXarhi2Va86sgxsF
|
||||
QcZEKfu5tgD0r00vXzikoSjn3qA5JW5FW07F1pGP4bF5f9J3CZbQyOjTSWMmmfTm
|
||||
2d2BURWzaDiJN9twY2yjzkoOMuPdXXvovg7KxLcQerKT+FbKbq8DySJX2rnOA77k
|
||||
UG4c9BGf/L1uBkAT8dpHLk6Uf5BfmypxUkydSWT1xfTDnw1MqxO0MsLlAHOR3J7c
|
||||
oW9kLcOLuCQn1hBEwfZv7VSWBkGXSmKfp0LLIxAFgRtv+Dh+rcMMRdJgKr1V3FU+
|
||||
rZ1+ZAfYiBpQJFPjv70vx+rGEgS801D3PJxBZUEy4Ic4ZYaKNhK9x9PRQuWcIBuW
|
||||
6eTe/6lKWZeyxCumLLdiS75mF2oTcBaWeoc3QxrPRV15eDKeYJMbhnUai/7lSrhs
|
||||
EWCkKR1RivgF4slYmtNE5ZPGZ/d61zjwn2xi4xNJVs8q9WRPMpHp0vCyMwARAQAB
|
||||
tDFGZWRvcmEgKDMzKSA8ZmVkb3JhLTMzLXByaW1hcnlAZmVkb3JhcHJvamVjdC5v
|
||||
cmc+iQI4BBMBAgAiBQJeMAb7AhsPBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAK
|
||||
CRBJ/XdJlXD/MZm2D/9kriL43vd3+0DNMeA82n2v9mSR2PQqKny39xNlYPyy/1yZ
|
||||
P/KXoa4NYSCA971LSd7lv4n/h5bEKgGHxZfttfOzOnWMVSSTfjRyM/df/NNzTUEV
|
||||
7ORA5GW18g8PEtS7uRxVBf3cLvWu5q+8jmqES5HqTAdGVcuIFQeBXFN8Gy1Jinuz
|
||||
AH8rJSdkUeZ0cehWbERq80BWM9dhad5dW+/+Gv0foFBvP15viwhWqajr8V0B8es+
|
||||
2/tHI0k86FAujV5i0rrXl5UOoLilO57QQNDZH/qW9GsHwVI+2yecLstpUNLq+EZC
|
||||
GqTZCYoxYRpl0gAMbDLztSL/8Bc0tJrCRG3tavJotFYlgUK60XnXlQzRkh9rgsfT
|
||||
EXbQifWdQMMogzjCJr0hzJ+V1d0iozdUxB2ZEgTjukOvatkB77DY1FPZRkSFIQs+
|
||||
fdcjazDIBLIxwJu5QwvTNW8lOLnJ46g4sf1WJoUdNTbR0BaC7HHj1inVWi0p7IuN
|
||||
66EPGzJOSjLK+vW+J0ncPDEgLCV74RF/0nR5fVTdrmiopPrzFuguHf9S9gYI3Zun
|
||||
Yl8FJUu4kRO6JPPTicUXWX+8XZmE94aK14RCJL23nOSi8T1eW8JLW43dCBRO8QUE
|
||||
Aso1t2pypm/1zZexJdOV8yGME3g5l2W6PLgpz58DBECgqc/kda+VWgEAp7rO2A==
|
||||
=EPL3
|
||||
-----END PGP PUBLIC KEY BLOCK-----
|
||||
EOF
|
||||
${ssh_cmd} ostree remote add --set=gpgkeypath=/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-33-primary --contenturl=mirrorlist=https://ostree.fedoraproject.org/mirrorlist fedora-coreos-33 https://ostree.fedoraproject.org
|
||||
fi
|
||||
|
||||
# NOTE(flwang): 1. Either deploy or rebase for only one upgrade
|
||||
|
|
@ -155,3 +180,10 @@ elif [ "$new_ostree_remote" != "" ] && [ "$current_ostree_remote" != "$new_ostre
|
|||
${ssh_cmd} rpm-ostree rebase $new_ostree_remote
|
||||
shutdown --reboot --no-wall -t 1
|
||||
fi
|
||||
|
||||
# We need to double check if this is a k8s version upgrade only action, if so, the command should exit to avoid run the later
|
||||
# cert rotate scripts.
|
||||
if [ "${new_kube_tag}" != "${KUBE_TAG}" ]; then
|
||||
echo "END: Upgrade k8s versions and operating system"
|
||||
exit 0
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -320,11 +320,13 @@ class FedoraKubernetesDriver(KubernetesDriver):
|
|||
heat_params = {}
|
||||
try:
|
||||
kube_tag = cluster_template.labels["kube_tag"]
|
||||
image_digest = cluster_template.labels.get("kube_image_digest")
|
||||
kube_tag_params = {
|
||||
"kube_tag": kube_tag,
|
||||
"kube_version": kube_tag,
|
||||
"master_kube_tag": kube_tag,
|
||||
"minion_kube_tag": kube_tag,
|
||||
"kube_image_digest": image_digest,
|
||||
}
|
||||
heat_params.update(kube_tag_params)
|
||||
except KeyError:
|
||||
|
|
|
|||
|
|
@ -124,7 +124,7 @@ storage:
|
|||
done
|
||||
|
||||
/usr/bin/update-ca-trust
|
||||
mkdir /etc/kubernetes/
|
||||
mkdir -p /etc/kubernetes/
|
||||
cp /etc/pki/tls/certs/ca-bundle.crt /etc/kubernetes/ca-bundle.crt
|
||||
|
||||
HTTP_PROXY="__HTTP_PROXY__"
|
||||
|
|
|
|||
|
|
@ -1515,6 +1515,7 @@ resources:
|
|||
ostree_commit: {get_param: ostree_commit}
|
||||
use_podman: {get_param: use_podman}
|
||||
selinux_mode: {get_param: selinux_mode}
|
||||
kube_image_digest: {get_param: kube_image_digest}
|
||||
container_runtime: {get_param: container_runtime}
|
||||
containerd_version: {get_param: containerd_version}
|
||||
containerd_tarball_url: {get_param: containerd_tarball_url}
|
||||
|
|
|
|||
|
|
@ -1060,6 +1060,7 @@ resources:
|
|||
- "\n"
|
||||
-
|
||||
- "#!/bin/bash"
|
||||
- "source /etc/bashrc"
|
||||
- get_file: ../../common/templates/kubernetes/fragments/upgrade-kubernetes.sh
|
||||
- get_file: ../../common/templates/kubernetes/fragments/make-cert.sh
|
||||
- get_file: ../../common/templates/kubernetes/fragments/rotate-kubernetes-ca-certs-master.sh
|
||||
|
|
|
|||
|
|
@ -334,6 +334,12 @@ parameters:
|
|||
description: >
|
||||
Choose SELinux mode
|
||||
|
||||
kube_image_digest:
|
||||
type: string
|
||||
description: >
|
||||
The digest of the image which should match the given kube_tag
|
||||
default: ''
|
||||
|
||||
container_runtime:
|
||||
type: string
|
||||
description: The container runtime to install
|
||||
|
|
@ -601,6 +607,7 @@ resources:
|
|||
group: script
|
||||
inputs:
|
||||
- name: kube_tag_input
|
||||
- name: kube_image_digest_input
|
||||
- name: ostree_remote_input
|
||||
- name: ostree_commit_input
|
||||
- name: kube_service_account_key_input
|
||||
|
|
@ -610,6 +617,7 @@ resources:
|
|||
- "\n"
|
||||
-
|
||||
- "#!/bin/bash"
|
||||
- "source /etc/bashrc"
|
||||
- get_file: ../../common/templates/kubernetes/fragments/upgrade-kubernetes.sh
|
||||
- get_file: ../../common/templates/kubernetes/fragments/make-cert-client.sh
|
||||
- get_file: ../../common/templates/kubernetes/fragments/rotate-kubernetes-ca-certs-worker.sh
|
||||
|
|
@ -623,6 +631,7 @@ resources:
|
|||
actions: ['UPDATE']
|
||||
input_values:
|
||||
kube_tag_input: {get_param: kube_tag}
|
||||
kube_image_digest_input: {get_param: kube_image_digest}
|
||||
ostree_remote_input: {get_param: ostree_remote}
|
||||
ostree_commit_input: {get_param: ostree_commit}
|
||||
kube_service_account_key_input: {get_param: kube_service_account_key}
|
||||
|
|
|
|||
|
|
@ -114,7 +114,7 @@
|
|||
"name": "root"
|
||||
},
|
||||
"contents": {
|
||||
"source": "data:,%23!%2Fbin%2Fbash%0A%0Aset%20-x%0Aset%20-e%0Aset%20%2Bu%0A%0Auntil%20%5B%20-f%20%2Fetc%2Fpki%2Fca-trust%2Fsource%2Fanchors%2Fopenstack-ca.pem%20%5D%0Ado%0A%20%20%20%20echo%20%22waiting%20for%20%2Fetc%2Fpki%2Fca-trust%2Fsource%2Fanchors%2Fopenstack-ca.pem%22%0A%20%20%20%20sleep%203s%0Adone%0A%0A%2Fusr%2Fbin%2Fupdate-ca-trust%0Amkdir%20%2Fetc%2Fkubernetes%2F%0Acp%20%2Fetc%2Fpki%2Ftls%2Fcerts%2Fca-bundle.crt%20%2Fetc%2Fkubernetes%2Fca-bundle.crt%0A%0AHTTP_PROXY%3D%22__HTTP_PROXY__%22%0AHTTPS_PROXY%3D%22__HTTPS_PROXY__%22%0ANO_PROXY%3D%22__NO_PROXY__%22%0A%0Aif%20%5B%20-n%20%22%24%7BHTTP_PROXY%7D%22%20%5D%3B%20then%0A%20%20%20%20export%20HTTP_PROXY%0A%20%20%20%20echo%20%22http_proxy%3D%24%7BHTTP_PROXY%7D%22%20%3E%3E%20%2Fetc%2Fenvironment%0Afi%0A%0Aif%20%5B%20-n%20%22%24%7BHTTPS_PROXY%7D%22%20%5D%3B%20then%0A%20%20%20%20export%20HTTPS_PROXY%0A%20%20%20%20echo%20%22https_proxy%3D%24%7BHTTPS_PROXY%7D%22%20%3E%3E%20%2Fetc%2Fenvironment%0Afi%0A%0Aif%20%5B%20-n%20%22%24%7BNO_PROXY%7D%22%20%5D%3B%20then%0A%20%20%20%20export%20NO_PROXY%0A%20%20%20%20echo%20%22no_proxy%3D%24%7BNO_PROXY%7D%22%20%3E%3E%20%2Fetc%2Fenvironment%0Afi%0A%0A%23%20Create%20a%20keypair%20for%20the%20heat-container-agent%20to%0A%23%20access%20the%20node%20over%20ssh.%20It%20is%20useful%20to%20operate%0A%23%20in%20host%20mount%20namespace%20and%20apply%20configuration.%0Aid%0Amkdir%20-p%20%2Fsrv%2Fmagnum%2F.ssh%0Achmod%200700%20%2Fsrv%2Fmagnum%2F.ssh%0A%23touch%20%2Fsrv%2Fmagnum%2F.ssh%2Fheat_agent_rsa%0Assh-keygen%20-q%20-t%20rsa%20-N%20''%20-f%20%2Ftmp%2Fheat_agent_rsa%0Amv%20%2Ftmp%2Fheat_agent_rsa%20%2Fsrv%2Fmagnum%2F.ssh%2Fheat_agent_rsa%0Amv%20%2Ftmp%2Fheat_agent_rsa.pub%20%2Fsrv%2Fmagnum%2F.ssh%2Fheat_agent_rsa.pub%0Achmod%200400%20%2Fsrv%2Fmagnum%2F.ssh%2Fheat_agent_rsa%0Achmod%200400%20%2Fsrv%2Fmagnum%2F.ssh%2Fheat_agent_rsa.pub%0A%23%20Add%20the%20public%20to%20the%20host%20authorized_keys%20file.%0Amkdir%20-p%20%2Froot%2F.ssh%0Achmod%200700%20%2Froot%2F.ssh%0Acat%20%2Fsrv%2Fmagnum%2F.ssh%2Fheat_agent_rsa.pub%20%3E%20%2Froot%2F.ssh%2Fauthorized_keys%0A%23%20Add%20localost%20to%20know_hosts%0Assh-keyscan%20127.0.0.1%20%3E%20%2Fsrv%2Fmagnum%2F.ssh%2Fknown_hosts%0A%23%20ssh%20configguration%20file%2C%20to%20be%20specified%20with%20ssh%20-F%0Acat%20%3E%20%2Fsrv%2Fmagnum%2F.ssh%2Fconfig%20%3C%3CEOF%0AHost%20localhost%0A%20%20%20%20%20HostName%20127.0.0.1%0A%20%20%20%20%20User%20root%0A%20%20%20%20%20IdentityFile%20%2Fsrv%2Fmagnum%2F.ssh%2Fheat_agent_rsa%0A%20%20%20%20%20UserKnownHostsFile%20%2Fsrv%2Fmagnum%2F.ssh%2Fknown_hosts%0AEOF%0A%0Ased%20-i%20'%2F%5EPermitRootLogin%2F%20s%2F%20.*%2F%20without-password%2F'%20%2Fetc%2Fssh%2Fsshd_config%0A%23%20Security%20enhancement%3A%20Disable%20password%20authentication%0Ased%20-i%20'%2F%5EPasswordAuthentication%20yes%2F%20s%2F%20yes%2F%20no%2F'%20%2Fetc%2Fssh%2Fsshd_config%0A%0Asystemctl%20restart%20sshd%0A"
|
||||
"source": "data:,%23!%2Fbin%2Fbash%0A%0Aset%20-x%0Aset%20-e%0Aset%20%2Bu%0A%0Auntil%20%5B%20-f%20%2Fetc%2Fpki%2Fca-trust%2Fsource%2Fanchors%2Fopenstack-ca.pem%20%5D%0Ado%0A%20%20%20%20echo%20%22waiting%20for%20%2Fetc%2Fpki%2Fca-trust%2Fsource%2Fanchors%2Fopenstack-ca.pem%22%0A%20%20%20%20sleep%203s%0Adone%0A%0A%2Fusr%2Fbin%2Fupdate-ca-trust%0Amkdir%20-p%20%2Fetc%2Fkubernetes%2F%0Acp%20%2Fetc%2Fpki%2Ftls%2Fcerts%2Fca-bundle.crt%20%2Fetc%2Fkubernetes%2Fca-bundle.crt%0A%0AHTTP_PROXY%3D%22__HTTP_PROXY__%22%0AHTTPS_PROXY%3D%22__HTTPS_PROXY__%22%0ANO_PROXY%3D%22__NO_PROXY__%22%0A%0Aif%20%5B%20-n%20%22%24%7BHTTP_PROXY%7D%22%20%5D%3B%20then%0A%20%20%20%20export%20HTTP_PROXY%0A%20%20%20%20echo%20%22http_proxy%3D%24%7BHTTP_PROXY%7D%22%20%3E%3E%20%2Fetc%2Fenvironment%0Afi%0A%0Aif%20%5B%20-n%20%22%24%7BHTTPS_PROXY%7D%22%20%5D%3B%20then%0A%20%20%20%20export%20HTTPS_PROXY%0A%20%20%20%20echo%20%22https_proxy%3D%24%7BHTTPS_PROXY%7D%22%20%3E%3E%20%2Fetc%2Fenvironment%0Afi%0A%0Aif%20%5B%20-n%20%22%24%7BNO_PROXY%7D%22%20%5D%3B%20then%0A%20%20%20%20export%20NO_PROXY%0A%20%20%20%20echo%20%22no_proxy%3D%24%7BNO_PROXY%7D%22%20%3E%3E%20%2Fetc%2Fenvironment%0Afi%0A%0A%23%20Create%20a%20keypair%20for%20the%20heat-container-agent%20to%0A%23%20access%20the%20node%20over%20ssh.%20It%20is%20useful%20to%20operate%0A%23%20in%20host%20mount%20namespace%20and%20apply%20configuration.%0Aid%0Amkdir%20-p%20%2Fsrv%2Fmagnum%2F.ssh%0Achmod%200700%20%2Fsrv%2Fmagnum%2F.ssh%0A%23touch%20%2Fsrv%2Fmagnum%2F.ssh%2Fheat_agent_rsa%0Assh-keygen%20-q%20-t%20rsa%20-N%20''%20-f%20%2Ftmp%2Fheat_agent_rsa%0Amv%20%2Ftmp%2Fheat_agent_rsa%20%2Fsrv%2Fmagnum%2F.ssh%2Fheat_agent_rsa%0Amv%20%2Ftmp%2Fheat_agent_rsa.pub%20%2Fsrv%2Fmagnum%2F.ssh%2Fheat_agent_rsa.pub%0Achmod%200400%20%2Fsrv%2Fmagnum%2F.ssh%2Fheat_agent_rsa%0Achmod%200400%20%2Fsrv%2Fmagnum%2F.ssh%2Fheat_agent_rsa.pub%0A%23%20Add%20the%20public%20to%20the%20host%20authorized_keys%20file.%0Amkdir%20-p%20%2Froot%2F.ssh%0Achmod%200700%20%2Froot%2F.ssh%0Acat%20%2Fsrv%2Fmagnum%2F.ssh%2Fheat_agent_rsa.pub%20%3E%20%2Froot%2F.ssh%2Fauthorized_keys%0A%23%20Add%20localost%20to%20know_hosts%0Assh-keyscan%20127.0.0.1%20%3E%20%2Fsrv%2Fmagnum%2F.ssh%2Fknown_hosts%0A%23%20ssh%20configguration%20file%2C%20to%20be%20specified%20with%20ssh%20-F%0Acat%20%3E%20%2Fsrv%2Fmagnum%2F.ssh%2Fconfig%20%3C%3CEOF%0AHost%20localhost%0A%20%20%20%20%20HostName%20127.0.0.1%0A%20%20%20%20%20User%20root%0A%20%20%20%20%20IdentityFile%20%2Fsrv%2Fmagnum%2F.ssh%2Fheat_agent_rsa%0A%20%20%20%20%20UserKnownHostsFile%20%2Fsrv%2Fmagnum%2F.ssh%2Fknown_hosts%0AEOF%0A%0Ased%20-i%20'%2F%5EPermitRootLogin%2F%20s%2F%20.*%2F%20without-password%2F'%20%2Fetc%2Fssh%2Fsshd_config%0A%23%20Security%20enhancement%3A%20Disable%20password%20authentication%0Ased%20-i%20'%2F%5EPasswordAuthentication%20yes%2F%20s%2F%20yes%2F%20no%2F'%20%2Fetc%2Fssh%2Fsshd_config%0A%0Asystemctl%20restart%20sshd%0A"
|
||||
},
|
||||
"mode": 448
|
||||
},
|
||||
|
|
|
|||
Loading…
Reference in New Issue