Add k8s cert rotation script to sysinv pkg
This commit is derived from https://review.opendev.org/#/c/692276
This script checks the cert expiration date and rotates them if they
expires within 90 days. After cert renewed, all the k8s master
component configurations will be updated.
An alarm will be sent to fm to notify the administrator to
reboot the controllers or renew the certs manually if the automatic
process fails.
Change-Id: I286c38758ded2f661498367c7adf4dd59e603b83
Partial-Bug: 1838659
Signed-off-by: Mingyuan Qi <mingyuan.qi@intel.com>
(cherry picked from commit 7b64d87a26)
This commit is contained in:
Mingyuan Qi
parent
9418550fb1
commit
31ce7c575f
|
|
@ -99,6 +99,7 @@ install -d -m 755 %{buildroot}%{local_bindir}
|
|||
install -p -D -m 755 sysinv/cmd/partition_info.sh %{buildroot}%{local_bindir}/partition_info.sh
|
||||
install -p -D -m 755 sysinv/cmd/manage-partitions %{buildroot}%{local_bindir}/manage-partitions
|
||||
install -p -D -m 755 sysinv/cmd/query_pci_id %{buildroot}%{local_bindir}/query_pci_id
|
||||
install -p -D -m 700 sysinv/cmd/kube-cert-rotation.sh %{buildroot}%{local_bindir}/kube-cert-rotation.sh
|
||||
|
||||
%clean
|
||||
echo "CLEAN CALLED"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,152 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Copyright (C) 2019 Intel Corporation
|
||||
#
|
||||
|
||||
#
|
||||
# This script is to rotate kubernetes cluster certificates automatically
|
||||
#
|
||||
|
||||
# Expiration date of k8s certs
|
||||
CERT_LASTDATE=$(openssl x509 -in /etc/kubernetes/pki/apiserver.crt -text | grep 'Not After' | awk -F ' : ' '{print $2}')
|
||||
|
||||
if [ "x${CERT_LASTDATE}" != "x" ]; then
|
||||
CERT_LASTDATE_S=$(date -d "${CERT_LASTDATE}" +%s)
|
||||
CURRENT_DATE_S=$(date +%s)
|
||||
DAY_LEFT_S=$((${CERT_LASTDATE_S}-${CURRENT_DATE_S}))
|
||||
fi
|
||||
|
||||
# Renew certificates 90 days before expiration
|
||||
ERR=0
|
||||
declare -r NINETY_DAYS_S=$((90*24*3600))
|
||||
if [ ${DAY_LEFT_S} -lt ${NINETY_DAYS_S} ]; then
|
||||
# Same expiration date of apiserver, apiserver-kubelet-client and front-proxy-client
|
||||
if [ ${ERR} -eq 0 ]; then
|
||||
kubeadm alpha certs renew apiserver
|
||||
if [ $? -ne 0 ]; then
|
||||
ERR=1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ${ERR} -eq 0 ]; then
|
||||
kubeadm alpha certs renew apiserver-kubelet-client
|
||||
if [ $? -ne 0 ]; then
|
||||
ERR=1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ${ERR} -eq 0 ]; then
|
||||
kubeadm alpha certs renew front-proxy-client
|
||||
if [ $? -ne 0 ]; then
|
||||
ERR=1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Update cluster configuration files using the renewed certificates
|
||||
|
||||
if [ ${ERR} -eq 0 ]; then
|
||||
ADVERTISE_ADDR=$(kubectl get endpoints kubernetes -o jsonpath='{.subsets[0].addresses[0].ip}')
|
||||
else
|
||||
ADVERTISE_ADDR=""
|
||||
fi
|
||||
|
||||
if [ "x${ADVERTISE_ADDR}" != "x" ]; then
|
||||
# Update admin.conf
|
||||
if [ ${ERR} -eq 0 ]; then
|
||||
kubeadm alpha kubeconfig user --client-name=kubernetes-admin --apiserver-advertise-address=${ADVERTISE_ADDR} --org system:masters > /tmp/admin.conf
|
||||
if [ $? -eq 0 ]; then
|
||||
mv /tmp/admin.conf /etc/kubernetes/admin.conf
|
||||
if [ $? -ne 0 ]; then
|
||||
ERR=1
|
||||
fi
|
||||
else
|
||||
ERR=1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Update controller-manager.conf
|
||||
if [ ${ERR} -eq 0 ]; then
|
||||
kubeadm alpha kubeconfig user --client-name=system:kube-controller-manager --apiserver-advertise-address=${ADVERTISE_ADDR} --cert-dir /etc/kubernetes/pki/ > /tmp/controller-manager.conf
|
||||
if [ $? -eq 0 ]; then
|
||||
mv /tmp/controller-manager.conf /etc/kubernetes/controller-manager.conf
|
||||
if [ $? -ne 0 ]; then
|
||||
ERR=1
|
||||
fi
|
||||
else
|
||||
ERR=1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Update scheduler.conf
|
||||
if [ ${ERR} -eq 0 ]; then
|
||||
kubeadm alpha kubeconfig user --client-name=system:kube-scheduler --apiserver-advertise-address=${ADVERTISE_ADDR} --cert-dir /etc/kubernetes/pki/ > /tmp/scheduler.conf
|
||||
if [ $? -eq 0 ]; then
|
||||
mv /tmp/scheduler.conf /etc/kubernetes/scheduler.conf
|
||||
if [ $? -ne 0 ]; then
|
||||
ERR=1
|
||||
fi
|
||||
else
|
||||
ERR=1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Update kubelet.conf
|
||||
# This block could be removed once this issue is resolved. https://github.com/kubernetes/kubeadm/issues/1753
|
||||
if [ ${ERR} -eq 0 ]; then
|
||||
kubeadm alpha kubeconfig user --client-name=system:node:${HOSTNAME} --apiserver-advertise-address=${ADVERTISE_ADDR} --org system:nodes > /tmp/kubelet.conf
|
||||
if [ $? -eq 0 ]; then
|
||||
mv /tmp/kubelet.conf /etc/kubernetes/kubelet.conf
|
||||
if [ $? -ne 0 ]; then
|
||||
ERR=1
|
||||
fi
|
||||
else
|
||||
ERR=1
|
||||
fi
|
||||
fi
|
||||
else
|
||||
ERR=1
|
||||
fi
|
||||
|
||||
# Restart docker container of k8s components to refresh the configurations within container
|
||||
if [ ${ERR} -eq 0 ]; then
|
||||
docker ps | awk '/k8s_kube-apiserver/{print$1}' | xargs docker restart > /dev/null
|
||||
if [ $? -ne 0 ]; then
|
||||
ERR=2
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ${ERR} -eq 0 ]; then
|
||||
docker ps | awk '/k8s_kube-controller-manager/{print$1}' | xargs docker restart > /dev/null
|
||||
if [ $? -ne 0 ]; then
|
||||
ERR=2
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ${ERR} -eq 0 ]; then
|
||||
docker ps | awk '/k8s_kube-scheduler/{print$1}' | xargs docker restart > /dev/null
|
||||
if [ $? -ne 0 ]; then
|
||||
ERR=2
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ${ERR} -eq 0 ]; then
|
||||
systemctl daemon-reload
|
||||
systemctl restart kubelet
|
||||
if [ $? -ne 0 ]; then
|
||||
ERR=2
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ${ERR} -eq 2 ]; then
|
||||
# Notify admin to lock and unlock this master node if restart k8s components failed
|
||||
fmClientCli -c "### ###250.003###set###host###host=${HOSTNAME}### ###major###Kubernetes certificates on host ${HOSTNAME} have been renewed but not updated.###operational-violation### ###Lock and unlock host ${HOSTNAME} to update config.### ### ###"
|
||||
elif [ ${ERR} -eq 1 ]; then
|
||||
# Notify admin to rotate kube cert manually if cert renew or config failed
|
||||
fmClientCli -c "### ###250.003###set###host###host=${HOSTNAME}### ###major###Kubernetes certificates automatic rotation failed on host ${HOSTNAME}###operational-violation### ###Rotate kubernetes certificates manually, lock and unlock host ${HOSTNAME} to update config.### ### ###"
|
||||
else
|
||||
# Clear the alarm if cert rotation completed
|
||||
fmClientCli -d "###250.003###host=${HOSTNAME}###"
|
||||
fi
|
||||
fi
|
||||
Loading…
Reference in New Issue