From 7b64d87a26fac2a5bcb2fce31ffa31b92b0e6f17 Mon Sep 17 00:00:00 2001 From: Mingyuan Qi Date: Thu, 12 Dec 2019 05:57:59 +0000 Subject: [PATCH] Add k8s cert rotation script to sysinv pkg This commit is derived from https://review.opendev.org/#/c/692276 This script checks the cert expiration date and rotates them if they expires within 90 days. After cert renewed, all the k8s master component configurations will be updated. An alarm will be sent to fm to notify the administrator to reboot the controllers or renew the certs manually if the automatic process fails. Change-Id: I286c38758ded2f661498367c7adf4dd59e603b83 Partial-Bug: 1838659 Signed-off-by: Mingyuan Qi --- sysinv/sysinv/centos/sysinv.spec | 1 + .../sysinv/sysinv/cmd/kube-cert-rotation.sh | 152 ++++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 sysinv/sysinv/sysinv/sysinv/cmd/kube-cert-rotation.sh diff --git a/sysinv/sysinv/centos/sysinv.spec b/sysinv/sysinv/centos/sysinv.spec index 9977d7aca0..0091a3a75f 100644 --- a/sysinv/sysinv/centos/sysinv.spec +++ b/sysinv/sysinv/centos/sysinv.spec @@ -102,6 +102,7 @@ install -d -m 755 %{buildroot}%{local_bindir} install -p -D -m 755 sysinv/cmd/partition_info.sh %{buildroot}%{local_bindir}/partition_info.sh install -p -D -m 755 sysinv/cmd/manage-partitions %{buildroot}%{local_bindir}/manage-partitions install -p -D -m 755 sysinv/cmd/query_pci_id %{buildroot}%{local_bindir}/query_pci_id +install -p -D -m 700 sysinv/cmd/kube-cert-rotation.sh %{buildroot}%{local_bindir}/kube-cert-rotation.sh %clean echo "CLEAN CALLED" diff --git a/sysinv/sysinv/sysinv/sysinv/cmd/kube-cert-rotation.sh b/sysinv/sysinv/sysinv/sysinv/cmd/kube-cert-rotation.sh new file mode 100644 index 0000000000..91795b742a --- /dev/null +++ b/sysinv/sysinv/sysinv/sysinv/cmd/kube-cert-rotation.sh @@ -0,0 +1,152 @@ +#!/bin/bash +# +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2019 Intel Corporation +# + +# +# This script is to rotate kubernetes cluster certificates automatically +# + +# Expiration date of k8s certs +CERT_LASTDATE=$(openssl x509 -in /etc/kubernetes/pki/apiserver.crt -text | grep 'Not After' | awk -F ' : ' '{print $2}') + +if [ "x${CERT_LASTDATE}" != "x" ]; then + CERT_LASTDATE_S=$(date -d "${CERT_LASTDATE}" +%s) + CURRENT_DATE_S=$(date +%s) + DAY_LEFT_S=$((${CERT_LASTDATE_S}-${CURRENT_DATE_S})) +fi + +# Renew certificates 90 days before expiration +ERR=0 +declare -r NINETY_DAYS_S=$((90*24*3600)) +if [ ${DAY_LEFT_S} -lt ${NINETY_DAYS_S} ]; then + # Same expiration date of apiserver, apiserver-kubelet-client and front-proxy-client + if [ ${ERR} -eq 0 ]; then + kubeadm alpha certs renew apiserver + if [ $? -ne 0 ]; then + ERR=1 + fi + fi + + if [ ${ERR} -eq 0 ]; then + kubeadm alpha certs renew apiserver-kubelet-client + if [ $? -ne 0 ]; then + ERR=1 + fi + fi + + if [ ${ERR} -eq 0 ]; then + kubeadm alpha certs renew front-proxy-client + if [ $? -ne 0 ]; then + ERR=1 + fi + fi + + # Update cluster configuration files using the renewed certificates + + if [ ${ERR} -eq 0 ]; then + ADVERTISE_ADDR=$(kubectl get endpoints kubernetes -o jsonpath='{.subsets[0].addresses[0].ip}') + else + ADVERTISE_ADDR="" + fi + + if [ "x${ADVERTISE_ADDR}" != "x" ]; then + # Update admin.conf + if [ ${ERR} -eq 0 ]; then + kubeadm alpha kubeconfig user --client-name=kubernetes-admin --apiserver-advertise-address=${ADVERTISE_ADDR} --org system:masters > /tmp/admin.conf + if [ $? -eq 0 ]; then + mv /tmp/admin.conf /etc/kubernetes/admin.conf + if [ $? -ne 0 ]; then + ERR=1 + fi + else + ERR=1 + fi + fi + + # Update controller-manager.conf + if [ ${ERR} -eq 0 ]; then + kubeadm alpha kubeconfig user --client-name=system:kube-controller-manager --apiserver-advertise-address=${ADVERTISE_ADDR} --cert-dir /etc/kubernetes/pki/ > /tmp/controller-manager.conf + if [ $? -eq 0 ]; then + mv /tmp/controller-manager.conf /etc/kubernetes/controller-manager.conf + if [ $? -ne 0 ]; then + ERR=1 + fi + else + ERR=1 + fi + fi + + # Update scheduler.conf + if [ ${ERR} -eq 0 ]; then + kubeadm alpha kubeconfig user --client-name=system:kube-scheduler --apiserver-advertise-address=${ADVERTISE_ADDR} --cert-dir /etc/kubernetes/pki/ > /tmp/scheduler.conf + if [ $? -eq 0 ]; then + mv /tmp/scheduler.conf /etc/kubernetes/scheduler.conf + if [ $? -ne 0 ]; then + ERR=1 + fi + else + ERR=1 + fi + fi + + # Update kubelet.conf + # This block could be removed once this issue is resolved. https://github.com/kubernetes/kubeadm/issues/1753 + if [ ${ERR} -eq 0 ]; then + kubeadm alpha kubeconfig user --client-name=system:node:${HOSTNAME} --apiserver-advertise-address=${ADVERTISE_ADDR} --org system:nodes > /tmp/kubelet.conf + if [ $? -eq 0 ]; then + mv /tmp/kubelet.conf /etc/kubernetes/kubelet.conf + if [ $? -ne 0 ]; then + ERR=1 + fi + else + ERR=1 + fi + fi + else + ERR=1 + fi + + # Restart docker container of k8s components to refresh the configurations within container + if [ ${ERR} -eq 0 ]; then + docker ps | awk '/k8s_kube-apiserver/{print$1}' | xargs docker restart > /dev/null + if [ $? -ne 0 ]; then + ERR=2 + fi + fi + + if [ ${ERR} -eq 0 ]; then + docker ps | awk '/k8s_kube-controller-manager/{print$1}' | xargs docker restart > /dev/null + if [ $? -ne 0 ]; then + ERR=2 + fi + fi + + if [ ${ERR} -eq 0 ]; then + docker ps | awk '/k8s_kube-scheduler/{print$1}' | xargs docker restart > /dev/null + if [ $? -ne 0 ]; then + ERR=2 + fi + fi + + if [ ${ERR} -eq 0 ]; then + systemctl daemon-reload + systemctl restart kubelet + if [ $? -ne 0 ]; then + ERR=2 + fi + fi + + if [ ${ERR} -eq 2 ]; then + # Notify admin to lock and unlock this master node if restart k8s components failed + fmClientCli -c "### ###250.003###set###host###host=${HOSTNAME}### ###major###Kubernetes certificates on host ${HOSTNAME} have been renewed but not updated.###operational-violation### ###Lock and unlock host ${HOSTNAME} to update config.### ### ###" + elif [ ${ERR} -eq 1 ]; then + # Notify admin to rotate kube cert manually if cert renew or config failed + fmClientCli -c "### ###250.003###set###host###host=${HOSTNAME}### ###major###Kubernetes certificates automatic rotation failed on host ${HOSTNAME}###operational-violation### ###Rotate kubernetes certificates manually, lock and unlock host ${HOSTNAME} to update config.### ### ###" + else + # Clear the alarm if cert rotation completed + fmClientCli -d "###250.003###host=${HOSTNAME}###" + fi +fi