From a71ce32bf6ba7916e1baf004b0050df469107850 Mon Sep 17 00:00:00 2001 From: Andy Ning Date: Thu, 17 Jun 2021 20:58:24 -0400 Subject: [PATCH] Kube rootca update abort changes This change updated k8s master and worker rootca runtime classes to support kube rootca update abort procedure. We also added a fix to cert-mon errors when update start again after abort. When root ca update start again after abort at the stage of host certs get updated, cert-mon will generate TLS errors at the stage of trustbothcas. This is fixed by moving the signing ca cert to the top of the admin.conf when update-certs phase completes on the host. Story: 2008675 Task: 43067 Signed-off-by: Andy Ning Change-Id: I79d4298694bcba96b89602e93adca49a71d284b6 --- .../modules/platform/manifests/kubernetes.pp | 55 +++++++++---------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/puppet-manifests/src/modules/platform/manifests/kubernetes.pp b/puppet-manifests/src/modules/platform/manifests/kubernetes.pp index 2afed291..5ffef077 100644 --- a/puppet-manifests/src/modules/platform/manifests/kubernetes.pp +++ b/puppet-manifests/src/modules/platform/manifests/kubernetes.pp @@ -46,9 +46,6 @@ class platform::kubernetes::params ( $controller_manager_key = undef, $kubelet_cert = undef, $kubelet_key = undef, - # The file holding the original root CA cert/key before update - $rootca_certfile_old = '/etc/kubernetes/pki/ca_old.crt', - $rootca_keyfile_old = '/etc/kubernetes/pki/ca_old.key', # The file holding the root CA cert/key to update to $rootca_certfile_new = '/etc/kubernetes/pki/ca_new.crt', $rootca_keyfile_new = '/etc/kubernetes/pki/ca_new.key', @@ -841,14 +838,8 @@ class platform::kubernetes::duplex_migration::runtime { class platform::kubernetes::master::rootca::trustbothcas::runtime inherits ::platform::kubernetes::params { - # Backup the original root CA cert - file { $rootca_certfile_old: - ensure => file, - source => $rootca_certfile, - replace => false, - } # Create the new root CA cert file - -> file { $rootca_certfile_new: + file { $rootca_certfile_new: ensure => file, content => base64('decode', $rootca_cert), } @@ -859,7 +850,7 @@ class platform::kubernetes::master::rootca::trustbothcas::runtime } # Append the new cert to the current cert -> exec { 'append_ca_cert': - command => "cat ${rootca_certfile_old} ${rootca_certfile_new} > ${rootca_certfile}", + command => "cat ${rootca_certfile_new} >> ${rootca_certfile}", } # update admin.conf with both old and new certs -> exec { 'update_admin_conf': @@ -909,16 +900,10 @@ class platform::kubernetes::master::rootca::trustbothcas::runtime class platform::kubernetes::worker::rootca::trustbothcas::runtime inherits ::platform::kubernetes::params { - $cluster = generate('/bin/bash', '-c', "/bin/sed -e '/- cluster/,/name:/!d' /etc/kubernetes/kubelet.conf \\ - | grep 'name:' | awk '{printf \"%s\", \$2}'") - # Backup the original root CA cert - file { $rootca_certfile_old: - ensure => file, - source => $rootca_certfile, - replace => false, - } + $cluster = generate('/bin/bash', '-c', "/bin/sed -e '/- cluster/,/name:/!d' /etc/kubernetes/kubelet.conf \ + | grep 'name:' | awk '{printf \"%s\", \$2}'") # Create the new root CA cert file - -> file { $rootca_certfile_new: + file { $rootca_certfile_new: ensure => file, content => base64('decode', $rootca_cert), } @@ -929,7 +914,8 @@ class platform::kubernetes::worker::rootca::trustbothcas::runtime } # Append the new cert to the current cert -> exec { 'append_ca_cert': - command => "cat ${rootca_certfile_old} ${rootca_certfile_new} > ${rootca_certfile}", + command => "cat ${rootca_certfile_new} >> ${rootca_certfile}", + unless => "grep -v '[BEGIN|END] CERTIFICATE' ${rootca_certfile_new} | awk /./ | grep -f ${rootca_certfile} &>/dev/null" } # Update kubelet.conf with both old and new certs -> exec { 'update_kubelet_conf': @@ -1007,10 +993,6 @@ class platform::kubernetes::master::rootca::trustnewca::runtime -> exec { 'remove_new_key_file': command => "/bin/rm -f ${rootca_keyfile_new}", } - # Remove the old cert file - -> exec { 'remove_old_cert_file': - command => "/bin/rm -f ${rootca_certfile_old}", - } } class platform::kubernetes::worker::rootca::trustnewca::runtime @@ -1028,10 +1010,6 @@ class platform::kubernetes::worker::rootca::trustnewca::runtime command => "/bin/mv -f ${rootca_keyfile_new} ${rootca_keyfile}", onlyif => "/usr/bin/test -e ${rootca_keyfile_new}", } - # Remove the old cert file - -> exec { 'remove_old_cert_file': - command => "/bin/rm -f ${rootca_certfile_old}", - } # Update kubelet.conf with the new cert -> exec { 'update_kubelet_conf': environment => [ 'KUBECONFIG=/etc/kubernetes/kubelet.conf' ], @@ -1231,6 +1209,25 @@ class platform::kubernetes::master::rootca::updatecerts::runtime command => "/usr/bin/kill -s SIGHUP $(pidof kubelet)" } + # Moving the signing ca cert in ca.crt and admin.conf to be the first in the bundle. + # This is neccessary for cert-mon, since it only uses the first ca cert in admin.conf + # to verify server certificate from apiserver. + # Remove the new ca cert from the bundle first + -> exec { 'remove_new_ca_cert_from_bottom': + command => "tac ${rootca_certfile} | sed '0,/-----BEGIN CERTIFICATE-----/d' | tac - > /tmp/kube_rootca_update/ca_tmp.crt" + } + + # Create the ca.crt with the new ca cert at the top of the bundle + -> exec { 'prepend_new_ca_cert_at_top': + command => "cat ${rootca_certfile_new} /tmp/kube_rootca_update/ca_tmp.crt > ${rootca_certfile}" + } + + # Update admin.conf with the newly create ca.crt + -> exec { 'update_admin_conf_with_new_cert_at_top': + environment => [ 'KUBECONFIG=/etc/kubernetes/admin.conf' ], + command => "kubectl config set-cluster kubernetes --certificate-authority ${rootca_certfile} --embed-certs", + } + # Removing temporary directory for files along this configuration process -> exec { 'remove_kube_rootca_update_dir': command => '/usr/bin/rm -rf /tmp/kube_rootca_update',