1301 lines
48 KiB
Puppet
1301 lines
48 KiB
Puppet
class platform::kubernetes::params (
|
|
$enabled = true,
|
|
# K8S version we are upgrading to (None if not in an upgrade)
|
|
$upgrade_to_version = undef,
|
|
# K8S version running on a host
|
|
$version = undef,
|
|
$kubeadm_version = undef,
|
|
$kubelet_version = undef,
|
|
$node_ip = undef,
|
|
$service_domain = undef,
|
|
$dns_service_ip = undef,
|
|
$host_labels = [],
|
|
$k8s_cpuset = undef,
|
|
$k8s_nodeset = undef,
|
|
$k8s_platform_cpuset = undef,
|
|
$k8s_reserved_mem = undef,
|
|
$k8s_all_reserved_cpuset = undef,
|
|
$k8s_cpu_mgr_policy = 'none',
|
|
$k8s_topology_mgr_policy = 'best-effort',
|
|
$k8s_cni_bin_dir = '/usr/libexec/cni',
|
|
$k8s_vol_plugin_dir = '/usr/libexec/kubernetes/kubelet-plugins/volume/exec/',
|
|
$k8s_pod_max_pids = '65535',
|
|
$join_cmd = undef,
|
|
$oidc_issuer_url = undef,
|
|
$oidc_client_id = undef,
|
|
$oidc_username_claim = undef,
|
|
$oidc_groups_claim = undef,
|
|
$admission_plugins = undef,
|
|
$etcd_cafile = undef,
|
|
$etcd_certfile = undef,
|
|
$etcd_keyfile = undef,
|
|
$etcd_servers = undef,
|
|
$rootca_certfile = '/etc/kubernetes/pki/ca.crt',
|
|
$rootca_keyfile = '/etc/kubernetes/pki/ca.key',
|
|
$rootca_cert = undef,
|
|
$rootca_key = undef,
|
|
$admin_cert = undef,
|
|
$admin_key = undef,
|
|
$apiserver_cert = undef,
|
|
$apiserver_key = undef,
|
|
$apiserver_kubelet_cert = undef,
|
|
$apiserver_kubelet_key = undef,
|
|
$scheduler_cert = undef,
|
|
$scheduler_key = undef,
|
|
$controller_manager_cert = undef,
|
|
$controller_manager_key = undef,
|
|
$kubelet_cert = undef,
|
|
$kubelet_key = undef,
|
|
# The file holding the original root CA cert/key before update
|
|
$rootca_certfile_old = '/etc/kubernetes/pki/ca_old.crt',
|
|
$rootca_keyfile_old = '/etc/kubernetes/pki/ca_old.key',
|
|
# The file holding the root CA cert/key to update to
|
|
$rootca_certfile_new = '/etc/kubernetes/pki/ca_new.crt',
|
|
$rootca_keyfile_new = '/etc/kubernetes/pki/ca_new.key',
|
|
) { }
|
|
|
|
class platform::kubernetes::configuration {
|
|
|
|
if 'kube-ignore-isol-cpus=enabled' in $::platform::kubernetes::params::host_labels {
|
|
$ensure = 'present'
|
|
} else {
|
|
$ensure = 'absent'
|
|
}
|
|
|
|
file { '/etc/kubernetes/ignore_isolcpus':
|
|
ensure => $ensure,
|
|
owner => 'root',
|
|
group => 'root',
|
|
mode => '0644',
|
|
}
|
|
}
|
|
|
|
class platform::kubernetes::bindmounts {
|
|
include ::platform::kubernetes::params
|
|
|
|
$kubeadm_version = $::platform::kubernetes::params::kubeadm_version
|
|
$kubelet_version = $::platform::kubernetes::params::kubelet_version
|
|
|
|
# In the following two bind mounts, the "remounts" option *must* be
|
|
# set to 'false' otherwise it doesn't work reliably. In my testing
|
|
# (as of July 2021) it will update /etc/fstab but any existing
|
|
# mounts will be left untouched. This sort of makes sense, as
|
|
# the mount man page specifies that the "remount" option does not
|
|
# change device or mount point and we may want to change the device.
|
|
|
|
notice("setting stage1 bind mount, kubeadm_version is ${kubeadm_version}")
|
|
mount { '/usr/local/kubernetes/current/stage1':
|
|
ensure => mounted,
|
|
device => "/usr/local/kubernetes/${kubeadm_version}/stage1",
|
|
fstype => 'none',
|
|
options => 'rw,bind',
|
|
remounts => false,
|
|
}
|
|
|
|
notice("setting stage2 bind mount, kubelet_version is ${kubelet_version}")
|
|
mount { '/usr/local/kubernetes/current/stage2':
|
|
ensure => mounted,
|
|
device => "/usr/local/kubernetes/${kubelet_version}/stage2",
|
|
fstype => 'none',
|
|
options => 'rw,bind',
|
|
remounts => false,
|
|
}
|
|
}
|
|
|
|
class platform::kubernetes::cgroup::params (
|
|
$cgroup_root = '/sys/fs/cgroup',
|
|
$cgroup_name = 'k8s-infra',
|
|
$controllers = ['cpuset', 'cpu', 'cpuacct', 'memory', 'systemd', 'pids'],
|
|
) {}
|
|
|
|
class platform::kubernetes::cgroup
|
|
inherits ::platform::kubernetes::cgroup::params {
|
|
include ::platform::kubernetes::params
|
|
|
|
$k8s_cpuset = $::platform::kubernetes::params::k8s_cpuset
|
|
$k8s_nodeset = $::platform::kubernetes::params::k8s_nodeset
|
|
|
|
# Default to float across all cpus and numa nodes
|
|
if !defined('$k8s_cpuset') {
|
|
$k8s_cpuset = generate('/bin/cat', '/sys/devices/system/cpu/online')
|
|
notice("System default cpuset ${k8s_cpuset}.")
|
|
}
|
|
if !defined('$k8s_nodeset') {
|
|
$k8s_nodeset = generate('/bin/cat', '/sys/devices/system/node/online')
|
|
notice("System default nodeset ${k8s_nodeset}.")
|
|
}
|
|
|
|
# Create kubelet cgroup for the minimal set of required controllers.
|
|
# NOTE: The kubernetes cgroup_manager_linux func Exists() checks that
|
|
# specific subsystem cgroup paths actually exist on the system. The
|
|
# particular cgroup cgroupRoot must exist for the following controllers:
|
|
# "cpu", "cpuacct", "cpuset", "memory", "systemd", "pids".
|
|
# Reference:
|
|
# https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/cm/cgroup_manager_linux.go
|
|
# systemd automatically mounts cgroups and controllers, so don't need
|
|
# to do that here.
|
|
notice("Create ${cgroup_root}/${controllers}/${cgroup_name}")
|
|
$controllers.each |String $controller| {
|
|
$cgroup_dir = "${cgroup_root}/${controller}/${cgroup_name}"
|
|
file { $cgroup_dir :
|
|
ensure => directory,
|
|
owner => 'root',
|
|
group => 'root',
|
|
mode => '0700',
|
|
}
|
|
|
|
# Modify k8s cpuset resources to reflect platform configured cores.
|
|
# NOTE: Using 'exec' here instead of 'file' resource type with 'content'
|
|
# tag to update contents under /sys, since puppet tries to create files
|
|
# with temp names in the same directory, and the kernel only allows
|
|
# specific filenames to be created in these particular directories.
|
|
# This causes puppet to fail if we use the 'content' tag.
|
|
# NOTE: Child cgroups cpuset must be subset of parent. In the case where
|
|
# child directories already exist and we change the parent's cpuset to
|
|
# be a subset of what the children have, will cause the command to fail
|
|
# with "-bash: echo: write error: device or resource busy".
|
|
if $controller == 'cpuset' {
|
|
$cgroup_mems = "${cgroup_dir}/cpuset.mems"
|
|
$cgroup_cpus = "${cgroup_dir}/cpuset.cpus"
|
|
$cgroup_tasks = "${cgroup_dir}/tasks"
|
|
|
|
notice("Set ${cgroup_name} nodeset: ${k8s_nodeset}, cpuset: ${k8s_cpuset}")
|
|
File[ $cgroup_dir ]
|
|
-> exec { "Create ${cgroup_mems}" :
|
|
command => "/bin/echo ${k8s_nodeset} > ${cgroup_mems} || :",
|
|
}
|
|
-> exec { "Create ${cgroup_cpus}" :
|
|
command => "/bin/echo ${k8s_cpuset} > ${cgroup_cpus} || :",
|
|
}
|
|
-> file { $cgroup_tasks :
|
|
ensure => file,
|
|
owner => 'root',
|
|
group => 'root',
|
|
mode => '0644',
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
class platform::kubernetes::kubeadm {
|
|
|
|
include ::platform::docker::params
|
|
include ::platform::kubernetes::params
|
|
include ::platform::params
|
|
|
|
# Update kubeadm/kubelet bindmounts if needed.
|
|
require platform::kubernetes::bindmounts
|
|
|
|
$node_ip = $::platform::kubernetes::params::node_ip
|
|
$host_labels = $::platform::kubernetes::params::host_labels
|
|
$k8s_platform_cpuset = $::platform::kubernetes::params::k8s_platform_cpuset
|
|
$k8s_reserved_mem = $::platform::kubernetes::params::k8s_reserved_mem
|
|
$k8s_all_reserved_cpuset = $::platform::kubernetes::params::k8s_all_reserved_cpuset
|
|
$k8s_cni_bin_dir = $::platform::kubernetes::params::k8s_cni_bin_dir
|
|
$k8s_vol_plugin_dir = $::platform::kubernetes::params::k8s_vol_plugin_dir
|
|
$k8s_cpu_mgr_policy = $::platform::kubernetes::params::k8s_cpu_mgr_policy
|
|
$k8s_topology_mgr_policy = $::platform::kubernetes::params::k8s_topology_mgr_policy
|
|
$k8s_pod_max_pids = $::platform::kubernetes::params::k8s_pod_max_pids
|
|
|
|
$iptables_file = "net.bridge.bridge-nf-call-ip6tables = 1
|
|
net.bridge.bridge-nf-call-iptables = 1"
|
|
|
|
# Configure kubelet cpumanager options
|
|
$opts_sys_res = join(['--system-reserved=',
|
|
"memory=${k8s_reserved_mem}Mi"])
|
|
|
|
if ($::personality == 'controller' and
|
|
$::platform::params::distributed_cloud_role == 'systemcontroller') {
|
|
$opts = '--cpu-manager-policy=none'
|
|
$k8s_cpu_manager_opts = join([$opts,
|
|
$opts_sys_res], ' ')
|
|
} else {
|
|
if !$::platform::params::virtual_system {
|
|
if str2bool($::is_worker_subfunction)
|
|
and !('openstack-compute-node=enabled' in $host_labels) {
|
|
# Enable TopologyManager for hosts with the worker subfunction.
|
|
# Exceptions are:
|
|
# - DC System controllers
|
|
# - Virtualized nodes (lab environment only)
|
|
|
|
$opts = join(['--feature-gates TopologyManager=true',
|
|
"--cpu-manager-policy=${k8s_cpu_mgr_policy}",
|
|
"--topology-manager-policy=${k8s_topology_mgr_policy}"], ' ')
|
|
|
|
if $k8s_cpu_mgr_policy == 'none' {
|
|
$k8s_reserved_cpus = $k8s_platform_cpuset
|
|
} else {
|
|
# The union of platform, isolated, and vswitch
|
|
$k8s_reserved_cpus = $k8s_all_reserved_cpuset
|
|
}
|
|
|
|
$opts_res_cpus = "--reserved-cpus=${k8s_reserved_cpus}"
|
|
$k8s_cpu_manager_opts = join([$opts,
|
|
$opts_sys_res,
|
|
$opts_res_cpus], ' ')
|
|
} else {
|
|
$opts = '--cpu-manager-policy=none'
|
|
$k8s_cpu_manager_opts = join([$opts,
|
|
$opts_sys_res], ' ')
|
|
|
|
}
|
|
} else {
|
|
$k8s_cpu_manager_opts = '--cpu-manager-policy=none'
|
|
}
|
|
}
|
|
|
|
# Enable kubelet extra parameters that are node specific such as
|
|
# cpumanager
|
|
file { '/etc/sysconfig/kubelet':
|
|
ensure => file,
|
|
content => template('platform/kubelet.conf.erb'),
|
|
}
|
|
# The cpu_manager_state file is regenerated when cpumanager starts or
|
|
# changes allocations so it is safe to remove before kubelet starts.
|
|
# This file persists so cpumanager's DefaultCPUSet becomes inconsistent
|
|
# when we offline/online CPUs or change the number of reserved cpus.
|
|
-> exec { 'remove cpu_manager_state':
|
|
command => 'rm -f /var/lib/kubelet/cpu_manager_state || true',
|
|
}
|
|
|
|
# Update iptables config. This is required based on:
|
|
# https://kubernetes.io/docs/tasks/tools/install-kubeadm
|
|
# This probably belongs somewhere else - initscripts package?
|
|
file { '/etc/sysctl.d/k8s.conf':
|
|
ensure => file,
|
|
content => $iptables_file,
|
|
owner => 'root',
|
|
group => 'root',
|
|
mode => '0644',
|
|
}
|
|
-> exec { 'update kernel parameters for iptables':
|
|
command => 'sysctl --system',
|
|
}
|
|
|
|
# Create manifests directory required by kubelet
|
|
-> file { '/etc/kubernetes/manifests':
|
|
ensure => directory,
|
|
owner => 'root',
|
|
group => 'root',
|
|
mode => '0700',
|
|
}
|
|
# A seperate enable is required since we have modified the service resource
|
|
# to never enable services.
|
|
-> exec { 'enable-kubelet':
|
|
command => '/usr/bin/systemctl enable kubelet.service',
|
|
}
|
|
# Start kubelet if it is standard controller.
|
|
if !str2bool($::is_worker_subfunction) {
|
|
File['/etc/kubernetes/manifests']
|
|
-> service { 'kubelet':
|
|
enable => true,
|
|
}
|
|
}
|
|
}
|
|
|
|
class platform::kubernetes::master::init
|
|
inherits ::platform::kubernetes::params {
|
|
|
|
include ::platform::params
|
|
include ::platform::docker::params
|
|
include ::platform::dockerdistribution::params
|
|
|
|
if str2bool($::is_initial_k8s_config) {
|
|
# This allows subsequent node installs
|
|
# Notes regarding ::is_initial_k8s_config check:
|
|
# - Ensures block is only run for new node installs (e.g. controller-1)
|
|
# or reinstalls. This part is needed only once;
|
|
# - Ansible configuration is independently configuring Kubernetes. A retry
|
|
# in configuration by puppet leads to failed manifest application.
|
|
# This flag is created by Ansible on controller-0;
|
|
# - Ansible replay is not impacted by flag creation.
|
|
|
|
$local_registry_auth = "${::platform::dockerdistribution::params::registry_username}:${::platform::dockerdistribution::params::registry_password}" # lint:ignore:140chars
|
|
|
|
exec { 'pre pull k8s images':
|
|
command => "kubeadm --kubeconfig=/etc/kubernetes/admin.conf config images list --kubernetes-version ${version} --image-repository registry.local:9001/k8s.gcr.io | xargs -i crictl pull --creds ${local_registry_auth} {}", # lint:ignore:140chars
|
|
logoutput => true,
|
|
}
|
|
|
|
-> exec { 'configure master node':
|
|
command => $join_cmd,
|
|
logoutput => true,
|
|
}
|
|
|
|
# Update ownership/permissions for file created by "kubeadm init".
|
|
# We want it readable by sysinv and sysadmin.
|
|
-> file { '/etc/kubernetes/admin.conf':
|
|
ensure => file,
|
|
owner => 'root',
|
|
group => $::platform::params::protected_group_name,
|
|
mode => '0640',
|
|
}
|
|
|
|
# Add a bash profile script to set a k8s env variable
|
|
-> file {'bash_profile_k8s':
|
|
ensure => present,
|
|
path => '/etc/profile.d/kubeconfig.sh',
|
|
mode => '0644',
|
|
source => "puppet:///modules/${module_name}/kubeconfig.sh"
|
|
}
|
|
|
|
# Remove the taint from the master node
|
|
-> exec { 'remove taint from master node':
|
|
command => "kubectl --kubeconfig=/etc/kubernetes/admin.conf taint node ${::platform::params::hostname} node-role.kubernetes.io/master- || true", # lint:ignore:140chars
|
|
logoutput => true,
|
|
}
|
|
|
|
# Add kubelet service override
|
|
-> file { '/etc/systemd/system/kubelet.service.d/kube-stx-override.conf':
|
|
ensure => file,
|
|
content => template('platform/kube-stx-override.conf.erb'),
|
|
owner => 'root',
|
|
group => 'root',
|
|
mode => '0644',
|
|
}
|
|
|
|
# set kubelet monitored by pmond
|
|
-> file { '/etc/pmon.d/kubelet.conf':
|
|
ensure => file,
|
|
content => template('platform/kubelet-pmond-conf.erb'),
|
|
owner => 'root',
|
|
group => 'root',
|
|
mode => '0644',
|
|
}
|
|
|
|
# Reload systemd
|
|
-> exec { 'perform systemctl daemon reload for kubelet override':
|
|
command => 'systemctl daemon-reload',
|
|
logoutput => true,
|
|
}
|
|
|
|
# Initial kubernetes config done on node
|
|
-> file { '/etc/platform/.initial_k8s_config_complete':
|
|
ensure => present,
|
|
}
|
|
}
|
|
|
|
# Run kube-cert-rotation daily
|
|
cron { 'kube-cert-rotation':
|
|
ensure => 'present',
|
|
command => '/usr/bin/kube-cert-rotation.sh',
|
|
environment => 'PATH=/bin:/usr/bin:/usr/sbin',
|
|
minute => '10',
|
|
hour => '*/24',
|
|
user => 'root',
|
|
}
|
|
}
|
|
|
|
class platform::kubernetes::master
|
|
inherits ::platform::kubernetes::params {
|
|
|
|
contain ::platform::kubernetes::kubeadm
|
|
contain ::platform::kubernetes::cgroup
|
|
contain ::platform::kubernetes::master::init
|
|
contain ::platform::kubernetes::coredns
|
|
contain ::platform::kubernetes::firewall
|
|
contain ::platform::kubernetes::configuration
|
|
|
|
Class['::platform::sysctl::controller::reserve_ports'] -> Class[$name]
|
|
Class['::platform::etcd'] -> Class[$name]
|
|
Class['::platform::docker::config'] -> Class[$name]
|
|
Class['::platform::containerd::config'] -> Class[$name]
|
|
# Ensure DNS is configured as name resolution is required when
|
|
# kubeadm init is run.
|
|
Class['::platform::dns'] -> Class[$name]
|
|
Class['::platform::kubernetes::configuration']
|
|
-> Class['::platform::kubernetes::kubeadm']
|
|
-> Class['::platform::kubernetes::cgroup']
|
|
-> Class['::platform::kubernetes::master::init']
|
|
-> Class['::platform::kubernetes::coredns']
|
|
-> Class['::platform::kubernetes::firewall']
|
|
}
|
|
|
|
class platform::kubernetes::worker::init
|
|
inherits ::platform::kubernetes::params {
|
|
|
|
Class['::platform::docker::config'] -> Class[$name]
|
|
Class['::platform::containerd::config'] -> Class[$name]
|
|
Class['::platform::filesystem::kubelet'] -> Class[$name]
|
|
|
|
if str2bool($::is_initial_config) {
|
|
include ::platform::dockerdistribution::params
|
|
# Pull pause image tag from kubeadm required images list for this version
|
|
# kubeadm config images list does not use the --kubeconfig argument
|
|
# and admin.conf will not exist on a pure worker, and kubelet.conf will not
|
|
# exist until after a join.
|
|
$local_registry_auth = "${::platform::dockerdistribution::params::registry_username}:${::platform::dockerdistribution::params::registry_password}" # lint:ignore:140chars
|
|
exec { 'load k8s pause image by containerd':
|
|
# splitting this command over multiple lines appears to break puppet-lint
|
|
command => "kubeadm config images list --kubernetes-version ${version} --image-repository=registry.local:9001/k8s.gcr.io 2>/dev/null | grep k8s.gcr.io/pause: | xargs -i crictl pull --creds ${local_registry_auth} {}", # lint:ignore:140chars
|
|
logoutput => true,
|
|
before => Exec['configure worker node'],
|
|
}
|
|
}
|
|
|
|
# Configure the worker node. Only do this once, so check whether the
|
|
# kubelet.conf file has already been created (by the join).
|
|
exec { 'configure worker node':
|
|
command => $join_cmd,
|
|
logoutput => true,
|
|
unless => 'test -f /etc/kubernetes/kubelet.conf',
|
|
}
|
|
|
|
# Add kubelet service override
|
|
-> file { '/etc/systemd/system/kubelet.service.d/kube-stx-override.conf':
|
|
ensure => file,
|
|
content => template('platform/kube-stx-override.conf.erb'),
|
|
owner => 'root',
|
|
group => 'root',
|
|
mode => '0644',
|
|
}
|
|
|
|
# set kubelet monitored by pmond
|
|
-> file { '/etc/pmon.d/kubelet.conf':
|
|
ensure => file,
|
|
content => template('platform/kubelet-pmond-conf.erb'),
|
|
owner => 'root',
|
|
group => 'root',
|
|
mode => '0644',
|
|
}
|
|
|
|
# Reload systemd
|
|
-> exec { 'perform systemctl daemon reload for kubelet override':
|
|
command => 'systemctl daemon-reload',
|
|
logoutput => true,
|
|
}
|
|
}
|
|
|
|
class platform::kubernetes::worker::pci
|
|
(
|
|
$pcidp_resources = undef,
|
|
) {
|
|
include ::platform::kubernetes::params
|
|
|
|
file { '/etc/pcidp':
|
|
ensure => 'directory',
|
|
owner => 'root',
|
|
group => 'root',
|
|
mode => '0700',
|
|
}
|
|
-> file { '/etc/pcidp/config.json':
|
|
ensure => present,
|
|
owner => 'root',
|
|
group => 'root',
|
|
mode => '0644',
|
|
content => template('platform/pcidp.conf.erb'),
|
|
}
|
|
}
|
|
|
|
class platform::kubernetes::worker::pci::runtime {
|
|
include ::platform::kubernetes::worker::pci
|
|
include ::platform::kubernetes::worker::sriovdp
|
|
}
|
|
|
|
class platform::kubernetes::worker::sriovdp {
|
|
include ::platform::kubernetes::params
|
|
include ::platform::params
|
|
$host_labels = $::platform::kubernetes::params::host_labels
|
|
if ($::personality == 'controller') and
|
|
str2bool($::is_worker_subfunction)
|
|
and ('sriovdp=enabled' in $host_labels) {
|
|
exec { 'Delete sriov device plugin pod if present':
|
|
path => '/usr/bin:/usr/sbin:/bin',
|
|
command => 'kubectl --kubeconfig=/etc/kubernetes/admin.conf delete pod -n kube-system --selector=app=sriovdp --field-selector spec.nodeName=$(hostname) --timeout=360s', # lint:ignore:140chars
|
|
onlyif => 'kubectl --kubeconfig=/etc/kubernetes/admin.conf get pods -n kube-system --selector=app=sriovdp --field-selector spec.nodeName=$(hostname) | grep kube-sriov-device-plugin', # lint:ignore:140chars
|
|
logoutput => true,
|
|
}
|
|
}
|
|
}
|
|
|
|
class platform::kubernetes::worker
|
|
inherits ::platform::kubernetes::params {
|
|
|
|
# Worker configuration is not required on AIO hosts, since the master
|
|
# will already be configured and includes support for running pods.
|
|
if $::personality != 'controller' {
|
|
contain ::platform::kubernetes::kubeadm
|
|
contain ::platform::kubernetes::cgroup
|
|
contain ::platform::kubernetes::worker::init
|
|
contain ::platform::kubernetes::configuration
|
|
|
|
Class['::platform::kubernetes::configuration']
|
|
-> Class['::platform::kubernetes::kubeadm']
|
|
-> Class['::platform::kubernetes::cgroup']
|
|
-> Class['::platform::kubernetes::worker::init']
|
|
}
|
|
|
|
# Enable kubelet on AIO and worker nodes.
|
|
Class['::platform::compute::allocate']
|
|
-> service { 'kubelet':
|
|
enable => true,
|
|
}
|
|
|
|
# TODO: The following exec is a workaround. Once kubernetes becomes the
|
|
# default installation, /etc/pmon.d/libvirtd.conf needs to be removed from
|
|
# the load.
|
|
exec { 'Update PMON libvirtd.conf':
|
|
command => "/bin/sed -i 's#mode = passive#mode = ignore #' /etc/pmon.d/libvirtd.conf",
|
|
onlyif => '/usr/bin/test -e /etc/pmon.d/libvirtd.conf'
|
|
}
|
|
|
|
contain ::platform::kubernetes::worker::pci
|
|
}
|
|
|
|
class platform::kubernetes::aio
|
|
inherits ::platform::kubernetes::params {
|
|
|
|
include ::platform::kubernetes::master
|
|
include ::platform::kubernetes::worker
|
|
|
|
Class['::platform::kubernetes::master']
|
|
-> Class['::platform::kubernetes::worker']
|
|
-> Class[$name]
|
|
}
|
|
|
|
class platform::kubernetes::gate {
|
|
if $::platform::params::system_type != 'All-in-one' {
|
|
Class['::platform::kubernetes::master'] -> Class[$name]
|
|
} else {
|
|
Class['::platform::kubernetes::aio'] -> Class[$name]
|
|
}
|
|
}
|
|
|
|
class platform::kubernetes::coredns::duplex {
|
|
# For duplex and multi-node system, restrict the dns pod to master nodes
|
|
exec { 'restrict coredns to master nodes':
|
|
command => 'kubectl --kubeconfig=/etc/kubernetes/admin.conf -n kube-system patch deployment coredns -p \'{"spec":{"template":{"spec":{"nodeSelector":{"node-role.kubernetes.io/master":""}}}}}\'', # lint:ignore:140chars
|
|
logoutput => true,
|
|
}
|
|
|
|
-> exec { 'Use anti-affinity for coredns pods':
|
|
command => 'kubectl --kubeconfig=/etc/kubernetes/admin.conf -n kube-system patch deployment coredns -p \'{"spec":{"template":{"spec":{"affinity":{"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchExpressions":[{"key":"k8s-app","operator":"In","values":["kube-dns"]}]},"topologyKey":"kubernetes.io/hostname"}]}}}}}}\'', # lint:ignore:140chars
|
|
logoutput => true,
|
|
}
|
|
}
|
|
|
|
class platform::kubernetes::coredns::simplex {
|
|
# For simplex system, 1 coredns is enough
|
|
exec { '1 coredns for simplex mode':
|
|
command => 'kubectl --kubeconfig=/etc/kubernetes/admin.conf -n kube-system scale --replicas=1 deployment coredns',
|
|
logoutput => true,
|
|
}
|
|
}
|
|
|
|
class platform::kubernetes::coredns {
|
|
|
|
include ::platform::params
|
|
|
|
if str2bool($::is_initial_k8s_config) {
|
|
if $::platform::params::system_mode != 'simplex' {
|
|
contain ::platform::kubernetes::coredns::duplex
|
|
} else {
|
|
contain ::platform::kubernetes::coredns::simplex
|
|
}
|
|
}
|
|
}
|
|
|
|
# TODO: remove port 9001 once we have a public docker image registry using standard ports.
|
|
# add 5000 as the default port for private registry
|
|
class platform::kubernetes::firewall::params (
|
|
$transport = 'tcp',
|
|
$table = 'nat',
|
|
$dports = [80, 443, 9001, 5000],
|
|
$chain = 'POSTROUTING',
|
|
$jump = 'SNAT',
|
|
) {}
|
|
|
|
class platform::kubernetes::firewall
|
|
inherits ::platform::kubernetes::firewall::params {
|
|
|
|
include ::platform::params
|
|
include ::platform::network::oam::params
|
|
include ::platform::network::mgmt::params
|
|
include ::platform::docker::params
|
|
|
|
# add http_proxy and https_proxy port to k8s firewall
|
|
# in order to allow worker node access public network via proxy
|
|
if $::platform::docker::params::http_proxy {
|
|
$http_proxy_str_array = split($::platform::docker::params::http_proxy, ':')
|
|
$http_proxy_port = $http_proxy_str_array[length($http_proxy_str_array) - 1]
|
|
if $http_proxy_port =~ /^\d+$/ {
|
|
$http_proxy_port_val = $http_proxy_port
|
|
}
|
|
}
|
|
|
|
if $::platform::docker::params::https_proxy {
|
|
$https_proxy_str_array = split($::platform::docker::params::https_proxy, ':')
|
|
$https_proxy_port = $https_proxy_str_array[length($https_proxy_str_array) - 1]
|
|
if $https_proxy_port =~ /^\d+$/ {
|
|
$https_proxy_port_val = $https_proxy_port
|
|
}
|
|
}
|
|
|
|
if defined('$http_proxy_port_val') {
|
|
if defined('$https_proxy_port_val') and ($http_proxy_port_val != $https_proxy_port_val) {
|
|
$dports = $dports << $http_proxy_port_val << $https_proxy_port_val
|
|
} else {
|
|
$dports = $dports << $http_proxy_port_val
|
|
}
|
|
} elsif defined('$https_proxy_port_val') {
|
|
$dports = $dports << $https_proxy_port_val
|
|
}
|
|
|
|
$system_mode = $::platform::params::system_mode
|
|
$oam_float_ip = $::platform::network::oam::params::controller_address
|
|
$oam_interface = $::platform::network::oam::params::interface_name
|
|
$mgmt_subnet = $::platform::network::mgmt::params::subnet_network
|
|
$mgmt_prefixlen = $::platform::network::mgmt::params::subnet_prefixlen
|
|
|
|
$s_mgmt_subnet = "${mgmt_subnet}/${mgmt_prefixlen}"
|
|
$d_mgmt_subnet = "! ${s_mgmt_subnet}"
|
|
|
|
if $system_mode != 'simplex' {
|
|
platform::firewall::rule { 'kubernetes-nat':
|
|
service_name => 'kubernetes',
|
|
table => $table,
|
|
chain => $chain,
|
|
proto => $transport,
|
|
jump => $jump,
|
|
ports => $dports,
|
|
host => $s_mgmt_subnet,
|
|
destination => $d_mgmt_subnet,
|
|
outiface => $oam_interface,
|
|
tosource => $oam_float_ip,
|
|
}
|
|
}
|
|
}
|
|
|
|
class platform::kubernetes::pre_pull_control_plane_images
|
|
inherits ::platform::kubernetes::params {
|
|
|
|
include ::platform::dockerdistribution::params
|
|
|
|
# Update kubeadm bindmount if needed
|
|
require platform::kubernetes::bindmounts
|
|
|
|
$local_registry_auth = "${::platform::dockerdistribution::params::registry_username}:${::platform::dockerdistribution::params::registry_password}" # lint:ignore:140chars
|
|
|
|
exec { 'pre pull images':
|
|
command => "kubeadm --kubeconfig=/etc/kubernetes/admin.conf config images list --kubernetes-version ${upgrade_to_version} --image-repository=registry.local:9001/k8s.gcr.io | xargs -i crictl pull --creds ${local_registry_auth} {}", # lint:ignore:140chars
|
|
logoutput => true,
|
|
}
|
|
}
|
|
|
|
class platform::kubernetes::upgrade_first_control_plane
|
|
inherits ::platform::kubernetes::params {
|
|
|
|
include ::platform::params
|
|
|
|
# Update kubeadm bindmount if needed.
|
|
require platform::kubernetes::bindmounts
|
|
|
|
# The --allow-*-upgrades options allow us to upgrade to any k8s release if necessary
|
|
exec { 'upgrade first control plane':
|
|
command => "kubeadm --kubeconfig=/etc/kubernetes/admin.conf upgrade apply ${version} --allow-experimental-upgrades --allow-release-candidate-upgrades -y", # lint:ignore:140chars
|
|
logoutput => true,
|
|
}
|
|
|
|
if $::platform::params::system_mode != 'simplex' {
|
|
# For duplex and multi-node system, restrict the coredns pod to master nodes
|
|
exec { 'restrict coredns to master nodes':
|
|
command => 'kubectl --kubeconfig=/etc/kubernetes/admin.conf -n kube-system patch deployment coredns -p \'{"spec":{"template":{"spec":{"nodeSelector":{"node-role.kubernetes.io/master":""}}}}}\'', # lint:ignore:140chars
|
|
logoutput => true,
|
|
require => Exec['upgrade first control plane']
|
|
}
|
|
-> exec { 'Use anti-affinity for coredns pods':
|
|
command => 'kubectl --kubeconfig=/etc/kubernetes/admin.conf -n kube-system patch deployment coredns -p \'{"spec":{"template":{"spec":{"affinity":{"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchExpressions":[{"key":"k8s-app","operator":"In","values":["kube-dns"]}]},"topologyKey":"kubernetes.io/hostname"}]}}}}}}\'', # lint:ignore:140chars
|
|
logoutput => true,
|
|
}
|
|
} else {
|
|
# For simplex system, 1 coredns is enough
|
|
exec { '1 coredns for simplex mode':
|
|
command => 'kubectl --kubeconfig=/etc/kubernetes/admin.conf -n kube-system scale --replicas=1 deployment coredns',
|
|
logoutput => true,
|
|
require => Exec['upgrade first control plane']
|
|
}
|
|
}
|
|
}
|
|
|
|
class platform::kubernetes::upgrade_control_plane
|
|
inherits ::platform::kubernetes::params {
|
|
|
|
# Update kubeadm bindmount if needed.
|
|
require platform::kubernetes::bindmounts
|
|
|
|
# control plane is only upgraded on a controller (which has admin.conf)
|
|
exec { 'upgrade control plane':
|
|
command => 'kubeadm --kubeconfig=/etc/kubernetes/admin.conf upgrade node',
|
|
logoutput => true,
|
|
}
|
|
}
|
|
|
|
class platform::kubernetes::master::upgrade_kubelet
|
|
inherits ::platform::kubernetes::params {
|
|
|
|
# Update kubeadm/kubelet bindmounts if needed.
|
|
require platform::kubernetes::bindmounts
|
|
|
|
exec { 'restart kubelet':
|
|
command => '/usr/local/sbin/pmon-restart kubelet'
|
|
}
|
|
}
|
|
|
|
class platform::kubernetes::worker::upgrade_kubelet
|
|
inherits ::platform::kubernetes::params {
|
|
|
|
include ::platform::dockerdistribution::params
|
|
|
|
# Update kubeadm/kubelet bindmounts if needed.
|
|
require platform::kubernetes::bindmounts
|
|
|
|
# workers use kubelet.conf rather than admin.conf
|
|
$local_registry_auth = "${::platform::dockerdistribution::params::registry_username}:${::platform::dockerdistribution::params::registry_password}" # lint:ignore:140chars
|
|
|
|
# Pull the pause image tag from kubeadm required images list for this version
|
|
exec { 'pull pause image':
|
|
# spltting this command over multiple lines will break puppet-lint for later violations
|
|
command => "kubeadm --kubeconfig=/etc/kubernetes/kubelet.conf config images list --kubernetes-version ${upgrade_to_version} --image-repository=registry.local:9001/k8s.gcr.io 2>/dev/null | grep k8s.gcr.io/pause: | xargs -i crictl pull --creds ${local_registry_auth} {}", # lint:ignore:140chars
|
|
logoutput => true,
|
|
before => Exec['upgrade kubelet'],
|
|
}
|
|
|
|
exec { 'upgrade kubelet':
|
|
command => 'kubeadm --kubeconfig=/etc/kubernetes/kubelet.conf upgrade node',
|
|
logoutput => true,
|
|
}
|
|
|
|
-> exec { 'restart kubelet':
|
|
command => '/usr/local/sbin/pmon-restart kubelet'
|
|
}
|
|
}
|
|
|
|
class platform::kubernetes::master::change_apiserver_parameters (
|
|
$etcd_cafile = $platform::kubernetes::params::etcd_cafile,
|
|
$etcd_certfile = $platform::kubernetes::params::etcd_certfile,
|
|
$etcd_keyfile = $platform::kubernetes::params::etcd_keyfile,
|
|
$etcd_servers = $platform::kubernetes::params::etcd_servers,
|
|
) inherits ::platform::kubernetes::params {
|
|
|
|
$configmap_temp_file = '/tmp/cluster_configmap.yaml'
|
|
$configview_temp_file = '/tmp/kubeadm_config_view.yaml'
|
|
|
|
exec { 'update kube-apiserver params':
|
|
command => template('platform/kube-apiserver-change-params.erb')
|
|
}
|
|
}
|
|
|
|
class platform::kubernetes::certsans::runtime
|
|
inherits ::platform::kubernetes::params {
|
|
include ::platform::params
|
|
include ::platform::network::mgmt::params
|
|
include ::platform::network::oam::params
|
|
include ::platform::network::cluster_host::params
|
|
|
|
if $::platform::network::mgmt::params::subnet_version == $::platform::params::ipv6 {
|
|
$localhost_address = '::1'
|
|
} else {
|
|
$localhost_address = '127.0.0.1'
|
|
}
|
|
|
|
if $::platform::params::system_mode == 'simplex' {
|
|
$certsans = "\"${platform::network::cluster_host::params::controller_address}, \
|
|
${localhost_address}, \
|
|
${platform::network::oam::params::controller_address}\""
|
|
} else {
|
|
$certsans = "\"${platform::network::cluster_host::params::controller_address}, \
|
|
${localhost_address}, \
|
|
${platform::network::oam::params::controller_address}, \
|
|
${platform::network::oam::params::controller0_address}, \
|
|
${platform::network::oam::params::controller1_address}\""
|
|
}
|
|
|
|
exec { 'update kube-apiserver certSANs':
|
|
provider => shell,
|
|
command => template('platform/kube-apiserver-update-certSANs.erb')
|
|
}
|
|
}
|
|
|
|
# The duplex_migration class is applied as part of SX to DX migration
|
|
class platform::kubernetes::duplex_migration::runtime::post {
|
|
file { '/var/run/.kubernetes_duplex_migration_complete':
|
|
ensure => present,
|
|
}
|
|
}
|
|
|
|
class platform::kubernetes::duplex_migration::runtime {
|
|
contain ::platform::kubernetes::coredns::duplex
|
|
|
|
# Update replicas to 2 for duplex
|
|
Class['::platform::kubernetes::coredns::duplex']
|
|
-> exec { '2 coredns for duplex mode':
|
|
command => 'kubectl --kubeconfig=/etc/kubernetes/admin.conf -n kube-system scale --replicas=2 deployment coredns',
|
|
logoutput => true,
|
|
}
|
|
|
|
class { '::platform::kubernetes::duplex_migration::runtime::post':
|
|
stage => post,
|
|
}
|
|
}
|
|
|
|
class platform::kubernetes::master::rootca::trustbothcas::runtime
|
|
inherits ::platform::kubernetes::params {
|
|
|
|
# Backup the original root CA cert
|
|
file { $rootca_certfile_old:
|
|
ensure => file,
|
|
source => $rootca_certfile,
|
|
replace => false,
|
|
}
|
|
# Create the new root CA cert file
|
|
-> file { $rootca_certfile_new:
|
|
ensure => file,
|
|
content => base64('decode', $rootca_cert),
|
|
}
|
|
# Create new root CA key file
|
|
-> file { $rootca_keyfile_new:
|
|
ensure => file,
|
|
content => base64('decode', $rootca_key),
|
|
}
|
|
# Append the new cert to the current cert
|
|
-> exec { 'append_ca_cert':
|
|
command => "cat ${rootca_certfile_old} ${rootca_certfile_new} > ${rootca_certfile}",
|
|
}
|
|
# update admin.conf with both old and new certs
|
|
-> exec { 'update_admin_conf':
|
|
environment => [ 'KUBECONFIG=/etc/kubernetes/admin.conf' ],
|
|
command => 'kubectl config set-cluster kubernetes --certificate-authority /etc/kubernetes/pki/ca.crt --embed-certs',
|
|
}
|
|
# Restart apiserver to trust both old and new certs
|
|
-> exec { 'restart_apiserver':
|
|
command => "/usr/bin/kill -s SIGHUP $(pidof kube-apiserver)",
|
|
}
|
|
# Update scheduler.conf with both old and new certs
|
|
-> exec { 'update_scheduler_conf':
|
|
environment => [ 'KUBECONFIG=/etc/kubernetes/scheduler.conf' ],
|
|
command => 'kubectl config set-cluster kubernetes --certificate-authority /etc/kubernetes/pki/ca.crt --embed-certs',
|
|
}
|
|
# Restart scheduler to trust both old and new certs
|
|
-> exec { 'restart_scheduler':
|
|
command => "/usr/bin/kill -s SIGHUP $(pidof kube-scheduler)"
|
|
}
|
|
# Update controller-manager.conf with both old and new certs
|
|
-> exec { 'update_controller-manager_conf':
|
|
environment => [ 'KUBECONFIG=/etc/kubernetes/controller-manager.conf' ],
|
|
command => 'kubectl config set-cluster kubernetes --certificate-authority /etc/kubernetes/pki/ca.crt --embed-certs',
|
|
}
|
|
# Update kube-controller-manager.yaml with new cert and key
|
|
-> exec { 'update_controller-manager_yaml':
|
|
command => "/bin/sed -i \\
|
|
-e 's|cluster-signing-cert-file=.*|cluster-signing-cert-file=/etc/kubernetes/pki/ca_new.crt|' \\
|
|
-e 's|cluster-signing-key-file=.*|cluster-signing-key-file=/etc/kubernetes/pki/ca_new.key|' \\
|
|
/etc/kubernetes/manifests/kube-controller-manager.yaml"
|
|
}
|
|
|
|
# Update kubelet.conf with both old and new certs
|
|
$cluster = generate('/bin/bash', '-c', "/bin/sed -e '/- cluster/,/name:/!d' /etc/kubernetes/kubelet.conf \\
|
|
| grep 'name:' | awk '{printf \"%s\", \$2}'")
|
|
exec { 'update_kubelet_conf':
|
|
environment => [ 'KUBECONFIG=/etc/kubernetes/kubelet.conf' ],
|
|
command => "kubectl config set-cluster ${cluster} --certificate-authority /etc/kubernetes/pki/ca.crt --embed-certs",
|
|
require => Exec['append_ca_cert'],
|
|
}
|
|
# Restart kubelet to truct both certs
|
|
-> exec { 'restart_kubelet':
|
|
command => '/usr/bin/systemctl restart kubelet',
|
|
}
|
|
}
|
|
|
|
class platform::kubernetes::worker::rootca::trustbothcas::runtime
|
|
inherits ::platform::kubernetes::params {
|
|
|
|
$cluster = generate('/bin/bash', '-c', "/bin/sed -e '/- cluster/,/name:/!d' /etc/kubernetes/kubelet.conf \\
|
|
| grep 'name:' | awk '{printf \"%s\", \$2}'")
|
|
# Backup the original root CA cert
|
|
file { $rootca_certfile_old:
|
|
ensure => file,
|
|
source => $rootca_certfile,
|
|
replace => false,
|
|
}
|
|
# Create the new root CA cert file
|
|
-> file { $rootca_certfile_new:
|
|
ensure => file,
|
|
content => base64('decode', $rootca_cert),
|
|
}
|
|
# Create new root CA key file
|
|
-> file { $rootca_keyfile_new:
|
|
ensure => file,
|
|
content => base64('decode', $rootca_key),
|
|
}
|
|
# Append the new cert to the current cert
|
|
-> exec { 'append_ca_cert':
|
|
command => "cat ${rootca_certfile_old} ${rootca_certfile_new} > ${rootca_certfile}",
|
|
}
|
|
# Update kubelet.conf with both old and new certs
|
|
-> exec { 'update_kubelet_conf':
|
|
environment => [ 'KUBECONFIG=/etc/kubernetes/kubelet.conf' ],
|
|
command => "kubectl config set-cluster ${cluster} --certificate-authority /etc/kubernetes/pki/ca.crt --embed-certs",
|
|
}
|
|
# Restart kubelet to trust both certs
|
|
-> exec { 'restart_kubelet':
|
|
command => '/usr/bin/systemctl restart kubelet',
|
|
}
|
|
}
|
|
|
|
class platform::kubernetes::master::rootca::trustnewca::runtime
|
|
inherits ::platform::kubernetes::params {
|
|
# Copy the new root CA cert in place
|
|
exec { 'put_new_ca_cert_in_place':
|
|
command => "/bin/cp ${rootca_certfile_new} ${rootca_certfile}",
|
|
}
|
|
# Copy the new root CA key in place
|
|
-> exec { 'put_new_ca_key_in_place':
|
|
command => "/bin/cp ${rootca_keyfile_new} ${rootca_keyfile}",
|
|
}
|
|
# Update admin.conf to remove the old CA cert
|
|
-> exec { 'update_admin_conf':
|
|
environment => [ 'KUBECONFIG=/etc/kubernetes/admin.conf' ],
|
|
command => "kubectl config set-cluster kubernetes --certificate-authority ${rootca_certfile} --embed-certs",
|
|
}
|
|
# Restart sysinv-conductor since it uses admin.conf
|
|
-> exec { 'restart_sysinv_conductor':
|
|
command => 'sm-restart-safe service sysinv-conductor',
|
|
}
|
|
# Restart cert-mon since it uses admin.conf
|
|
-> exec { 'restart_cert_mon':
|
|
command => 'sm-restart-safe service cert-mon',
|
|
}
|
|
# Restart kube-apiserver to pick up the new cert
|
|
-> exec { 'restart_apiserver':
|
|
command => "/usr/bin/kill -s SIGHUP $(pidof kube-apiserver)",
|
|
}
|
|
# Update kube-controller-manager.yaml with the new cert and key,
|
|
# this also restart controller-manager
|
|
-> exec { 'update_controller-manager_yaml':
|
|
command => "/bin/sed -i \\
|
|
-e 's|cluster-signing-cert-file=.*|cluster-signing-cert-file=${rootca_certfile}|' \\
|
|
-e 's|cluster-signing-key-file=.*|cluster-signing-key-file=${rootca_keyfile}|' \\
|
|
/etc/kubernetes/manifests/kube-controller-manager.yaml",
|
|
}
|
|
# Update scheduler.conf with the new cert
|
|
-> exec { 'update_scheduler_conf':
|
|
environment => [ 'KUBECONFIG=/etc/kubernetes/scheduler.conf' ],
|
|
command => "kubectl config set-cluster kubernetes --certificate-authority ${rootca_certfile} --embed-certs",
|
|
}
|
|
# Restart scheduler to trust the new cert
|
|
-> exec { 'restart_scheduler':
|
|
command => "/usr/bin/kill -s SIGHUP $(pidof kube-scheduler)",
|
|
}
|
|
# Update kubelet.conf with the new cert
|
|
$cluster = generate('/bin/bash', '-c', "/bin/sed -e '/- cluster/,/name:/!d' /etc/kubernetes/kubelet.conf \
|
|
| grep 'name:' | awk '{printf \"%s\", \$2}'")
|
|
|
|
exec { 'update_kubelet_conf':
|
|
environment => [ 'KUBECONFIG=/etc/kubernetes/kubelet.conf' ],
|
|
command => "kubectl config set-cluster ${cluster} --certificate-authority ${rootca_certfile} --embed-certs",
|
|
require => Exec['put_new_ca_key_in_place'],
|
|
}
|
|
# Restart kubelet to trust only the new cert
|
|
-> exec { 'restart_kubelet':
|
|
command => '/usr/bin/systemctl restart kubelet',
|
|
}
|
|
# Remove the new cert file
|
|
-> exec { 'remove_new_cert_file':
|
|
command => "/bin/rm -f ${rootca_certfile_new}",
|
|
}
|
|
# Remove the new key file
|
|
-> exec { 'remove_new_key_file':
|
|
command => "/bin/rm -f ${rootca_keyfile_new}",
|
|
}
|
|
# Remove the old cert file
|
|
-> exec { 'remove_old_cert_file':
|
|
command => "/bin/rm -f ${rootca_certfile_old}",
|
|
}
|
|
}
|
|
|
|
class platform::kubernetes::worker::rootca::trustnewca::runtime
|
|
inherits ::platform::kubernetes::params {
|
|
|
|
$cluster = generate('/bin/bash', '-c', "/bin/sed -e '/- cluster/,/name:/!d' /etc/kubernetes/kubelet.conf \
|
|
| grep 'name:' | awk '{printf \"%s\", \$2}'")
|
|
# Replace the current root CA cert with the new one
|
|
exec { 'replace_ca_cert_with_new_one':
|
|
command => "/bin/mv -f ${rootca_certfile_new} ${rootca_certfile}",
|
|
onlyif => "/usr/bin/test -e ${rootca_certfile_new}",
|
|
}
|
|
# Replace the current root CA key with the new one
|
|
-> exec { 'replace_ca_key_with_new_one':
|
|
command => "/bin/mv -f ${rootca_keyfile_new} ${rootca_keyfile}",
|
|
onlyif => "/usr/bin/test -e ${rootca_keyfile_new}",
|
|
}
|
|
# Remove the old cert file
|
|
-> exec { 'remove_old_cert_file':
|
|
command => "/bin/rm -f ${rootca_certfile_old}",
|
|
}
|
|
# Update kubelet.conf with the new cert
|
|
-> exec { 'update_kubelet_conf':
|
|
environment => [ 'KUBECONFIG=/etc/kubernetes/kubelet.conf' ],
|
|
command => "kubectl config set-cluster ${cluster} --certificate-authority ${rootca_certfile} --embed-certs",
|
|
}
|
|
# Restart kubelet to trust only the new cert
|
|
-> exec { 'restart_kubelet':
|
|
command => '/usr/bin/systemctl restart kubelet',
|
|
}
|
|
}
|
|
|
|
class platform::kubernetes::master::rootca::pods::trustbothcas::runtime
|
|
inherits ::platform::kubernetes::params {
|
|
exec { 'update_pods_trustbothcas':
|
|
environment => [ 'KUBECONFIG=/etc/kubernetes/admin.conf' ],
|
|
provider => shell,
|
|
command => template('platform/kube-rootca-update-pods.erb'),
|
|
timeout => 600,
|
|
}
|
|
}
|
|
|
|
class platform::kubernetes::master::rootca::pods::trustnewca::runtime
|
|
inherits ::platform::kubernetes::params {
|
|
exec { 'update_pods_trustnewca':
|
|
environment => [ 'KUBECONFIG=/etc/kubernetes/admin.conf' ],
|
|
provider => shell,
|
|
command => template('platform/kube-rootca-update-pods.erb'),
|
|
timeout => 600,
|
|
}
|
|
}
|
|
|
|
class platform::kubernetes::master::rootca::updatecerts::runtime
|
|
inherits ::platform::kubernetes::params {
|
|
|
|
# Create directory to use crt and key from secret in kubernetes components configuration
|
|
file { '/tmp/kube_rootca_update':
|
|
ensure => directory,
|
|
owner => 'root',
|
|
group => 'root',
|
|
mode => '0640',
|
|
}
|
|
|
|
# Create the new k8s admin cert file
|
|
-> file { '/tmp/kube_rootca_update/kubernetes-admin.crt':
|
|
ensure => file,
|
|
content => base64('decode', $admin_cert),
|
|
owner => 'root',
|
|
group => 'root',
|
|
mode => '0640',
|
|
}
|
|
|
|
# Create the new k8s admin key file
|
|
-> file { '/tmp/kube_rootca_update/kubernetes-admin.key':
|
|
ensure => file,
|
|
content => base64('decode', $admin_key),
|
|
owner => 'root',
|
|
group => 'root',
|
|
mode => '0640',
|
|
}
|
|
|
|
# Update admin.conf with new cert/key
|
|
-> exec { 'update_admin_conf_credentials':
|
|
environment => [ 'KUBECONFIG=/etc/kubernetes/admin.conf' ],
|
|
command => "kubectl config set-credentials kubernetes-admin --client-key /tmp/kube_rootca_update/kubernetes-admin.key \
|
|
--client-certificate /tmp/kube_rootca_update/kubernetes-admin.crt --embed-certs",
|
|
}
|
|
|
|
# Copy the new apiserver.crt, apiserver.key to replace the ones in /etc/kubernetes/pki/ directory
|
|
# Create the new k8s apiserver cert file
|
|
-> file { '/etc/kubernetes/pki/apiserver.crt':
|
|
ensure => file,
|
|
content => base64('decode', $apiserver_cert),
|
|
replace => true,
|
|
}
|
|
|
|
# Create the new k8s apiserver key file
|
|
-> file { '/etc/kubernetes/pki/apiserver.key':
|
|
ensure => file,
|
|
content => base64('decode', $apiserver_key),
|
|
replace => true,
|
|
}
|
|
|
|
# Copy the new apiserver-kubelet-client.crt, apiserver-kubelet-client.key to replace the ones in /etc/kubernetes/pki/ directory
|
|
# Create the new k8s apiserver-kubelet-client cert file
|
|
-> file { '/etc/kubernetes/pki/apiserver-kubelet-client.crt':
|
|
ensure => file,
|
|
content => base64('decode', $apiserver_kubelet_cert),
|
|
}
|
|
|
|
# Create the new k8s apiserver-kubelet-client key file
|
|
-> file { '/etc/kubernetes/pki/apiserver-kubelet-client.key':
|
|
ensure => file,
|
|
content => base64('decode', $apiserver_kubelet_key),
|
|
}
|
|
|
|
# Create the new kube scheduler crt file
|
|
-> file { '/tmp/kube_rootca_update/kube-scheduler.crt':
|
|
ensure => file,
|
|
content => base64('decode', $scheduler_cert),
|
|
owner => 'root',
|
|
group => 'root',
|
|
mode => '0640',
|
|
}
|
|
|
|
# Create the new kube scheduler key file
|
|
-> file { '/tmp/kube_rootca_update/kube-scheduler.key':
|
|
ensure => file,
|
|
content => base64('decode', $scheduler_key),
|
|
owner => 'root',
|
|
group => 'root',
|
|
mode => '0640',
|
|
}
|
|
|
|
# Update scheduler.conf with the new client cert
|
|
-> exec { 'scheduler_conf':
|
|
environment => [ 'KUBECONFIG=/etc/kubernetes/scheduler.conf' ],
|
|
command => "kubectl config set-credentials system:kube-scheduler --client-key /tmp/kube_rootca_update/kube-scheduler.key \
|
|
--client-certificate /tmp/kube_rootca_update/kube-scheduler.crt --embed-certs",
|
|
}
|
|
|
|
# Restart scheduler
|
|
-> exec { 'restart_scheduler':
|
|
command => "/usr/bin/kill -s SIGHUP $(pidof kube-scheduler)"
|
|
}
|
|
|
|
# Create the new k8s controller-manager crt file
|
|
-> file { '/tmp/kube_rootca_update/kube-controller-manager.crt':
|
|
ensure => file,
|
|
content => base64('decode', $controller_manager_cert),
|
|
owner => 'root',
|
|
group => 'root',
|
|
mode => '0640',
|
|
}
|
|
|
|
# Create the new k8s controller-manager key file
|
|
-> file { '/tmp/kube_rootca_update/kube-controller-manager.key':
|
|
ensure => file,
|
|
content => base64('decode', $controller_manager_key),
|
|
owner => 'root',
|
|
group => 'root',
|
|
mode => '0640',
|
|
}
|
|
|
|
# Update controller-manager.conf with the new client cert/key
|
|
-> exec { 'controller-manager_conf':
|
|
environment => [ 'KUBECONFIG=/etc/kubernetes/controller-manager.conf' ],
|
|
command => "kubectl config set-credentials system:kube-controller-manager \
|
|
--client-key /tmp/kube_rootca_update/kube-controller-manager.key \
|
|
--client-certificate /tmp/kube_rootca_update/kube-controller-manager.crt --embed-certs",
|
|
}
|
|
|
|
# Restart kube-controller-manager
|
|
-> exec { 'restart_controller-manager':
|
|
command => "/usr/bin/kill -s SIGHUP $(pidof kube-controller-manager)"
|
|
}
|
|
|
|
# Create the new kubelet client crt file
|
|
-> file { "/tmp/kube_rootca_update/${::platform::params::hostname}.crt":
|
|
ensure => file,
|
|
content => base64('decode', $kubelet_cert),
|
|
owner => 'root',
|
|
group => 'root',
|
|
mode => '0640',
|
|
}
|
|
|
|
# Create the new kubelet client key file
|
|
-> file { "/tmp/kube_rootca_update/${::platform::params::hostname}.key":
|
|
ensure => file,
|
|
content => base64('decode', $kubelet_key),
|
|
owner => 'root',
|
|
group => 'root',
|
|
mode => '0640',
|
|
}
|
|
|
|
# Append the cert and key to a pem file
|
|
-> exec { 'append_kubelet_client_cert_and_key':
|
|
command => "cat /tmp/kube_rootca_update/${::platform::params::hostname}.crt \
|
|
/tmp/kube_rootca_update/${::platform::params::hostname}.key > /tmp/kube_rootca_update/kubelet-client-cert-with-key.pem",
|
|
}
|
|
|
|
# Copy the new apiserver.crt, apiserver.key to replace the ones in /etc/kubernetes/pki/ directory
|
|
-> file { '/var/lib/kubelet/pki/kubelet-client-cert-with-key.pem':
|
|
ensure => file,
|
|
source => '/tmp/kube_rootca_update/kubelet-client-cert-with-key.pem',
|
|
replace => true,
|
|
}
|
|
|
|
# add link to new kubelet client cert
|
|
-> file { '/var/lib/kubelet/pki/kubelet-client-current.pem':
|
|
ensure => 'link',
|
|
target => '/var/lib/kubelet/pki/kubelet-client-cert-with-key.pem',
|
|
replace => true,
|
|
}
|
|
|
|
# Restart kubelet
|
|
-> exec { 'restart_kubelet-client':
|
|
command => "/usr/bin/kill -s SIGHUP $(pidof kubelet)"
|
|
}
|
|
|
|
# Removing temporary directory for files along this configuration process
|
|
-> exec { 'remove_kube_rootca_update_dir':
|
|
command => '/usr/bin/rm -rf /tmp/kube_rootca_update',
|
|
}
|
|
}
|
|
|
|
class platform::kubernetes::worker::rootca::updatecerts::runtime
|
|
inherits ::platform::kubernetes::params {
|
|
file { '/tmp/kube_rootca_update':
|
|
ensure => directory,
|
|
owner => 'root',
|
|
group => 'root',
|
|
mode => '0640',
|
|
}
|
|
|
|
# Create the new k8s kubelet client cert file
|
|
-> file { "/tmp/kube_rootca_update/${::platform::params::hostname}.crt":
|
|
ensure => file,
|
|
content => base64('decode', $kubelet_cert),
|
|
owner => 'root',
|
|
group => 'root',
|
|
mode => '0640',
|
|
}
|
|
|
|
# Create the new k8s kubelet client key file
|
|
-> file { "/tmp/kube_rootca_update/${::platform::params::hostname}.key":
|
|
ensure => file,
|
|
content => base64('decode', $kubelet_key),
|
|
owner => 'root',
|
|
group => 'root',
|
|
mode => '0640',
|
|
}
|
|
|
|
# Append the new cert and key files
|
|
-> exec { 'append_kubelet_client_cert_and_key':
|
|
command => "cat /tmp/kube_rootca_update/${::platform::params::hostname}.crt \
|
|
/tmp/kube_rootca_update/${::platform::params::hostname}.key > /tmp/kube_rootca_update/kubelet-client-cert-with-key.pem",
|
|
}
|
|
|
|
# Copy kubelet cert and key file to replace the one in /var/lib/kubelet/pki/ directory
|
|
-> file { '/var/lib/kubelet/pki/kubelet-client-cert-with-key.pem':
|
|
ensure => file,
|
|
source => '/tmp/kube_rootca_update/kubelet-client-cert-with-key.pem',
|
|
replace => true,
|
|
}
|
|
|
|
# Remove the current kubelet-client reference
|
|
-> exec { 'remove_current_kubelet_cert_link':
|
|
command => '/usr/bin/rm -rf /var/lib/kubelet/pki/kubelet-client-current.pem',
|
|
}
|
|
|
|
# add link to new kubelet client cert
|
|
-> file { '/var/lib/kubelet/pki/kubelet-client-current.pem':
|
|
ensure => 'link',
|
|
target => '/var/lib/kubelet/pki/kubelet-client-cert-with-key.pem',
|
|
}
|
|
|
|
# Restart kubelet
|
|
-> exec { 'restart_kubelet-client':
|
|
command => "/usr/bin/kill -s SIGHUP $(pidof kubelet)"
|
|
}
|
|
|
|
# Removing temporary directory for files along this configuration process
|
|
-> exec { 'remove_kube_rootca_update_dir':
|
|
command => '/usr/bin/rm -rf /tmp/kube_rootca_update',
|
|
}
|
|
}
|