From 32b5a57e89e84961f7dbcc631330d43bfdab96bc Mon Sep 17 00:00:00 2001 From: Ruslan Aliev Date: Wed, 14 May 2025 20:59:52 -0500 Subject: [PATCH] Use kubeadm for bootstrap k8s Change-Id: I6e441b2e793427880b44a2475f5d896eb41087e6 Signed-off-by: Ruslan Aliev --- charts/kubeadm/Chart.yaml | 18 + charts/kubeadm/requirements.yaml | 18 + charts/kubeadm/templates/bin/_anchor.tpl | 374 ++++++++++++++ .../templates/bin/_kubelet_restart.tpl | 29 ++ charts/kubeadm/templates/bin/_pre_stop.tpl | 19 + .../templates/configmap-apiserver-misc.yaml | 34 ++ charts/kubeadm/templates/configmap-bin.yaml | 32 ++ charts/kubeadm/templates/configmap-certs.yaml | 44 ++ charts/kubeadm/templates/configmap-jobs.yaml | 28 + .../kubeadm/templates/configmap-kubeadm.yaml | 32 ++ .../templates/configmap-kubeconfig.yaml | 28 + .../kubeadm/templates/configmap-kubelet.yaml | 31 ++ .../kubeadm/templates/configmap-patches.yaml | 27 + .../kubeadm/templates/daemonset-masters.yaml | 179 +++++++ .../kubeadm/templates/daemonset-workers.yaml | 145 ++++++ .../templates/jobs/_kubelet_restart.yaml.tpl | 46 ++ .../kubeadm/_cluster_config.yaml.tpl | 66 +++ .../templates/kubeadm/_join_config.yaml.tpl | 43 ++ .../kubeadm/_upgrade_config.yaml.tpl | 29 ++ .../templates/kubeconfig/_kubeconfig.yaml.tpl | 34 ++ .../kubelet/_kubelet-service.yaml.tpl | 35 ++ .../templates/kubelet/_kubelet.yaml.tpl | 23 + .../kubelet/_kubelet_config.yaml.tpl | 33 ++ charts/kubeadm/templates/rbac.yaml | 67 +++ charts/kubeadm/values.yaml | 484 ++++++++++++++++++ promenade/control/join_scripts.py | 11 - promenade/templates/include/up.sh | 7 +- promenade/templates/include/utils.sh | 2 + .../roles/common/etc/default/kubelet | 24 +- .../roles/common/etc/kubernetes/admin.conf | 22 + .../etc/kubernetes/apiserver/acconfig.yaml | 7 + .../kubernetes/apiserver/audit-policy.yaml | 1 + .../apiserver/encryption_provider.yaml | 6 + .../etc/kubernetes/apiserver/eventconfig.yaml | 8 + .../etc/kubernetes/controller-manager.conf | 21 + .../kubernetes/{kubeconfig => kubelet.conf} | 2 +- .../common/etc/kubernetes/kubelet/config.yaml | 4 +- .../common/etc/kubernetes/pki/cluster-ca.pem | 1 - .../common/etc/kubernetes/scheduler.conf | 21 + .../common/etc/systemd/system/kubelet.service | 18 +- .../roles/common/usr/local/bin/haproxy | 6 +- .../roles/common/var/lib/kubelet/config.yaml | 3 + .../etc/kubernetes/kubeadm/init-config.yaml | 39 ++ .../etc/kubernetes/kubeadm/join-config.yaml | 36 ++ promenade/templates/scripts/genesis.sh | 22 +- promenade/templates/scripts/join.sh | 3 +- tools/helm_install.sh | 5 +- 47 files changed, 2127 insertions(+), 40 deletions(-) create mode 100644 charts/kubeadm/Chart.yaml create mode 100644 charts/kubeadm/requirements.yaml create mode 100644 charts/kubeadm/templates/bin/_anchor.tpl create mode 100644 charts/kubeadm/templates/bin/_kubelet_restart.tpl create mode 100644 charts/kubeadm/templates/bin/_pre_stop.tpl create mode 100644 charts/kubeadm/templates/configmap-apiserver-misc.yaml create mode 100644 charts/kubeadm/templates/configmap-bin.yaml create mode 100644 charts/kubeadm/templates/configmap-certs.yaml create mode 100644 charts/kubeadm/templates/configmap-jobs.yaml create mode 100644 charts/kubeadm/templates/configmap-kubeadm.yaml create mode 100644 charts/kubeadm/templates/configmap-kubeconfig.yaml create mode 100644 charts/kubeadm/templates/configmap-kubelet.yaml create mode 100644 charts/kubeadm/templates/configmap-patches.yaml create mode 100644 charts/kubeadm/templates/daemonset-masters.yaml create mode 100644 charts/kubeadm/templates/daemonset-workers.yaml create mode 100644 charts/kubeadm/templates/jobs/_kubelet_restart.yaml.tpl create mode 100644 charts/kubeadm/templates/kubeadm/_cluster_config.yaml.tpl create mode 100644 charts/kubeadm/templates/kubeadm/_join_config.yaml.tpl create mode 100644 charts/kubeadm/templates/kubeadm/_upgrade_config.yaml.tpl create mode 100644 charts/kubeadm/templates/kubeconfig/_kubeconfig.yaml.tpl create mode 100644 charts/kubeadm/templates/kubelet/_kubelet-service.yaml.tpl create mode 100644 charts/kubeadm/templates/kubelet/_kubelet.yaml.tpl create mode 100644 charts/kubeadm/templates/kubelet/_kubelet_config.yaml.tpl create mode 100644 charts/kubeadm/templates/rbac.yaml create mode 100644 charts/kubeadm/values.yaml create mode 100644 promenade/templates/roles/common/etc/kubernetes/admin.conf create mode 100644 promenade/templates/roles/common/etc/kubernetes/apiserver/acconfig.yaml create mode 100644 promenade/templates/roles/common/etc/kubernetes/apiserver/audit-policy.yaml create mode 100644 promenade/templates/roles/common/etc/kubernetes/apiserver/encryption_provider.yaml create mode 100644 promenade/templates/roles/common/etc/kubernetes/apiserver/eventconfig.yaml create mode 100644 promenade/templates/roles/common/etc/kubernetes/controller-manager.conf rename promenade/templates/roles/common/etc/kubernetes/{kubeconfig => kubelet.conf} (87%) delete mode 100644 promenade/templates/roles/common/etc/kubernetes/pki/cluster-ca.pem create mode 100644 promenade/templates/roles/common/etc/kubernetes/scheduler.conf create mode 100644 promenade/templates/roles/common/var/lib/kubelet/config.yaml create mode 100644 promenade/templates/roles/genesis/etc/kubernetes/kubeadm/init-config.yaml create mode 100644 promenade/templates/roles/join/etc/kubernetes/kubeadm/join-config.yaml diff --git a/charts/kubeadm/Chart.yaml b/charts/kubeadm/Chart.yaml new file mode 100644 index 00000000..719282b2 --- /dev/null +++ b/charts/kubeadm/Chart.yaml @@ -0,0 +1,18 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +description: A chart for managing Kubernetes cluster via Kubeadm +name: kubeadm +version: 0.1.0 diff --git a/charts/kubeadm/requirements.yaml b/charts/kubeadm/requirements.yaml new file mode 100644 index 00000000..d7891bca --- /dev/null +++ b/charts/kubeadm/requirements.yaml @@ -0,0 +1,18 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - name: helm-toolkit + repository: file://../deps/helm-toolkit + version: ">= 0.1.0" diff --git a/charts/kubeadm/templates/bin/_anchor.tpl b/charts/kubeadm/templates/bin/_anchor.tpl new file mode 100644 index 00000000..0da5e846 --- /dev/null +++ b/charts/kubeadm/templates/bin/_anchor.tpl @@ -0,0 +1,374 @@ +#!/bin/bash +# Copyright 2025 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xeuo pipefail + +# directories +HOST_DIR="{{ .Values.host_dir }}" +KUBERNETES_DIR="/etc/kubernetes" +CERT_DIR="${HOST_DIR}${KUBERNETES_DIR}/pki" + +# flow controls +READY_TO_START=false +FIRST_POD=false # remove +KUBEADM_ACTION_REQUIRED=$([ $NODE_ROLE == "master" ] && echo "true" || echo "false") +KUBELET_RESTART_REQUIRED=false + +KUBERNETES_VERSION="{{ .Values.kubeadm.cluster_config.kubernetesVersion }}" +KUBEADM_ANNOTATION="last-applied-kubeadm-cfg-sha256" +KUBELET_ANNOTATION="last-applied-kubelet-cfg-sha256" + +LAST_APPLIED_KUBEADM_CONFIG_SHA256=$([ $NODE_ROLE == "master" ] && kubectl get node $NODE_NAME -o jsonpath="{.metadata.annotations.$KUBEADM_ANNOTATION}" || echo "") +LAST_APPLIED_KUBELET_CONFIG_SHA256=$(kubectl get node $NODE_NAME -o jsonpath="{.metadata.annotations.$KUBELET_ANNOTATION}") + +# substitute from values +MASTERS_DS_NAME="{{ .Values.service.name }}-masters-anchor" +WORKERS_DS_NAME="{{ .Values.service.name }}-workers-anchor" +CURRENT_DS_NAME=$([ $NODE_ROLE == "master" ] && echo "$MASTERS_DS_NAME" || echo "$WORKERS_DS_NAME") + +MASTERS_POD_LABELS="component=kubernetes-kubeadm-anchor" +WORKERS_POD_LABELS="component=kubernetes-kubeadm-workers-anchor" +CURRENT_POD_LABELS=$([ $NODE_ROLE == "master" ] && echo "$MASTERS_POD_LABELS" || echo "$WORKERS_POD_LABELS") + +kubeadm() { + $(which kubeadm) --rootfs "$HOST_DIR" --v=5 $@ +} + +annotate_node() { + kubectl annotate node --overwrite "$NODE_NAME" "$1=$2" +} + +sync_configs() { + sync_kubeconfigs + sync_kubelet_configs + sync_binaries + + if [[ $NODE_ROLE == "master" ]]; then + sync_control_plane_certs + sync_apiserver_misc_configs + sync_kubeadm_configs + sync_patches + fi + + cleanup_old_configs +} + +compare_copy_file() { + src=$1 + delete_src=false + if [ ! -f "$1" ]; then + src=$(mktemp) + cat - > $src + delete_src=true + fi + dst=$2 + + if [ ! -e "${dst}" ] || ! cmp -s $src $dst; then + [[ -d "$(dirname ${dst})" ]] || mkdir -p $(dirname "${dst}") + chmod a+r $(dirname "${dst}") + + cp "${src}" "${dst}" + chmod a+r "${dst}" + +{{- if .Values.kubelet.restart }} + if [[ "${dst}" == "${HOST_DIR}/etc/default/kubelet" || + "${dst}" == "${HOST_DIR}/etc/systemd/system/kubelet.service" || + "${dst}" == "${HOST_DIR}/var/lib/kubelet/config.yaml" ]]; then + KUBELET_RESTART_REQUIRED=true + fi +{{- end }} + + fi + + if [[ $delete_src == true ]]; then + rm $src + fi +} + +move_if_exists() { + path="$1" + move_to="$2" + + if [ -e "$path" ]; then mv "$path" "$move_to"; fi +} + +remove_if_exists() { + path="$1" + if [ -e "$path" ]; then + if [ -d "$path" ]; then rm -rf $path; else rm -f "$path"; fi + fi +} + +sync_kubeconfigs() { + cluster_ip="{{ .Values.kubeconfig.cluster_ip }}" + cluster_port="{{ .Values.kubeconfig.cluster_port }}" + cluster_name="{{ .Values.kubeconfig.cluster_name }}" + {{ range $name, $data := .Values.kubeconfig.configs }} + if [[ "{{ $name }}" == "kubelet" || $NODE_ROLE == "master" ]]; then + CLUSTER_IP="$cluster_ip" CLUSTER_PORT="$cluster_port" \ + CERT_AUTH="{{ $data.ca }}" CLUSTER_NAME="$cluster_name" USER="{{ $name }}" \ + CLIENT_CERT="{{ $data.client_cert }}" CLIENT_KEY="{{ $data.client_key }}" \ + envsubst < /tmp/kubeconfig/kubeconfig.yaml.tpl | compare_copy_file - "$HOST_DIR{{ $data.path }}" + fi + {{- end }} + + if [ $NODE_ROLE == "master" ] && ! kubectl get cm -n kube-public cluster-info; then + kubeadm init phase bootstrap-token --kubeconfig /etc/kubernetes/admin.conf + fi + + echo "kubeconfigs are synced" +} + +sync_control_plane_certs() { + if [[ $NODE_ROLE != "master" ]]; then return; fi + for file in /tmp/certs/*; do + file_name="$(basename $file)" + if [[ "$file_name" == etcd-* ]]; then + compare_copy_file "$file" "$CERT_DIR/etcd/${file_name/etcd-/}" + else + compare_copy_file "$file" "$CERT_DIR/$file_name" + fi + done +} + +sync_kubelet_certs() { + echo "placeholder" +} + +sync_etcd_certs() { + echo "placeholder" +} + +sync_patches() { + #remove_if_exists "${HOST_DIR}${KUBERNETES_DIR}/kubeadm/patches/" + if [[ $NODE_ROLE != "master" ]]; then return; fi + for file in /tmp/patches/*; do + compare_copy_file "$file" "${HOST_DIR}${KUBERNETES_DIR}/kubeadm/patches/$(basename $file)" + done +} + +sync_apiserver_misc_configs() { + if [[ $NODE_ROLE == "master" ]]; then + for file in /tmp/apiserver-misc/*; do + compare_copy_file "$file" "${HOST_DIR}${KUBERNETES_DIR}/apiserver/$(basename $file)" + done + fi +} + +sync_kubelet_configs() { + envsubst < /tmp/kubelet/default-kubelet | compare_copy_file - "${HOST_DIR}/etc/default/kubelet" + compare_copy_file /tmp/kubelet/kubelet.service "${HOST_DIR}/etc/systemd/system/kubelet.service" + compare_copy_file /tmp/kubelet/kubelet "${HOST_DIR}/var/lib/kubelet/config.yaml" +} + +sync_kubeadm_configs() { + #ETCD_ENABLED=$(kubectl get nodes -l kubernetes-etcd=enabled --no-headers -o custom-columns=":metadata.name" | grep -q "$NODE_NAME" && echo "true" || echo "false") + envsubst < /tmp/kubeadm/join-config.yaml | compare_copy_file - "${HOST_DIR}${KUBERNETES_DIR}/kubeadm/join_config.yaml" + envsubst < /tmp/kubeadm/upgrade-config.yaml | compare_copy_file - "${HOST_DIR}${KUBERNETES_DIR}/kubeadm/upgrade_config.yaml" +} + +sync_binaries() { + envsubst < "/tmp/bin/kubelet_restart.sh" | compare_copy_file - "$HOST_DIR/usr/local/bin/kubelet_restart.sh" + chmod a+x "$HOST_DIR/usr/local/bin/kubelet_restart.sh" + + if [[ ! -e "$HOST_DIR/usr/local/bin/kubelet" || $(kubelet --version) != $($HOST_DIR/usr/local/bin/kubelet --version) ]]; then +{{- if .Values.kubelet.restart }} + KUBELET_RESTART_REQUIRED=true +{{- end }} + + install -v -b --suffix=-backup /usr/bin/kubelet $HOST_DIR/usr/local/bin/ + fi + + if [[ $NODE_ROLE == "master" ]]; then + if [[ ! -e "$HOST_DIR/usr/local/bin/kubeadm" || "$(kubeadm version -o short)" != "$($HOST_DIR/usr/local/bin/kubeadm version -o short)" ]]; then + install -v -b --suffix=-backup /usr/bin/kubeadm $HOST_DIR/usr/local/bin/ + fi + + if [[ ! -e "$HOST_DIR/usr/local/bin/kubectl" || $(kubectl version --client | grep Client) != $($HOST_DIR/usr/local/bin/kubectl version --client | grep Client) ]]; then + install -v -b --suffix=-backup /usr/bin/kubectl $HOST_DIR/usr/local/bin/ + fi + fi +} + +# to remove +ready_to_start() { + pod_list=$(kubectl get pods -n kube-system -l $CURRENT_POD_LABELS --sort-by=.metadata.creationTimestamp --no-headers -o custom-columns=":metadata.name") + for pod in $pod_list; do + if [ $pod == $POD_NAME ]; then + READY_TO_START="true" + break + fi + + if ! kubectl wait --for=condition=ready pod -n kube-system $pod --timeout=0; then + break + fi + done +} + +daemonset_status() { + if [[ $NODE_ROLE == "worker" ]] && ! kubectl rollout status "ds/$MASTERS_DS_NAME" -n kube-system -w=false | grep -q "successfully rolled out"; then + echo "waiting for masters nodes to be ready..." + return + fi + + generation=0 + observed_generation=0 + updated_number_scheduled=0 + desired_number_scheduled=0 + number_available=0 + + read -r generation observed_generation updated_number_scheduled desired_number_scheduled number_available <<<$(kubectl get ds -n kube-system "$CURRENT_DS_NAME" -o jsonpath='{ .metadata.generation } { .status.observedGeneration } { .status.updatedNumberScheduled } { .status.desiredNumberScheduled } { .status.numberAvailable }') + while [ $generation -gt $observed_generation ]; do + echo "Waiting for daemon set spec update to be observed..." + read -r generation observed_generation updated_number_scheduled desired_number_scheduled number_available <<<$(kubectl get ds -n kube-system "$CURRENT_DS_NAME" -o jsonpath='{ .metadata.generation } { .status.observedGeneration } { .status.updatedNumberScheduled } { .status.desiredNumberScheduled } { .status.numberAvailable }') + sleep 5 + done + + if [ -z $number_available ]; then + number_available=0 + fi + + if [ $updated_number_scheduled -lt $desired_number_scheduled ]; then + echo "Waiting for daemon set rollout to finish: ${updated_number_scheduled} out of ${desired_number_scheduled} new pods have been updated..." + # add verification that the pod has to be single (unready) + READY_TO_START="true" + if [ $updated_number_scheduled -eq 1 ]; then + echo "first in sequence" + FIRST_POD=true + fi + elif [ $number_available -lt $desired_number_scheduled ]; then + echo "Waiting for daemon set rollout to finish: ${number_available} of ${desired_number_scheduled} updated pods are available..." + if [[ $(($desired_number_scheduled-$number_available)) -eq 1 ]]; then + READY_TO_START="true" + else + ready_to_start + fi + else + echo "Daemon set successfully rolled out" + READY_TO_START="true" + KUBEADM_ACTION_REQUIRED=false + fi +} + +cleanup_old_configs() { +{{- range $file := .Values.const.files_to_delete }} + remove_if_exists "${HOST_DIR}{{ $file }}" +{{- end }} +} + +restart_kubelet() { + # kubelet_restart.yaml has to be renamed to node_action.yaml + kubectl drain "${NODE_NAME}" --pod-selector '!kubelet-restart' --ignore-daemonsets --delete-emptydir-data + job_name=$(envsubst < "/tmp/jobs/kubelet_restart.yaml.tpl" | kubectl create -f - | grep created | awk '{print $1}') + kubectl wait -n kube-system --for=condition=complete $job_name --timeout=600s + + kubectl uncordon "${NODE_NAME}" + kubectl wait node --for=condition=ready "${NODE_NAME}" --timeout=60s + + sleep 60 + + if [[ $(kubectl get node "${NODE_NAME}" -o jsonpath='{.status.nodeInfo.kubeletVersion}') != $(kubelet --version | awk '{print $2}') ]]; then + echo "kubelet version mismatch" + exit 1 + fi + + sleep 60 +} + +is_upgrade_required() { + IFS='|' read -r last_cluster_config_sha256 last_kubelet_config_sha256 kubelet_current_version <<<$(kubectl get node $NODE_NAME -o jsonpath="{.metadata.annotations.$KUBEADM_ANNOTATION}|{.metadata.annotations.$KUBELET_ANNOTATION}|{.status.nodeInfo.kubeletVersion}") + + if [[ $NODE_ROLE == "master" ]]; then + if [ -z $last_cluster_config_sha256 ]; then return 0; fi + current_cluster_config_sha256=$(sha256sum < /tmp/kubeadm/ClusterConfiguration | cut -d ' ' -f 1) + if [[ $last_cluster_config_sha256 != $current_cluster_config_sha256 ]]; then + return 0 + fi + fi + +{{- if .Values.kubelet.restart }} + if [ -z $last_kubelet_config_sha256 ]; then return 0; fi + current_kubelet_config_sha256=$(sha256sum < /tmp/kubelet/kubelet | cut -d ' ' -f 1) + + if [[ $last_kubelet_config_sha256 != $current_kubelet_config_sha256 || $kubelet_current_version != $KUBERNETES_VERSION ]]; then + KUBELET_RESTART_REQUIRED=true + return 0 + fi +{{- end }} + + return 1 +} + +verlte() { + printf '%s\n' "$1" "$2" | sort -C -V +} + +verlt() { + ! verlte "$2" "$1" +} + +kubeadm_action() { + if is_upgrade_required; then + if [[ ! -f "${HOST_DIR}${KUBERNETES_DIR}/manifests/kube-apiserver.yaml" || + ! -f "${HOST_DIR}${KUBERNETES_DIR}/manifests/kube-controller-manager.yaml" || + ! -f "${HOST_DIR}${KUBERNETES_DIR}/manifests/kube-scheduler.yaml" ]]; then + rm -f "${HOST_DIR}${KUBERNETES_DIR}/manifests/kube-*" || true + kubeadm join --config "${KUBERNETES_DIR}/kubeadm/join_config.yaml" + else + kubeadm upgrade node --config "${KUBERNETES_DIR}/kubeadm/upgrade_config.yaml" + sync_kubeconfigs + fi + fi +} + +rollout_restart() { + if kubectl rollout status "ds/$CURRENT_DS_NAME" -n kube-system -w=false | grep -q "successfully rolled out"; then + kubectl rollout restart "ds/$CURRENT_DS_NAME" -n kube-system + fi +} + +while [[ $READY_TO_START != true ]]; do + daemonset_status + sleep 5 +done + +sync_configs + +if [[ $(kubeadm version -o short) != "$KUBERNETES_VERSION" ]]; then + echo "Desired kubernetes version mismatch with provided binaries version, please update kubernetes version in values to $(kubeadm version -o short)" + exit 1 +fi + +if [[ $KUBEADM_ACTION_REQUIRED == true ]]; then + kubeadm_action + annotate_node "$KUBEADM_ANNOTATION" "$(sha256sum < /tmp/kubeadm/ClusterConfiguration | cut -d ' ' -f 1)" +fi + +if [[ $KUBELET_RESTART_REQUIRED == true ]]; then + restart_kubelet + annotate_node "$KUBELET_ANNOTATION" "$(sha256sum < /tmp/kubelet/kubelet | cut -d ' ' -f 1)" +fi + +touch /tmp/done + +# main loop +while true; do + if [ -e /tmp/stop ] || is_upgrade_required; then + echo "Stopping" + rollout_restart + break + fi + sleep 15 +done diff --git a/charts/kubeadm/templates/bin/_kubelet_restart.tpl b/charts/kubeadm/templates/bin/_kubelet_restart.tpl new file mode 100644 index 00000000..42d824b6 --- /dev/null +++ b/charts/kubeadm/templates/bin/_kubelet_restart.tpl @@ -0,0 +1,29 @@ +#!/bin/bash +# Copyright 2025 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xeuo pipefail + +echo "perform daemon-reload" +systemctl daemon-reload +{{- if .Values.kubelet.restart }} +echo "restarting kubelet service" +systemctl restart kubelet.service +{{- end }} + +while ! systemctl is-active --quiet kubelet.service; do + sleep 5 +done + +exit 0 diff --git a/charts/kubeadm/templates/bin/_pre_stop.tpl b/charts/kubeadm/templates/bin/_pre_stop.tpl new file mode 100644 index 00000000..5e6342b5 --- /dev/null +++ b/charts/kubeadm/templates/bin/_pre_stop.tpl @@ -0,0 +1,19 @@ +#!/bin/sh +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +touch /tmp/stop +sleep {{ .Values.anchor.period }} diff --git a/charts/kubeadm/templates/configmap-apiserver-misc.yaml b/charts/kubeadm/templates/configmap-apiserver-misc.yaml new file mode 100644 index 00000000..ed3d91df --- /dev/null +++ b/charts/kubeadm/templates/configmap-apiserver-misc.yaml @@ -0,0 +1,34 @@ +{{/* +Copyright 2017 AT&T Intellectual Property. All other rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_apiserver_misc }} +{{- $envAll := . }} + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.service.name }}-apiserver-misc +data: +{{/* Dynamically added apiserver config files */}} +{{- range $key, $val := .Values.apiserver.config_files }} +{{- if and (hasKey $val "file") (hasKey $val "content") }} + {{ $val.file }}: | +{{- if kindIs "string" $val.content }} +{{ indent 4 $val.content }} +{{- else }} +{{ toYaml $val.content | indent 4 }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/kubeadm/templates/configmap-bin.yaml b/charts/kubeadm/templates/configmap-bin.yaml new file mode 100644 index 00000000..4c452680 --- /dev/null +++ b/charts/kubeadm/templates/configmap-bin.yaml @@ -0,0 +1,32 @@ +{{/* +Copyright 2017 AT&T Intellectual Property. All other rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.service.name }}-bin +data: + anchor: |+ +{{ tuple "bin/_anchor.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + kubelet_restart.sh: |+ +{{ tuple "bin/_kubelet_restart.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + pre_stop: |+ +{{ tuple "bin/_pre_stop.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} diff --git a/charts/kubeadm/templates/configmap-certs.yaml b/charts/kubeadm/templates/configmap-certs.yaml new file mode 100644 index 00000000..7192b6d5 --- /dev/null +++ b/charts/kubeadm/templates/configmap-certs.yaml @@ -0,0 +1,44 @@ +{{/* +Copyright 2025 AT&T Intellectual Property. All other rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_certs }} +{{- $envAll := . }} + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.service.name }}-certs +data: + apiserver.crt: {{ .Values.secrets.apiserver.crt | quote }} + apiserver.key: {{ .Values.secrets.apiserver.key | quote }} + apiserver-etcd-client.crt: {{ .Values.secrets.apiserver.etcd_client.crt | quote }} + apiserver-etcd-client.key: {{ .Values.secrets.apiserver.etcd_client.key | quote }} + apiserver-kubelet-client.crt: {{ .Values.secrets.apiserver.kubelet_client.crt | quote }} + apiserver-kubelet-client.key: {{ .Values.secrets.apiserver.kubelet_client.key | quote }} + ca.crt: {{ .Values.secrets.ca.crt | quote }} + ca.key: {{ .Values.secrets.ca.key | quote }} + controller-manager.pem: {{ .Values.secrets.controller_manager.crt | quote }} + controller-manager-key.pem: {{ .Values.secrets.controller_manager.key | quote }} + etcd-ca.crt: {{ .Values.secrets.etcd.ca.crt | quote }} + etcd-healthcheck-client.crt: {{ .Values.secrets.etcd.healthcheck_client.crt | quote }} + etcd-healthcheck-client.key: {{ .Values.secrets.etcd.healthcheck_client.key | quote }} + front-proxy-ca.crt: {{ .Values.secrets.front_proxy_ca.crt | quote }} + front-proxy-ca.key: {{ .Values.secrets.front_proxy_ca.key | quote }} + front-proxy-client.crt: {{ .Values.secrets.front_proxy_client.crt | quote }} + front-proxy-client.key: {{ .Values.secrets.front_proxy_client.key | quote }} + sa.key: {{ .Values.secrets.sa.key | quote }} + sa.pub: {{ .Values.secrets.sa.pub | quote }} + scheduler.pem: {{ .Values.secrets.scheduler.crt | quote }} + scheduler-key.pem: {{ .Values.secrets.scheduler.key | quote }} +{{- end }} diff --git a/charts/kubeadm/templates/configmap-jobs.yaml b/charts/kubeadm/templates/configmap-jobs.yaml new file mode 100644 index 00000000..9621f38e --- /dev/null +++ b/charts/kubeadm/templates/configmap-jobs.yaml @@ -0,0 +1,28 @@ +{{/* +Copyright 2017 AT&T Intellectual Property. All other rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_jobs }} +{{- $envAll := . }} + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.service.name }}-jobs +data: + kubelet_restart.yaml.tpl: | +{{ tuple "jobs/_kubelet_restart.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} diff --git a/charts/kubeadm/templates/configmap-kubeadm.yaml b/charts/kubeadm/templates/configmap-kubeadm.yaml new file mode 100644 index 00000000..fad10744 --- /dev/null +++ b/charts/kubeadm/templates/configmap-kubeadm.yaml @@ -0,0 +1,32 @@ +{{/* +Copyright 2017 AT&T Intellectual Property. All other rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_kubeadm }} +{{- $envAll := . }} + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kubeadm-config +data: + ClusterConfiguration: | +{{ toYaml .Values.kubeadm.cluster_config | indent 4 }} + join-config.yaml: | +{{ tuple "kubeadm/_join_config.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + upgrade-config.yaml: | +{{ tuple "kubeadm/_upgrade_config.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} diff --git a/charts/kubeadm/templates/configmap-kubeconfig.yaml b/charts/kubeadm/templates/configmap-kubeconfig.yaml new file mode 100644 index 00000000..9ee68807 --- /dev/null +++ b/charts/kubeadm/templates/configmap-kubeconfig.yaml @@ -0,0 +1,28 @@ +{{/* +Copyright 2017 AT&T Intellectual Property. All other rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_kubeconfig }} +{{- $envAll := . }} + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.service.name }}-kubeconfig +data: + kubeconfig.yaml.tpl: | +{{ tuple "kubeconfig/_kubeconfig.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} diff --git a/charts/kubeadm/templates/configmap-kubelet.yaml b/charts/kubeadm/templates/configmap-kubelet.yaml new file mode 100644 index 00000000..879dccc8 --- /dev/null +++ b/charts/kubeadm/templates/configmap-kubelet.yaml @@ -0,0 +1,31 @@ +{{/* +Copyright 2017 AT&T Intellectual Property. All other rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_kubelet }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kubelet-config +data: + default-kubelet: | +{{ tuple "kubelet/_kubelet.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + kubelet.service: | +{{ tuple "kubelet/_kubelet-service.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + kubelet: | +{{ toYaml .Values.kubelet.config | indent 4 }} +{{- end }} diff --git a/charts/kubeadm/templates/configmap-patches.yaml b/charts/kubeadm/templates/configmap-patches.yaml new file mode 100644 index 00000000..27c8359c --- /dev/null +++ b/charts/kubeadm/templates/configmap-patches.yaml @@ -0,0 +1,27 @@ +{{/* +Copyright 2017 AT&T Intellectual Property. All other rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_patches }} +{{- $envAll := . }} + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.service.name }}-patches +data: +{{- range $file_name, $data := .Values.kubeadm.patches }} + {{ $file_name }}: | +{{ toYaml $data | indent 4 }} +{{- end }} +{{- end }} diff --git a/charts/kubeadm/templates/daemonset-masters.yaml b/charts/kubeadm/templates/daemonset-masters.yaml new file mode 100644 index 00000000..d6dd385d --- /dev/null +++ b/charts/kubeadm/templates/daemonset-masters.yaml @@ -0,0 +1,179 @@ +{{/* +Copyright 2017 AT&T Intellectual Property. All other rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.daemonset_masters }} +{{- $envAll := . }} +{{- $mounts_kubernetes_apiserver := .Values.pod.mounts.kubernetes_apiserver.kubernetes_apiserver }} +{{- $mounts_kubernetes_apiserver_init := .Values.pod.mounts.kubernetes_apiserver.init_container }} +{{- $labels := tuple $envAll "kubernetes" "kubernetes-kubeadm-anchor" | include "helm-toolkit.snippets.kubernetes_metadata_labels" -}} + +--- +apiVersion: "apps/v1" +kind: DaemonSet +metadata: + name: {{ .Values.service.name }}-masters-anchor + labels: +{{ $labels | indent 4 }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + selector: + matchLabels: +{{ $labels | indent 6 }} +{{ tuple $envAll "kubernetes-apiserver-anchor" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }} + template: + metadata: + labels: +{{ $labels | indent 8 }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-kubeconfig-hash: {{ tuple "configmap-kubeconfig.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "kubernetes_apiserver_anchor" "containerNames" (list "anchor") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "kubernetes_apiserver_anchor" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: {{ .Values.labels.common.node_selector_key }} + operator: In + values: + - {{ .Values.labels.common.node_selector_value }} + - key: {{ .Values.labels.masters.node_selector_key }} + operator: In + values: + - {{ .Values.labels.masters.node_selector_value }} + dnsPolicy: {{ .Values.anchor.dns_policy }} + serviceAccountName: kubeadm + hostNetwork: true + priorityClassName: system-node-critical + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: CriticalAddonsOnly + operator: Exists + terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.kubernetes_apiserver.timeout }} + containers: + - name: anchor + image: {{ .Values.images.tags.anchor }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.anchor_pod | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "kubernetes_apiserver_anchor" "container" "anchor" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NODE_ROLE + value: "master" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + command: ["/bin/bash","-c"] + args: ["/tmp/bin/anchor"] + lifecycle: + preStop: + exec: + command: + - /tmp/bin/pre_stop + + readinessProbe: + exec: + command: + - /bin/bash + - -c + - |- + cat /tmp/done +# [ "$(curl -s -S -o /dev/null \ +# --cert "/host{{ .Values.apiserver.host_etc_path }}/pki/apiserver.pem" \ +# --key "/host{{ .Values.apiserver.host_etc_path }}/pki/apiserver-key.pem" \ +# --cacert "/host{{ .Values.apiserver.host_etc_path }}/pki/cluster-ca.pem" \ +# "https://localhost:{{ .Values.network.kubernetes_apiserver.port }}/healthz" \ +# -w "%{http_code}")" = "200" ] +# exit $? +# initialDelaySeconds: 10 +# periodSeconds: 5 +# timeoutSeconds: 5 + + volumeMounts: + - name: host + mountPath: {{ .Values.host_dir }} + mountPropagation: HostToContainer + - name: {{ .Values.service.name }}-certs + mountPath: /tmp/certs + - name: {{ .Values.service.name }}-patches + mountPath: /tmp/patches + - name: {{ .Values.service.name }}-bin + mountPath: /tmp/bin + - name: {{ .Values.service.name }}-kubeconfig + mountPath: /tmp/kubeconfig + - name: {{ .Values.service.name }}-apiserver-misc + mountPath: /tmp/apiserver-misc + - name: kubelet-config + mountPath: /tmp/kubelet + - name: kubeadm-config + mountPath: /tmp/kubeadm + - name: pod-tmp + mountPath: /tmp +{{ if $mounts_kubernetes_apiserver.volumeMounts }}{{ toYaml $mounts_kubernetes_apiserver.volumeMounts | indent 12 }}{{ end }} + volumes: + - name: {{ .Values.service.name }}-bin + configMap: + name: {{ .Values.service.name }}-bin + defaultMode: 0555 + - name: {{ .Values.service.name }}-apiserver-misc + configMap: + name: {{ .Values.service.name }}-apiserver-misc + defaultMode: 0555 + - name: {{ .Values.service.name }}-certs + configMap: + name: {{ .Values.service.name }}-certs + defaultMode: 0444 + - name: {{ .Values.service.name }}-patches + configMap: + name: {{ .Values.service.name }}-patches + defaultMode: 0444 + - name: host + hostPath: + path: / + - name: {{ .Values.service.name }}-kubeconfig + configMap: + name: {{ .Values.service.name }}-kubeconfig + defaultMode: 0444 + - name: kubelet-config + configMap: + name: kubelet-config + defaultMode: 0444 + - name: kubeadm-config + configMap: + name: kubeadm-config + defaultMode: 0444 + - name: pod-tmp + emptyDir: {} +{{ if $mounts_kubernetes_apiserver.volumes }}{{ toYaml $mounts_kubernetes_apiserver.volumes | indent 8 }}{{ end }} + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + maxSurge: 0 +{{- end }} diff --git a/charts/kubeadm/templates/daemonset-workers.yaml b/charts/kubeadm/templates/daemonset-workers.yaml new file mode 100644 index 00000000..9092dc58 --- /dev/null +++ b/charts/kubeadm/templates/daemonset-workers.yaml @@ -0,0 +1,145 @@ +{{/* +Copyright 2017 AT&T Intellectual Property. All other rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.daemonset_workers }} +{{- $envAll := . }} +{{- $mounts_kubernetes_apiserver := .Values.pod.mounts.kubernetes_apiserver.kubernetes_apiserver }} +{{- $mounts_kubernetes_apiserver_init := .Values.pod.mounts.kubernetes_apiserver.init_container }} +{{- $labels := tuple $envAll "kubernetes" "kubernetes-kubeadm-workers-anchor" | include "helm-toolkit.snippets.kubernetes_metadata_labels" -}} + +--- +apiVersion: "apps/v1" +kind: DaemonSet +metadata: + name: {{ .Values.service.name }}-workers-anchor + labels: +{{ $labels | indent 4 }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + selector: + matchLabels: +{{ $labels | indent 6 }} +{{ tuple $envAll "kubernetes-apiserver-anchor" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }} + template: + metadata: + labels: +{{ $labels | indent 8 }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-kubeconfig-hash: {{ tuple "configmap-kubeconfig.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "kubernetes_apiserver_anchor" "containerNames" (list "anchor") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "kubernetes_apiserver_anchor" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: {{ .Values.labels.common.node_selector_key }} + operator: In + values: + - {{ .Values.labels.common.node_selector_value }} + - key: {{ .Values.labels.masters.node_selector_key }} + operator: DoesNotExist + dnsPolicy: {{ .Values.anchor.dns_policy }} + serviceAccountName: kubeadm + hostNetwork: true + priorityClassName: system-node-critical + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: CriticalAddonsOnly + operator: Exists + terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.kubernetes_apiserver.timeout }} + containers: + - name: anchor + image: {{ .Values.images.tags.anchor }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.anchor_pod | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "kubernetes_apiserver_anchor" "container" "anchor" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NODE_ROLE + value: "worker" + - name: PKI_PATH + value: /host{{ .Values.apiserver.host_etc_path }}/pki + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + command: ["/bin/bash","-c"] + args: ["/tmp/bin/anchor"] + lifecycle: + preStop: + exec: + command: + - /tmp/bin/pre_stop + + readinessProbe: + exec: + command: + - /bin/bash + - -c + - |- + cat /tmp/done + + volumeMounts: + - name: host + mountPath: {{ .Values.host_dir }} + mountPropagation: HostToContainer + - name: {{ .Values.service.name }}-bin + mountPath: /tmp/bin + - name: {{ .Values.service.name }}-kubeconfig + mountPath: /tmp/kubeconfig + - name: kubelet-config + mountPath: /tmp/kubelet + - name: pod-tmp + mountPath: /tmp +{{ if $mounts_kubernetes_apiserver.volumeMounts }}{{ toYaml $mounts_kubernetes_apiserver.volumeMounts | indent 12 }}{{ end }} + volumes: + - name: host + hostPath: + path: / + - name: {{ .Values.service.name }}-bin + configMap: + name: {{ .Values.service.name }}-bin + defaultMode: 0555 + - name: {{ .Values.service.name }}-kubeconfig + configMap: + name: {{ .Values.service.name }}-kubeconfig + defaultMode: 0444 + - name: kubelet-config + configMap: + name: kubelet-config + defaultMode: 0444 + - name: pod-tmp + emptyDir: {} +{{ if $mounts_kubernetes_apiserver.volumes }}{{ toYaml $mounts_kubernetes_apiserver.volumes | indent 8 }}{{ end }} + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + maxSurge: 0 +{{- end }} diff --git a/charts/kubeadm/templates/jobs/_kubelet_restart.yaml.tpl b/charts/kubeadm/templates/jobs/_kubelet_restart.yaml.tpl new file mode 100644 index 00000000..2d7149af --- /dev/null +++ b/charts/kubeadm/templates/jobs/_kubelet_restart.yaml.tpl @@ -0,0 +1,46 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + generateName: kubelet-restart-$NODE_NAME- + labels: + "kubelet-restart": "true" +spec: + template: + metadata: + labels: + "kubelet-restart": "true" + annotations: + container.apparmor.security.beta.kubernetes.io/kubelet-restart: runtime/default + spec: + restartPolicy: Never + serviceAccountName: kubeadm + serviceAccount: kubeadm + hostNetwork: true + enableServiceLinks: true + hostPID: true + hostIPC: true + nodeName: $NODE_NAME + containers: + - name: kubelet-restart + image: {{ .Values.images.tags.anchor }} + imagePullPolicy: Always + resources: + limits: + cpu: '8' + memory: 8Gi + requests: + cpu: 100m + memory: 64Mi + securityContext: + privileged: true + command: + - nsenter + - '--target' + - '1' + - '--mount' + - '--uts' + - '--ipc' + - '--net' + - '--pid' + - kubelet_restart.sh diff --git a/charts/kubeadm/templates/kubeadm/_cluster_config.yaml.tpl b/charts/kubeadm/templates/kubeadm/_cluster_config.yaml.tpl new file mode 100644 index 00000000..d41dba71 --- /dev/null +++ b/charts/kubeadm/templates/kubeadm/_cluster_config.yaml.tpl @@ -0,0 +1,66 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: kubeadm.k8s.io/v1beta4 +kind: ClusterConfiguration +kubernetesVersion: {{ .Values.kubernetes.version }} +apiServer: +{{- if .Values.apiserver.extraArgs }} + extraArgs: +{{ toYaml .Values.apiserver.extraArgs | indent 4 }} +{{- end }} +{{- if .Values.apiserver.extraVolumes }} + extraVolumes: +{{ toYaml .Values.apiserver.extraVolumes | indent 4 }} +{{- end }} +caCertificateValidityPeriod: 87600h0m0s +certificateValidityPeriod: 8760h0m0s +certificatesDir: "/etc/kubernetes/pki" +clusterName: kubernetes +controlPlaneEndpoint: "127.0.0.1:6553" +controllerManager: + extraArgs: + - name: node-monitor-period + value: "5s" + - name: node-monitor-grace-period + value: "20s" + - name: terminated-pod-gc-threshold + value: "1000" + - name: configure-cloud-routes + value: "false" + - name: v + value: "2" + extraVolumes: [] +dns: + disabled: true +encryptionAlgorithm: RSA-2048 +etcd: +{{ toYaml .Values.etcd | indent 2 }} +imageRepository: docker-open-nc.zc1.cti.att.com/upstream-local/kubernetes +networking: + dnsDomain: cluster.local + podSubnet: "10.97.0.0/16" + serviceSubnet: "10.96.0.0/16" +proxy: + disabled: true +scheduler: + extraArgs: + - name: secure-port + value: "10259" + - name: v + value: "2" + extraVolumes: + - hostPath: "/etc/kubernetes/pki" + mountPath: "/etc/kubernetes/pki" + name: pki-certs + pathType: DirectoryOrCreate diff --git a/charts/kubeadm/templates/kubeadm/_join_config.yaml.tpl b/charts/kubeadm/templates/kubeadm/_join_config.yaml.tpl new file mode 100644 index 00000000..a79a46b7 --- /dev/null +++ b/charts/kubeadm/templates/kubeadm/_join_config.yaml.tpl @@ -0,0 +1,43 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: kubeadm.k8s.io/v1beta4 +kind: JoinConfiguration +caCertPath: /etc/kubernetes/pki/ca.crt +controlPlane: + localAPIEndpoint: + advertiseAddress: "${HOST_IP}" + bindPort: {{ .Values.network.kubernetes_apiserver.port }} +discovery: + file: + kubeConfigPath: /etc/kubernetes/admin.conf +nodeRegistration: + imagePullPolicy: Always + imagePullSerial: false + name: ${NODE_NAME} + ignorePreflightErrors: + - Service-Kubelet + - FileAvailable--etc-kubernetes-kubelet.conf + - SystemVerification + - Port-10250 + - Port-6443 + - Port-10259 + - Port-10257 + - Port-2379 + - Port-2380 + - DirAvailable--var-lib-etcd + taints: [] +skipPhases: +{{ toYaml .Values.kubeadm.join_config.skip_phases }} +patches: + directory: /etc/kubernetes/kubeadm/patches diff --git a/charts/kubeadm/templates/kubeadm/_upgrade_config.yaml.tpl b/charts/kubeadm/templates/kubeadm/_upgrade_config.yaml.tpl new file mode 100644 index 00000000..72fbd413 --- /dev/null +++ b/charts/kubeadm/templates/kubeadm/_upgrade_config.yaml.tpl @@ -0,0 +1,29 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: kubeadm.k8s.io/v1beta4 +kind: UpgradeConfiguration +node: + certificateRenewal: false + dryRun: false + etcdUpgrade: false + ignorePreflightErrors: + - SystemVerification + - CoreDNSUnsupportedPlugins + - CoreDNSMigration + patches: + directory: /etc/kubernetes/kubeadm/patches + skipPhases: +{{ toYaml .Values.kubeadm.upgrade_config.skip_phases | indent 2 }} + imagePullPolicy: Always + imagePullSerial: false diff --git a/charts/kubeadm/templates/kubeconfig/_kubeconfig.yaml.tpl b/charts/kubeadm/templates/kubeconfig/_kubeconfig.yaml.tpl new file mode 100644 index 00000000..b28ce563 --- /dev/null +++ b/charts/kubeadm/templates/kubeconfig/_kubeconfig.yaml.tpl @@ -0,0 +1,34 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v1 +clusters: +- cluster: + server: https://${CLUSTER_IP}:${CLUSTER_PORT} + certificate-authority: ${CERT_AUTH} + name: ${CLUSTER_NAME} +contexts: +- context: + cluster: ${CLUSTER_NAME} + user: ${USER} + name: ${USER}@${CLUSTER_NAME} +current-context: ${USER}@${CLUSTER_NAME} +kind: Config +preferences: {} +users: +- name: ${USER} + user: + client-certificate: ${CLIENT_CERT} + client-key: ${CLIENT_KEY} diff --git a/charts/kubeadm/templates/kubelet/_kubelet-service.yaml.tpl b/charts/kubeadm/templates/kubelet/_kubelet-service.yaml.tpl new file mode 100644 index 00000000..9b2820a1 --- /dev/null +++ b/charts/kubeadm/templates/kubelet/_kubelet-service.yaml.tpl @@ -0,0 +1,35 @@ +# Copyright 2025 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[Unit] +Description=Kubernetes Worker Process +Requires=network-online.target +After=network-online.target containerd.service + +[Service] +Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf" +Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" +# This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically +EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env +# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use +# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file. +EnvironmentFile=-/etc/default/kubelet +ExecStart=/usr/local/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS +Restart=always +StartLimitInterval=0 +RestartSec=10 +KillMode=process + +[Install] +WantedBy=multi-user.target diff --git a/charts/kubeadm/templates/kubelet/_kubelet.yaml.tpl b/charts/kubeadm/templates/kubelet/_kubelet.yaml.tpl new file mode 100644 index 00000000..962eef30 --- /dev/null +++ b/charts/kubeadm/templates/kubelet/_kubelet.yaml.tpl @@ -0,0 +1,23 @@ +# Copyright 2025 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +KUBELET_EXTRA_ARGS=" +--hostname-override=${NODE_NAME} +--node-ip=${HOST_IP} +{{- if .Values.kubelet.arguments }} +{{- range $arg := .Values.kubelet.arguments }} +{{ $arg }} +{{- end }} +{{- end }} +" diff --git a/charts/kubeadm/templates/kubelet/_kubelet_config.yaml.tpl b/charts/kubeadm/templates/kubelet/_kubelet_config.yaml.tpl new file mode 100644 index 00000000..f94345da --- /dev/null +++ b/charts/kubeadm/templates/kubelet/_kubelet_config.yaml.tpl @@ -0,0 +1,33 @@ +# Copyright 2025 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +authentication: + anonymous: + enabled: false + webhook: + enabled: true + x509: + clientCAFile: "/etc/kubernetes/pki/kubelet-client-ca.pem" +authorization: + mode: AlwaysAllow +clusterDNS: + - {{ .Values.network.dns.service_ip }} +clusterDomain: {{ .Values.network.dns.cluster_domain }} +staticPodPath: "/etc/kubernetes/manifests" +tlsCertFile: "/etc/kubernetes/pki/kubelet.pem" +tlsPrivateKeyFile: "/etc/kubernetes/pki/kubelet-key.pem" +{{ if .Values.kubelet.config_file_overrides }} +{{- toYaml .Values.kubelet.config_file_overrides }} +{{- end }} diff --git a/charts/kubeadm/templates/rbac.yaml b/charts/kubeadm/templates/rbac.yaml new file mode 100644 index 00000000..bd445068 --- /dev/null +++ b/charts/kubeadm/templates/rbac.yaml @@ -0,0 +1,67 @@ +{{/* +Copyright 2018 AT&T Intellectual Property. All other rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.rbac }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubeadm + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kubeadm + namespace: kube-system +rules: + - apiGroups: [""] + resources: + - nodes + - pods + - pods/eviction + verbs: + - "*" + - apiGroups: [""] + resources: + - configmaps + verbs: + - "*" + - apiGroups: + - "" + - apps + resources: + - daemonsets + verbs: + - "*" + - apiGroups: [ "batch" ] + resources: [ "jobs" ] + verbs: [ "*" ] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kubeadm + namespace: kube-system +subjects: + - kind: ServiceAccount + name: kubeadm + namespace: kube-system +roleRef: + kind: ClusterRole + name: kubeadm + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/charts/kubeadm/values.yaml b/charts/kubeadm/values.yaml new file mode 100644 index 00000000..6104dcf3 --- /dev/null +++ b/charts/kubeadm/values.yaml @@ -0,0 +1,484 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +release_group: null + +kubernetes: + version: v1.33.3 + +host_dir: "/host" + +kubeconfig: + cluster_name: kubernetes + cluster_ip: "127.0.0.1" + cluster_port: 6553 + configs: + admin: + path: /etc/kubernetes/admin.conf + ca: /etc/kubernetes/admin/pki/cluster-ca.pem + client_cert: /etc/kubernetes/admin/pki/admin.pem + client_key: /etc/kubernetes/admin/pki/admin-key.pem + controller-manager: + path: /etc/kubernetes/controller-manager.conf + ca: pki/ca.crt + client_cert: pki/controller-manager.pem + client_key: pki/controller-manager-key.pem + kubelet: + path: /etc/kubernetes/kubelet.conf + ca: /etc/kubernetes/pki/kubelet-client-ca.pem + client_cert: /etc/kubernetes/pki/kubelet.pem + client_key: /etc/kubernetes/pki/kubelet-key.pem + scheduler: + path: /etc/kubernetes/scheduler.conf + ca: pki/ca.crt + client_cert: pki/scheduler.pem + client_key: pki/scheduler-key.pem + +# NOTE(mark-burnett): These values are not really configurable -- they live +# here to keep the templates cleaner. +const: + + files_to_copy: {} + # NOTE(mark-burnett): These are (host dest): (container source) pairs + + files_to_delete: + - /etc/kubernetes/manifests/kubernetes-apiserver.yaml + - /etc/kubernetes/manifests/kubernetes-controller-manager.yaml + - /etc/kubernetes/manifests/kubernetes-scheduler.yaml + - /etc/kubernetes/kubelet + - /etc/kubernetes/controller-manager + - /etc/kubernetes/scheduler + - /etc/kubernetes/apiserver/apiserver-proxy-cert.pem + - /etc/kubernetes/apiserver/apiserver-proxy-key.pem + - /etc/kubernetes/apiserver/agg-api-ca.pem + - /etc/kubernetes/apiserver/kubeconfig + - /etc/kubernetes/apiserver/pki + - /etc/kubernetes/kubeconfig + - /etc/kubernetes/pki/cluster-ca.pem + - /opt/kubernetes/bin/kubelet + +images: + tags: + dep_check: docker-open-nc.zc1.cti.att.com/airship/kubernetes-entrypoint@sha256:9c43e6965e8cdac750c256cbf161d8ab6549d6f7e4fbf08d73d2de78174e2e94 + anchor: docker-open-nc.zc1.cti.att.com/airship/porthole/kubeadm-utility@sha256:fec3c6ed2834fc5cf88c031a4e3705286ff33c1d917c578c3c02ccbb81d0a36c + apiserver: registry.k8s.io/kube-apiserver-amd64:v1.32.1 + key_rotate: quay.io/airshipit/porthole-compute-utility:latest-ubuntu_focal + pull_policy: "IfNotPresent" + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +labels: + common: + node_selector_key: kubeadm + node_selector_value: enabled + masters: + node_selector_key: kubernetes-apiserver + node_selector_value: enabled + etcd: + node_selector_key: kubernetes-etcd + node_selector_value: enabled + job: + node_selector_key: kubelet-restart + node_selector_value: true + +anchor: + dns_policy: Default + enable_cleanup: true + kubelet: + manifest_path: /etc/kubernetes/manifests + period: 15 + +# TODO(sh8121att): Add dynamic rendering of the admission controller list allowing a base list +# and each conf entry to enable additional AC plugins +conf: {} + +kubeadm: + cluster_config: + apiVersion: kubeadm.k8s.io/v1beta4 + kind: ClusterConfiguration + + + join_config: + skip_phases: + - preflight + - control-plane-prepare/download-certs + - control-plane-prepare/certs + - control-plane-prepare/kubeconfig + - kubelet-start + - control-plane-join + - wait-control-plane + + upgrade_config: + skip_phases: + - kubelet-config + - addon + + patches: + kube-apiserver.yaml: + spec: + containers: + - name: kube-apiserver + resources: + limits: + cpu: '8' + memory: 8Gi + requests: + cpu: 100m + memory: 128Mi + kube-controller-manager.yaml: + spec: + containers: + - name: kube-controller-manager + resources: + limits: + cpu: '8' + memory: 8Gi + requests: + cpu: 100m + memory: 128Mi + kube-scheduler.yaml: + spec: + containers: + - name: kube-scheduler + resources: + limits: + cpu: '8' + memory: 8Gi + requests: + cpu: 100m + memory: 128Mi + +etcd: + external: + endpoints: + - "https://172.29.0.11:2379" + - "https://172.29.0.12:2379" + - "https://172.29.0.17:2379" + caFile: "/etc/kubernetes/pki/etcd/ca.crt" + certFile: "/etc/kubernetes/pki/etcd/healthcheck-client.crt" + keyFile: "/etc/kubernetes/pki/etcd/healthcheck-client.key" + +apiserver: + extraArgs: + - name: admission-control-config-file + value: "/etc/kubernetes/apiserver/acconfig.yaml" + - name: anonymous-auth + value: "true" + - name: audit-log-maxbackup + value: "10" + - name: audit-log-maxsize + value: "100" + - name: audit-log-path + value: "/var/log/kubernetes/apiserver/kube-apiserver-audit.log" + - name: audit-policy-file + value: "/etc/kubernetes/apiserver/audit-policy.yaml" + - name: bind-address + value: "0.0.0.0" + - name: enable-admission-plugins + value: "PodSecurity,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit" + - name: encryption-provider-config + value: "/etc/kubernetes/apiserver/encryption_provider.yaml" + - name: kubelet-certificate-authority + value: "/etc/kubernetes/pki/ca.crt" + - name: requestheader-allowed-names + value: "aggregator" + - name: service-node-port-range + value: "30000-32767" + - name: tls-cipher-suites + value: "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA" + - name: tls-min-version + value: "VersionTLS12" + - name: v + value: "2" + extraVolumes: + - hostPath: "/etc/kubernetes/apiserver" + mountPath: "/etc/kubernetes/apiserver" + name: apiserver + pathType: DirectoryOrCreate + - hostPath: "/var/log/kubernetes/apiserver" + mountPath: "/var/log/kubernetes/apiserver" + name: audit-logs + pathType: DirectoryOrCreate + etcd: + endpoints: https://kubernetes-etcd.kube-system.svc.cluster.local + host_etc_path: /etc/kubernetes/apiserver + config_files: {} + logging: + # Which messages to log. + # Valid values include any number from 0 to 9. + # Default 5(Trace level verbosity). + log_level: 5 +#XXX another possible configuration +# tls: +# tls-cipher-suites: "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA" +# # https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/ +# #Possible values: VersionTLS10, VersionTLS11, VersionTLS12 +# tls-min-version: 'VersionTLS12' + +kubelet: + arguments: + - --maximum-dead-containers-per-container=1 + config: + apiVersion: kubelet.config.k8s.io/v1beta1 + kind: KubeletConfiguration + restart: true + config_file_overrides: + cgroupDriver: systemd + cgroupRoot: /kube_whitelist + containerLogMaxFiles: 2 + containerRuntimeEndpoint: unix:///run/containerd/containerd.sock + eventRecordQPS: 50 + evictionMaxPodGracePeriod: -1 + kubeAPIBurst: 40 + kubeAPIQPS: 20 + maxPods: 250 + nodeStatusUpdateFrequency: 5s + podPidsLimit: 10000 + podsPerCore: 10 + readOnlyPort: 0 + runtimeRequestTimeout: 15m + seccompDefault: true + tlsCipherSuites: + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA + - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA + - TLS_RSA_WITH_AES_128_GCM_SHA256 + - TLS_RSA_WITH_AES_256_GCM_SHA384 + - TLS_RSA_WITH_AES_128_CBC_SHA + - TLS_RSA_WITH_AES_256_CBC_SHA + + files_to_copy: + /etc/default/kubelet: /tmp/kubelet/kubelet-defaults + /etc/systemd/system/kubelet.service: /tmp/kubelet/kubelet-service + /var/lib/kubelet/config: /tmp/kubelet/kubelet-config + +network: + dns: + service_ip: "10.96.0.10" + cluster_domain: "cluster.local" + kubernetes_apiserver: + ingress: + public: true + classes: + namespace: "nginx-cluster" + cluster: "nginx-cluster" + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + nginx.ingress.kubernetes.io/proxy-read-timeout: "120" + nginx.ingress.kubernetes.io/ssl-redirect: "true" + nginx.ingress.kubernetes.io/secure-backends: "true" + name: kubernetes-apiserver + port: 6443 + node_port: + enabled: false + port: 31943 + +service: + name: kubeadm-service + ip: null + +secrets: + ca: + crt: placeholder + key: placeholder + + sa: + key: placeholder + pub: placeholder + + front_proxy_ca: + crt: placeholder + key: placeholder + + front_proxy_client: + crt: placeholder + key: placeholder + + apiserver: + crt: placeholder + key: placeholder + etcd_client: + crt: placeholder + key: placeholder + kubelet_client: + crt: placeholder + key: placeholder + + controller_manager: + crt: placeholder + key: placeholder + + scheduler: + crt: placeholder + key: placeholder + + etcd: + ca: + crt: placeholder + healthcheck_client: + crt: placeholder + key: placeholder + + kubelet: {} + +# typically overriden by environmental +# values, but should include all endpoints +# required by this chart +endpoints: + cluster_domain_suffix: cluster.local + kubernetes_apiserver: + name: kubernetes-apiserver + hosts: + default: kubernetes-apiserver + port: + https: + default: 6443 + public: 443 + path: + default: / + scheme: + default: https + public: https + host_fqdn_override: + default: null + # NOTE: this chart supports TLS for fqdn over-ridden public + # endpoints using the following format: + # public: + # host: null + # tls: + # crt: null + # key: null + +pod: + mandatory_access_control: + type: apparmor + kubernetes_apiserver_anchor: + anchor: runtime/default + kube-apiserver: + init: runtime/default + apiserver-key-rotate: runtime/default + apiserver: + apiserver: runtime/default + security_context: + kubernetes_apiserver_anchor: + pod: + runAsUser: 65534 + container: + anchor: + runAsUser: 0 + readOnlyRootFilesystem: false + key_rotate: + pod: + runAsUser: 65534 + container: + apiserver_key_rotate: + runAsUser: 0 + readOnlyRootFilesystem: false + apiserver: + pod: + runAsUser: 65534 + container: + apiserver: + runAsUser: 0 + readOnlyRootFilesystem: false + mounts: + # .pod.mounts.kubernetes_apiserver is for the anchor daemonset + kubernetes_apiserver: + init_container: null + kubernetes_apiserver: + # .pod.mounts.apiserver is for the apiserver static pod + apiserver: + apiserver: +# Example mounts for audit logging, refer to .conf.auditpolicy above. +# volumeMounts: +# - name: audit-logs +# mountPath: /var/log/audit +# mountPropagation: HostToContainer +# readOnly: false +# volumes: +# - name: audit-logs +# hostPath: +# path: /var/log/audit +# type: DirectoryOrCreate + lifecycle: + upgrades: + daemonsets: + pod_replacement_strategy: RollingUpdate + kubernetes-apiserver-anchor: + enabled: false + min_ready_seconds: 0 + max_unavailable: 1 + termination_grace_period: + kubernetes_apiserver: + timeout: 3600 + resources: + enabled: true + anchor_pod: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + key_rotate: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + kubernetes_apiserver: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + probes: + apiserver: + apiserver: + liveness: + enabled: true + params: + failureThreshold: 3 + initialDelaySeconds: 60 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 10 + readiness: + enabled: true + params: + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 5 + env: + apiserver: + +manifests: + configmap_apiserver_misc: true + configmap_bin: true + configmap_certs: true + configmap_kubeconfig: true + configmap_jobs: true + configmap_kubelet: true + configmap_kubeadm: true + configmap_patches: true + daemonset_masters: true + daemonset_workers: true + rbac: true diff --git a/promenade/control/join_scripts.py b/promenade/control/join_scripts.py index 1a9fa595..3b647dcb 100644 --- a/promenade/control/join_scripts.py +++ b/promenade/control/join_scripts.py @@ -14,7 +14,6 @@ import falcon import kubernetes -import re from promenade.control.base import BaseResource from promenade.builder import Builder @@ -49,7 +48,6 @@ class JoinScriptsResource(BaseResource): static_labels = _get_param_list(req, 'labels.static') join_ips = _get_join_ips() - role = _get_role(hostname) try: config = Configuration.from_design_ref( @@ -80,7 +78,6 @@ class JoinScriptsResource(BaseResource): 'ip': ip, 'external_ip': external_ip, 'join_ips': join_ips, - 'role': role, 'labels': { 'dynamic': dynamic_labels, 'static': static_labels, @@ -106,14 +103,6 @@ def _get_join_ips(): return list(map(_extract_ip, response.items)) -def _get_role(hostname): - pattern = re.compile(r'^[a-z]{3}\d{2}r\d{2}(?P[co])\d{3}$') - m = pattern.match(hostname) - if not m: - return "unknown" - return "controller" if m.group("role") == "o" else "worker" - - def _extract_ip(item): for address in item.status.addresses: if address.type == 'InternalIP': diff --git a/promenade/templates/include/up.sh b/promenade/templates/include/up.sh index 310c53d2..ab0ea3ae 100644 --- a/promenade/templates/include/up.sh +++ b/promenade/templates/include/up.sh @@ -153,5 +153,8 @@ fi if systemctl -q is-enabled containerd > /dev/null 2>&1; then systemctl restart containerd || true fi -systemctl enable kubelet -systemctl restart kubelet + +# Start haproxy and kubelet +haproxy & +sleep 10 +systemctl enable --now kubelet diff --git a/promenade/templates/include/utils.sh b/promenade/templates/include/utils.sh index acb9df57..81fe8ac8 100644 --- a/promenade/templates/include/utils.sh +++ b/promenade/templates/include/utils.sh @@ -187,7 +187,9 @@ function register_annotations { NODE=$1 SEC=$2 shift 2 + ANNOTATIONS="$@" + end=$(($(date +%s) + $SEC)) while true; do if kubectl annotate node --overwrite $NODE $ANNOTATIONS ; then diff --git a/promenade/templates/roles/common/etc/default/kubelet b/promenade/templates/roles/common/etc/default/kubelet index cf1e00dc..45ea6c6f 100644 --- a/promenade/templates/roles/common/etc/default/kubelet +++ b/promenade/templates/roles/common/etc/default/kubelet @@ -1,6 +1,17 @@ -{%- if 'kubeadm-enabled=true' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %} -KUBELET_EXTRA_ARGS="" -{%- else %} +# Copyright 2025 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + KUBELET_EXTRA_ARGS=" --hostname-override={{ config.get_first('Genesis:hostname', 'KubernetesNode:hostname') }} --node-ip={{ config.get_first('Genesis:ip', 'KubernetesNode:ip') }} @@ -9,10 +20,15 @@ KUBELET_EXTRA_ARGS=" {%- elif config['KubernetesNode:labels.static'] is defined %} --node-labels={{ config['KubernetesNode:labels.static'] | join(',') }} {%- endif %} +{%- if 'kubeadm=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %} +{%- for argument in config.get(schema='armada/Chart/v1', name='kubernetes-kubeadm', jsonpath='values.kubelet.arguments', default=[]) %} +{{ argument }} +{%- endfor %} +{%- else %} {%- if config['Kubelet:arguments'] is defined %} {%- for argument in config['Kubelet:arguments'] %} {{ argument }} {%- endfor %} {%- endif %} -" {%- endif %} +" diff --git a/promenade/templates/roles/common/etc/kubernetes/admin.conf b/promenade/templates/roles/common/etc/kubernetes/admin.conf new file mode 100644 index 00000000..ec37c57a --- /dev/null +++ b/promenade/templates/roles/common/etc/kubernetes/admin.conf @@ -0,0 +1,22 @@ +{%- if 'kubeadm=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %} +--- +apiVersion: v1 +clusters: +- cluster: + server: https://127.0.0.1:6553 + certificate-authority: /etc/kubernetes/admin/pki/cluster-ca.pem + name: kubernetes +contexts: +- context: + cluster: kubernetes + user: admin + name: admin@kubernetes +current-context: admin@kubernetes +kind: Config +preferences: {} +users: +- name: admin + user: + client-certificate: /etc/kubernetes/admin/pki/admin.pem + client-key: /etc/kubernetes/admin/pki/admin-key.pem +{%- endif %} diff --git a/promenade/templates/roles/common/etc/kubernetes/apiserver/acconfig.yaml b/promenade/templates/roles/common/etc/kubernetes/apiserver/acconfig.yaml new file mode 100644 index 00000000..d837d99c --- /dev/null +++ b/promenade/templates/roles/common/etc/kubernetes/apiserver/acconfig.yaml @@ -0,0 +1,7 @@ +{%- if 'kubeadm=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') and 'kubernetes-apiserver=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %}--- +kind: AdmissionConfiguration +apiVersion: apiserver.k8s.io/v1alpha1 +plugins: + - name: EventRateLimit + path: eventconfig.yaml +{%- endif %} \ No newline at end of file diff --git a/promenade/templates/roles/common/etc/kubernetes/apiserver/audit-policy.yaml b/promenade/templates/roles/common/etc/kubernetes/apiserver/audit-policy.yaml new file mode 100644 index 00000000..ce4ee342 --- /dev/null +++ b/promenade/templates/roles/common/etc/kubernetes/apiserver/audit-policy.yaml @@ -0,0 +1 @@ +{%- if 'kubeadm=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') and 'kubernetes-apiserver=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %}{{ config.get(schema='nc/AuditPolicy/v1', name='k8s-audit-policy') | toyaml | trim }}{%- endif %} diff --git a/promenade/templates/roles/common/etc/kubernetes/apiserver/encryption_provider.yaml b/promenade/templates/roles/common/etc/kubernetes/apiserver/encryption_provider.yaml new file mode 100644 index 00000000..e46c3775 --- /dev/null +++ b/promenade/templates/roles/common/etc/kubernetes/apiserver/encryption_provider.yaml @@ -0,0 +1,6 @@ +{%- if 'kubeadm=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') and 'kubernetes-apiserver=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %}--- +kind: EncryptionConfiguration +apiVersion: apiserver.config.k8s.io/v1 +resources: +{{ config.get_path('EncryptionPolicy:etcd', {}) | toyaml | indent(2, true) }} +{%- endif %} diff --git a/promenade/templates/roles/common/etc/kubernetes/apiserver/eventconfig.yaml b/promenade/templates/roles/common/etc/kubernetes/apiserver/eventconfig.yaml new file mode 100644 index 00000000..fbcb4f1c --- /dev/null +++ b/promenade/templates/roles/common/etc/kubernetes/apiserver/eventconfig.yaml @@ -0,0 +1,8 @@ +{%- if 'kubeadm=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') and 'kubernetes-apiserver=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %}--- +kind: Configuration +apiVersion: eventratelimit.admission.k8s.io/v1alpha1 +limits: + - type: Server + qps: 1000 # reduce to 100 + burst: 10000 # reduce to 1000 +{%- endif %} diff --git a/promenade/templates/roles/common/etc/kubernetes/controller-manager.conf b/promenade/templates/roles/common/etc/kubernetes/controller-manager.conf new file mode 100644 index 00000000..4a67cd50 --- /dev/null +++ b/promenade/templates/roles/common/etc/kubernetes/controller-manager.conf @@ -0,0 +1,21 @@ +{%- if 'kubeadm=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') and 'kubernetes-apiserver=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %}--- +apiVersion: v1 +clusters: +- cluster: + server: https://127.0.0.1:6553 + certificate-authority: pki/ca.crt + name: kubernetes +contexts: +- context: + cluster: kubernetes + user: controller-manager + name: controller-manager@kubernetes +current-context: controller-manager@kubernetes +kind: Config +preferences: {} +users: +- name: controller-manager + user: + client-certificate: pki/controller-manager.pem + client-key: pki/controller-manager-key.pem +{%- endif %} diff --git a/promenade/templates/roles/common/etc/kubernetes/kubeconfig b/promenade/templates/roles/common/etc/kubernetes/kubelet.conf similarity index 87% rename from promenade/templates/roles/common/etc/kubernetes/kubeconfig rename to promenade/templates/roles/common/etc/kubernetes/kubelet.conf index 85a93672..43fa7bba 100644 --- a/promenade/templates/roles/common/etc/kubernetes/kubeconfig +++ b/promenade/templates/roles/common/etc/kubernetes/kubelet.conf @@ -3,7 +3,7 @@ apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6553 - certificate-authority: pki/cluster-ca.pem + certificate-authority: pki/kubelet-client-ca.pem name: kubernetes contexts: - context: diff --git a/promenade/templates/roles/common/etc/kubernetes/kubelet/config.yaml b/promenade/templates/roles/common/etc/kubernetes/kubelet/config.yaml index 569e8b06..96fac0c9 100644 --- a/promenade/templates/roles/common/etc/kubernetes/kubelet/config.yaml +++ b/promenade/templates/roles/common/etc/kubernetes/kubelet/config.yaml @@ -1,3 +1 @@ -{%- if 'kubeadm-enabled=true' not in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %} -{% include "kubelet-config.yaml" with context %} -{%- endif %} \ No newline at end of file +{%- if 'kubeadm=enabled' not in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %}{% include "kubelet-config.yaml" with context %}{%- endif %} diff --git a/promenade/templates/roles/common/etc/kubernetes/pki/cluster-ca.pem b/promenade/templates/roles/common/etc/kubernetes/pki/cluster-ca.pem deleted file mode 100644 index 0c922732..00000000 --- a/promenade/templates/roles/common/etc/kubernetes/pki/cluster-ca.pem +++ /dev/null @@ -1 +0,0 @@ -{{ config.get(schema='deckhand/CertificateAuthority/v1', name='kubernetes') }} diff --git a/promenade/templates/roles/common/etc/kubernetes/scheduler.conf b/promenade/templates/roles/common/etc/kubernetes/scheduler.conf new file mode 100644 index 00000000..1d1140ad --- /dev/null +++ b/promenade/templates/roles/common/etc/kubernetes/scheduler.conf @@ -0,0 +1,21 @@ +{%- if 'kubeadm=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') and 'kubernetes-apiserver=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %}--- +apiVersion: v1 +clusters: +- cluster: + server: https://127.0.0.1:6553 + certificate-authority: pki/ca.crt + name: kubernetes +contexts: +- context: + cluster: kubernetes + user: scheduler + name: scheduler@kubernetes +current-context: scheduler@kubernetes +kind: Config +preferences: {} +users: +- name: scheduler + user: + client-certificate: pki/scheduler.pem + client-key: pki/scheduler-key.pem +{%- endif %} diff --git a/promenade/templates/roles/common/etc/systemd/system/kubelet.service b/promenade/templates/roles/common/etc/systemd/system/kubelet.service index 4f7e3399..52598528 100644 --- a/promenade/templates/roles/common/etc/systemd/system/kubelet.service +++ b/promenade/templates/roles/common/etc/systemd/system/kubelet.service @@ -1,14 +1,28 @@ +# Copyright 2025 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + [Unit] Description=Kubernetes Worker Process Requires=network-online.target After=network-online.target containerd.service [Service] -{%- if 'kubeadm-enabled=true' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %} +{%- if 'kubeadm=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %} Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf" Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" {%- else %} -Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubeconfig" +Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf" Environment="KUBELET_CONFIG_ARGS=--config=/etc/kubernetes/kubelet/config.yaml" {%- endif %} # This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically diff --git a/promenade/templates/roles/common/usr/local/bin/haproxy b/promenade/templates/roles/common/usr/local/bin/haproxy index 415baf24..88945cbb 100755 --- a/promenade/templates/roles/common/usr/local/bin/haproxy +++ b/promenade/templates/roles/common/usr/local/bin/haproxy @@ -1,17 +1,19 @@ #!/bin/bash -set -exuo pipefail +set -euo pipefail IMAGE_HAPROXY="{{ config['HostSystem:images.haproxy'] }}" while true; do - if systemctl is-active kubelet && [[ -e /etc/kubernetes/manifests/haproxy.yaml ]]; then + if systemctl -q is-active kubelet && [[ -e /etc/kubernetes/manifests/haproxy.yaml ]]; then if docker ps | grep -q haproxy; then + echo "Stopping auxiliary haproxy..." docker stop -t 10 haproxy fi break else if ! docker ps | grep -q haproxy; then + echo "Starting auxiliary haproxy..." docker run --rm -d --net host -v /etc/promenade/haproxy:/usr/local/etc/haproxy:ro \ --name haproxy "$IMAGE_HAPROXY" fi diff --git a/promenade/templates/roles/common/var/lib/kubelet/config.yaml b/promenade/templates/roles/common/var/lib/kubelet/config.yaml new file mode 100644 index 00000000..e5239d8f --- /dev/null +++ b/promenade/templates/roles/common/var/lib/kubelet/config.yaml @@ -0,0 +1,3 @@ +{% if 'kubeadm=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') -%} +{{- config.get(schema='armada/Chart/v1', name='kubernetes-kubeadm', jsonpath='values.kubelet.config') | toyaml | trim }} +{% endif -%} diff --git a/promenade/templates/roles/genesis/etc/kubernetes/kubeadm/init-config.yaml b/promenade/templates/roles/genesis/etc/kubernetes/kubeadm/init-config.yaml new file mode 100644 index 00000000..2d491a22 --- /dev/null +++ b/promenade/templates/roles/genesis/etc/kubernetes/kubeadm/init-config.yaml @@ -0,0 +1,39 @@ +{% if 'kubeadm=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') -%} +apiVersion: kubeadm.k8s.io/v1beta4 +bootstrapTokens: [] +kind: InitConfiguration +localAPIEndpoint: + advertiseAddress: "{{ config['Genesis:ip'] }}" + bindPort: 6443 +nodeRegistration: + ignorePreflightErrors: + - IsPrivilegedUser + - ExternalEtcdVersion + - Port-10250 + imagePullPolicy: IfNotPresent + imagePullSerial: false + name: "{{ config['Genesis:hostname'] }}" + criSocket: "unix:///run/containerd/containerd.sock" + taints: [] +skipPhases: +- certs +- kubeconfig +- etcd +- kubelet-start +- upload-config +- upload-certs +- mark-control-plane +- kubelet-finalize +- addon +- show-join-command +timeouts: + controlPlaneComponentHealthCheck: 4m0s + discovery: 5m0s + etcdAPICall: 2m0s + kubeletHealthCheck: 4m0s + kubernetesAPICall: 1m0s + tlsBootstrap: 5m0s + upgradeManifests: 5m0s +--- +{{ config.get(schema='armada/Chart/v1', name='kubernetes-kubeadm', jsonpath='values.kubeadm.cluster_config') | toyaml | trim }} +{%- endif %} diff --git a/promenade/templates/roles/join/etc/kubernetes/kubeadm/join-config.yaml b/promenade/templates/roles/join/etc/kubernetes/kubeadm/join-config.yaml new file mode 100644 index 00000000..e164f3b9 --- /dev/null +++ b/promenade/templates/roles/join/etc/kubernetes/kubeadm/join-config.yaml @@ -0,0 +1,36 @@ +{%- if 'kubeadm=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %}--- +apiVersion: kubeadm.k8s.io/v1beta4 +caCertPath: /etc/kubernetes/pki/ca.crt +{%- if 'kubernetes-apiserver=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %} +controlPlane: + localAPIEndpoint: + advertiseAddress: "{{ config['KubernetesNode:ip'] }}" + bindPort: 6443 +{%- endif %} +discovery: + file: + kubeConfigPath: /etc/kubernetes/admin.conf +kind: JoinConfiguration +nodeRegistration: + imagePullPolicy: IfNotPresent + imagePullSerial: false + name: {{ config['KubernetesNode:hostname'] }} + criSocket: "unix:///run/containerd/containerd.sock" + ignorePreflightErrors: + - FileAvailable--etc-kubernetes-kubelet.conf + - Port-10250 + taints: [] +skipPhases: +{%- if 'kubernetes-apiserver=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %} + - control-plane-prepare/download-certs + - control-plane-prepare/certs + - control-plane-prepare/kubeconfig + - kubelet-start + - control-plane-join +{%- else %} + - control-plane-prepare + - kubelet-start + - control-plane-join + - wait-control-plane +{%- endif %} +{%- endif %} diff --git a/promenade/templates/scripts/genesis.sh b/promenade/templates/scripts/genesis.sh index 0b42fecb..e4514527 100644 --- a/promenade/templates/scripts/genesis.sh +++ b/promenade/templates/scripts/genesis.sh @@ -4,7 +4,14 @@ {% include "up.sh" with context %} -haproxy & +{%- if 'kubeadm=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %} +set +x +log +log === Bootstrapping node using kubeadm === +set -x +kubeadm init --config /etc/kubernetes/kubeadm/init-config.yaml --v=5 2>&1 | tee /var/log/kubeadm.log +register_annotations {{ config['Genesis:hostname'] }} 3600 "kubeadm.alpha.kubernetes.io/cri-socket={{ config.get(schema='armada/Chart/v1', name='kubernetes-kubeadm', jsonpath='values.kubelet.config.containerRuntimeEndpoint') }}" +{%- endif %} mkdir -p /var/log/armada touch /var/log/armada/bootstrap-armada.log @@ -38,7 +45,7 @@ set +x log log === Deploying bootstrap manifest via Armada === if [[ ! -e /etc/kubernetes/manifests/bootstrap-armada.yaml ]]; then -touch /etc/kubernetes/manifests/bootstrap-armada.yaml + touch /etc/kubernetes/manifests/bootstrap-armada.yaml fi {%- if config['Genesis:armada_helm_bootstrap'] is sameas true %} helm armadachart-install armada-bootstrap /etc/genesis/armada/assets/manifest.yaml @@ -73,20 +80,15 @@ while true; do break fi done -set -x -# Terminate background job (tear down exit trap?) -kill %1 -{%- if config['Genesis:enable_operator'] is sameas true %} -kill %2 -{%- endif %} - -set +x log log === Waiting for Node to be Ready === set -x wait_for_node_ready {{ config['Genesis:hostname'] }} 3600 +# Terminate background job (tear down exit trap?) +kill $(jobs -p) + {% include "cleanup.sh" with context %} set +x diff --git a/promenade/templates/scripts/join.sh b/promenade/templates/scripts/join.sh index f574e83c..f52c88ea 100644 --- a/promenade/templates/scripts/join.sh +++ b/promenade/templates/scripts/join.sh @@ -17,8 +17,6 @@ trap delete_kubectl EXIT {% include "up.sh" with context %} -haproxy & - set +x log log === Waiting for Node to be Ready === @@ -34,6 +32,7 @@ register_labels {{ config['KubernetesNode:hostname'] }} 3600 {{ config['Kubernet {%- endif %} sleep 60 +wait $(jobs -p) set +x log diff --git a/tools/helm_install.sh b/tools/helm_install.sh index bab8ef59..77abbae9 100755 --- a/tools/helm_install.sh +++ b/tools/helm_install.sh @@ -17,7 +17,8 @@ set -x HELM=$1 -HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.17.3-linux-amd64.tar.gz"} +OS=$(uname -s | awk '{print tolower($0)}') +HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.17.3-$OS-amd64.tar.gz"} function install_helm_binary { @@ -33,7 +34,7 @@ function install_helm_binary { curl -o "${TMP_DIR}/helm.tar.gz" "${HELM_ARTIFACT_URL}" cd ${TMP_DIR} tar -xvzf helm.tar.gz - cp "${TMP_DIR}/linux-amd64/helm" "${HELM}" + cp "${TMP_DIR}/$OS-amd64/helm" "${HELM}" else echo "Cannot write to ${HELM}" exit -1