Use kubeadm for bootstrap k8s

Change-Id: I6e441b2e793427880b44a2475f5d896eb41087e6
Signed-off-by: Ruslan Aliev <raliev@mirantis.com>

charts/kubeadm/Chart.yaml

@@ -0,0 +1,18 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+description: A chart for managing Kubernetes cluster via Kubeadm
+name: kubeadm
+version: 0.1.0

charts/kubeadm/requirements.yaml

@@ -0,0 +1,18 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+dependencies:
+  - name: helm-toolkit
+    repository: file://../deps/helm-toolkit
+    version: ">= 0.1.0"

charts/kubeadm/templates/bin/_anchor.tpl

@@ -0,0 +1,374 @@
+#!/bin/bash
+# Copyright 2025 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -xeuo pipefail
+
+# directories
+HOST_DIR="{{ .Values.host_dir }}"
+KUBERNETES_DIR="/etc/kubernetes"
+CERT_DIR="${HOST_DIR}${KUBERNETES_DIR}/pki"
+
+# flow controls
+READY_TO_START=false
+FIRST_POD=false # remove
+KUBEADM_ACTION_REQUIRED=$([ $NODE_ROLE == "master" ] && echo "true" || echo "false")
+KUBELET_RESTART_REQUIRED=false
+
+KUBERNETES_VERSION="{{ .Values.kubeadm.cluster_config.kubernetesVersion }}"
+KUBEADM_ANNOTATION="last-applied-kubeadm-cfg-sha256"
+KUBELET_ANNOTATION="last-applied-kubelet-cfg-sha256"
+
+LAST_APPLIED_KUBEADM_CONFIG_SHA256=$([ $NODE_ROLE == "master" ] && kubectl get node $NODE_NAME -o jsonpath="{.metadata.annotations.$KUBEADM_ANNOTATION}" || echo "")
+LAST_APPLIED_KUBELET_CONFIG_SHA256=$(kubectl get node $NODE_NAME -o jsonpath="{.metadata.annotations.$KUBELET_ANNOTATION}")
+
+# substitute from values
+MASTERS_DS_NAME="{{ .Values.service.name }}-masters-anchor"
+WORKERS_DS_NAME="{{ .Values.service.name }}-workers-anchor"
+CURRENT_DS_NAME=$([ $NODE_ROLE == "master" ] && echo "$MASTERS_DS_NAME" || echo "$WORKERS_DS_NAME")
+
+MASTERS_POD_LABELS="component=kubernetes-kubeadm-anchor"
+WORKERS_POD_LABELS="component=kubernetes-kubeadm-workers-anchor"
+CURRENT_POD_LABELS=$([ $NODE_ROLE == "master" ] && echo "$MASTERS_POD_LABELS" || echo "$WORKERS_POD_LABELS")
+
+kubeadm() {
+  $(which kubeadm) --rootfs "$HOST_DIR" --v=5 $@
+}
+
+annotate_node() {
+  kubectl annotate node --overwrite "$NODE_NAME" "$1=$2"
+}
+
+sync_configs() {
+  sync_kubeconfigs
+  sync_kubelet_configs
+  sync_binaries
+
+  if [[ $NODE_ROLE == "master" ]]; then
+    sync_control_plane_certs
+    sync_apiserver_misc_configs
+    sync_kubeadm_configs
+    sync_patches
+  fi
+
+  cleanup_old_configs
+}
+
+compare_copy_file() {
+  src=$1
+  delete_src=false
+  if [ ! -f "$1" ]; then
+    src=$(mktemp)
+    cat - > $src
+    delete_src=true
+  fi
+  dst=$2
+
+  if [ ! -e "${dst}" ] || ! cmp -s $src $dst; then
+    [[ -d "$(dirname ${dst})" ]] || mkdir -p $(dirname "${dst}")
+    chmod a+r $(dirname "${dst}")
+
+    cp "${src}" "${dst}"
+    chmod a+r "${dst}"
+
+{{- if .Values.kubelet.restart }}
+    if [[ "${dst}" == "${HOST_DIR}/etc/default/kubelet" ||
+          "${dst}" == "${HOST_DIR}/etc/systemd/system/kubelet.service" ||
+          "${dst}" == "${HOST_DIR}/var/lib/kubelet/config.yaml" ]]; then
+      KUBELET_RESTART_REQUIRED=true
+    fi
+{{- end }}
+
+  fi
+
+  if [[ $delete_src == true ]]; then
+    rm $src
+  fi
+}
+
+move_if_exists() {
+  path="$1"
+  move_to="$2"
+
+  if [ -e "$path" ]; then mv "$path" "$move_to"; fi
+}
+
+remove_if_exists() {
+  path="$1"
+  if [ -e "$path" ]; then
+    if [ -d "$path" ]; then rm -rf $path; else rm -f "$path"; fi
+  fi
+}
+
+sync_kubeconfigs() {
+  cluster_ip="{{ .Values.kubeconfig.cluster_ip }}"
+  cluster_port="{{ .Values.kubeconfig.cluster_port }}"
+  cluster_name="{{ .Values.kubeconfig.cluster_name }}"
+  {{ range $name, $data := .Values.kubeconfig.configs }}
+  if [[ "{{ $name }}" == "kubelet" || $NODE_ROLE == "master" ]]; then
+    CLUSTER_IP="$cluster_ip" CLUSTER_PORT="$cluster_port" \
+      CERT_AUTH="{{ $data.ca }}" CLUSTER_NAME="$cluster_name" USER="{{ $name }}" \
+      CLIENT_CERT="{{ $data.client_cert }}" CLIENT_KEY="{{ $data.client_key }}" \
+      envsubst < /tmp/kubeconfig/kubeconfig.yaml.tpl | compare_copy_file - "$HOST_DIR{{ $data.path }}"
+  fi
+  {{- end }}
+
+  if [ $NODE_ROLE == "master" ] && ! kubectl get cm -n kube-public cluster-info; then
+    kubeadm init phase bootstrap-token --kubeconfig /etc/kubernetes/admin.conf
+  fi
+
+  echo "kubeconfigs are synced"
+}
+
+sync_control_plane_certs() {
+  if [[ $NODE_ROLE != "master" ]]; then return; fi
+  for file in /tmp/certs/*; do
+    file_name="$(basename $file)"
+    if [[ "$file_name" == etcd-* ]]; then
+      compare_copy_file "$file" "$CERT_DIR/etcd/${file_name/etcd-/}"
+    else
+      compare_copy_file "$file" "$CERT_DIR/$file_name"
+    fi
+  done
+}
+
+sync_kubelet_certs() {
+  echo "placeholder"
+}
+
+sync_etcd_certs() {
+  echo "placeholder"
+}
+
+sync_patches() {
+  #remove_if_exists "${HOST_DIR}${KUBERNETES_DIR}/kubeadm/patches/"
+  if [[ $NODE_ROLE != "master" ]]; then return; fi
+  for file in /tmp/patches/*; do
+    compare_copy_file "$file" "${HOST_DIR}${KUBERNETES_DIR}/kubeadm/patches/$(basename $file)"
+  done
+}
+
+sync_apiserver_misc_configs() {
+  if [[ $NODE_ROLE == "master" ]]; then
+    for file in /tmp/apiserver-misc/*; do
+      compare_copy_file  "$file" "${HOST_DIR}${KUBERNETES_DIR}/apiserver/$(basename $file)"
+    done
+  fi
+}
+
+sync_kubelet_configs() {
+  envsubst < /tmp/kubelet/default-kubelet | compare_copy_file - "${HOST_DIR}/etc/default/kubelet"
+  compare_copy_file /tmp/kubelet/kubelet.service "${HOST_DIR}/etc/systemd/system/kubelet.service"
+  compare_copy_file /tmp/kubelet/kubelet "${HOST_DIR}/var/lib/kubelet/config.yaml"
+}
+
+sync_kubeadm_configs() {
+  #ETCD_ENABLED=$(kubectl get nodes -l kubernetes-etcd=enabled --no-headers -o custom-columns=":metadata.name" | grep -q "$NODE_NAME" && echo "true" || echo "false")
+  envsubst < /tmp/kubeadm/join-config.yaml | compare_copy_file - "${HOST_DIR}${KUBERNETES_DIR}/kubeadm/join_config.yaml"
+  envsubst < /tmp/kubeadm/upgrade-config.yaml | compare_copy_file - "${HOST_DIR}${KUBERNETES_DIR}/kubeadm/upgrade_config.yaml"
+}
+
+sync_binaries() {
+  envsubst < "/tmp/bin/kubelet_restart.sh" | compare_copy_file - "$HOST_DIR/usr/local/bin/kubelet_restart.sh"
+  chmod a+x "$HOST_DIR/usr/local/bin/kubelet_restart.sh"
+
+  if [[ ! -e "$HOST_DIR/usr/local/bin/kubelet" || $(kubelet --version) != $($HOST_DIR/usr/local/bin/kubelet --version) ]]; then
+{{- if .Values.kubelet.restart }}
+    KUBELET_RESTART_REQUIRED=true
+{{- end }}
+
+    install -v -b --suffix=-backup /usr/bin/kubelet $HOST_DIR/usr/local/bin/
+  fi
+
+  if [[ $NODE_ROLE == "master" ]]; then
+    if [[ ! -e "$HOST_DIR/usr/local/bin/kubeadm" || "$(kubeadm version -o short)" != "$($HOST_DIR/usr/local/bin/kubeadm version -o short)" ]]; then
+      install -v -b --suffix=-backup /usr/bin/kubeadm $HOST_DIR/usr/local/bin/
+    fi
+
+    if [[ ! -e "$HOST_DIR/usr/local/bin/kubectl" || $(kubectl version --client | grep Client) != $($HOST_DIR/usr/local/bin/kubectl version --client | grep Client) ]]; then
+      install -v -b --suffix=-backup /usr/bin/kubectl $HOST_DIR/usr/local/bin/
+    fi
+  fi
+}
+
+# to remove
+ready_to_start() {
+  pod_list=$(kubectl get pods -n kube-system -l $CURRENT_POD_LABELS --sort-by=.metadata.creationTimestamp --no-headers -o custom-columns=":metadata.name")
+  for pod in $pod_list; do
+    if [ $pod == $POD_NAME ]; then
+      READY_TO_START="true"
+      break
+    fi
+
+    if ! kubectl wait --for=condition=ready pod -n kube-system $pod --timeout=0; then
+      break
+    fi
+  done
+}
+
+daemonset_status() {
+  if [[ $NODE_ROLE == "worker" ]] && ! kubectl rollout status "ds/$MASTERS_DS_NAME" -n kube-system -w=false | grep -q "successfully rolled out"; then
+    echo "waiting for masters nodes to be ready..."
+    return
+  fi
+
+  generation=0
+  observed_generation=0
+  updated_number_scheduled=0
+  desired_number_scheduled=0
+  number_available=0
+
+  read -r generation observed_generation updated_number_scheduled desired_number_scheduled number_available <<<$(kubectl get ds -n kube-system "$CURRENT_DS_NAME" -o jsonpath='{ .metadata.generation } { .status.observedGeneration } { .status.updatedNumberScheduled } { .status.desiredNumberScheduled } { .status.numberAvailable }')
+  while [ $generation -gt $observed_generation ]; do
+    echo "Waiting for daemon set spec update to be observed..."
+    read -r generation observed_generation updated_number_scheduled desired_number_scheduled number_available <<<$(kubectl get ds -n kube-system "$CURRENT_DS_NAME" -o jsonpath='{ .metadata.generation } { .status.observedGeneration } { .status.updatedNumberScheduled } { .status.desiredNumberScheduled } { .status.numberAvailable }')
+    sleep 5
+  done
+
+  if [ -z $number_available ]; then
+    number_available=0
+  fi
+
+	if [ $updated_number_scheduled -lt $desired_number_scheduled ]; then
+		echo "Waiting for daemon set rollout to finish: ${updated_number_scheduled} out of ${desired_number_scheduled} new pods have been updated..."
+	  # add verification that the pod has to be single (unready)
+	  READY_TO_START="true"
+	  if [ $updated_number_scheduled -eq 1 ]; then
+	    echo "first in sequence"
+	    FIRST_POD=true
+	  fi
+	elif [ $number_available -lt $desired_number_scheduled ]; then
+		echo "Waiting for daemon set rollout to finish: ${number_available} of ${desired_number_scheduled} updated pods are available..."
+	  if [[ $(($desired_number_scheduled-$number_available)) -eq 1 ]]; then
+	    READY_TO_START="true"
+	  else
+	    ready_to_start
+	  fi
+	else
+		echo "Daemon set successfully rolled out"
+		READY_TO_START="true"
+		KUBEADM_ACTION_REQUIRED=false
+  fi
+}
+
+cleanup_old_configs() {
+{{- range $file := .Values.const.files_to_delete }}
+  remove_if_exists "${HOST_DIR}{{ $file }}"
+{{- end }}
+}
+
+restart_kubelet() {
+  # kubelet_restart.yaml has to be renamed to node_action.yaml
+  kubectl drain "${NODE_NAME}" --pod-selector '!kubelet-restart' --ignore-daemonsets --delete-emptydir-data
+  job_name=$(envsubst < "/tmp/jobs/kubelet_restart.yaml.tpl" | kubectl create -f - | grep created | awk '{print $1}')
+  kubectl wait -n kube-system --for=condition=complete $job_name --timeout=600s
+
+  kubectl uncordon "${NODE_NAME}"
+  kubectl wait node --for=condition=ready "${NODE_NAME}" --timeout=60s
+
+  sleep 60
+
+  if [[ $(kubectl get node "${NODE_NAME}" -o jsonpath='{.status.nodeInfo.kubeletVersion}') != $(kubelet --version | awk '{print $2}') ]]; then
+    echo "kubelet version mismatch"
+    exit 1
+  fi
+
+  sleep 60
+}
+
+is_upgrade_required() {
+  IFS='|' read -r last_cluster_config_sha256 last_kubelet_config_sha256 kubelet_current_version <<<$(kubectl get node $NODE_NAME -o jsonpath="{.metadata.annotations.$KUBEADM_ANNOTATION}|{.metadata.annotations.$KUBELET_ANNOTATION}|{.status.nodeInfo.kubeletVersion}")
+
+  if [[ $NODE_ROLE == "master" ]]; then
+    if [ -z $last_cluster_config_sha256 ]; then return 0; fi
+    current_cluster_config_sha256=$(sha256sum < /tmp/kubeadm/ClusterConfiguration | cut -d ' ' -f 1)
+    if [[ $last_cluster_config_sha256 != $current_cluster_config_sha256 ]]; then
+      return 0
+    fi
+  fi
+
+{{- if .Values.kubelet.restart }}
+  if [ -z $last_kubelet_config_sha256 ]; then return 0; fi
+  current_kubelet_config_sha256=$(sha256sum < /tmp/kubelet/kubelet | cut -d ' ' -f 1)
+
+  if [[ $last_kubelet_config_sha256 != $current_kubelet_config_sha256 || $kubelet_current_version != $KUBERNETES_VERSION ]]; then
+    KUBELET_RESTART_REQUIRED=true
+    return 0
+  fi
+{{- end }}
+
+  return 1
+}
+
+verlte() {
+    printf '%s\n' "$1" "$2" | sort -C -V
+}
+
+verlt() {
+    ! verlte "$2" "$1"
+}
+
+kubeadm_action() {
+  if is_upgrade_required; then
+   if [[ ! -f "${HOST_DIR}${KUBERNETES_DIR}/manifests/kube-apiserver.yaml" ||
+         ! -f "${HOST_DIR}${KUBERNETES_DIR}/manifests/kube-controller-manager.yaml" ||
+         ! -f "${HOST_DIR}${KUBERNETES_DIR}/manifests/kube-scheduler.yaml"  ]]; then
+     rm -f "${HOST_DIR}${KUBERNETES_DIR}/manifests/kube-*" || true
+     kubeadm join --config "${KUBERNETES_DIR}/kubeadm/join_config.yaml"
+   else
+     kubeadm upgrade node --config "${KUBERNETES_DIR}/kubeadm/upgrade_config.yaml"
+     sync_kubeconfigs
+   fi
+  fi
+}
+
+rollout_restart() {
+  if kubectl rollout status "ds/$CURRENT_DS_NAME" -n kube-system -w=false | grep -q "successfully rolled out"; then
+    kubectl rollout restart "ds/$CURRENT_DS_NAME" -n kube-system
+  fi
+}
+
+while [[ $READY_TO_START != true ]]; do
+  daemonset_status
+  sleep 5
+done
+
+sync_configs
+
+if [[ $(kubeadm version -o short) != "$KUBERNETES_VERSION" ]]; then
+  echo "Desired kubernetes version mismatch with provided binaries version, please update kubernetes version in values to $(kubeadm version -o short)"
+  exit 1
+fi
+
+if [[ $KUBEADM_ACTION_REQUIRED == true ]]; then
+  kubeadm_action
+  annotate_node "$KUBEADM_ANNOTATION" "$(sha256sum < /tmp/kubeadm/ClusterConfiguration | cut -d ' ' -f 1)"
+fi
+
+if [[ $KUBELET_RESTART_REQUIRED == true ]]; then
+  restart_kubelet
+  annotate_node "$KUBELET_ANNOTATION" "$(sha256sum < /tmp/kubelet/kubelet | cut -d ' ' -f 1)"
+fi
+
+touch /tmp/done
+
+# main loop
+while true; do
+  if [ -e /tmp/stop ] || is_upgrade_required; then
+    echo "Stopping"
+    rollout_restart
+    break
+  fi
+  sleep 15
+done

charts/kubeadm/templates/bin/_kubelet_restart.tpl

@@ -0,0 +1,29 @@
+#!/bin/bash
+# Copyright 2025 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -xeuo pipefail
+
+echo "perform daemon-reload"
+systemctl daemon-reload
+{{- if .Values.kubelet.restart }}
+echo "restarting kubelet service"
+systemctl restart kubelet.service
+{{- end }}
+
+while ! systemctl is-active --quiet kubelet.service; do
+  sleep 5
+done
+
+exit 0

charts/kubeadm/templates/bin/_pre_stop.tpl

@@ -0,0 +1,19 @@
+#!/bin/sh
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+touch /tmp/stop
+sleep {{ .Values.anchor.period }}

charts/kubeadm/templates/configmap-apiserver-misc.yaml

@@ -0,0 +1,34 @@
+{{/*
+Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+   http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.configmap_apiserver_misc }}
+{{- $envAll := . }}
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.service.name }}-apiserver-misc
+data:
+{{/* Dynamically added apiserver config files */}}
+{{- range $key, $val := .Values.apiserver.config_files }}
+{{- if and (hasKey $val "file") (hasKey $val "content") }}
+  {{ $val.file }}: |
+{{- if kindIs "string" $val.content }}
+{{ indent 4 $val.content }}
+{{- else }}
+{{ toYaml $val.content | indent 4 }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}

charts/kubeadm/templates/configmap-bin.yaml

@@ -0,0 +1,32 @@
+{{/*
+Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.configmap_bin }}
+{{- $envAll := . }}
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.service.name }}-bin
+data:
+  anchor: |+
+{{ tuple "bin/_anchor.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  kubelet_restart.sh: |+
+{{ tuple "bin/_kubelet_restart.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  pre_stop: |+
+{{ tuple "bin/_pre_stop.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{- end }}

charts/kubeadm/templates/configmap-certs.yaml

@@ -0,0 +1,44 @@
+{{/*
+Copyright 2025 AT&T Intellectual Property.  All other rights reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+   http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.configmap_certs }}
+{{- $envAll := . }}
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.service.name }}-certs
+data:
+  apiserver.crt: {{ .Values.secrets.apiserver.crt | quote }}
+  apiserver.key: {{ .Values.secrets.apiserver.key | quote }}
+  apiserver-etcd-client.crt: {{ .Values.secrets.apiserver.etcd_client.crt | quote }}
+  apiserver-etcd-client.key: {{ .Values.secrets.apiserver.etcd_client.key | quote }}
+  apiserver-kubelet-client.crt: {{ .Values.secrets.apiserver.kubelet_client.crt | quote }}
+  apiserver-kubelet-client.key: {{ .Values.secrets.apiserver.kubelet_client.key | quote }}
+  ca.crt: {{ .Values.secrets.ca.crt | quote }}
+  ca.key: {{ .Values.secrets.ca.key | quote }}
+  controller-manager.pem: {{ .Values.secrets.controller_manager.crt | quote }}
+  controller-manager-key.pem: {{ .Values.secrets.controller_manager.key | quote }}
+  etcd-ca.crt: {{ .Values.secrets.etcd.ca.crt | quote }}
+  etcd-healthcheck-client.crt: {{ .Values.secrets.etcd.healthcheck_client.crt | quote }}
+  etcd-healthcheck-client.key: {{ .Values.secrets.etcd.healthcheck_client.key | quote }}
+  front-proxy-ca.crt: {{ .Values.secrets.front_proxy_ca.crt | quote }}
+  front-proxy-ca.key: {{ .Values.secrets.front_proxy_ca.key | quote }}
+  front-proxy-client.crt: {{ .Values.secrets.front_proxy_client.crt | quote }}
+  front-proxy-client.key: {{ .Values.secrets.front_proxy_client.key | quote }}
+  sa.key: {{ .Values.secrets.sa.key | quote }}
+  sa.pub: {{ .Values.secrets.sa.pub | quote }}
+  scheduler.pem: {{ .Values.secrets.scheduler.crt | quote }}
+  scheduler-key.pem: {{ .Values.secrets.scheduler.key | quote }}
+{{- end }}

charts/kubeadm/templates/configmap-jobs.yaml

@@ -0,0 +1,28 @@
+{{/*
+Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.configmap_jobs }}
+{{- $envAll := . }}
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.service.name }}-jobs
+data:
+  kubelet_restart.yaml.tpl: |
+{{ tuple "jobs/_kubelet_restart.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{- end }}

charts/kubeadm/templates/configmap-kubeadm.yaml

@@ -0,0 +1,32 @@
+{{/*
+Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.configmap_kubeadm }}
+{{- $envAll := . }}
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kubeadm-config
+data:
+  ClusterConfiguration: |
+{{ toYaml .Values.kubeadm.cluster_config | indent 4 }}
+  join-config.yaml: |
+{{ tuple "kubeadm/_join_config.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  upgrade-config.yaml: |
+{{ tuple "kubeadm/_upgrade_config.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{- end }}

charts/kubeadm/templates/configmap-kubeconfig.yaml

@@ -0,0 +1,28 @@
+{{/*
+Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.configmap_kubeconfig }}
+{{- $envAll := . }}
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.service.name }}-kubeconfig
+data:
+  kubeconfig.yaml.tpl: |
+{{ tuple "kubeconfig/_kubeconfig.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{- end }}

charts/kubeadm/templates/configmap-kubelet.yaml

@@ -0,0 +1,31 @@
+{{/*
+Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.configmap_kubelet }}
+{{- $envAll := . }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kubelet-config
+data:
+  default-kubelet: |
+{{ tuple "kubelet/_kubelet.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  kubelet.service: |
+{{ tuple "kubelet/_kubelet-service.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  kubelet: |
+{{ toYaml .Values.kubelet.config | indent 4 }}
+{{- end }}

charts/kubeadm/templates/configmap-patches.yaml

@@ -0,0 +1,27 @@
+{{/*
+Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+   http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.configmap_patches }}
+{{- $envAll := . }}
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.service.name }}-patches
+data:
+{{- range $file_name, $data := .Values.kubeadm.patches }}
+  {{ $file_name }}: |
+{{ toYaml $data | indent 4 }}
+{{- end }}
+{{- end }}

charts/kubeadm/templates/daemonset-masters.yaml

@@ -0,0 +1,179 @@
+{{/*
+Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.daemonset_masters }}
+{{- $envAll := . }}
+{{- $mounts_kubernetes_apiserver := .Values.pod.mounts.kubernetes_apiserver.kubernetes_apiserver }}
+{{- $mounts_kubernetes_apiserver_init := .Values.pod.mounts.kubernetes_apiserver.init_container }}
+{{- $labels := tuple $envAll "kubernetes" "kubernetes-kubeadm-anchor" | include "helm-toolkit.snippets.kubernetes_metadata_labels" -}}
+
+---
+apiVersion: "apps/v1"
+kind: DaemonSet
+metadata:
+  name: {{ .Values.service.name }}-masters-anchor
+  labels:
+{{ $labels | indent 4 }}
+  annotations:
+    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+spec:
+  selector:
+    matchLabels:
+{{ $labels | indent 6 }}
+{{ tuple $envAll "kubernetes-apiserver-anchor" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }}
+  template:
+    metadata:
+      labels:
+{{ $labels | indent 8 }}
+      annotations:
+        {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+        configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
+        configmap-kubeconfig-hash: {{ tuple "configmap-kubeconfig.yaml" . | include "helm-toolkit.utils.hash" }}
+{{ dict "envAll" $envAll "podName" "kubernetes_apiserver_anchor" "containerNames" (list "anchor") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
+    spec:
+{{ dict "envAll" $envAll "application" "kubernetes_apiserver_anchor" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: {{ .Values.labels.common.node_selector_key }}
+                    operator: In
+                    values:
+                      - {{ .Values.labels.common.node_selector_value }}
+                  - key: {{ .Values.labels.masters.node_selector_key }}
+                    operator: In
+                    values:
+                      - {{ .Values.labels.masters.node_selector_value }}
+      dnsPolicy: {{ .Values.anchor.dns_policy }}
+      serviceAccountName: kubeadm
+      hostNetwork: true
+      priorityClassName: system-node-critical
+      tolerations:
+        - key: node-role.kubernetes.io/master
+          effect: NoSchedule
+        - key: CriticalAddonsOnly
+          operator: Exists
+      terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.kubernetes_apiserver.timeout }}
+      containers:
+        - name: anchor
+          image: {{ .Values.images.tags.anchor }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+{{ tuple $envAll $envAll.Values.pod.resources.anchor_pod | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ dict "envAll" $envAll "application" "kubernetes_apiserver_anchor" "container" "anchor" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          env:
+          - name: HOST_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.hostIP
+          - name: NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
+          - name: NODE_ROLE
+            value: "master"
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          command: ["/bin/bash","-c"]
+          args: ["/tmp/bin/anchor"]
+          lifecycle:
+            preStop:
+              exec:
+                command:
+                  - /tmp/bin/pre_stop
+
+          readinessProbe:
+            exec:
+              command:
+              - /bin/bash
+              - -c
+              - |-
+                cat /tmp/done
+#                [ "$(curl -s -S -o /dev/null \
+#                    --cert "/host{{ .Values.apiserver.host_etc_path }}/pki/apiserver.pem" \
+#                    --key "/host{{ .Values.apiserver.host_etc_path }}/pki/apiserver-key.pem" \
+#                    --cacert "/host{{ .Values.apiserver.host_etc_path }}/pki/cluster-ca.pem" \
+#                    "https://localhost:{{ .Values.network.kubernetes_apiserver.port }}/healthz" \
+#                    -w "%{http_code}")" = "200" ]
+#                exit $?
+#            initialDelaySeconds: 10
+#            periodSeconds: 5
+#            timeoutSeconds: 5
+
+          volumeMounts:
+            - name: host
+              mountPath: {{ .Values.host_dir }}
+              mountPropagation: HostToContainer
+            - name: {{ .Values.service.name }}-certs
+              mountPath: /tmp/certs
+            - name: {{ .Values.service.name }}-patches
+              mountPath: /tmp/patches
+            - name: {{ .Values.service.name }}-bin
+              mountPath: /tmp/bin
+            - name: {{ .Values.service.name }}-kubeconfig
+              mountPath: /tmp/kubeconfig
+            - name: {{ .Values.service.name }}-apiserver-misc
+              mountPath: /tmp/apiserver-misc
+            - name: kubelet-config
+              mountPath: /tmp/kubelet
+            - name: kubeadm-config
+              mountPath: /tmp/kubeadm
+            - name: pod-tmp
+              mountPath: /tmp
+{{ if $mounts_kubernetes_apiserver.volumeMounts }}{{ toYaml $mounts_kubernetes_apiserver.volumeMounts | indent 12 }}{{ end }}
+      volumes:
+        - name: {{ .Values.service.name }}-bin
+          configMap:
+            name: {{ .Values.service.name }}-bin
+            defaultMode: 0555
+        - name: {{ .Values.service.name }}-apiserver-misc
+          configMap:
+            name: {{ .Values.service.name }}-apiserver-misc
+            defaultMode: 0555
+        - name: {{ .Values.service.name }}-certs
+          configMap:
+            name: {{ .Values.service.name }}-certs
+            defaultMode: 0444
+        - name: {{ .Values.service.name }}-patches
+          configMap:
+            name: {{ .Values.service.name }}-patches
+            defaultMode: 0444
+        - name: host
+          hostPath:
+            path: /
+        - name: {{ .Values.service.name }}-kubeconfig
+          configMap:
+            name: {{ .Values.service.name }}-kubeconfig
+            defaultMode: 0444
+        - name: kubelet-config
+          configMap:
+            name: kubelet-config
+            defaultMode: 0444
+        - name: kubeadm-config
+          configMap:
+            name: kubeadm-config
+            defaultMode: 0444
+        - name: pod-tmp
+          emptyDir: {}
+{{ if $mounts_kubernetes_apiserver.volumes }}{{ toYaml $mounts_kubernetes_apiserver.volumes | indent 8 }}{{ end }}
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+      maxSurge: 0
+{{- end }}

charts/kubeadm/templates/daemonset-workers.yaml

@@ -0,0 +1,145 @@
+{{/*
+Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.daemonset_workers }}
+{{- $envAll := . }}
+{{- $mounts_kubernetes_apiserver := .Values.pod.mounts.kubernetes_apiserver.kubernetes_apiserver }}
+{{- $mounts_kubernetes_apiserver_init := .Values.pod.mounts.kubernetes_apiserver.init_container }}
+{{- $labels := tuple $envAll "kubernetes" "kubernetes-kubeadm-workers-anchor" | include "helm-toolkit.snippets.kubernetes_metadata_labels" -}}
+
+---
+apiVersion: "apps/v1"
+kind: DaemonSet
+metadata:
+  name: {{ .Values.service.name }}-workers-anchor
+  labels:
+{{ $labels | indent 4 }}
+  annotations:
+    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+spec:
+  selector:
+    matchLabels:
+{{ $labels | indent 6 }}
+{{ tuple $envAll "kubernetes-apiserver-anchor" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }}
+  template:
+    metadata:
+      labels:
+{{ $labels | indent 8 }}
+      annotations:
+        {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+        configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
+        configmap-kubeconfig-hash: {{ tuple "configmap-kubeconfig.yaml" . | include "helm-toolkit.utils.hash" }}
+{{ dict "envAll" $envAll "podName" "kubernetes_apiserver_anchor" "containerNames" (list "anchor") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
+    spec:
+{{ dict "envAll" $envAll "application" "kubernetes_apiserver_anchor" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: {{ .Values.labels.common.node_selector_key }}
+                    operator: In
+                    values:
+                      - {{ .Values.labels.common.node_selector_value }}
+                  - key: {{ .Values.labels.masters.node_selector_key }}
+                    operator: DoesNotExist
+      dnsPolicy: {{ .Values.anchor.dns_policy }}
+      serviceAccountName: kubeadm
+      hostNetwork: true
+      priorityClassName: system-node-critical
+      tolerations:
+        - key: node-role.kubernetes.io/master
+          effect: NoSchedule
+        - key: CriticalAddonsOnly
+          operator: Exists
+      terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.kubernetes_apiserver.timeout }}
+      containers:
+        - name: anchor
+          image: {{ .Values.images.tags.anchor }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+{{ tuple $envAll $envAll.Values.pod.resources.anchor_pod | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ dict "envAll" $envAll "application" "kubernetes_apiserver_anchor" "container" "anchor" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          env:
+          - name: HOST_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.hostIP
+          - name: NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
+          - name: NODE_ROLE
+            value: "worker"
+          - name: PKI_PATH
+            value: /host{{ .Values.apiserver.host_etc_path }}/pki
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          command: ["/bin/bash","-c"]
+          args: ["/tmp/bin/anchor"]
+          lifecycle:
+            preStop:
+              exec:
+                command:
+                  - /tmp/bin/pre_stop
+
+          readinessProbe:
+            exec:
+              command:
+              - /bin/bash
+              - -c
+              - |-
+                cat /tmp/done
+
+          volumeMounts:
+            - name: host
+              mountPath: {{ .Values.host_dir }}
+              mountPropagation: HostToContainer
+            - name: {{ .Values.service.name }}-bin
+              mountPath: /tmp/bin
+            - name: {{ .Values.service.name }}-kubeconfig
+              mountPath: /tmp/kubeconfig
+            - name: kubelet-config
+              mountPath: /tmp/kubelet
+            - name: pod-tmp
+              mountPath: /tmp
+{{ if $mounts_kubernetes_apiserver.volumeMounts }}{{ toYaml $mounts_kubernetes_apiserver.volumeMounts | indent 12 }}{{ end }}
+      volumes:
+        - name: host
+          hostPath:
+            path: /
+        - name: {{ .Values.service.name }}-bin
+          configMap:
+            name: {{ .Values.service.name }}-bin
+            defaultMode: 0555
+        - name: {{ .Values.service.name }}-kubeconfig
+          configMap:
+            name: {{ .Values.service.name }}-kubeconfig
+            defaultMode: 0444
+        - name: kubelet-config
+          configMap:
+            name: kubelet-config
+            defaultMode: 0444
+        - name: pod-tmp
+          emptyDir: {}
+{{ if $mounts_kubernetes_apiserver.volumes }}{{ toYaml $mounts_kubernetes_apiserver.volumes | indent 8 }}{{ end }}
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+      maxSurge: 0
+{{- end }}

charts/kubeadm/templates/jobs/_kubelet_restart.yaml.tpl

@@ -0,0 +1,46 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  generateName: kubelet-restart-$NODE_NAME-
+  labels:
+    "kubelet-restart": "true"
+spec:
+  template:
+    metadata:
+      labels:
+        "kubelet-restart": "true"
+      annotations:
+        container.apparmor.security.beta.kubernetes.io/kubelet-restart: runtime/default
+    spec:
+      restartPolicy: Never
+      serviceAccountName: kubeadm
+      serviceAccount: kubeadm
+      hostNetwork: true
+      enableServiceLinks: true
+      hostPID: true
+      hostIPC: true
+      nodeName: $NODE_NAME
+      containers:
+        - name: kubelet-restart
+          image: {{ .Values.images.tags.anchor }}
+          imagePullPolicy: Always
+          resources:
+            limits:
+              cpu: '8'
+              memory: 8Gi
+            requests:
+              cpu: 100m
+              memory: 64Mi
+          securityContext:
+            privileged: true
+          command:
+            - nsenter
+            - '--target'
+            - '1'
+            - '--mount'
+            - '--uts'
+            - '--ipc'
+            - '--net'
+            - '--pid'
+            - kubelet_restart.sh

charts/kubeadm/templates/kubeadm/_cluster_config.yaml.tpl

@@ -0,0 +1,66 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+apiVersion: kubeadm.k8s.io/v1beta4
+kind: ClusterConfiguration
+kubernetesVersion: {{ .Values.kubernetes.version }}
+apiServer:
+{{- if .Values.apiserver.extraArgs }}
+  extraArgs:
+{{ toYaml .Values.apiserver.extraArgs | indent 4 }}
+{{- end }}
+{{- if .Values.apiserver.extraVolumes }}
+  extraVolumes:
+{{ toYaml .Values.apiserver.extraVolumes | indent 4 }}
+{{- end }}
+caCertificateValidityPeriod: 87600h0m0s
+certificateValidityPeriod: 8760h0m0s
+certificatesDir: "/etc/kubernetes/pki"
+clusterName: kubernetes
+controlPlaneEndpoint: "127.0.0.1:6553"
+controllerManager:
+  extraArgs:
+  - name: node-monitor-period
+    value: "5s"
+  - name: node-monitor-grace-period
+    value: "20s"
+  - name: terminated-pod-gc-threshold
+    value: "1000"
+  - name: configure-cloud-routes
+    value: "false"
+  - name: v
+    value: "2"
+  extraVolumes: []
+dns:
+  disabled: true
+encryptionAlgorithm: RSA-2048
+etcd:
+{{ toYaml .Values.etcd | indent 2 }}
+imageRepository: docker-open-nc.zc1.cti.att.com/upstream-local/kubernetes
+networking:
+  dnsDomain: cluster.local
+  podSubnet: "10.97.0.0/16"
+  serviceSubnet: "10.96.0.0/16"
+proxy:
+  disabled: true
+scheduler:
+  extraArgs:
+  - name: secure-port
+    value: "10259"
+  - name: v
+    value: "2"
+  extraVolumes:
+  - hostPath: "/etc/kubernetes/pki"
+    mountPath: "/etc/kubernetes/pki"
+    name: pki-certs
+    pathType: DirectoryOrCreate

charts/kubeadm/templates/kubeadm/_join_config.yaml.tpl

@@ -0,0 +1,43 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+apiVersion: kubeadm.k8s.io/v1beta4
+kind: JoinConfiguration
+caCertPath: /etc/kubernetes/pki/ca.crt
+controlPlane:
+  localAPIEndpoint:
+    advertiseAddress: "${HOST_IP}"
+    bindPort: {{ .Values.network.kubernetes_apiserver.port }}
+discovery:
+  file:
+    kubeConfigPath: /etc/kubernetes/admin.conf
+nodeRegistration:
+  imagePullPolicy: Always
+  imagePullSerial: false
+  name: ${NODE_NAME}
+  ignorePreflightErrors:
+  - Service-Kubelet
+  - FileAvailable--etc-kubernetes-kubelet.conf
+  - SystemVerification
+  - Port-10250
+  - Port-6443
+  - Port-10259
+  - Port-10257
+  - Port-2379
+  - Port-2380
+  - DirAvailable--var-lib-etcd
+  taints: []
+skipPhases:
+{{ toYaml .Values.kubeadm.join_config.skip_phases }}
+patches:
+  directory: /etc/kubernetes/kubeadm/patches

charts/kubeadm/templates/kubeadm/_upgrade_config.yaml.tpl

@@ -0,0 +1,29 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+apiVersion: kubeadm.k8s.io/v1beta4
+kind: UpgradeConfiguration
+node:
+  certificateRenewal: false
+  dryRun: false
+  etcdUpgrade: false
+  ignorePreflightErrors:
+  - SystemVerification
+  - CoreDNSUnsupportedPlugins
+  - CoreDNSMigration
+  patches:
+    directory: /etc/kubernetes/kubeadm/patches
+  skipPhases:
+{{ toYaml .Values.kubeadm.upgrade_config.skip_phases | indent 2 }}
+  imagePullPolicy: Always
+  imagePullSerial: false

charts/kubeadm/templates/kubeconfig/_kubeconfig.yaml.tpl

@@ -0,0 +1,34 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+apiVersion: v1
+clusters:
+- cluster:
+    server: https://${CLUSTER_IP}:${CLUSTER_PORT}
+    certificate-authority: ${CERT_AUTH}
+  name: ${CLUSTER_NAME}
+contexts:
+- context:
+    cluster: ${CLUSTER_NAME}
+    user: ${USER}
+  name: ${USER}@${CLUSTER_NAME}
+current-context: ${USER}@${CLUSTER_NAME}
+kind: Config
+preferences: {}
+users:
+- name: ${USER}
+  user:
+    client-certificate: ${CLIENT_CERT}
+    client-key: ${CLIENT_KEY}

charts/kubeadm/templates/kubelet/_kubelet-service.yaml.tpl

@@ -0,0 +1,35 @@
+# Copyright 2025 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[Unit]
+Description=Kubernetes Worker Process
+Requires=network-online.target
+After=network-online.target containerd.service
+
+[Service]
+Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
+Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
+# This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically
+EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
+# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use
+# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file.
+EnvironmentFile=-/etc/default/kubelet
+ExecStart=/usr/local/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
+Restart=always
+StartLimitInterval=0
+RestartSec=10
+KillMode=process
+
+[Install]
+WantedBy=multi-user.target

charts/kubeadm/templates/kubelet/_kubelet.yaml.tpl

@@ -0,0 +1,23 @@
+# Copyright 2025 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+KUBELET_EXTRA_ARGS="
+--hostname-override=${NODE_NAME}
+--node-ip=${HOST_IP}
+{{- if .Values.kubelet.arguments }}
+{{- range $arg := .Values.kubelet.arguments }}
+{{ $arg }}
+{{- end }}
+{{- end }}
+"

charts/kubeadm/templates/kubelet/_kubelet_config.yaml.tpl

@@ -0,0 +1,33 @@
+# Copyright 2025 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+authentication:
+  anonymous:
+    enabled: false
+  webhook:
+    enabled: true
+  x509:
+    clientCAFile: "/etc/kubernetes/pki/kubelet-client-ca.pem"
+authorization:
+  mode: AlwaysAllow
+clusterDNS:
+  - {{ .Values.network.dns.service_ip }}
+clusterDomain: {{ .Values.network.dns.cluster_domain }}
+staticPodPath: "/etc/kubernetes/manifests"
+tlsCertFile: "/etc/kubernetes/pki/kubelet.pem"
+tlsPrivateKeyFile: "/etc/kubernetes/pki/kubelet-key.pem"
+{{ if .Values.kubelet.config_file_overrides }}
+{{- toYaml .Values.kubelet.config_file_overrides }}
+{{- end }}

charts/kubeadm/templates/rbac.yaml

@@ -0,0 +1,67 @@
+{{/*
+Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.rbac }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kubeadm
+  namespace: kube-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kubeadm
+  namespace: kube-system
+rules:
+  - apiGroups: [""]
+    resources:
+      - nodes
+      - pods
+      - pods/eviction
+    verbs:
+      - "*"
+  - apiGroups: [""]
+    resources:
+      - configmaps
+    verbs:
+      - "*"
+  - apiGroups:
+      - ""
+      - apps
+    resources:
+      - daemonsets
+    verbs:
+      - "*"
+  - apiGroups: [ "batch" ]
+    resources: [ "jobs" ]
+    verbs: [ "*" ]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kubeadm
+  namespace: kube-system
+subjects:
+  - kind: ServiceAccount
+    name: kubeadm
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: kubeadm
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}

charts/kubeadm/values.yaml

@@ -0,0 +1,484 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+release_group: null
+
+kubernetes:
+  version: v1.33.3
+
+host_dir: "/host"
+
+kubeconfig:
+  cluster_name: kubernetes
+  cluster_ip: "127.0.0.1"
+  cluster_port: 6553
+  configs:
+    admin:
+      path: /etc/kubernetes/admin.conf
+      ca: /etc/kubernetes/admin/pki/cluster-ca.pem
+      client_cert: /etc/kubernetes/admin/pki/admin.pem
+      client_key: /etc/kubernetes/admin/pki/admin-key.pem
+    controller-manager:
+      path: /etc/kubernetes/controller-manager.conf
+      ca: pki/ca.crt
+      client_cert: pki/controller-manager.pem
+      client_key: pki/controller-manager-key.pem
+    kubelet:
+      path: /etc/kubernetes/kubelet.conf
+      ca: /etc/kubernetes/pki/kubelet-client-ca.pem
+      client_cert: /etc/kubernetes/pki/kubelet.pem
+      client_key: /etc/kubernetes/pki/kubelet-key.pem
+    scheduler:
+      path: /etc/kubernetes/scheduler.conf
+      ca: pki/ca.crt
+      client_cert: pki/scheduler.pem
+      client_key: pki/scheduler-key.pem
+
+# NOTE(mark-burnett): These values are not really configurable -- they live
+# here to keep the templates cleaner.
+const:
+
+  files_to_copy: {}
+    # NOTE(mark-burnett): These are (host dest): (container source) pairs
+
+  files_to_delete:
+    - /etc/kubernetes/manifests/kubernetes-apiserver.yaml
+    - /etc/kubernetes/manifests/kubernetes-controller-manager.yaml
+    - /etc/kubernetes/manifests/kubernetes-scheduler.yaml
+    - /etc/kubernetes/kubelet
+    - /etc/kubernetes/controller-manager
+    - /etc/kubernetes/scheduler
+    - /etc/kubernetes/apiserver/apiserver-proxy-cert.pem
+    - /etc/kubernetes/apiserver/apiserver-proxy-key.pem
+    - /etc/kubernetes/apiserver/agg-api-ca.pem
+    - /etc/kubernetes/apiserver/kubeconfig
+    - /etc/kubernetes/apiserver/pki
+    - /etc/kubernetes/kubeconfig
+    - /etc/kubernetes/pki/cluster-ca.pem
+    - /opt/kubernetes/bin/kubelet
+
+images:
+  tags:
+    dep_check: docker-open-nc.zc1.cti.att.com/airship/kubernetes-entrypoint@sha256:9c43e6965e8cdac750c256cbf161d8ab6549d6f7e4fbf08d73d2de78174e2e94
+    anchor: docker-open-nc.zc1.cti.att.com/airship/porthole/kubeadm-utility@sha256:fec3c6ed2834fc5cf88c031a4e3705286ff33c1d917c578c3c02ccbb81d0a36c
+    apiserver: registry.k8s.io/kube-apiserver-amd64:v1.32.1
+    key_rotate: quay.io/airshipit/porthole-compute-utility:latest-ubuntu_focal
+  pull_policy: "IfNotPresent"
+  local_registry:
+    active: false
+    exclude:
+      - dep_check
+      - image_repo_sync
+
+labels:
+  common:
+    node_selector_key: kubeadm
+    node_selector_value: enabled
+  masters:
+    node_selector_key: kubernetes-apiserver
+    node_selector_value: enabled
+  etcd:
+    node_selector_key: kubernetes-etcd
+    node_selector_value: enabled
+  job:
+    node_selector_key: kubelet-restart
+    node_selector_value: true
+
+anchor:
+  dns_policy: Default
+  enable_cleanup: true
+  kubelet:
+    manifest_path: /etc/kubernetes/manifests
+  period: 15
+
+# TODO(sh8121att): Add dynamic rendering of the admission controller list allowing a base list
+# and each conf entry to enable additional AC plugins
+conf: {}
+
+kubeadm:
+  cluster_config:
+    apiVersion: kubeadm.k8s.io/v1beta4
+    kind: ClusterConfiguration
+
+
+  join_config:
+    skip_phases:
+      - preflight
+      - control-plane-prepare/download-certs
+      - control-plane-prepare/certs
+      - control-plane-prepare/kubeconfig
+      - kubelet-start
+      - control-plane-join
+      - wait-control-plane
+
+  upgrade_config:
+    skip_phases:
+      - kubelet-config
+      - addon
+
+  patches:
+    kube-apiserver.yaml:
+      spec:
+        containers:
+          - name: kube-apiserver
+            resources:
+              limits:
+                cpu: '8'
+                memory: 8Gi
+              requests:
+                cpu: 100m
+                memory: 128Mi
+    kube-controller-manager.yaml:
+      spec:
+        containers:
+          - name: kube-controller-manager
+            resources:
+              limits:
+                cpu: '8'
+                memory: 8Gi
+              requests:
+                cpu: 100m
+                memory: 128Mi
+    kube-scheduler.yaml:
+      spec:
+        containers:
+          - name: kube-scheduler
+            resources:
+              limits:
+                cpu: '8'
+                memory: 8Gi
+              requests:
+                cpu: 100m
+                memory: 128Mi
+
+etcd:
+  external:
+    endpoints:
+      - "https://172.29.0.11:2379"
+      - "https://172.29.0.12:2379"
+      - "https://172.29.0.17:2379"
+    caFile: "/etc/kubernetes/pki/etcd/ca.crt"
+    certFile: "/etc/kubernetes/pki/etcd/healthcheck-client.crt"
+    keyFile: "/etc/kubernetes/pki/etcd/healthcheck-client.key"
+
+apiserver:
+  extraArgs:
+    - name: admission-control-config-file
+      value: "/etc/kubernetes/apiserver/acconfig.yaml"
+    - name: anonymous-auth
+      value: "true"
+    - name: audit-log-maxbackup
+      value: "10"
+    - name: audit-log-maxsize
+      value: "100"
+    - name: audit-log-path
+      value: "/var/log/kubernetes/apiserver/kube-apiserver-audit.log"
+    - name: audit-policy-file
+      value: "/etc/kubernetes/apiserver/audit-policy.yaml"
+    - name: bind-address
+      value: "0.0.0.0"
+    - name: enable-admission-plugins
+      value: "PodSecurity,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit"
+    - name: encryption-provider-config
+      value: "/etc/kubernetes/apiserver/encryption_provider.yaml"
+    - name: kubelet-certificate-authority
+      value: "/etc/kubernetes/pki/ca.crt"
+    - name: requestheader-allowed-names
+      value: "aggregator"
+    - name: service-node-port-range
+      value: "30000-32767"
+    - name: tls-cipher-suites
+      value: "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA"
+    - name: tls-min-version
+      value: "VersionTLS12"
+    - name: v
+      value: "2"
+  extraVolumes:
+    - hostPath: "/etc/kubernetes/apiserver"
+      mountPath: "/etc/kubernetes/apiserver"
+      name: apiserver
+      pathType: DirectoryOrCreate
+    - hostPath: "/var/log/kubernetes/apiserver"
+      mountPath: "/var/log/kubernetes/apiserver"
+      name: audit-logs
+      pathType: DirectoryOrCreate
+  etcd:
+    endpoints: https://kubernetes-etcd.kube-system.svc.cluster.local
+  host_etc_path: /etc/kubernetes/apiserver
+  config_files: {}
+  logging:
+    # Which messages to log.
+    # Valid values include any number from 0 to 9.
+    # Default 5(Trace level verbosity).
+    log_level: 5
+#XXX another possible configuration
+#  tls:
+#    tls-cipher-suites: "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA"
+#    # https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/
+#    #Possible values: VersionTLS10, VersionTLS11, VersionTLS12
+#    tls-min-version: 'VersionTLS12'
+
+kubelet:
+  arguments:
+    - --maximum-dead-containers-per-container=1
+  config:
+    apiVersion: kubelet.config.k8s.io/v1beta1
+    kind: KubeletConfiguration
+  restart: true
+  config_file_overrides:
+    cgroupDriver: systemd
+    cgroupRoot: /kube_whitelist
+    containerLogMaxFiles: 2
+    containerRuntimeEndpoint: unix:///run/containerd/containerd.sock
+    eventRecordQPS: 50
+    evictionMaxPodGracePeriod: -1
+    kubeAPIBurst: 40
+    kubeAPIQPS: 20
+    maxPods: 250
+    nodeStatusUpdateFrequency: 5s
+    podPidsLimit: 10000
+    podsPerCore: 10
+    readOnlyPort: 0
+    runtimeRequestTimeout: 15m
+    seccompDefault: true
+    tlsCipherSuites:
+      - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
+      - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256
+      - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
+      - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA
+      - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA
+      - TLS_RSA_WITH_AES_128_GCM_SHA256
+      - TLS_RSA_WITH_AES_256_GCM_SHA384
+      - TLS_RSA_WITH_AES_128_CBC_SHA
+      - TLS_RSA_WITH_AES_256_CBC_SHA
+
+  files_to_copy:
+    /etc/default/kubelet: /tmp/kubelet/kubelet-defaults
+    /etc/systemd/system/kubelet.service: /tmp/kubelet/kubelet-service
+    /var/lib/kubelet/config: /tmp/kubelet/kubelet-config
+
+network:
+  dns:
+    service_ip: "10.96.0.10"
+    cluster_domain: "cluster.local"
+  kubernetes_apiserver:
+    ingress:
+      public: true
+      classes:
+        namespace: "nginx-cluster"
+        cluster: "nginx-cluster"
+      annotations:
+        nginx.ingress.kubernetes.io/rewrite-target: /
+        nginx.ingress.kubernetes.io/proxy-read-timeout: "120"
+        nginx.ingress.kubernetes.io/ssl-redirect: "true"
+        nginx.ingress.kubernetes.io/secure-backends: "true"
+    name: kubernetes-apiserver
+    port: 6443
+    node_port:
+      enabled: false
+      port: 31943
+
+service:
+  name: kubeadm-service
+  ip: null
+
+secrets:
+  ca:
+    crt: placeholder
+    key: placeholder
+
+  sa:
+    key: placeholder
+    pub: placeholder
+
+  front_proxy_ca:
+    crt: placeholder
+    key: placeholder
+
+  front_proxy_client:
+    crt: placeholder
+    key: placeholder
+
+  apiserver:
+    crt: placeholder
+    key: placeholder
+    etcd_client:
+      crt: placeholder
+      key: placeholder
+    kubelet_client:
+      crt: placeholder
+      key: placeholder
+
+  controller_manager:
+    crt: placeholder
+    key: placeholder
+
+  scheduler:
+    crt: placeholder
+    key: placeholder
+
+  etcd:
+    ca:
+      crt: placeholder
+    healthcheck_client:
+      crt: placeholder
+      key: placeholder
+
+  kubelet: {}
+
+# typically overriden by environmental
+# values, but should include all endpoints
+# required by this chart
+endpoints:
+  cluster_domain_suffix: cluster.local
+  kubernetes_apiserver:
+    name: kubernetes-apiserver
+    hosts:
+      default: kubernetes-apiserver
+    port:
+      https:
+        default: 6443
+        public: 443
+    path:
+      default: /
+    scheme:
+      default: https
+      public: https
+    host_fqdn_override:
+      default: null
+      # NOTE: this chart supports TLS for fqdn over-ridden public
+      # endpoints using the following format:
+      # public:
+      #   host: null
+      #   tls:
+      #     crt: null
+      #     key: null
+
+pod:
+  mandatory_access_control:
+    type: apparmor
+    kubernetes_apiserver_anchor:
+      anchor: runtime/default
+    kube-apiserver:
+      init: runtime/default
+      apiserver-key-rotate: runtime/default
+    apiserver:
+      apiserver: runtime/default
+  security_context:
+    kubernetes_apiserver_anchor:
+      pod:
+        runAsUser: 65534
+      container:
+        anchor:
+          runAsUser: 0
+          readOnlyRootFilesystem: false
+    key_rotate:
+      pod:
+        runAsUser: 65534
+      container:
+        apiserver_key_rotate:
+          runAsUser: 0
+          readOnlyRootFilesystem: false
+    apiserver:
+      pod:
+        runAsUser: 65534
+      container:
+        apiserver:
+          runAsUser: 0
+          readOnlyRootFilesystem: false
+  mounts:
+    # .pod.mounts.kubernetes_apiserver is for the anchor daemonset
+    kubernetes_apiserver:
+      init_container: null
+      kubernetes_apiserver:
+    # .pod.mounts.apiserver is for the apiserver static pod
+    apiserver:
+      apiserver:
+# Example mounts for audit logging, refer to .conf.auditpolicy above.
+#        volumeMounts:
+#          - name: audit-logs
+#            mountPath: /var/log/audit
+#            mountPropagation: HostToContainer
+#            readOnly: false
+#        volumes:
+#          - name: audit-logs
+#            hostPath:
+#              path: /var/log/audit
+#              type: DirectoryOrCreate
+  lifecycle:
+    upgrades:
+      daemonsets:
+        pod_replacement_strategy: RollingUpdate
+        kubernetes-apiserver-anchor:
+          enabled: false
+          min_ready_seconds: 0
+          max_unavailable: 1
+    termination_grace_period:
+      kubernetes_apiserver:
+        timeout: 3600
+  resources:
+    enabled: true
+    anchor_pod:
+      requests:
+        memory: "128Mi"
+        cpu: "100m"
+      limits:
+        memory: "1024Mi"
+        cpu: "2000m"
+    key_rotate:
+      requests:
+        memory: "128Mi"
+        cpu: "100m"
+      limits:
+        memory: "1024Mi"
+        cpu: "2000m"
+    kubernetes_apiserver:
+      requests:
+        memory: "128Mi"
+        cpu: "100m"
+      limits:
+        memory: "1024Mi"
+        cpu: "2000m"
+  probes:
+    apiserver:
+      apiserver:
+        liveness:
+          enabled: true
+          params:
+            failureThreshold: 3
+            initialDelaySeconds: 60
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 10
+        readiness:
+          enabled: true
+          params:
+            initialDelaySeconds: 10
+            periodSeconds: 5
+            timeoutSeconds: 5
+  env:
+    apiserver:
+
+manifests:
+  configmap_apiserver_misc: true
+  configmap_bin: true
+  configmap_certs: true
+  configmap_kubeconfig: true
+  configmap_jobs: true
+  configmap_kubelet: true
+  configmap_kubeadm: true
+  configmap_patches: true
+  daemonset_masters: true
+  daemonset_workers: true
+  rbac: true

promenade/control/join_scripts.py

@@ -14,7 +14,6 @@
 
 import falcon
 import kubernetes
-import re
 
 from promenade.control.base import BaseResource
 from promenade.builder import Builder
@@ -49,7 +48,6 @@ class JoinScriptsResource(BaseResource):
         static_labels = _get_param_list(req, 'labels.static')
 
         join_ips = _get_join_ips()
-        role = _get_role(hostname)
 
         try:
             config = Configuration.from_design_ref(
@@ -80,7 +78,6 @@ class JoinScriptsResource(BaseResource):
                 'ip': ip,
                 'external_ip': external_ip,
                 'join_ips': join_ips,
-                'role': role,
                 'labels': {
                     'dynamic': dynamic_labels,
                     'static': static_labels,
@@ -106,14 +103,6 @@ def _get_join_ips():
     return list(map(_extract_ip, response.items))
 
 
-def _get_role(hostname):
-    pattern = re.compile(r'^[a-z]{3}\d{2}r\d{2}(?P<role>[co])\d{3}$')
-    m = pattern.match(hostname)
-    if not m:
-        return "unknown"
-    return "controller" if m.group("role") == "o" else "worker"
-
-
 def _extract_ip(item):
     for address in item.status.addresses:
         if address.type == 'InternalIP':

promenade/templates/include/up.sh

@@ -153,5 +153,8 @@ fi
 if systemctl -q is-enabled containerd > /dev/null 2>&1; then
   systemctl restart containerd || true
 fi
-systemctl enable kubelet
+
-systemctl restart kubelet
+# Start haproxy and kubelet
+haproxy &
+sleep 10
+systemctl enable --now kubelet

promenade/templates/include/utils.sh

@@ -187,7 +187,9 @@ function register_annotations {
     NODE=$1
     SEC=$2
     shift 2
+
     ANNOTATIONS="$@"
+
     end=$(($(date +%s) + $SEC))
     while true; do
         if kubectl annotate node --overwrite $NODE $ANNOTATIONS ; then

promenade/templates/roles/common/etc/default/kubelet

@@ -1,6 +1,17 @@
-{%- if 'kubeadm-enabled=true' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %}
+# Copyright 2025 AT&T Intellectual Property.  All other rights reserved.
-KUBELET_EXTRA_ARGS=""
+#
-{%- else %}
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 KUBELET_EXTRA_ARGS="
 --hostname-override={{ config.get_first('Genesis:hostname', 'KubernetesNode:hostname') }}
 --node-ip={{ config.get_first('Genesis:ip', 'KubernetesNode:ip') }}
@@ -9,10 +20,15 @@ KUBELET_EXTRA_ARGS="
 {%- elif config['KubernetesNode:labels.static'] is defined %}
 --node-labels={{ config['KubernetesNode:labels.static'] | join(',') }}
 {%- endif %}
+{%- if 'kubeadm=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %}
+{%- for argument in config.get(schema='armada/Chart/v1', name='kubernetes-kubeadm', jsonpath='values.kubelet.arguments', default=[]) %}
+{{ argument }}
+{%- endfor %}
+{%- else %}
 {%- if config['Kubelet:arguments'] is defined %}
 {%- for argument in config['Kubelet:arguments'] %}
 {{ argument }}
 {%- endfor %}
 {%- endif %}
-"
 {%- endif %}
+"

promenade/templates/roles/common/etc/kubernetes/admin.conf

@@ -0,0 +1,22 @@
+{%- if 'kubeadm=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %}
+---
+apiVersion: v1
+clusters:
+- cluster:
+    server: https://127.0.0.1:6553
+    certificate-authority: /etc/kubernetes/admin/pki/cluster-ca.pem
+  name: kubernetes
+contexts:
+- context:
+    cluster: kubernetes
+    user: admin
+  name: admin@kubernetes
+current-context: admin@kubernetes
+kind: Config
+preferences: {}
+users:
+- name: admin
+  user:
+    client-certificate: /etc/kubernetes/admin/pki/admin.pem
+    client-key: /etc/kubernetes/admin/pki/admin-key.pem
+{%- endif %}

promenade/templates/roles/common/etc/kubernetes/apiserver/acconfig.yaml

@@ -0,0 +1,7 @@
+{%- if 'kubeadm=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') and 'kubernetes-apiserver=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %}---
+kind: AdmissionConfiguration
+apiVersion: apiserver.k8s.io/v1alpha1
+plugins:
+  - name: EventRateLimit
+    path: eventconfig.yaml
+{%- endif %}

promenade/templates/roles/common/etc/kubernetes/apiserver/audit-policy.yaml

@@ -0,0 +1 @@
+{%- if 'kubeadm=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') and 'kubernetes-apiserver=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %}{{ config.get(schema='nc/AuditPolicy/v1', name='k8s-audit-policy') | toyaml | trim }}{%- endif %}

promenade/templates/roles/common/etc/kubernetes/apiserver/encryption_provider.yaml

@@ -0,0 +1,6 @@
+{%- if 'kubeadm=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') and 'kubernetes-apiserver=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %}---
+kind: EncryptionConfiguration
+apiVersion: apiserver.config.k8s.io/v1
+resources:
+{{ config.get_path('EncryptionPolicy:etcd', {}) | toyaml | indent(2, true) }}
+{%- endif %}

promenade/templates/roles/common/etc/kubernetes/apiserver/eventconfig.yaml

@@ -0,0 +1,8 @@
+{%- if 'kubeadm=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') and 'kubernetes-apiserver=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %}---
+kind: Configuration
+apiVersion: eventratelimit.admission.k8s.io/v1alpha1
+limits:
+  - type: Server
+    qps: 1000 # reduce to 100
+    burst: 10000 # reduce to 1000
+{%- endif %}

promenade/templates/roles/common/etc/kubernetes/controller-manager.conf

@@ -0,0 +1,21 @@
+{%- if 'kubeadm=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') and 'kubernetes-apiserver=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %}---
+apiVersion: v1
+clusters:
+- cluster:
+    server: https://127.0.0.1:6553
+    certificate-authority: pki/ca.crt
+  name: kubernetes
+contexts:
+- context:
+    cluster: kubernetes
+    user: controller-manager
+  name: controller-manager@kubernetes
+current-context: controller-manager@kubernetes
+kind: Config
+preferences: {}
+users:
+- name: controller-manager
+  user:
+    client-certificate: pki/controller-manager.pem
+    client-key: pki/controller-manager-key.pem
+{%- endif %}

promenade/templates/roles/common/etc/kubernetes/kubelet.conf

@@ -3,7 +3,7 @@ apiVersion: v1
 clusters:
 - cluster:
     server: https://127.0.0.1:6553
-    certificate-authority: pki/cluster-ca.pem
+    certificate-authority: pki/kubelet-client-ca.pem
   name: kubernetes
 contexts:
 - context:

promenade/templates/roles/common/etc/kubernetes/kubelet/config.yaml

@@ -1,3 +1 @@
-{%- if 'kubeadm-enabled=true' not in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %}
+{%- if 'kubeadm=enabled' not in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %}{% include "kubelet-config.yaml" with context %}{%- endif %}
-{% include "kubelet-config.yaml" with context %}
-{%- endif %}

promenade/templates/roles/common/etc/kubernetes/pki/cluster-ca.pem

@@ -1 +0,0 @@
-{{ config.get(schema='deckhand/CertificateAuthority/v1', name='kubernetes') }}

promenade/templates/roles/common/etc/kubernetes/scheduler.conf

@@ -0,0 +1,21 @@
+{%- if 'kubeadm=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') and 'kubernetes-apiserver=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %}---
+apiVersion: v1
+clusters:
+- cluster:
+    server: https://127.0.0.1:6553
+    certificate-authority: pki/ca.crt
+  name: kubernetes
+contexts:
+- context:
+    cluster: kubernetes
+    user: scheduler
+  name: scheduler@kubernetes
+current-context: scheduler@kubernetes
+kind: Config
+preferences: {}
+users:
+- name: scheduler
+  user:
+    client-certificate: pki/scheduler.pem
+    client-key: pki/scheduler-key.pem
+{%- endif %}

promenade/templates/roles/common/etc/systemd/system/kubelet.service

@@ -1,14 +1,28 @@
+# Copyright 2025 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 [Unit]
 Description=Kubernetes Worker Process
 Requires=network-online.target
 After=network-online.target containerd.service
 
 [Service]
-{%- if 'kubeadm-enabled=true' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %}
+{%- if 'kubeadm=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %}
 Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
 Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
 {%- else %}
-Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubeconfig"
+Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf"
 Environment="KUBELET_CONFIG_ARGS=--config=/etc/kubernetes/kubelet/config.yaml"
 {%- endif %}
 # This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically

promenade/templates/roles/common/usr/local/bin/haproxy

@@ -1,17 +1,19 @@
 #!/bin/bash
 
-set -exuo pipefail
+set -euo pipefail
 
 IMAGE_HAPROXY="{{ config['HostSystem:images.haproxy'] }}"
 
 while true; do
-    if systemctl is-active kubelet && [[ -e /etc/kubernetes/manifests/haproxy.yaml ]]; then
+    if systemctl -q is-active kubelet && [[ -e /etc/kubernetes/manifests/haproxy.yaml ]]; then
         if docker ps | grep -q haproxy; then
+          echo "Stopping auxiliary haproxy..."
           docker stop -t 10 haproxy
         fi
         break
     else
         if ! docker ps | grep -q haproxy; then
+          echo "Starting auxiliary haproxy..."
           docker run --rm -d --net host -v /etc/promenade/haproxy:/usr/local/etc/haproxy:ro \
                 --name haproxy "$IMAGE_HAPROXY"
         fi

promenade/templates/roles/common/var/lib/kubelet/config.yaml

@@ -0,0 +1,3 @@
+{% if 'kubeadm=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') -%}
+{{- config.get(schema='armada/Chart/v1', name='kubernetes-kubeadm', jsonpath='values.kubelet.config') | toyaml | trim }}
+{% endif -%}

promenade/templates/roles/genesis/etc/kubernetes/kubeadm/init-config.yaml

@@ -0,0 +1,39 @@
+{% if 'kubeadm=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') -%}
+apiVersion: kubeadm.k8s.io/v1beta4
+bootstrapTokens: []
+kind: InitConfiguration
+localAPIEndpoint:
+  advertiseAddress: "{{ config['Genesis:ip'] }}"
+  bindPort: 6443
+nodeRegistration:
+  ignorePreflightErrors:
+  - IsPrivilegedUser
+  - ExternalEtcdVersion
+  - Port-10250
+  imagePullPolicy: IfNotPresent
+  imagePullSerial: false
+  name: "{{ config['Genesis:hostname'] }}"
+  criSocket: "unix:///run/containerd/containerd.sock"
+  taints: []
+skipPhases:
+- certs
+- kubeconfig
+- etcd
+- kubelet-start
+- upload-config
+- upload-certs
+- mark-control-plane
+- kubelet-finalize
+- addon
+- show-join-command
+timeouts:
+  controlPlaneComponentHealthCheck: 4m0s
+  discovery: 5m0s
+  etcdAPICall: 2m0s
+  kubeletHealthCheck: 4m0s
+  kubernetesAPICall: 1m0s
+  tlsBootstrap: 5m0s
+  upgradeManifests: 5m0s
+---
+{{ config.get(schema='armada/Chart/v1', name='kubernetes-kubeadm', jsonpath='values.kubeadm.cluster_config') | toyaml | trim }}
+{%- endif %}

promenade/templates/roles/join/etc/kubernetes/kubeadm/join-config.yaml

@@ -0,0 +1,36 @@
+{%- if 'kubeadm=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %}---
+apiVersion: kubeadm.k8s.io/v1beta4
+caCertPath: /etc/kubernetes/pki/ca.crt
+{%- if 'kubernetes-apiserver=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %}
+controlPlane:
+  localAPIEndpoint:
+    advertiseAddress: "{{ config['KubernetesNode:ip'] }}"
+    bindPort: 6443
+{%- endif %}
+discovery:
+  file:
+    kubeConfigPath: /etc/kubernetes/admin.conf
+kind: JoinConfiguration
+nodeRegistration:
+  imagePullPolicy: IfNotPresent
+  imagePullSerial: false
+  name: {{ config['KubernetesNode:hostname'] }}
+  criSocket: "unix:///run/containerd/containerd.sock"
+  ignorePreflightErrors:
+    - FileAvailable--etc-kubernetes-kubelet.conf
+    - Port-10250
+  taints: []
+skipPhases:
+{%- if 'kubernetes-apiserver=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %}
+  - control-plane-prepare/download-certs
+  - control-plane-prepare/certs
+  - control-plane-prepare/kubeconfig
+  - kubelet-start
+  - control-plane-join
+{%- else %}
+  - control-plane-prepare
+  - kubelet-start
+  - control-plane-join
+  - wait-control-plane
+{%- endif %}
+{%- endif %}

promenade/templates/scripts/genesis.sh

@@ -4,7 +4,14 @@
 
 {% include "up.sh" with context %}
 
-haproxy &
+{%- if 'kubeadm=enabled' in config.get_first('Genesis:labels.dynamic', 'KubernetesNode:labels.dynamic') %}
+set +x
+log
+log === Bootstrapping node using kubeadm ===
+set -x
+kubeadm init --config /etc/kubernetes/kubeadm/init-config.yaml --v=5 2>&1 | tee /var/log/kubeadm.log
+register_annotations {{ config['Genesis:hostname'] }} 3600 "kubeadm.alpha.kubernetes.io/cri-socket={{ config.get(schema='armada/Chart/v1', name='kubernetes-kubeadm', jsonpath='values.kubelet.config.containerRuntimeEndpoint') }}"
+{%- endif %}
 
 mkdir -p /var/log/armada
 touch /var/log/armada/bootstrap-armada.log
@@ -38,7 +45,7 @@ set +x
 log
 log === Deploying bootstrap manifest via Armada ===
 if [[ ! -e /etc/kubernetes/manifests/bootstrap-armada.yaml ]]; then
-touch /etc/kubernetes/manifests/bootstrap-armada.yaml
+  touch /etc/kubernetes/manifests/bootstrap-armada.yaml
 fi
 {%- if config['Genesis:armada_helm_bootstrap'] is sameas true %}
 helm armadachart-install armada-bootstrap /etc/genesis/armada/assets/manifest.yaml
@@ -73,20 +80,15 @@ while true; do
         break
     fi
 done
-set -x
 
-# Terminate background job (tear down exit trap?)
-kill %1
-{%- if config['Genesis:enable_operator'] is sameas true %}
-kill %2
-{%- endif %}
-
-set +x
 log
 log === Waiting for Node to be Ready ===
 set -x
 wait_for_node_ready {{ config['Genesis:hostname'] }} 3600
 
+# Terminate background job (tear down exit trap?)
+kill $(jobs -p)
+
 {% include "cleanup.sh" with context %}
 
 set +x

promenade/templates/scripts/join.sh

@@ -17,8 +17,6 @@ trap delete_kubectl EXIT
 
 {% include "up.sh" with context %}
 
-haproxy &
-
 set +x
 log
 log === Waiting for Node to be Ready ===
@@ -34,6 +32,7 @@ register_labels {{ config['KubernetesNode:hostname'] }} 3600 {{ config['Kubernet
 {%- endif %}
 
 sleep 60
+wait $(jobs -p)
 
 set +x
 log

tools/helm_install.sh

@@ -17,7 +17,8 @@
 set -x
 
 HELM=$1
-HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.17.3-linux-amd64.tar.gz"}
+OS=$(uname -s | awk '{print tolower($0)}')
+HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.17.3-$OS-amd64.tar.gz"}
 
 
 function install_helm_binary {
@@ -33,7 +34,7 @@ function install_helm_binary {
     curl -o "${TMP_DIR}/helm.tar.gz" "${HELM_ARTIFACT_URL}"
     cd ${TMP_DIR}
     tar -xvzf helm.tar.gz
-    cp "${TMP_DIR}/linux-amd64/helm" "${HELM}"
+    cp "${TMP_DIR}/$OS-amd64/helm" "${HELM}"
   else
     echo "Cannot write to ${HELM}"
     exit -1