9e0ab1871a
* Adds retries to parts of the teardown script. * Caches tar_urls from Promenade config. Change-Id: I009eff2479cf486d03bececc975aba6c08b9a8f6
47 lines
1.2 KiB
Plaintext
Executable File
47 lines
1.2 KiB
Plaintext
Executable File
{% include "header.sh" with context %}
|
|
|
|
{% set hostname = config.get_first('Genesis:hostname', 'KubernetesNode:hostname') -%}
|
|
|
|
set -xe
|
|
|
|
if [ $(kubectl get nodes | grep '\bReady\b' | wc -l) -lt 2 ]; then
|
|
echo Not enough live nodes to proceed with teardown. 1>&2
|
|
exit 1
|
|
fi
|
|
|
|
# Evict pods
|
|
kubectl drain --force --timeout 3600s --grace-period 1800 --ignore-daemonsets --delete-local-data {{ hostname }} || true
|
|
|
|
LABELS=()
|
|
for attempt in $(seq 5); do
|
|
LABELS=($(kubectl get node {{ hostname }} -o 'jsonpath={.metadata.labels}' | sed 's/map\[//' | sed 's/\]//' | tr ' ' '\n' | sed 's/:.*/-/g' | grep -v 'kubernetes.io'))
|
|
if [[ ! -z "${LABELS[@]}" ]]; then
|
|
log "Got labels for host."
|
|
break
|
|
fi
|
|
log "Didn't get labels for host, retrying.."
|
|
sleep 5
|
|
done
|
|
|
|
if [[ -z "${LABELS[@]}" ]]; then
|
|
log "Failed to get labels for host. Aborting teardown."
|
|
exit 1
|
|
fi
|
|
|
|
for attempt in $(seq 5); do
|
|
if kubectl label node {{ hostname }} "${LABELS[@]}"; then
|
|
break
|
|
fi
|
|
log "Failed to remove labels from node, retrying.."
|
|
sleep 5
|
|
done
|
|
|
|
sleep 30
|
|
|
|
# Stop kubelet
|
|
systemctl stop kubelet
|
|
systemctl disable kubelet
|
|
|
|
# Stop remaining containers processes
|
|
docker rm -f $(docker ps -aq) || true
|