Use Kubernetes lease endpoint reconciler
In the resiliency gate: * Enable the --endpoint-reconciler-type=least option for the apiserver. * Extract etcd validation into its own stages. * Test joining a node while one control plane node is down. Change-Id: Id89b0816e91ab6427c5e2f4833ad4ec4e1e3d133 Depends-On: I2150d40e917567a4072a1565c1b96089f3d6fd2b
This commit is contained in:
parent
faf5a9a2d0
commit
15658a088e
@ -9,6 +9,13 @@ metadata:
|
|||||||
data:
|
data:
|
||||||
hostname: n0
|
hostname: n0
|
||||||
ip: 192.168.77.10
|
ip: 192.168.77.10
|
||||||
|
apiserver:
|
||||||
|
command_prefix:
|
||||||
|
- /apiserver
|
||||||
|
- --authorization-mode=Node,RBAC
|
||||||
|
- --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds
|
||||||
|
- --service-cluster-ip-range=10.96.0.0/16
|
||||||
|
- --endpoint-reconciler-type=lease
|
||||||
armada:
|
armada:
|
||||||
target_manifest: cluster-bootstrap
|
target_manifest: cluster-bootstrap
|
||||||
labels:
|
labels:
|
||||||
|
@ -716,6 +716,12 @@ data:
|
|||||||
upgrade:
|
upgrade:
|
||||||
no_hooks: true
|
no_hooks: true
|
||||||
values:
|
values:
|
||||||
|
command_prefix:
|
||||||
|
- /apiserver
|
||||||
|
- --authorization-mode=Node,RBAC
|
||||||
|
- --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds
|
||||||
|
- --service-cluster-ip-range=10.96.0.0/16
|
||||||
|
- --endpoint-reconciler-type=lease
|
||||||
apiserver:
|
apiserver:
|
||||||
etcd:
|
etcd:
|
||||||
endpoints: https://127.0.0.1:2378
|
endpoints: https://127.0.0.1:2378
|
||||||
|
@ -14,7 +14,7 @@ validate_etcd_membership() {
|
|||||||
|
|
||||||
# NOTE(mark-burnett): Wait a moment for disks in test environment to settle.
|
# NOTE(mark-burnett): Wait a moment for disks in test environment to settle.
|
||||||
sleep 10
|
sleep 10
|
||||||
log Validating "${CLUSTER}" etcd membership via "${VM}"
|
log Validating "${CLUSTER}" etcd membership via "${VM}" for members: "${EXPECTED_MEMBERS[@]}"
|
||||||
FOUND_MEMBERS=$(etcdctl_member_list "${CLUSTER}" "${VM}" | tr '\n' ' ' | sed 's/ $//')
|
FOUND_MEMBERS=$(etcdctl_member_list "${CLUSTER}" "${VM}" | tr '\n' ' ' | sed 's/ $//')
|
||||||
|
|
||||||
if [[ "x${EXPECTED_MEMBERS}" != "x${FOUND_MEMBERS}" ]]; then
|
if [[ "x${EXPECTED_MEMBERS}" != "x${FOUND_MEMBERS}" ]]; then
|
||||||
|
22
tools/g2/manifests/one.json
Normal file
22
tools/g2/manifests/one.json
Normal file
@ -0,0 +1,22 @@
|
|||||||
|
{
|
||||||
|
"configuration": [
|
||||||
|
"promenade/schemas"
|
||||||
|
],
|
||||||
|
"stages": [
|
||||||
|
{
|
||||||
|
"name": "Gate Setup",
|
||||||
|
"script": "gate-setup.sh"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Create VMs",
|
||||||
|
"script": "create-vms.sh"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"vm": {
|
||||||
|
"memory": 20480,
|
||||||
|
"names": [
|
||||||
|
"n0"
|
||||||
|
],
|
||||||
|
"vcpus": 4
|
||||||
|
}
|
||||||
|
}
|
@ -48,7 +48,14 @@
|
|||||||
"-l", "kubernetes-controller-manager=enabled",
|
"-l", "kubernetes-controller-manager=enabled",
|
||||||
"-l", "kubernetes-etcd=enabled",
|
"-l", "kubernetes-etcd=enabled",
|
||||||
"-l", "kubernetes-scheduler=enabled",
|
"-l", "kubernetes-scheduler=enabled",
|
||||||
"-l", "ucp-control-plane=enabled",
|
"-l", "ucp-control-plane=enabled"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Check initial etcd cluster",
|
||||||
|
"script": "check-etcd-health.sh",
|
||||||
|
"arguments": [
|
||||||
|
"-w", "10",
|
||||||
"-e", "kubernetes n0 n0 n1 n2",
|
"-e", "kubernetes n0 n0 n1 n2",
|
||||||
"-e", "calico n0 n0 n1 n2"
|
"-e", "calico n0 n0 n1 n2"
|
||||||
]
|
]
|
||||||
@ -61,6 +68,14 @@
|
|||||||
"-n", "n3"
|
"-n", "n3"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "Power off n2",
|
||||||
|
"script": "power-down-node.sh",
|
||||||
|
"arguments": [
|
||||||
|
"-s",
|
||||||
|
"-n", "n2"
|
||||||
|
]
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "Update Generated Certs",
|
"name": "Update Generated Certs",
|
||||||
"script": "generate-certificates.sh",
|
"script": "generate-certificates.sh",
|
||||||
@ -80,7 +95,21 @@
|
|||||||
"-l", "kubernetes-controller-manager=enabled",
|
"-l", "kubernetes-controller-manager=enabled",
|
||||||
"-l", "kubernetes-etcd=enabled",
|
"-l", "kubernetes-etcd=enabled",
|
||||||
"-l", "kubernetes-scheduler=enabled",
|
"-l", "kubernetes-scheduler=enabled",
|
||||||
"-l", "ucp-control-plane=enabled",
|
"-l", "ucp-control-plane=enabled"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Power up n2",
|
||||||
|
"script": "power-up-node.sh",
|
||||||
|
"arguments": [
|
||||||
|
"-n", "n2"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Check full etcd cluster",
|
||||||
|
"script": "check-etcd-health.sh",
|
||||||
|
"arguments": [
|
||||||
|
"-w", "30",
|
||||||
"-e", "kubernetes n0 n0 n1 n2 n3",
|
"-e", "kubernetes n0 n0 n1 n2 n3",
|
||||||
"-e", "calico n0 n0 n1 n2 n3"
|
"-e", "calico n0 n0 n1 n2 n3"
|
||||||
]
|
]
|
||||||
@ -91,7 +120,13 @@
|
|||||||
"arguments": [
|
"arguments": [
|
||||||
"-v", "n1",
|
"-v", "n1",
|
||||||
"-n", "n0",
|
"-n", "n0",
|
||||||
"-r",
|
"-r"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Check post-teardown etcd cluster",
|
||||||
|
"script": "check-etcd-health.sh",
|
||||||
|
"arguments": [
|
||||||
"-e", "kubernetes n1 n1 n2 n3",
|
"-e", "kubernetes n1 n1 n2 n3",
|
||||||
"-e", "calico n1 n1 n2 n3"
|
"-e", "calico n1 n1 n2 n3"
|
||||||
]
|
]
|
||||||
@ -102,7 +137,13 @@
|
|||||||
"arguments": [
|
"arguments": [
|
||||||
"-v", "n1",
|
"-v", "n1",
|
||||||
"-n", "n0",
|
"-n", "n0",
|
||||||
"-l", "ucp-control-plane=enabled",
|
"-l", "ucp-control-plane=enabled"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Check final etcd cluster",
|
||||||
|
"script": "check-etcd-health.sh",
|
||||||
|
"arguments": [
|
||||||
"-e", "kubernetes n1 n1 n2 n3",
|
"-e", "kubernetes n1 n1 n2 n3",
|
||||||
"-e", "calico n1 n1 n2 n3"
|
"-e", "calico n1 n1 n2 n3"
|
||||||
]
|
]
|
||||||
|
38
tools/g2/stages/check-etcd-health.sh
Executable file
38
tools/g2/stages/check-etcd-health.sh
Executable file
@ -0,0 +1,38 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
source "${GATE_UTILS}"
|
||||||
|
|
||||||
|
declare -a ETCD_CLUSTERS
|
||||||
|
|
||||||
|
WAIT_BEFORE_CHECK=0
|
||||||
|
|
||||||
|
while getopts "e:w:" opt; do
|
||||||
|
case "${opt}" in
|
||||||
|
e)
|
||||||
|
ETCD_CLUSTERS+=("${OPTARG}")
|
||||||
|
;;
|
||||||
|
w)
|
||||||
|
WAIT_BEFORE_CHECK="${OPTARG}"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Unknown option"
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
shift $((OPTIND-1))
|
||||||
|
|
||||||
|
if [ $# -gt 0 ]; then
|
||||||
|
echo "Unknown arguments specified: ${*}"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
log Waiting "${WAIT_BEFORE_CHECK}" seconds before checking cluster health.
|
||||||
|
sleep "${WAIT_BEFORE_CHECK}"
|
||||||
|
|
||||||
|
for etcd_validation_string in "${ETCD_CLUSTERS[@]}"; do
|
||||||
|
IFS=' ' read -a etcd_validation_args <<<"${etcd_validation_string}"
|
||||||
|
validate_etcd_membership "${etcd_validation_args[@]}"
|
||||||
|
done
|
@ -4,7 +4,6 @@ set -eu
|
|||||||
|
|
||||||
source "${GATE_UTILS}"
|
source "${GATE_UTILS}"
|
||||||
|
|
||||||
declare -a ETCD_CLUSTERS
|
|
||||||
declare -a LABELS
|
declare -a LABELS
|
||||||
declare -a NODES
|
declare -a NODES
|
||||||
|
|
||||||
@ -12,11 +11,8 @@ GET_KEYSTONE_TOKEN=0
|
|||||||
USE_DECKHAND=0
|
USE_DECKHAND=0
|
||||||
DECKHAND_REVISION=''
|
DECKHAND_REVISION=''
|
||||||
|
|
||||||
while getopts "d:e:l:n:tv:" opt; do
|
while getopts "d:l:n:tv:" opt; do
|
||||||
case "${opt}" in
|
case "${opt}" in
|
||||||
e)
|
|
||||||
ETCD_CLUSTERS+=("${OPTARG}")
|
|
||||||
;;
|
|
||||||
d)
|
d)
|
||||||
USE_DECKHAND=1
|
USE_DECKHAND=1
|
||||||
DECKHAND_REVISION=${OPTARG}
|
DECKHAND_REVISION=${OPTARG}
|
||||||
@ -48,7 +44,6 @@ fi
|
|||||||
|
|
||||||
SCRIPT_DIR="${TEMP_DIR}/curled-scripts"
|
SCRIPT_DIR="${TEMP_DIR}/curled-scripts"
|
||||||
|
|
||||||
echo Etcd Clusters: "${ETCD_CLUSTERS[@]}"
|
|
||||||
echo Labels: "${LABELS[@]}"
|
echo Labels: "${LABELS[@]}"
|
||||||
echo Nodes: "${NODES[@]}"
|
echo Nodes: "${NODES[@]}"
|
||||||
|
|
||||||
@ -86,10 +81,3 @@ for NAME in "${NODES[@]}"; do
|
|||||||
rsync_cmd "${SCRIPT_DIR}/join-${NAME}.sh" "${NAME}:/root/promenade/"
|
rsync_cmd "${SCRIPT_DIR}/join-${NAME}.sh" "${NAME}:/root/promenade/"
|
||||||
ssh_cmd "${NAME}" "/root/promenade/join-${NAME}.sh" 2>&1 | tee -a "${LOG_FILE}"
|
ssh_cmd "${NAME}" "/root/promenade/join-${NAME}.sh" 2>&1 | tee -a "${LOG_FILE}"
|
||||||
done
|
done
|
||||||
|
|
||||||
sleep 10
|
|
||||||
|
|
||||||
for etcd_validation_string in "${ETCD_CLUSTERS[@]}"; do
|
|
||||||
IFS=' ' read -a etcd_validation_args <<<"${etcd_validation_string}"
|
|
||||||
validate_etcd_membership "${etcd_validation_args[@]}"
|
|
||||||
done
|
|
||||||
|
31
tools/g2/stages/power-down-node.sh
Executable file
31
tools/g2/stages/power-down-node.sh
Executable file
@ -0,0 +1,31 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
source "${GATE_UTILS}"
|
||||||
|
|
||||||
|
declare -a NODES
|
||||||
|
SYNC_BEFORE_STOP=0
|
||||||
|
|
||||||
|
while getopts "n:s" opt; do
|
||||||
|
case "${opt}" in
|
||||||
|
n)
|
||||||
|
NODES+=("${OPTARG}")
|
||||||
|
;;
|
||||||
|
s)
|
||||||
|
SYNC_BEFORE_STOP=1
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Unknown option"
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
shift $((OPTIND-1))
|
||||||
|
|
||||||
|
for node in "${NODES[@]}"; do
|
||||||
|
if [[ $SYNC_BEFORE_STOP == 1 ]]; then
|
||||||
|
ssh_cmd "${node}" sync
|
||||||
|
fi
|
||||||
|
vm_stop "${node}"
|
||||||
|
done
|
24
tools/g2/stages/power-up-node.sh
Executable file
24
tools/g2/stages/power-up-node.sh
Executable file
@ -0,0 +1,24 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
source "${GATE_UTILS}"
|
||||||
|
|
||||||
|
declare -a NODES
|
||||||
|
|
||||||
|
while getopts "n:s" opt; do
|
||||||
|
case "${opt}" in
|
||||||
|
n)
|
||||||
|
NODES+=("${OPTARG}")
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Unknown option"
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
shift $((OPTIND-1))
|
||||||
|
|
||||||
|
for node in "${NODES[@]}"; do
|
||||||
|
vm_start "${node}"
|
||||||
|
done
|
@ -4,16 +4,12 @@ set -eu
|
|||||||
|
|
||||||
source "${GATE_UTILS}"
|
source "${GATE_UTILS}"
|
||||||
|
|
||||||
declare -a ETCD_CLUSTERS
|
|
||||||
declare -a NODES
|
declare -a NODES
|
||||||
|
|
||||||
RECREATE=0
|
RECREATE=0
|
||||||
|
|
||||||
while getopts "e:n:rv:" opt; do
|
while getopts "n:rv:" opt; do
|
||||||
case "${opt}" in
|
case "${opt}" in
|
||||||
e)
|
|
||||||
ETCD_CLUSTERS+=("${OPTARG}")
|
|
||||||
;;
|
|
||||||
n)
|
n)
|
||||||
NODES+=("${OPTARG}")
|
NODES+=("${OPTARG}")
|
||||||
;;
|
;;
|
||||||
@ -44,8 +40,3 @@ for NAME in "${NODES[@]}"; do
|
|||||||
vm_create "${NAME}"
|
vm_create "${NAME}"
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
for etcd_validation_string in "${ETCD_CLUSTERS[@]}"; do
|
|
||||||
IFS=' ' read -a etcd_validation_args <<<"${etcd_validation_string}"
|
|
||||||
validate_etcd_membership "${etcd_validation_args[@]}"
|
|
||||||
done
|
|
||||||
|
Loading…
Reference in New Issue
Block a user