Remove healthcheck sidecar from etcd
Also make a switch from auxiliary to permanent cluster. Signed-off-by: Ruslan Aliev <raliev@mirantis.com> Change-Id: I7918072a6ba5a6b22b359d1616def8c31425462d
This commit is contained in:
Ruslan Aliev
parent
a58678d5d2
commit
905f1a74fd
@ -61,6 +61,7 @@ cleanup_host () {
|
||||
firstrun=true
|
||||
saddness_duration=0
|
||||
while true; do
|
||||
date
|
||||
# TODO(mark-burnett) Need to monitor a file(s) when shutting down/starting
|
||||
# up so I don't try to take two actions on the node at once.
|
||||
{{- if .Values.bootstrapping.enabled }}
|
||||
@ -117,7 +118,10 @@ while true; do
|
||||
fi
|
||||
echo Successfully added $HOSTNAME to cluster members.
|
||||
# Refresh member list so we start with the right configuration.
|
||||
etcdctl member list > /tmp/members
|
||||
if ! etcdctl member list > /tmp/members; then
|
||||
echo Could not get a member list, trying again.
|
||||
continue
|
||||
fi
|
||||
elif grep $PEER_ENDPOINT /tmp/members | grep '\bunstarted\b'; then
|
||||
# This member is in the cluster but not started
|
||||
if [ $saddness_duration -ge {{ .Values.anchor.saddness_threshold }} ]
|
||||
|
||||
@ -16,22 +16,14 @@
|
||||
|
||||
{{- $envAll := . }}
|
||||
{{- define "etcdreadinessProbeTemplate" }}
|
||||
exec:
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |-
|
||||
etcdctl endpoint health
|
||||
exit $?
|
||||
# There's no designated service for readiness check via grpc
|
||||
# This probe will be unused
|
||||
grpc:
|
||||
port: {{ .Values.network.service_client.port }}
|
||||
{{- end }}
|
||||
{{- define "etcdlivenessProbeTemplate" }}
|
||||
exec:
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |-
|
||||
etcdctl endpoint status
|
||||
exit $?
|
||||
grpc:
|
||||
port: {{ .Values.network.service_client.port }}
|
||||
{{- end }}
|
||||
# Strip off "etcd" from service name to get the application name
|
||||
# Note that application can either be kubernetes or calico for now
|
||||
@ -123,36 +115,13 @@ spec:
|
||||
- name: MANIFEST_PATH
|
||||
value: /manifests/{{ .Values.service.name }}.yaml
|
||||
{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.etcd | indent 8 }}
|
||||
# {{ dict "envAll" $envAll "component" "etcd" "container" "etcd" "type" "readiness" "probeTemplate" (include "etcdreadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 6 }}
|
||||
{{ dict "envAll" $envAll "component" "etcd" "container" "etcd" "type" "liveness" "probeTemplate" (include "etcdlivenessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 6 }}
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /var/lib/etcd
|
||||
- name: etc
|
||||
mountPath: /etc/etcd
|
||||
- name: etcd-health-check
|
||||
image: {{ .Values.images.tags.etcdctl }}
|
||||
imagePullPolicy: {{ .Values.images.pull_policy }}
|
||||
{{ tuple $envAll $envAll.Values.pod.resources.etcd_pod_sidecar | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }}
|
||||
{{ dict "envAll" $envAll "application" "etcd" "container" "etcd" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }}
|
||||
env:
|
||||
- name: ETCDCTL_API
|
||||
value: "{{ .Values.etcd.etcdctl_api }}"
|
||||
- name: ETCDCTL_DIAL_TIMEOUT
|
||||
value: "3s"
|
||||
- name: ETCDCTL_ENDPOINTS
|
||||
value: "https://127.0.0.1:{{ .Values.network.service_client.target_port }}"
|
||||
- name: ETCDCTL_CACERT
|
||||
value: "/etc/etcd/tls/client-ca.pem"
|
||||
- name: ETCDCTL_CERT
|
||||
value: "/etc/etcd/tls/etcd-client.pem"
|
||||
- name: ETCDCTL_KEY
|
||||
value: "/etc/etcd/tls/etcd-client-key.pem"
|
||||
command: ["/bin/sh", "-c", "--"]
|
||||
args: ["while true; do sleep 30; done;"]
|
||||
{{ dict "envAll" $envAll "component" "etcd" "container" "etcd" "type" "readiness" "probeTemplate" (include "etcdreadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 6 }}
|
||||
{{ dict "envAll" $envAll "component" "etcd" "container" "etcd" "type" "liveness" "probeTemplate" (include "etcdlivenessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 6 }}
|
||||
volumeMounts:
|
||||
- name: etc
|
||||
mountPath: /etc/etcd
|
||||
volumes:
|
||||
- name: data
|
||||
hostPath:
|
||||
|
||||
@ -261,13 +261,6 @@ pod:
|
||||
limits:
|
||||
memory: "1024Mi"
|
||||
cpu: "2000m"
|
||||
etcd_pod_sidecar:
|
||||
requests:
|
||||
memory: "128Mi"
|
||||
cpu: "100m"
|
||||
limits:
|
||||
memory: "1024Mi"
|
||||
cpu: "2000m"
|
||||
test:
|
||||
limits:
|
||||
memory: "128Mi"
|
||||
|
||||
@ -55,29 +55,37 @@
|
||||
value: $(ETCD_KEY_FILE)
|
||||
{%- if config['Genesis:etcd.heartbeat_interval'] is defined %}
|
||||
- name: ETCD_HEARTBEAT_INTERVAL
|
||||
value: {{ config['Genesis:etcd.heartbeat_interval'] }}
|
||||
value: "{{ config['Genesis:etcd.heartbeat_interval'] }}"
|
||||
{%- endif %}
|
||||
{%- if config['Genesis:etcd.election_timeout'] is defined %}
|
||||
- name: ETCD_ELECTION_TIMEOUT
|
||||
value: {{ config['Genesis:etcd.election_timeout'] }}
|
||||
value: "{{ config['Genesis:etcd.election_timeout'] }}"
|
||||
{%- endif %}
|
||||
{%- if config['Genesis:etcd.snapshot_count'] is defined %}
|
||||
- name: ETCD_SNAPSHOT_COUNT
|
||||
value: {{ config['Genesis:etcd.snapshot_count'] }}
|
||||
value: "{{ config['Genesis:etcd.snapshot_count'] }}"
|
||||
{%- endif %}
|
||||
{%- if config['Genesis:etcd.cipher_suites'] is defined %}
|
||||
- name: ETCD_CIPHER_SUITES
|
||||
value: {{ config['Genesis:etcd.cipher_suites'] }}
|
||||
value: "{{ config['Genesis:etcd.cipher_suites'] }}"
|
||||
{%- endif %}
|
||||
{%- if config['Genesis:etcd.gomaxprocs'] is defined %}
|
||||
- name: GOMAXPROCS
|
||||
value: {{ config['Genesis:etcd.gomaxprocs'] }}
|
||||
value: "{{ config['Genesis:etcd.gomaxprocs'] }}"
|
||||
{%- endif %}
|
||||
ports:
|
||||
- name: client
|
||||
containerPort: {{ client_port }}
|
||||
- name: peer
|
||||
containerPort: {{ peer_port }}
|
||||
livenessProbe:
|
||||
grpc:
|
||||
port: {{ client_port }}
|
||||
initialDelaySeconds: 15
|
||||
timeoutSeconds: 10
|
||||
periodSeconds: 30
|
||||
successThreshold: 1
|
||||
failureThreshold: 10
|
||||
volumeMounts:
|
||||
- name: data-{{ etcd_name }}
|
||||
mountPath: /var/lib/etcd
|
||||
|
||||
@ -38,14 +38,62 @@ spec:
|
||||
MEMBER_NAME=$1
|
||||
MEMBER_ID=$(etcdctl member list | grep "${MEMBER_NAME}" | awk -F ', ' '{ print $1 }')
|
||||
if [ -n "${MEMBER_ID}" ]; then
|
||||
if is_leader "$MEMBER_ID"; then
|
||||
abdicate
|
||||
fi
|
||||
etcdctl member remove $MEMBER_ID
|
||||
sleep 10
|
||||
fi
|
||||
}
|
||||
|
||||
adjust_endpoints () {
|
||||
ENDPOINTS=$(echo $1 | sed 's/\//\\\//g')
|
||||
echo "Adjusting ETCD_ENDPOINTS to $1 in apiserver"
|
||||
# sed -i "/- name: ETCD_ENDPOINTS/{n;s/value: .*/value: \"$ENDPOINTS\"/}" /manifests/kubernetes-apiserver.yaml
|
||||
}
|
||||
|
||||
abdicate () {
|
||||
NEW_LEADER=$(etcdctl member list | grep '\bstarted\b' | grep -Ev '\\b(auxiliary-0|auxiliary-1)\\b' | head -1 | awk -F ', ' '{print $1}')
|
||||
if [ -n "$NEW_LEADER" ]; then
|
||||
if ! etcdctl move-leader "$NEW_LEADER"; then
|
||||
echo "Attempted abdication, but failed."
|
||||
return
|
||||
fi
|
||||
sleep 5
|
||||
return
|
||||
fi
|
||||
}
|
||||
|
||||
abdicate_if_needed () {
|
||||
AUX_MEMBERS=$(etcdctl member list | grep '\bstarted\b' | grep -E "\\b(auxiliary-0|auxiliary-1)\\b" | awk -F ', ' '{print $1}')
|
||||
for m in $AUX_MEMBERS; do
|
||||
if is_leader "$m"; then
|
||||
abdicate
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
is_leader () {
|
||||
MEMBER_ID="$1"
|
||||
MEMBER_EP=$(etcdctl member list | grep "$MEMBER_ID" | awk -F ', ' '{print $5}')
|
||||
IS_LEADER=$(ETCDCTL_ENDPOINTS="$MEMBER_EP" etcdctl endpoint status | awk -F ', ' '{ print $5 }')
|
||||
if [ "$IS_LEADER" = "true" ]; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
auxiliary_threshold="{{ config.get_first('Genesis:etcd.auxiliary_threshold', default=3) }}"
|
||||
# NOTE(sh8121att): If there are enough (a fully resilient contigent) non-auxiliary members,
|
||||
# then we are ready to remove the auxiliary members. Otherwise, wait.
|
||||
while [ ! "$(external_member_count)" -ge "$auxiliary_threshold" ]; do
|
||||
# abdicate_if_needed
|
||||
CURRENT_ENDPOINTS=$(etcdctl member list | awk -F ',' '{ if (output != "") output=output","$5; else output=$5} END {gsub(/ /, "", output); print output}')
|
||||
APISERVER_ENDPOINTS=$(awk '/- name: ETCD_ENDPOINTS/{getline; sub(/.*: "/, ""); sub(/".*/, ""); print}' /manifests/kubernetes-apiserver.yaml)
|
||||
if [ "$CURRENT_ENDPOINTS" != "$APISERVER_ENDPOINTS" ]; then
|
||||
adjust_endpoints $CURRENT_ENDPOINTS
|
||||
fi
|
||||
sleep 30
|
||||
done
|
||||
|
||||
@ -53,6 +101,11 @@ spec:
|
||||
# should be recovered by restarting this container.
|
||||
set -e
|
||||
|
||||
sleep 600
|
||||
# make a switch to permanent etcd cluster
|
||||
PERMANENT_ENDPOINTS=$(etcdctl member list | awk -F ',' '$3 ~! /auxiliary/ { if (output != "") output=output","$5; else output=$5} END {gsub(/ /, "", output); print output}' | sed 's/\//\\\//g')
|
||||
adjust_endpoints $PERMANENT_ENDPOINTS
|
||||
|
||||
remove_if_possible auxiliary-0
|
||||
remove_if_possible auxiliary-1
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user