(haproxy) Additional config safeguards
- Some reported cases that the haproxy config was corrupted during node reboots. Attempt to add additional safeguards of coordination between the anchor and the service pod. - Support nulling out a default entry in the service list - Add additional log statements in the anchor Change-Id: Ie673c50e1037d5dff2b9f67b14032e188183a5d9
This commit is contained in:
parent
19169bb458
commit
479d3cc402
@ -35,10 +35,11 @@ install_config() {
|
||||
SUCCESS=1
|
||||
# Inject global and default config
|
||||
mkdir -p $(dirname "$HAPROXY_CONF")
|
||||
cp "$HAPROXY_HEADER" "$NEXT_HAPROXY_CONF"
|
||||
cat "$HAPROXY_HEADER" > "$NEXT_HAPROXY_CONF"
|
||||
|
||||
{{- range $namespace, $services := $envAll.Values.conf.anchor.services }}
|
||||
{{- range $service, $svc_data := $services }}
|
||||
{{- if $svc_data }}
|
||||
{{- $fe_count = add $fe_count 1 }}
|
||||
|
||||
echo Constructing config for namespace=\"{{ $namespace }}\" service=\"{{ $service }}\"
|
||||
@ -52,6 +53,12 @@ install_config() {
|
||||
--namespace {{ $namespace }} \
|
||||
get endpoints {{ $service }} \
|
||||
-o 'jsonpath={.subsets[0].addresses[*].ip}')
|
||||
|
||||
if [ $? -ne 0]; then
|
||||
echo "Unable to retrieve service IPs for {{ $service }}, will retry configuration render."
|
||||
return 1
|
||||
fi
|
||||
|
||||
DEST_PORT=$(kubectl \
|
||||
--server "$KUBE_URL" \
|
||||
--certificate-authority "$KUBE_CA" \
|
||||
@ -59,11 +66,18 @@ install_config() {
|
||||
--namespace {{ $namespace }} \
|
||||
get endpoints {{ $service }} \
|
||||
-o 'jsonpath={.subsets[0].ports[0].port}')
|
||||
|
||||
if [ $? -ne 0]; then
|
||||
echo "Unable to retrieve service port for {{ $service }}, will retry configuration render."
|
||||
return 1
|
||||
fi
|
||||
|
||||
set -x
|
||||
|
||||
if [ "x$SERVICE_IPS" != "x" ]; then
|
||||
if [ "x$DEST_PORT" != "x" ]; then
|
||||
IDENTIFIER=$(echo "{{ $namespace }}-{{ $service }}")
|
||||
echo "Adding $IDENTIFIER to haproxy config"
|
||||
# Add frontend config
|
||||
echo >> "$NEXT_HAPROXY_CONF"
|
||||
echo "frontend ${IDENTIFIER}-fe" >> "$NEXT_HAPROXY_CONF"
|
||||
@ -86,6 +100,7 @@ install_config() {
|
||||
{{- end }}
|
||||
|
||||
for IP in $SERVICE_IPS; do
|
||||
echo "Adding backend $IP:$DEST_PORT"
|
||||
echo " server s$IP $IP:$DEST_PORT" {{ $svc_data.server_opts | quote }} >> "$NEXT_HAPROXY_CONF"
|
||||
done
|
||||
else
|
||||
@ -98,6 +113,7 @@ install_config() {
|
||||
fi
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
if [ $SUCCESS = 1 ]; then
|
||||
mkdir -p $(dirname "$HAPROXY_CONF")
|
||||
@ -115,7 +131,10 @@ install_config() {
|
||||
fi
|
||||
chmod -R go-rwx $(dirname "$HAPROXY_CONF")
|
||||
chown -R $RUNASUSER:$RUNASUSER $(dirname "$HAPROXY_CONF")
|
||||
return 0
|
||||
fi
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
validate_config() {
|
||||
@ -155,9 +174,9 @@ while true; do
|
||||
break
|
||||
fi
|
||||
|
||||
install_config
|
||||
|
||||
compare_copy_files
|
||||
if install_config; then
|
||||
compare_copy_files
|
||||
fi
|
||||
|
||||
sleep {{ .Values.conf.anchor.period }}
|
||||
done
|
||||
|
@ -38,21 +38,40 @@ spec:
|
||||
value: {{ .Values.conf.haproxy.container_config_dir }}/haproxy.cfg
|
||||
- name: LIVE_HAPROXY_CONF
|
||||
value: /tmp/live_haproxy.cfg
|
||||
- name: STAGE_HAPROXY_CONF
|
||||
value: /tmp/stage_haproxy.cfg
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
set -eux
|
||||
|
||||
while [ ! -s "$HAPROXY_CONF" ]; do
|
||||
echo Waiting for "HAPROXY_CONF"
|
||||
test_conf () {
|
||||
if [ ! -s "$HAPROXY_CONF" ]; then
|
||||
echo "New proposed config not found at $HAPROXY_CONF"
|
||||
return 1
|
||||
fi
|
||||
# this is a safety gate to avoid a race of the anchor
|
||||
# changing a possible new config between the validation
|
||||
# and installation
|
||||
echo "Staging proposed config for installation."
|
||||
cp "$HAPROXY_CONF" "$STAGE_HAPROXY_CONF"
|
||||
if [ ! haproxy -c -f "$STAGE_HAPROXY_CONF"]; then
|
||||
echo "Proposed config not valid."
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
while ! test_conf; do
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo vvv Starting with initial config vvv
|
||||
cat "$HAPROXY_CONF"
|
||||
cat "$STAGE_HAPROXY_CONF"
|
||||
echo
|
||||
cp "$HAPROXY_CONF" "$LIVE_HAPROXY_CONF"
|
||||
chmod 700 $LIVE_HAPROXY_CONF
|
||||
mv "$STAGE_HAPROXY_CONF" "$LIVE_HAPROXY_CONF"
|
||||
chmod 700 "$LIVE_HAPROXY_CONF"
|
||||
|
||||
# NOTE(mark-burnett): sleep for clearer log output
|
||||
sleep 1
|
||||
@ -64,7 +83,7 @@ spec:
|
||||
set +x
|
||||
while true; do
|
||||
if ! cmp -s "$HAPROXY_CONF" "$LIVE_HAPROXY_CONF"; then
|
||||
if ! haproxy -c -f "$HAPROXY_CONF"; then
|
||||
if ! test_conf; then
|
||||
echo New config file appears invalid, refusing to replace.
|
||||
else
|
||||
echo vvv Replacing old config vvv
|
||||
@ -72,10 +91,10 @@ spec:
|
||||
echo
|
||||
|
||||
echo vvv With new config vvv
|
||||
cat "$HAPROXY_CONF"
|
||||
cat "$STAGE_HAPROXY_CONF"
|
||||
echo
|
||||
|
||||
cat "$HAPROXY_CONF" > "$LIVE_HAPROXY_CONF"
|
||||
cp "$STAGE_HAPROXY_CONF" "$LIVE_HAPROXY_CONF"
|
||||
|
||||
# NOTE(mark-burnett): sleep for clearer log output
|
||||
sleep 1
|
||||
|
Loading…
Reference in New Issue
Block a user