openstack/octavia
Code Issues Proposed changes

Preserve haproxy server states during reloads

Browse Source 
haproxy doesn't keep the state of the servers (UP or DOWN) during
reloads, so a member with an ERROR operating_status may be updated
to ONLINE after updating a load balancer.

This commit adds the load-server-state-from-file option in haproxy
configuration files. It also adds an extra step in the haproxy systemd
service file to dump the current server-state in a file when reloading
the service.

Story: 2009142
Task: 43084

Change-Id: Ia8ec48ab858eeecf71ed429e5b7625fd7af9d8f6
(cherry picked from commit 1b15e1ef22)
This commit is contained in:
Gregory Thiemonge 2021-08-23 19:53:28 +02:00
parent 58eac3d91f
commit 24ebc652e8
10 changed files with 58 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
octavia/amphorae/backends/agent/api_server/loadbalancer.py
View File

@ -189,6 +189,8 @@ class Loadbalancer(object):
haproxy_pid=util.pid_path(lb_id),
haproxy_cmd=util.CONF.haproxy_amphora.haproxy_cmd,
haproxy_cfg=util.config_path(lb_id),
haproxy_state_file=util.state_file_path(lb_id),
haproxy_socket=util.haproxy_sock_path(lb_id),
haproxy_user_group_cfg=consts.HAPROXY_USER_GROUP_CFG,
respawn_count=util.CONF.haproxy_amphora.respawn_count,
respawn_interval=(util.CONF.haproxy_amphora.

1
octavia/amphorae/backends/agent/api_server/templates/systemd.conf.j2
View File

@ -13,6 +13,7 @@ Environment="CONFIG={{ haproxy_cfg }}" "USERCONFIG={{ haproxy_user_group_cfg }}"
ExecStartPre={{ haproxy_cmd }} -f $CONFIG -f $USERCONFIG -c -q -L {{ peer_name }}
ExecReload=/bin/sh -c "echo 'show servers state' | socat stdio unix-connect:{{ haproxy_socket }} > {{ haproxy_state_file }}"
ExecReload={{ haproxy_cmd }} -c -f $CONFIG -f $USERCONFIG -L {{ peer_name }}
ExecReload=/bin/kill -USR2 $MAINPID

4
octavia/amphorae/backends/agent/api_server/util.py
View File

@ -114,6 +114,10 @@ def config_path(lb_id):
return os.path.join(haproxy_dir(lb_id), 'haproxy.cfg')
def state_file_path(lb_id):
return os.path.join(haproxy_dir(lb_id), 'servers-state')
def get_haproxy_pid(lb_id):
with open(pid_path(lb_id), 'r', encoding='utf-8') as f:
return f.readline().rstrip()

4
octavia/common/jinja/haproxy/combined_listeners/jinja_cfg.py
View File

@ -162,11 +162,15 @@ class JinjaTemplater(object):
if not socket_path:
socket_path = '%s/%s.sock' % (self.base_amp_path,
listeners[0].load_balancer.id)
state_file_path = '%s/%s/servers-state' % (
self.base_amp_path,
listeners[0].load_balancer.id)
return self._get_template().render(
{'loadbalancer': loadbalancer,
'stats_sock': socket_path,
'log_http': self.log_http,
'log_server': self.log_server,
'state_file': state_file_path,
'administrative_log_facility':
CONF.amphora_agent.administrative_log_facility,
'user_log_facility': CONF.amphora_agent.user_log_facility,

1
octavia/common/jinja/haproxy/combined_listeners/templates/base.j2
View File

@ -20,6 +20,7 @@ global
log {{ log_http | default('/run/rsyslog/octavia/log', true)}} local{{ user_log_facility }}
log {{ log_server | default('/run/rsyslog/octavia/log', true)}} local{{ administrative_log_facility }} notice
stats socket {{ sock_path }} mode 0666 level user
server-state-file {{ state_file }}
{% if loadbalancer.global_connection_limit is defined %}
maxconn {{ loadbalancer.global_connection_limit }}
{% endif %}

1
octavia/common/jinja/haproxy/combined_listeners/templates/macros.j2
View File

@ -340,6 +340,7 @@ backend {{ pool.id }}:{{ listener.id }}
{% endif %}
{% endif %}
{% if pool.health_monitor and pool.health_monitor.enabled %}
load-server-state-from-file global
timeout check {{ pool.health_monitor.timeout }}s
{% if (pool.health_monitor.type ==
constants.HEALTH_MONITOR_HTTP or pool.health_monitor.type ==

2
octavia/tests/unit/amphorae/backends/agent/api_server/test_haproxy_compatibility.py
View File

@ -31,6 +31,8 @@ class HAProxyCompatTestCase(base.TestCase):
" log /run/rsyslog/octavia/log local1 notice\n"
" stats socket /var/lib/octavia/sample_loadbalancer_id_1.sock"
" mode 0666 level user\n"
" server-state-file /var/lib/octavia/sample_loadbalancer_id_1"
"/servers-state\n"
" maxconn {maxconn}\n\n"
"defaults\n"
" log global\n"

34
octavia/tests/unit/common/jinja/haproxy/combined_listeners/test_jinja_cfg.py
View File

@ -64,6 +64,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -118,6 +119,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -168,6 +170,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -223,6 +226,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -276,6 +280,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -329,6 +334,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -380,6 +386,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -415,6 +422,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -441,6 +449,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -477,6 +486,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -512,6 +522,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -540,6 +551,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -580,6 +592,7 @@ class TestHaproxyCfg(base.TestCase):
" mode tcp\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -616,6 +629,7 @@ class TestHaproxyCfg(base.TestCase):
" mode tcp\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option ssl-hello-chk\n"
" fullconn {maxconn}\n"
@ -683,6 +697,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option external-check\n"
" external-check command /var/lib/octavia/ping-wrapper.sh\n"
@ -743,6 +758,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.1\\r\\nHost:\\ "
"testlab.com\n"
@ -820,6 +836,7 @@ class TestHaproxyCfg(base.TestCase):
" balance roundrobin\n"
" stick-table type ip size 10k\n"
" stick on src\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -847,6 +864,7 @@ class TestHaproxyCfg(base.TestCase):
" stick-table type string len 64 size 10k\n"
" stick store-response res.cook(JSESSIONID)\n"
" stick match req.cook(JSESSIONID)\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -970,6 +988,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -986,6 +1005,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /healthmon.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -1007,6 +1027,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -1035,6 +1056,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -1065,6 +1087,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" fullconn {maxconn}\n"
" option allbackups\n"
@ -1092,6 +1115,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -1132,6 +1156,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -1171,6 +1196,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -1211,6 +1237,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -1248,6 +1275,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -1281,6 +1309,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -1311,6 +1340,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -1527,6 +1557,7 @@ class TestHaproxyCfg(base.TestCase):
" http-reuse safe\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" fullconn {maxconn}\n"
" option allbackups\n"
@ -1555,6 +1586,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" fullconn {maxconn}\n"
" option allbackups\n"
@ -1628,6 +1660,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -1643,6 +1676,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /healthmon.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"

5
octavia/tests/unit/common/sample_configs/sample_configs_combined.py
View File

@ -1213,6 +1213,7 @@ def sample_base_expected_config(frontend=None, logging=None, backend=None,
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -1246,5 +1247,7 @@ def sample_base_expected_config(frontend=None, logging=None, backend=None,
" log /run/rsyslog/octavia/log local0\n"
" log /run/rsyslog/octavia/log local1 notice\n"
" stats socket /var/lib/octavia/sample_loadbalancer_id_1.sock"
" mode 0666 level user\n" +
" mode 0666 level user\n"
" server-state-file /var/lib/octavia/sample_loadbalancer_id_1"
"/servers-state\n" +
global_opts + defaults + peers + frontend + logging + backend)

5
releasenotes/notes/fix-member-operating-status-on-reload-fe3688603bae8726.yaml Normal file
View File

@ -0,0 +1,5 @@
---
fixes:
- |
Fixed an issue with members in ERROR operating status that may have been
updated briefly to ONLINE during a Load Balancer configuration change.