openstack
/
octavia
Code Issues Proposed changes

Preserve haproxy server states during reloads 

haproxy doesn't keep the state of the servers (UP or DOWN) during
reloads, so a member with an ERROR operating_status may be updated
to ONLINE after updating a load balancer.

This commit adds the load-server-state-from-file option in haproxy
configuration files. It also adds an extra step in the haproxy systemd
service file to dump the current server-state in a file when reloading
the service.

Story: 2009142
Task: 43084

Conflicts:
	octavia/tests/unit/common/jinja/haproxy/combined_listeners/test_jinja_cfg.py

Change-Id: Ia8ec48ab858eeecf71ed429e5b7625fd7af9d8f6
(cherry picked from commit 1b15e1ef22)
(cherry picked from commit 24ebc652e8)
(cherry picked from commit 55bce250ca)
(cherry picked from commit 20a874e774)
(cherry picked from commit 1d8ac7e2c1)
This commit is contained in:
Gregory Thiemonge 2021-08-23 19:53:28 +02:00
parent 51c6d8bd90
commit 1bc1477809
10 changed files with 48 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
octavia/amphorae/backends/agent/api_server/loadbalancer.py
View File

@ -192,6 +192,8 @@ class Loadbalancer(object):
haproxy_pid=util.pid_path(lb_id),
haproxy_cmd=util.CONF.haproxy_amphora.haproxy_cmd,
haproxy_cfg=util.config_path(lb_id),
haproxy_state_file=util.state_file_path(lb_id),
haproxy_socket=util.haproxy_sock_path(lb_id),
haproxy_user_group_cfg=consts.HAPROXY_USER_GROUP_CFG,
respawn_count=util.CONF.haproxy_amphora.respawn_count,
respawn_interval=(util.CONF.haproxy_amphora.

1
octavia/amphorae/backends/agent/api_server/templates/systemd.conf.j2
View File

@ -13,6 +13,7 @@ Environment="CONFIG={{ haproxy_cfg }}" "USERCONFIG={{ haproxy_user_group_cfg }}"
ExecStartPre={{ haproxy_cmd }} -f $CONFIG -f $USERCONFIG -c -q -L {{ peer_name }}
ExecReload=/bin/sh -c "echo 'show servers state' | socat stdio unix-connect:{{ haproxy_socket }} > {{ haproxy_state_file }}"
ExecReload={{ haproxy_cmd }} -c -f $CONFIG -f $USERCONFIG -L {{ peer_name }}
ExecReload=/bin/kill -USR2 $MAINPID

4
octavia/amphorae/backends/agent/api_server/util.py
View File

@ -115,6 +115,10 @@ def config_path(lb_id):
return os.path.join(haproxy_dir(lb_id), 'haproxy.cfg')
def state_file_path(lb_id):
return os.path.join(haproxy_dir(lb_id), 'servers-state')
def get_haproxy_pid(lb_id):
with open(pid_path(lb_id), 'r') as f:
return f.readline().rstrip()

4
octavia/common/jinja/haproxy/combined_listeners/jinja_cfg.py
View File

@ -159,11 +159,15 @@ class JinjaTemplater(object):
if not socket_path:
socket_path = '%s/%s.sock' % (self.base_amp_path,
listeners[0].load_balancer.id)
state_file_path = '%s/%s/servers-state' % (
self.base_amp_path,
listeners[0].load_balancer.id)
return self._get_template().render(
{'loadbalancer': loadbalancer,
'stats_sock': socket_path,
'log_http': self.log_http,
'log_server': self.log_server,
'state_file': state_file_path,
'administrative_log_facility':
CONF.amphora_agent.administrative_log_facility,
'user_log_facility': CONF.amphora_agent.user_log_facility,

1
octavia/common/jinja/haproxy/combined_listeners/templates/base.j2
View File

@ -20,6 +20,7 @@ global
log {{ log_http | default('/run/rsyslog/octavia/log', true)}} local{{ user_log_facility }}
log {{ log_server | default('/run/rsyslog/octavia/log', true)}} local{{ administrative_log_facility }} notice
stats socket {{ sock_path }} mode 0666 level user
server-state-file {{ state_file }}
{% if loadbalancer.global_connection_limit is defined %}
maxconn {{ loadbalancer.global_connection_limit }}
{% endif %}

1
octavia/common/jinja/haproxy/combined_listeners/templates/macros.j2
View File

@ -276,6 +276,7 @@ backend {{ pool.id }}:{{ listener.id }}
{% endif %}
{% endif %}
{% if pool.health_monitor and pool.health_monitor.enabled %}
load-server-state-from-file global
timeout check {{ pool.health_monitor.timeout }}s
{% if (pool.health_monitor.type ==
constants.HEALTH_MONITOR_HTTP or pool.health_monitor.type ==

2
octavia/tests/unit/amphorae/backends/agent/api_server/test_haproxy_compatibility.py
View File

@ -32,6 +32,8 @@ class HAProxyCompatTestCase(base.TestCase):
" log /run/rsyslog/octavia/log local1 notice\n"
" stats socket /var/lib/octavia/sample_loadbalancer_id_1.sock"
" mode 0666 level user\n"
" server-state-file /var/lib/octavia/sample_loadbalancer_id_1"
"/servers-state\n"
" maxconn {maxconn}\n\n"
"defaults\n"
" log global\n"

24
octavia/tests/unit/common/jinja/haproxy/combined_listeners/test_jinja_cfg.py
View File

@ -61,6 +61,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -112,6 +113,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -146,6 +148,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -172,6 +175,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -208,6 +212,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -243,6 +248,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -271,6 +277,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -311,6 +318,7 @@ class TestHaproxyCfg(base.TestCase):
" mode tcp\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -347,6 +355,7 @@ class TestHaproxyCfg(base.TestCase):
" mode tcp\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option ssl-hello-chk\n"
" fullconn {maxconn}\n"
@ -414,6 +423,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option external-check\n"
" external-check command /var/lib/octavia/ping-wrapper.sh\n"
@ -474,6 +484,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.1\\r\\nHost:\\ "
"testlab.com\n"
@ -551,6 +562,7 @@ class TestHaproxyCfg(base.TestCase):
" balance roundrobin\n"
" stick-table type ip size 10k\n"
" stick on src\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -578,6 +590,7 @@ class TestHaproxyCfg(base.TestCase):
" stick-table type string len 64 size 10k\n"
" stick store-response res.cook(JSESSIONID)\n"
" stick match req.cook(JSESSIONID)\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -701,6 +714,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -717,6 +731,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /healthmon.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -738,6 +753,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -766,6 +782,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -796,6 +813,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" fullconn {maxconn}\n"
" option allbackups\n"
@ -822,6 +840,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -857,6 +876,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -1100,6 +1120,7 @@ class TestHaproxyCfg(base.TestCase):
" http-reuse safe\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" fullconn {maxconn}\n"
" option allbackups\n"
@ -1128,6 +1149,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" fullconn {maxconn}\n"
" option allbackups\n"
@ -1198,6 +1220,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -1213,6 +1236,7 @@ class TestHaproxyCfg(base.TestCase):
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /healthmon.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"

5
octavia/tests/unit/common/sample_configs/sample_configs_combined.py
View File

@ -1086,6 +1086,7 @@ def sample_base_expected_config(frontend=None, logging=None, backend=None,
" mode http\n"
" balance roundrobin\n"
" cookie SRV insert indirect nocache\n"
" load-server-state-from-file global\n"
" timeout check 31s\n"
" option httpchk GET /index.html HTTP/1.0\\r\\n\n"
" http-check expect rstatus 418\n"
@ -1119,5 +1120,7 @@ def sample_base_expected_config(frontend=None, logging=None, backend=None,
" log /run/rsyslog/octavia/log local0\n"
" log /run/rsyslog/octavia/log local1 notice\n"
" stats socket /var/lib/octavia/sample_loadbalancer_id_1.sock"
" mode 0666 level user\n" +
" mode 0666 level user\n"
" server-state-file /var/lib/octavia/sample_loadbalancer_id_1"
"/servers-state\n" +
global_opts + defaults + peers + frontend + logging + backend)

5
releasenotes/notes/fix-member-operating-status-on-reload-fe3688603bae8726.yaml Normal file
View File

@ -0,0 +1,5 @@
---
fixes:
- |
Fixed an issue with members in ERROR operating status that may have been
updated briefly to ONLINE during a Load Balancer configuration change.