openstack/neutron
Code Issues Proposed changes

Add configurable HAProxy timeouts for metadata proxy

Browse Source 
The HAProxy metadata proxy uses hardcoded timeout values (30-32s)
in the METADATA_HAPROXY_GLOBAL template. When backend services
such as dynamic vendordata take longer than these timeouts to
respond, HAProxy terminates the connection before the response
can be delivered to the instance.

Add a new [metadata_haproxy] configuration group with options
for timeout_connect, timeout_client, timeout_server,
timeout_http_request, and timeout_http_keep_alive. Default
values match the previous hardcoded values for backward
compatibility.

Closes-Bug: #2154609
Generated-By: claude-opus-4-6 (OpenCode)
Change-Id: If3c7ee9ea16e13047e5ec8b8463d022abca745b5
Signed-off-by: flux.adam@gmail.com
This commit is contained in:
Adam Harwell committed 2026-06-16 16:14:29 -04:00
1 parent 5c095ee6e1
commit fa2b68d907
13 files changed
+125 -17
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

neutron/agent/l3_agent.py
+3
View File
@@ -38,6 +38,9 @@ def register_opts(conf):
meta_conf.register_meta_conf_opts(meta_conf.METADATA_RATE_LIMITING_OPTS,
cfg=conf,
group=meta_conf.RATE_LIMITING_GROUP)
meta_conf.register_meta_conf_opts(meta_conf.METADATA_HAPROXY_OPTS,
cfg=conf,
group=meta_conf.HAPROXY_GROUP)
config.register_interface_driver_opts_helper(conf)
config.register_agent_state_opts_helper(conf)
config.register_interface_opts(conf)
neutron/agent/metadata/driver_base.py
+7 -1
View File
@@ -52,7 +52,7 @@ class HaproxyConfiguratorBase(metaclass=abc.ABCMeta):
def __init__(self, network_id, router_id, unix_socket_path, host, port,
user, group, state_path, pid_file, rate_limiting_config,
host_v6=None, bind_interface=None):
haproxy_config, host_v6=None, bind_interface=None):
self.network_id = network_id
self.router_id = router_id
if network_id is None and router_id is None:
@@ -68,6 +68,7 @@ class HaproxyConfiguratorBase(metaclass=abc.ABCMeta):
self.unix_socket_path = unix_socket_path
self.pidfile = pid_file
self.rate_limiting_config = rate_limiting_config
self.haproxy_config = haproxy_config
self.log_level = (
'debug' if logging.is_debug_enabled(cfg.CONF) else 'info')
# log-tag will cause entries to have the string pre-pended, so use
@@ -127,6 +128,10 @@ class HaproxyConfiguratorBase(metaclass=abc.ABCMeta):
'log_tag': self.log_tag,
'bind_v6_line': '',
}
for key in ('timeout_connect', 'timeout_client',
'timeout_server', 'timeout_http_request',
'timeout_http_keep_alive'):
cfg_info[key] = self.haproxy_config[key]
if self.host_v6 and self.bind_interface:
cfg_info['bind_v6_line'] = (
'bind {}:{} interface {}'.format(
@@ -233,6 +238,7 @@ class MetadataDriverBase(metaclass=abc.ABCMeta):
conf.state_path,
pid_file,
conf.metadata_rate_limiting,
conf.metadata_haproxy,
bind_address_v6,
bind_interface)
neutron/agent/ovn/extensions/metadata.py
+2
View File
@@ -110,6 +110,8 @@ class MetadataExtension(extension_manager.OVNAgentExtension,
ovn_meta.register_meta_conf_opts(meta_conf.METADATA_PROXY_HANDLER_OPTS)
ovn_meta.register_meta_conf_opts(meta_conf.METADATA_RATE_LIMITING_OPTS,
group=meta_conf.RATE_LIMITING_GROUP)
ovn_meta.register_meta_conf_opts(meta_conf.METADATA_HAPROXY_OPTS,
group=meta_conf.HAPROXY_GROUP)
def initialize(self, *args):
self._register_config_options()
neutron/agent/ovn/metadata_agent.py
+2
View File
@@ -34,6 +34,8 @@ def main():
ovn_meta.register_meta_conf_opts(meta.METADATA_PROXY_HANDLER_OPTS)
ovn_meta.register_meta_conf_opts(meta.METADATA_RATE_LIMITING_OPTS,
group=meta.RATE_LIMITING_GROUP)
ovn_meta.register_meta_conf_opts(meta.METADATA_HAPROXY_OPTS,
group=meta.HAPROXY_GROUP)
ovn_meta.register_meta_conf_opts(ovn_meta.OVS_OPTS, group='ovs')
config.init(sys.argv[1:])
config.setup_logging()
neutron/common/metadata.py
+5 -5
View File
@@ -64,11 +64,11 @@ defaults
option http-server-close
option forwardfor
retries 3
timeout http-request 30s
timeout connect 30s
timeout client 32s
timeout server 32s
timeout http-keep-alive 30s
timeout http-request %(timeout_http_request)ss
timeout connect %(timeout_connect)ss
timeout client %(timeout_client)ss
timeout server %(timeout_server)ss
timeout http-keep-alive %(timeout_http_keep_alive)ss
"""
RATE_LIMITED_CONFIG_TEMPLATE = """
neutron/conf/agent/dhcp.py
+3
View File
@@ -133,3 +133,6 @@ def register_agent_dhcp_opts(cfg=cfg.CONF):
meta_conf.register_meta_conf_opts(meta_conf.METADATA_RATE_LIMITING_OPTS,
cfg=cfg,
group=meta_conf.RATE_LIMITING_GROUP)
meta_conf.register_meta_conf_opts(meta_conf.METADATA_HAPROXY_OPTS,
cfg=cfg,
group=meta_conf.HAPROXY_GROUP)
neutron/conf/agent/metadata/config.py
+32
View File
@@ -24,6 +24,7 @@ GROUP_MODE = 'group'
ALL_MODE = 'all'
SOCKET_MODES = (DEDUCE_MODE, USER_MODE, GROUP_MODE, ALL_MODE)
RATE_LIMITING_GROUP = 'metadata_rate_limiting'
HAPROXY_GROUP = 'metadata_haproxy'
SHARED_OPTS = [
cfg.StrOpt('metadata_proxy_socket',
@@ -143,5 +144,36 @@ METADATA_RATE_LIMITING_OPTS = [
]
METADATA_HAPROXY_OPTS = [
cfg.IntOpt('timeout_connect',
default=30,
min=1,
help=_('Maximum time in seconds to wait for a connection '
'to the metadata proxy backend (Unix socket).')),
cfg.IntOpt('timeout_client',
default=32,
min=1,
help=_('Maximum inactivity time in seconds on the client '
'(instance) side of the metadata proxy.')),
cfg.IntOpt('timeout_server',
default=32,
min=1,
help=_('Maximum inactivity time in seconds on the server '
'(Nova metadata) side of the metadata proxy. Increase '
'this if metadata responses take too long, for example '
'when using dynamic vendordata.')),
cfg.IntOpt('timeout_http_request',
default=30,
min=1,
help=_('Maximum time in seconds to wait for a complete '
'HTTP request from the client.')),
cfg.IntOpt('timeout_http_keep_alive',
default=30,
min=1,
help=_('Maximum time in seconds to wait for a new HTTP '
'request on a keep-alive connection.')),
]
def register_meta_conf_opts(opts, cfg=cfg.CONF, group=None):
cfg.register_opts(opts, group=group)
neutron/conf/agent/ovn/metadata/config.py
+3 -1
View File
@@ -49,7 +49,9 @@ def list_metadata_agent_opts():
),
('ovs', OVS_OPTS),
(meta_conf.RATE_LIMITING_GROUP,
meta_conf.METADATA_RATE_LIMITING_OPTS)
meta_conf.METADATA_RATE_LIMITING_OPTS),
(meta_conf.HAPROXY_GROUP,
meta_conf.METADATA_HAPROXY_OPTS),
]
neutron/conf/agent/ovn/ovn_neutron_agent/config.py
+1
View File
@@ -49,6 +49,7 @@ def list_ovn_neutron_agent_opts():
),
(meta_conf.RATE_LIMITING_GROUP, meta_conf.METADATA_RATE_LIMITING_OPTS),
('ovn_evpn', evpn_conf.EVPN_OPTS),
(meta_conf.HAPROXY_GROUP, meta_conf.METADATA_HAPROXY_OPTS),
]
neutron/opts.py
+6 -2
View File
@@ -240,7 +240,9 @@ def list_dhcp_agent_opts():
neutron.conf.agent.dhcp.DNSMASQ_OPTS)
),
(meta_conf.RATE_LIMITING_GROUP,
meta_conf.METADATA_RATE_LIMITING_OPTS)
meta_conf.METADATA_RATE_LIMITING_OPTS),
(meta_conf.HAPROXY_GROUP,
meta_conf.METADATA_HAPROXY_OPTS),
]
@@ -258,7 +260,9 @@ def list_l3_agent_opts():
('network_log',
neutron.conf.services.logging.log_driver_opts),
(meta_conf.RATE_LIMITING_GROUP,
meta_conf.METADATA_RATE_LIMITING_OPTS)
meta_conf.METADATA_RATE_LIMITING_OPTS),
(meta_conf.HAPROXY_GROUP,
meta_conf.METADATA_HAPROXY_OPTS),
]
neutron/tests/unit/agent/metadata/test_driver.py
+25 -4
View File
@@ -112,6 +112,9 @@ class TestMetadataDriverProcess(base.BaseTestCase):
meta_conf.register_meta_conf_opts(
meta_conf.METADATA_RATE_LIMITING_OPTS, cfg.CONF,
group=meta_conf.RATE_LIMITING_GROUP)
meta_conf.register_meta_conf_opts(
meta_conf.METADATA_HAPROXY_OPTS, cfg.CONF,
group=meta_conf.HAPROXY_GROUP)
self.mock_conf_obsolete = mock.patch.object(
driver_base.HaproxyConfiguratorBase,
'is_config_file_obsolete').start()
@@ -164,7 +167,8 @@ class TestMetadataDriverProcess(base.BaseTestCase):
f.assert_not_called()
def _test_spawn_metadata_proxy(self, dad_failed=False, rate_limited=False,
is_config_file_obsolete=False):
is_config_file_obsolete=False,
haproxy_timeouts=None):
router_id = _uuid()
router_ns = 'qrouter-%s' % router_id
service_name = 'haproxy'
@@ -249,7 +253,15 @@ class TestMetadataDriverProcess(base.BaseTestCase):
'pidfile': self.PIDFILE,
'log_level': 'debug',
'log_tag': log_tag,
'bind_v6_line': bind_v6_line}
'bind_v6_line': bind_v6_line,
'timeout_connect': 30,
'timeout_client': 32,
'timeout_server': 32,
'timeout_http_request': 30,
'timeout_http_keep_alive': 30}
if haproxy_timeouts:
expected_params.update(haproxy_timeouts)
if dad_failed:
mock_del.assert_called_once_with(self.METADATA_DEFAULT_IPV6,
@@ -318,6 +330,15 @@ class TestMetadataDriverProcess(base.BaseTestCase):
def test_spawn_metadata_proxy_no_matching_configurations(self):
self._test_spawn_metadata_proxy(is_config_file_obsolete=True)
def test_spawn_metadata_proxy_custom_haproxy_timeouts(self):
cfg.CONF.set_override('timeout_server', 120,
group=meta_conf.HAPROXY_GROUP)
cfg.CONF.set_override('timeout_client', 120,
group=meta_conf.HAPROXY_GROUP)
self._test_spawn_metadata_proxy(
haproxy_timeouts={'timeout_server': 120,
'timeout_client': 120})
@mock.patch.object(driver_base.LOG, 'error')
def test_spawn_metadata_proxy_handles_process_exception(self, error_log):
process_instance = mock.Mock(active=False)
@@ -344,7 +365,7 @@ class TestMetadataDriverProcess(base.BaseTestCase):
metadata_driver.HaproxyConfigurator, _uuid(),
mock.ANY, mock.ANY, mock.ANY, mock.ANY,
self.EUNAME, self.EGNAME, mock.ANY, mock.ANY,
mock.ANY)
mock.ANY, mock.ANY)
def test_create_config_file_wrong_group(self):
with mock.patch('grp.getgrnam', side_effect=KeyError),\
@@ -354,7 +375,7 @@ class TestMetadataDriverProcess(base.BaseTestCase):
metadata_driver.HaproxyConfigurator, _uuid(),
mock.ANY, mock.ANY, mock.ANY, mock.ANY,
self.EUNAME, self.EGNAME, mock.ANY, mock.ANY,
mock.ANY)
mock.ANY, mock.ANY)
def test_destroy_monitored_metadata_proxy(self):
mproxy_process = mock.Mock(active=False)
neutron/tests/unit/agent/ovn/metadata/test_driver.py
+25 -4
View File
@@ -61,6 +61,9 @@ class TestMetadataDriverProcess(base.BaseTestCase):
ovn_meta_conf.register_meta_conf_opts(
meta_conf.METADATA_RATE_LIMITING_OPTS, cfg.CONF,
group=meta_conf.RATE_LIMITING_GROUP)
ovn_meta_conf.register_meta_conf_opts(
meta_conf.METADATA_HAPROXY_OPTS, cfg.CONF,
group=meta_conf.HAPROXY_GROUP)
ovn_conf.register_opts()
cfg.CONF.set_override('check_child_processes_interval', 0.1,
group='AGENT')
@@ -76,7 +79,17 @@ class TestMetadataDriverProcess(base.BaseTestCase):
return self._test_spawn_metadata_proxy(rate_limited=True)
def _test_spawn_metadata_proxy(self, rate_limited=False):
def test_spawn_metadata_proxy_custom_haproxy_timeouts(self):
cfg.CONF.set_override('timeout_server', 120,
group=meta_conf.HAPROXY_GROUP)
cfg.CONF.set_override('timeout_client', 120,
group=meta_conf.HAPROXY_GROUP)
return self._test_spawn_metadata_proxy(
haproxy_timeouts={'timeout_server': 120,
'timeout_client': 120})
def _test_spawn_metadata_proxy(self, rate_limited=False,
haproxy_timeouts=None):
datapath_id = _uuid()
metadata_ns = metadata_agent.NS_PREFIX + datapath_id
ip_class_path = 'neutron.agent.linux.ip_lib.IPWrapper'
@@ -148,7 +161,15 @@ class TestMetadataDriverProcess(base.BaseTestCase):
'pidfile': self.PIDFILE,
'log_level': 'debug',
'log_tag': log_tag,
'bind_v6_line': bind_v6_line}
'bind_v6_line': bind_v6_line,
'timeout_connect': 30,
'timeout_client': 32,
'timeout_server': 32,
'timeout_http_request': 30,
'timeout_http_keep_alive': 30}
if haproxy_timeouts:
expected_params.update(haproxy_timeouts)
if rate_limited:
expected_params.update(self.RATE_LIMIT_CONFIG,
@@ -211,7 +232,7 @@ class TestMetadataDriverProcess(base.BaseTestCase):
metadata_driver.HaproxyConfigurator, mock.ANY,
mock.ANY, mock.ANY, mock.ANY, mock.ANY,
self.EUNAME, self.EGNAME, mock.ANY, mock.ANY,
mock.ANY)
mock.ANY, mock.ANY)
def test_create_config_file_wrong_group(self):
with mock.patch('grp.getgrnam', side_effect=KeyError),\
@@ -221,4 +242,4 @@ class TestMetadataDriverProcess(base.BaseTestCase):
metadata_driver.HaproxyConfigurator, mock.ANY,
mock.ANY, mock.ANY, mock.ANY, mock.ANY,
self.EUNAME, self.EGNAME, mock.ANY, mock.ANY,
mock.ANY)
mock.ANY, mock.ANY)
releasenotes/notes/configurable-haproxy-metadata-timeouts-1fb08be09280a39e.yaml
+11
View File
@@ -0,0 +1,11 @@
---
features:
- |
Added a new ``[metadata_haproxy]`` configuration group with options to
control HAProxy timeout values for the metadata proxy. The following
options are now configurable: ``timeout_connect`` (default 30s),
``timeout_client`` (default 32s), ``timeout_server`` (default 32s),
``timeout_http_request`` (default 30s), and ``timeout_http_keep_alive``
(default 30s). Previously these values were hardcoded. Operators with
slow metadata backends (e.g. dynamic vendordata) may need to increase
``timeout_server`` to prevent premature connection termination.