Fix health monitor type "PING" to use icmp ping
Currently with Octavia, if the user specifies a health monitor of type "PING" we are still using a TCP connect to check for health. This patch fixes that to actually ping the member to validate health. Change-Id: I8a67efb7113ffa49b2805b37c3855373b17e5789 Story: 2001280 Task: 5826
This commit is contained in:
Michael Johnson
32
elements/haproxy-octavia/post-install.d/21-create-ping-wrapper
Executable file
32
elements/haproxy-octavia/post-install.d/21-create-ping-wrapper
Executable file
@ -0,0 +1,32 @@
|
||||
#!/bin/bash
|
||||
# Copyright 2017 Rackspace, US Inc.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
#
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
ping_cmd=$(command -v ping)
|
||||
ping6_cmd=$(command -v ping6)
|
||||
|
||||
cat > /var/lib/octavia/ping-wrapper.sh <<EOF
|
||||
#!/bin/bash
|
||||
if [[ \$HAPROXY_SERVER_ADDR =~ ":" ]]; then
|
||||
$ping6_cmd -q -n -w 1 -c 1 \$HAPROXY_SERVER_ADDR > /dev/null 2>&1
|
||||
else
|
||||
$ping_cmd -q -n -w 1 -c 1 \$HAPROXY_SERVER_ADDR > /dev/null 2>&1
|
||||
fi
|
||||
EOF
|
||||
|
||||
chmod 755 /var/lib/octavia/ping-wrapper.sh
|
||||
@ -0,0 +1,54 @@
|
||||
# Copyright 2017 Rackspace, US Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import logging
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_haproxy_versions():
|
||||
"""Get major and minor version number from haproxy
|
||||
|
||||
:returns major_version: The major version digit
|
||||
:returns minor_version: The minor version digit
|
||||
"""
|
||||
cmd = "haproxy -v"
|
||||
|
||||
version = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT)
|
||||
|
||||
version_re = re.search('.*version (.+?)\.(.+?)\..*',
|
||||
version.decode('utf-8'))
|
||||
|
||||
major_version = int(version_re.group(1))
|
||||
minor_version = int(version_re.group(2))
|
||||
|
||||
return major_version, minor_version
|
||||
|
||||
|
||||
def process_cfg_for_version_compat(haproxy_cfg):
|
||||
|
||||
major, minor = get_haproxy_versions()
|
||||
|
||||
# Versions less than 1.6 do not support external health checks
|
||||
# Removed those configuration times
|
||||
if major < 2 and minor < 6:
|
||||
LOG.warning("Found %(major)s.%(minor)s version of haproxy. "
|
||||
"Disabling external checks. Health monitor of type "
|
||||
"PING will revert to TCP.",
|
||||
{'major': major, 'minor': minor})
|
||||
haproxy_cfg = re.sub(r" * ?.*external-check ?.*\s", "", haproxy_cfg)
|
||||
|
||||
return haproxy_cfg
|
||||
@ -28,6 +28,7 @@ import six
|
||||
import webob
|
||||
from werkzeug import exceptions
|
||||
|
||||
from octavia.amphorae.backends.agent.api_server import haproxy_compatibility
|
||||
from octavia.amphorae.backends.agent.api_server import osutils
|
||||
from octavia.amphorae.backends.agent.api_server import util
|
||||
from octavia.amphorae.backends.utils import haproxy_query as query
|
||||
@ -124,6 +125,10 @@ class Listener(object):
|
||||
# result an error when haproxy starts.
|
||||
new_config = re.sub(r"\s+group\s.+", "", s_io.getvalue())
|
||||
|
||||
# Handle any haproxy version compatibility issues
|
||||
new_config = haproxy_compatibility.process_cfg_for_version_compat(
|
||||
new_config)
|
||||
|
||||
with os.fdopen(os.open(name, flags, mode), 'w') as file:
|
||||
file.write(new_config)
|
||||
|
||||
|
||||
@ -24,6 +24,7 @@ global
|
||||
{% if loadbalancer.global_connection_limit is defined %}
|
||||
maxconn {{ loadbalancer.global_connection_limit }}
|
||||
{% endif %}
|
||||
external-check
|
||||
|
||||
defaults
|
||||
log global
|
||||
|
||||
@ -227,6 +227,10 @@ backend {{ pool.id }}
|
||||
{% if pool.health_monitor.type == constants.HEALTH_MONITOR_TLS_HELLO %}
|
||||
option ssl-hello-chk
|
||||
{% endif %}
|
||||
{% if pool.health_monitor.type == constants.HEALTH_MONITOR_PING %}
|
||||
option external-check
|
||||
external-check command /var/lib/octavia/ping-wrapper.sh
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% if pool.protocol.lower() == constants.PROTOCOL_HTTP.lower() %}
|
||||
{% if listener.insert_headers.get('X-Forwarded-For',
|
||||
|
||||
@ -82,16 +82,20 @@ class TestServerTestCase(base.TestCase):
|
||||
self._test_haproxy(consts.INIT_UPSTART, consts.UBUNTU,
|
||||
mock_init_system)
|
||||
|
||||
@mock.patch('octavia.amphorae.backends.agent.api_server.'
|
||||
'haproxy_compatibility.get_haproxy_versions')
|
||||
@mock.patch('os.path.exists')
|
||||
@mock.patch('os.makedirs')
|
||||
@mock.patch('os.rename')
|
||||
@mock.patch('subprocess.check_output')
|
||||
def _test_haproxy(self, init_system, distro, mock_init_system,
|
||||
mock_subprocess, mock_rename,
|
||||
mock_makedirs, mock_exists):
|
||||
mock_makedirs, mock_exists, mock_get_version):
|
||||
|
||||
self.assertIn(distro, [consts.UBUNTU, consts.CENTOS])
|
||||
|
||||
mock_get_version.return_value = [1, 6]
|
||||
|
||||
flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
|
||||
mock_exists.return_value = True
|
||||
file_name = '/var/lib/octavia/123/haproxy.cfg.new'
|
||||
|
||||
@ -0,0 +1,106 @@
|
||||
# Copyright 2017 Rackspace, US Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import mock
|
||||
|
||||
from octavia.amphorae.backends.agent.api_server import haproxy_compatibility
|
||||
import octavia.tests.unit.base as base
|
||||
from octavia.tests.unit.common.sample_configs import sample_configs
|
||||
|
||||
|
||||
class HAProxyCompatTestCase(base.TestCase):
|
||||
def setUp(self):
|
||||
super(HAProxyCompatTestCase, self).setUp()
|
||||
self.old_haproxy_global = (
|
||||
"# Configuration for test-lb\n"
|
||||
"global\n"
|
||||
" daemon\n"
|
||||
" user nobody\n"
|
||||
" group nogroup\n"
|
||||
" log /dev/log local0\n"
|
||||
" log /dev/log local1 notice\n"
|
||||
" stats socket /var/lib/octavia/sample_listener_id_1.sock"
|
||||
" mode 0666 level user\n"
|
||||
" maxconn 98\n\n"
|
||||
"defaults\n"
|
||||
" log global\n"
|
||||
" retries 3\n"
|
||||
" option redispatch\n"
|
||||
" timeout connect 5000\n"
|
||||
" timeout client 50000\n"
|
||||
" timeout server 50000\n\n\n\n"
|
||||
"frontend sample_listener_id_1\n"
|
||||
" option httplog\n"
|
||||
" maxconn 98\n"
|
||||
" bind 10.0.0.2:80\n"
|
||||
" mode http\n"
|
||||
" default_backend sample_pool_id_1\n\n")
|
||||
self.backend_without_external = (
|
||||
"backend sample_pool_id_1\n"
|
||||
" mode http\n"
|
||||
" balance roundrobin\n"
|
||||
" cookie SRV insert indirect nocache\n"
|
||||
" timeout check 31\n"
|
||||
" option httpchk GET /index.html\n"
|
||||
" http-check expect rstatus 418\n"
|
||||
" fullconn 98\n"
|
||||
" server sample_member_id_1 10.0.0.99:82 weight 13 "
|
||||
"check inter 30s fall 3 rise 2 cookie sample_member_id_1\n"
|
||||
" server sample_member_id_2 10.0.0.98:82 weight 13 "
|
||||
"check inter 30s fall 3 rise 2 cookie sample_member_id_2\n")
|
||||
self.backend_with_external = (
|
||||
"backend sample_pool_id_1\n"
|
||||
" mode http\n"
|
||||
" balance roundrobin\n"
|
||||
" cookie SRV insert indirect nocache\n"
|
||||
" timeout check 31\n"
|
||||
" option httpchk GET /index.html\n"
|
||||
" http-check expect rstatus 418\n"
|
||||
" fullconn 98\n"
|
||||
" option external-check\n"
|
||||
" external-check command /var/lib/octavia/ping-wrapper.sh\n"
|
||||
" server sample_member_id_1 10.0.0.99:82 weight 13 "
|
||||
"check inter 30s fall 3 rise 2 cookie sample_member_id_1\n"
|
||||
" server sample_member_id_2 10.0.0.98:82 weight 13 "
|
||||
"check inter 30s fall 3 rise 2 cookie sample_member_id_2\n")
|
||||
|
||||
@mock.patch('subprocess.check_output')
|
||||
def test_get_haproxy_versions(self, mock_process):
|
||||
mock_process.return_value = (
|
||||
b"THIS-App version 1.6.3 2099/10/12\n"
|
||||
b"Some other data here <test@example.com>\n")
|
||||
major, minor = haproxy_compatibility.get_haproxy_versions()
|
||||
self.assertEqual(1, major)
|
||||
self.assertEqual(6, minor)
|
||||
|
||||
@mock.patch('octavia.amphorae.backends.agent.api_server.'
|
||||
'haproxy_compatibility.get_haproxy_versions')
|
||||
def test_process_cfg_for_version_compat(self, mock_get_version):
|
||||
# Test 1.6 version path, no change to config expected
|
||||
mock_get_version.return_value = [1, 6]
|
||||
test_config = sample_configs.sample_base_expected_config(
|
||||
backend=self.backend_with_external)
|
||||
result_config = haproxy_compatibility.process_cfg_for_version_compat(
|
||||
test_config)
|
||||
self.assertEqual(test_config, result_config)
|
||||
|
||||
# Test 1.5 version path, external-check should be removed
|
||||
mock_get_version.return_value = [1, 5]
|
||||
test_config = sample_configs.sample_base_expected_config(
|
||||
backend=self.backend_with_external)
|
||||
result_config = haproxy_compatibility.process_cfg_for_version_compat(
|
||||
test_config)
|
||||
expected_config = (self.old_haproxy_global +
|
||||
self.backend_without_external)
|
||||
self.assertEqual(expected_config, result_config)
|
||||
@ -727,7 +727,8 @@ def sample_base_expected_config(frontend=None, backend=None, peers=None):
|
||||
" log /dev/log local1 notice\n"
|
||||
" stats socket /var/lib/octavia/sample_listener_id_1.sock"
|
||||
" mode 0666 level user\n"
|
||||
" maxconn 98\n\n"
|
||||
" maxconn 98\n"
|
||||
" external-check\n\n"
|
||||
"defaults\n"
|
||||
" log global\n"
|
||||
" retries 3\n"
|
||||
|
||||
@ -0,0 +1,17 @@
|
||||
---
|
||||
issues:
|
||||
- |
|
||||
Amphora images with HAProxy older than 1.6 (CentOS 7, etc.) will still
|
||||
use health monitor type TCP when PING is selected by the user.
|
||||
upgrade:
|
||||
- |
|
||||
Amphora will need to be updated to a new image with this version of the
|
||||
agent and ping-wrapper.sh script prior to updating the Octavia controllers.
|
||||
If a load balancer is using a health monitor of type PING with an
|
||||
amphora image that has not been updated, the next configuration change to
|
||||
the load balancer will cause it to go into an ERROR state until it is
|
||||
failed over to an updated image.
|
||||
fixes:
|
||||
- |
|
||||
Fixed an issue where health monitors of type PING were really doing a
|
||||
TCP health check.
|
||||
Reference in New Issue
Block a user