|
|
|
@ -12,21 +12,26 @@
|
|
|
|
|
# License for the specific language governing permissions and limitations |
|
|
|
|
# under the License. |
|
|
|
|
# |
|
|
|
|
import os |
|
|
|
|
import queue |
|
|
|
|
from unittest import mock |
|
|
|
|
|
|
|
|
|
from oslo_config import cfg |
|
|
|
|
from oslo_config import fixture as oslo_fixture |
|
|
|
|
from oslo_utils import uuidutils |
|
|
|
|
import simplejson |
|
|
|
|
|
|
|
|
|
from octavia.amphorae.backends.health_daemon import health_daemon |
|
|
|
|
from octavia.common import constants |
|
|
|
|
from octavia.tests.common import utils as test_utils |
|
|
|
|
import octavia.tests.unit.base as base |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
LB_ID1 = uuidutils.generate_uuid() |
|
|
|
|
LISTENER_ID1 = uuidutils.generate_uuid() |
|
|
|
|
LISTENER_ID2 = uuidutils.generate_uuid() |
|
|
|
|
LISTENER_IDS = [LISTENER_ID1, LISTENER_ID2] |
|
|
|
|
AMPHORA_ID = uuidutils.generate_uuid() |
|
|
|
|
BASE_PATH = '/tmp/test' |
|
|
|
|
SAMPLE_POOL_STATUS = { |
|
|
|
|
'432fc8b3-d446-48d4-bb64-13beb90e22bc': { |
|
|
|
@ -45,46 +50,46 @@ SAMPLE_BOGUS_POOL_STATUS = {LISTENER_ID1: {
|
|
|
|
|
'302e33d9-dee1-4de9-98d5-36329a06fb58': |
|
|
|
|
'DOWN'}}} |
|
|
|
|
|
|
|
|
|
SAMPLE_STATS = ({'': '', 'status': 'OPEN', 'lastchg': '', |
|
|
|
|
'weight': '', 'slim': '2000', 'pid': '1', 'comp_byp': '0', |
|
|
|
|
'lastsess': '', 'rate_lim': '0', 'check_duration': '', |
|
|
|
|
'rate': '0', 'req_rate': '0', 'check_status': '', |
|
|
|
|
'econ': '', 'comp_out': '0', 'wredis': '', 'dresp': '0', |
|
|
|
|
'ereq': '0', 'tracked': '', 'comp_in': '0', |
|
|
|
|
'pxname': LISTENER_ID1, |
|
|
|
|
'dreq': '0', 'hrsp_5xx': '0', 'last_chk': '', |
|
|
|
|
'check_code': '', 'sid': '0', 'bout': '0', 'hrsp_1xx': '0', |
|
|
|
|
'qlimit': '', 'hrsp_other': '0', 'bin': '0', 'rtime': '', |
|
|
|
|
'smax': '0', 'req_tot': '0', 'lbtot': '', 'stot': '0', |
|
|
|
|
'wretr': '', 'req_rate_max': '0', 'ttime': '', 'iid': '2', |
|
|
|
|
'hrsp_4xx': '0', 'chkfail': '', 'hanafail': '', |
|
|
|
|
'downtime': '', 'qcur': '', 'eresp': '', 'comp_rsp': '0', |
|
|
|
|
'cli_abrt': '', 'ctime': '', 'qtime': '', 'srv_abrt': '', |
|
|
|
|
'throttle': '', 'last_agt': '', 'scur': '0', 'type': '0', |
|
|
|
|
'bck': '', 'qmax': '', 'rate_max': '0', 'hrsp_2xx': '0', |
|
|
|
|
'act': '', 'chkdown': '', 'svname': 'FRONTEND', |
|
|
|
|
'hrsp_3xx': '0'}, |
|
|
|
|
{'': '', 'status': 'no check', 'lastchg': '', 'weight': '1', |
|
|
|
|
'slim': '', 'pid': '1', 'comp_byp': '', 'lastsess': '-1', |
|
|
|
|
'rate_lim': '', 'check_duration': '', 'rate': '0', |
|
|
|
|
'req_rate': '', 'check_status': '', 'econ': '0', |
|
|
|
|
'comp_out': '', 'wredis': '0', 'dresp': '0', 'ereq': '', |
|
|
|
|
'tracked': '', 'comp_in': '', |
|
|
|
|
'pxname': '432fc8b3-d446-48d4-bb64-13beb90e22bc', |
|
|
|
|
'dreq': '', 'hrsp_5xx': '0', 'last_chk': '', |
|
|
|
|
'check_code': '', 'sid': '1', 'bout': '0', 'hrsp_1xx': '0', |
|
|
|
|
'qlimit': '', 'hrsp_other': '0', 'bin': '0', 'rtime': '0', |
|
|
|
|
'smax': '0', 'req_tot': '', 'lbtot': '0', 'stot': '0', |
|
|
|
|
'wretr': '0', 'req_rate_max': '', 'ttime': '0', 'iid': '3', |
|
|
|
|
'hrsp_4xx': '0', 'chkfail': '', 'hanafail': '0', |
|
|
|
|
'downtime': '', 'qcur': '0', 'eresp': '0', 'comp_rsp': '', |
|
|
|
|
'cli_abrt': '0', 'ctime': '0', 'qtime': '0', 'srv_abrt': '0', |
|
|
|
|
'throttle': '', 'last_agt': '', 'scur': '0', 'type': '2', |
|
|
|
|
'bck': '0', 'qmax': '0', 'rate_max': '0', 'hrsp_2xx': '0', |
|
|
|
|
'act': '1', 'chkdown': '', |
|
|
|
|
'svname': '302e33d9-dee1-4de9-98d5-36329a06fb58', |
|
|
|
|
'hrsp_3xx': '0'}, |
|
|
|
|
{'': '', 'status': 'UP', 'lastchg': '122', 'weight': '1', |
|
|
|
|
FRONTEND_STATS = {'': '', 'status': 'OPEN', 'lastchg': '', |
|
|
|
|
'weight': '', 'slim': '2000', 'pid': '1', 'comp_byp': '0', |
|
|
|
|
'lastsess': '', 'rate_lim': '0', 'check_duration': '', |
|
|
|
|
'rate': '0', 'req_rate': '0', 'check_status': '', |
|
|
|
|
'econ': '', 'comp_out': '0', 'wredis': '', 'dresp': '0', |
|
|
|
|
'ereq': '5', 'tracked': '', 'comp_in': '0', |
|
|
|
|
'pxname': LISTENER_ID1, |
|
|
|
|
'dreq': '0', 'hrsp_5xx': '0', 'last_chk': '', |
|
|
|
|
'check_code': '', 'sid': '0', 'bout': '10', 'hrsp_1xx': '0', |
|
|
|
|
'qlimit': '', 'hrsp_other': '0', 'bin': '5', 'rtime': '', |
|
|
|
|
'smax': '0', 'req_tot': '0', 'lbtot': '', 'stot': '0', |
|
|
|
|
'wretr': '', 'req_rate_max': '0', 'ttime': '', 'iid': '2', |
|
|
|
|
'hrsp_4xx': '0', 'chkfail': '', 'hanafail': '', |
|
|
|
|
'downtime': '', 'qcur': '', 'eresp': '', 'comp_rsp': '0', |
|
|
|
|
'cli_abrt': '', 'ctime': '', 'qtime': '', 'srv_abrt': '', |
|
|
|
|
'throttle': '', 'last_agt': '', 'scur': '0', 'type': '0', |
|
|
|
|
'bck': '', 'qmax': '', 'rate_max': '0', 'hrsp_2xx': '0', |
|
|
|
|
'act': '', 'chkdown': '', 'svname': 'FRONTEND', |
|
|
|
|
'hrsp_3xx': '0'} |
|
|
|
|
MEMBER_STATS = {'': '', 'status': 'no check', 'lastchg': '', 'weight': '1', |
|
|
|
|
'slim': '', 'pid': '1', 'comp_byp': '', 'lastsess': '-1', |
|
|
|
|
'rate_lim': '', 'check_duration': '', 'rate': '0', |
|
|
|
|
'req_rate': '', 'check_status': '', 'econ': '0', |
|
|
|
|
'comp_out': '', 'wredis': '0', 'dresp': '0', 'ereq': '', |
|
|
|
|
'tracked': '', 'comp_in': '', |
|
|
|
|
'pxname': '432fc8b3-d446-48d4-bb64-13beb90e22bc', |
|
|
|
|
'dreq': '', 'hrsp_5xx': '0', 'last_chk': '', |
|
|
|
|
'check_code': '', 'sid': '1', 'bout': '0', 'hrsp_1xx': '0', |
|
|
|
|
'qlimit': '', 'hrsp_other': '0', 'bin': '0', 'rtime': '0', |
|
|
|
|
'smax': '0', 'req_tot': '', 'lbtot': '0', 'stot': '0', |
|
|
|
|
'wretr': '0', 'req_rate_max': '', 'ttime': '0', 'iid': '3', |
|
|
|
|
'hrsp_4xx': '0', 'chkfail': '', 'hanafail': '0', |
|
|
|
|
'downtime': '', 'qcur': '0', 'eresp': '0', 'comp_rsp': '', |
|
|
|
|
'cli_abrt': '0', 'ctime': '0', 'qtime': '0', 'srv_abrt': '0', |
|
|
|
|
'throttle': '', 'last_agt': '', 'scur': '0', 'type': '2', |
|
|
|
|
'bck': '0', 'qmax': '0', 'rate_max': '0', 'hrsp_2xx': '0', |
|
|
|
|
'act': '1', 'chkdown': '', |
|
|
|
|
'svname': '302e33d9-dee1-4de9-98d5-36329a06fb58', |
|
|
|
|
'hrsp_3xx': '0'} |
|
|
|
|
BACKEND_STATS = {'': '', 'status': 'UP', 'lastchg': '122', 'weight': '1', |
|
|
|
|
'slim': '200', 'pid': '1', 'comp_byp': '0', 'lastsess': '-1', |
|
|
|
|
'rate_lim': '', 'check_duration': '', 'rate': '0', |
|
|
|
|
'req_rate': '', 'check_status': '', 'econ': '0', |
|
|
|
@ -101,14 +106,36 @@ SAMPLE_STATS = ({'': '', 'status': 'OPEN', 'lastchg': '',
|
|
|
|
|
'srv_abrt': '0', 'throttle': '', 'last_agt': '', 'scur': '0', |
|
|
|
|
'type': '1', 'bck': '0', 'qmax': '0', 'rate_max': '0', |
|
|
|
|
'hrsp_2xx': '0', 'act': '1', 'chkdown': '0', |
|
|
|
|
'svname': 'BACKEND', 'hrsp_3xx': '0'}) |
|
|
|
|
'svname': 'BACKEND', 'hrsp_3xx': '0'} |
|
|
|
|
SAMPLE_STATS = (FRONTEND_STATS, MEMBER_STATS, BACKEND_STATS) |
|
|
|
|
|
|
|
|
|
SAMPLE_STATS_MSG = { |
|
|
|
|
'listeners': { |
|
|
|
|
LISTENER_ID1: { |
|
|
|
|
'stats': { |
|
|
|
|
'totconns': 0, 'conns': 0, |
|
|
|
|
'tx': 0, 'rx': 0, 'ereq': 0}, |
|
|
|
|
'tx': 8, 'rx': 4, 'ereq': 5}, |
|
|
|
|
'status': 'OPEN'}, |
|
|
|
|
}, |
|
|
|
|
'pools': { |
|
|
|
|
'432fc8b3-d446-48d4-bb64-13beb90e22bc': { |
|
|
|
|
'members': {'302e33d9-dee1-4de9-98d5-36329a06fb58': 'DOWN'}, |
|
|
|
|
'status': 'UP'}, |
|
|
|
|
'3661ed10-99db-4d2c-bffb-99b60eb876ff': { |
|
|
|
|
'members': {'e657f950-a6a2-4d28-bffa-0c8a8c05f815': 'DOWN'}, |
|
|
|
|
'status': 'UP'}, |
|
|
|
|
}, |
|
|
|
|
'id': AMPHORA_ID, |
|
|
|
|
'seq': mock.ANY, |
|
|
|
|
'ver': health_daemon.MSG_VER |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
SAMPLE_MSG_HAPROXY_RESTART = { |
|
|
|
|
'listeners': { |
|
|
|
|
LISTENER_ID1: { |
|
|
|
|
'stats': { |
|
|
|
|
'totconns': 0, 'conns': 0, |
|
|
|
|
'tx': 10, 'rx': 5, 'ereq': 5}, |
|
|
|
|
'status': 'OPEN'}, |
|
|
|
|
}, |
|
|
|
|
'pools': { |
|
|
|
@ -119,8 +146,8 @@ SAMPLE_STATS_MSG = {
|
|
|
|
|
'members': {'e657f950-a6a2-4d28-bffa-0c8a8c05f815': 'DOWN'}, |
|
|
|
|
'status': 'UP'}, |
|
|
|
|
}, |
|
|
|
|
'id': None, |
|
|
|
|
'seq': 0, |
|
|
|
|
'id': AMPHORA_ID, |
|
|
|
|
'seq': mock.ANY, |
|
|
|
|
'ver': health_daemon.MSG_VER |
|
|
|
|
} |
|
|
|
|
|
|
|
|
@ -131,6 +158,10 @@ class TestHealthDaemon(base.TestCase):
|
|
|
|
|
super(TestHealthDaemon, self).setUp() |
|
|
|
|
conf = oslo_fixture.Config(cfg.CONF) |
|
|
|
|
conf.config(group="haproxy_amphora", base_path=BASE_PATH) |
|
|
|
|
conf.config(group="amphora_agent", amphora_id=AMPHORA_ID) |
|
|
|
|
file_name = os.path.join(BASE_PATH, "stats_counters") |
|
|
|
|
self.mock_open = self.useFixture( |
|
|
|
|
test_utils.OpenFixture(file_name)).mock_open |
|
|
|
|
|
|
|
|
|
@mock.patch('octavia.amphorae.backends.agent.' |
|
|
|
|
'api_server.util.get_loadbalancers') |
|
|
|
@ -296,18 +327,33 @@ class TestHealthDaemon(base.TestCase):
|
|
|
|
|
'health_daemon.list_sock_stat_files') |
|
|
|
|
def test_build_stats_message(self, mock_list_files, |
|
|
|
|
mock_get_stats, mock_is_running): |
|
|
|
|
mock_list_files.return_value = {LISTENER_ID1: 'TEST', |
|
|
|
|
LISTENER_ID2: 'TEST2'} |
|
|
|
|
health_daemon.COUNTERS = None |
|
|
|
|
health_daemon.COUNTERS_FILE = None |
|
|
|
|
lb1_stats_socket = '/var/lib/octavia/{0}/haproxy.sock'.format(LB_ID1) |
|
|
|
|
mock_list_files.return_value = {LB_ID1: lb1_stats_socket} |
|
|
|
|
|
|
|
|
|
mock_is_running.return_value = True |
|
|
|
|
mock_get_stats.return_value = SAMPLE_STATS, SAMPLE_POOL_STATUS |
|
|
|
|
|
|
|
|
|
msg = health_daemon.build_stats_message() |
|
|
|
|
with mock.patch('os.open'), mock.patch.object( |
|
|
|
|
os, 'fdopen', self.mock_open) as mock_fdopen: |
|
|
|
|
mock_fdopen().read.return_value = simplejson.dumps({ |
|
|
|
|
LISTENER_ID1: {'bin': 1, 'bout': 2}, |
|
|
|
|
}) |
|
|
|
|
msg = health_daemon.build_stats_message() |
|
|
|
|
|
|
|
|
|
self.assertEqual(SAMPLE_STATS_MSG, msg) |
|
|
|
|
|
|
|
|
|
mock_get_stats.assert_any_call('TEST') |
|
|
|
|
mock_get_stats.assert_any_call('TEST2') |
|
|
|
|
mock_get_stats.assert_any_call(lb1_stats_socket) |
|
|
|
|
mock_fdopen().write.assert_called_once_with(simplejson.dumps({ |
|
|
|
|
LISTENER_ID1: { |
|
|
|
|
'bin': int(FRONTEND_STATS['bin']), |
|
|
|
|
'bout': int(FRONTEND_STATS['bout']), |
|
|
|
|
'ereq': int(FRONTEND_STATS['ereq']), |
|
|
|
|
'stot': int(FRONTEND_STATS['stot']) |
|
|
|
|
|
|
|
|
|
} |
|
|
|
|
})) |
|
|
|
|
|
|
|
|
|
@mock.patch('octavia.amphorae.backends.agent.api_server.' |
|
|
|
|
'util.is_lb_running') |
|
|
|
@ -318,15 +364,19 @@ class TestHealthDaemon(base.TestCase):
|
|
|
|
|
def test_build_stats_message_no_listener(self, mock_list_files, |
|
|
|
|
mock_get_stats, |
|
|
|
|
mock_is_running): |
|
|
|
|
mock_list_files.return_value = {LISTENER_ID1: 'TEST', |
|
|
|
|
LISTENER_ID2: 'TEST2'} |
|
|
|
|
health_daemon.COUNTERS = None |
|
|
|
|
health_daemon.COUNTERS_FILE = None |
|
|
|
|
lb1_stats_socket = '/var/lib/octavia/{0}/haproxy.sock'.format(LB_ID1) |
|
|
|
|
mock_list_files.return_value = {LB_ID1: lb1_stats_socket} |
|
|
|
|
|
|
|
|
|
mock_is_running.side_effect = [True, False] |
|
|
|
|
mock_get_stats.return_value = SAMPLE_STATS, SAMPLE_POOL_STATUS |
|
|
|
|
mock_is_running.return_value = False |
|
|
|
|
|
|
|
|
|
health_daemon.build_stats_message() |
|
|
|
|
with mock.patch('os.open'), mock.patch.object( |
|
|
|
|
os, 'fdopen', self.mock_open) as mock_fdopen: |
|
|
|
|
health_daemon.build_stats_message() |
|
|
|
|
|
|
|
|
|
self.assertEqual(1, mock_get_stats.call_count) |
|
|
|
|
self.assertEqual(0, mock_get_stats.call_count) |
|
|
|
|
self.assertEqual(0, mock_fdopen().read.call_count) |
|
|
|
|
|
|
|
|
|
@mock.patch("octavia.amphorae.backends.utils.keepalivedlvs_query." |
|
|
|
|
"get_udp_listener_pool_status") |
|
|
|
@ -335,8 +385,10 @@ class TestHealthDaemon(base.TestCase):
|
|
|
|
|
@mock.patch("octavia.amphorae.backends.agent.api_server.util." |
|
|
|
|
"get_udp_listeners") |
|
|
|
|
def test_build_stats_message_with_udp_listener( |
|
|
|
|
self, mock_get_udp_listeners, mock_get_listener_stats, |
|
|
|
|
mock_get_pool_status): |
|
|
|
|
self, mock_get_udp_listeners, |
|
|
|
|
mock_get_listener_stats, mock_get_pool_status): |
|
|
|
|
health_daemon.COUNTERS = None |
|
|
|
|
health_daemon.COUNTERS_FILE = None |
|
|
|
|
udp_listener_id1 = uuidutils.generate_uuid() |
|
|
|
|
udp_listener_id2 = uuidutils.generate_uuid() |
|
|
|
|
udp_listener_id3 = uuidutils.generate_uuid() |
|
|
|
@ -346,10 +398,11 @@ class TestHealthDaemon(base.TestCase):
|
|
|
|
|
mock_get_udp_listeners.return_value = [udp_listener_id1, |
|
|
|
|
udp_listener_id2, |
|
|
|
|
udp_listener_id3] |
|
|
|
|
|
|
|
|
|
mock_get_listener_stats.return_value = { |
|
|
|
|
udp_listener_id1: { |
|
|
|
|
'status': constants.OPEN, |
|
|
|
|
'stats': {'bin': 6387472, 'stot': 5, 'bout': 7490, |
|
|
|
|
'stats': {'bin': 5, 'stot': 5, 'bout': 10, |
|
|
|
|
'ereq': 0, 'scur': 0}}, |
|
|
|
|
udp_listener_id3: { |
|
|
|
|
'status': constants.DOWN, |
|
|
|
@ -373,7 +426,7 @@ class TestHealthDaemon(base.TestCase):
|
|
|
|
|
udp_listener_id1: { |
|
|
|
|
'status': constants.OPEN, |
|
|
|
|
'stats': {'conns': 0, 'totconns': 5, 'ereq': 0, |
|
|
|
|
'rx': 6387472, 'tx': 7490}}, |
|
|
|
|
'rx': 4, 'tx': 8}}, |
|
|
|
|
udp_listener_id3: { |
|
|
|
|
'status': constants.DOWN, |
|
|
|
|
'stats': {'conns': 0, 'totconns': 0, 'ereq': 0, |
|
|
|
@ -384,10 +437,58 @@ class TestHealthDaemon(base.TestCase):
|
|
|
|
|
'members': { |
|
|
|
|
member_id1: constants.UP, |
|
|
|
|
member_id2: constants.UP}}}, |
|
|
|
|
'id': None, |
|
|
|
|
'id': AMPHORA_ID, |
|
|
|
|
'seq': mock.ANY, 'ver': health_daemon.MSG_VER} |
|
|
|
|
msg = health_daemon.build_stats_message() |
|
|
|
|
|
|
|
|
|
with mock.patch('os.open'), mock.patch.object( |
|
|
|
|
os, 'fdopen', self.mock_open) as mock_fdopen: |
|
|
|
|
mock_fdopen().read.return_value = simplejson.dumps({ |
|
|
|
|
udp_listener_id1: { |
|
|
|
|
'bin': 1, 'bout': 2, "ereq": 0, "stot": 0} |
|
|
|
|
}) |
|
|
|
|
msg = health_daemon.build_stats_message() |
|
|
|
|
|
|
|
|
|
self.assertEqual(expected, msg) |
|
|
|
|
mock_fdopen().write.assert_called_once_with(simplejson.dumps({ |
|
|
|
|
udp_listener_id1: {'bin': 5, 'bout': 10, 'ereq': 0, 'stot': 5}, |
|
|
|
|
udp_listener_id3: {'bin': 0, 'bout': 0, 'ereq': 0, 'stot': 0}, |
|
|
|
|
})) |
|
|
|
|
|
|
|
|
|
@mock.patch('octavia.amphorae.backends.agent.api_server.' |
|
|
|
|
'util.is_lb_running') |
|
|
|
|
@mock.patch('octavia.amphorae.backends.health_daemon.' |
|
|
|
|
'health_daemon.get_stats') |
|
|
|
|
@mock.patch('octavia.amphorae.backends.health_daemon.' |
|
|
|
|
'health_daemon.list_sock_stat_files') |
|
|
|
|
def test_haproxy_restart(self, mock_list_files, |
|
|
|
|
mock_get_stats, mock_is_running): |
|
|
|
|
health_daemon.COUNTERS = None |
|
|
|
|
health_daemon.COUNTERS_FILE = None |
|
|
|
|
lb1_stats_socket = '/var/lib/octavia/{0}/haproxy.sock'.format(LB_ID1) |
|
|
|
|
mock_list_files.return_value = {LB_ID1: lb1_stats_socket} |
|
|
|
|
|
|
|
|
|
mock_is_running.return_value = True |
|
|
|
|
mock_get_stats.return_value = SAMPLE_STATS, SAMPLE_POOL_STATUS |
|
|
|
|
|
|
|
|
|
with mock.patch('os.open'), mock.patch.object( |
|
|
|
|
os, 'fdopen', self.mock_open) as mock_fdopen: |
|
|
|
|
mock_fdopen().read.return_value = simplejson.dumps({ |
|
|
|
|
LISTENER_ID1: {'bin': 15, 'bout': 20}, |
|
|
|
|
}) |
|
|
|
|
msg = health_daemon.build_stats_message() |
|
|
|
|
|
|
|
|
|
self.assertEqual(SAMPLE_MSG_HAPROXY_RESTART, msg) |
|
|
|
|
|
|
|
|
|
mock_get_stats.assert_any_call(lb1_stats_socket) |
|
|
|
|
mock_fdopen().write.assert_called_once_with(simplejson.dumps({ |
|
|
|
|
LISTENER_ID1: { |
|
|
|
|
'bin': int(FRONTEND_STATS['bin']), |
|
|
|
|
'bout': int(FRONTEND_STATS['bout']), |
|
|
|
|
'ereq': int(FRONTEND_STATS['ereq']), |
|
|
|
|
'stot': int(FRONTEND_STATS['stot']) |
|
|
|
|
|
|
|
|
|
} |
|
|
|
|
})) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class FileNotFoundError(IOError): |
|
|
|
|