openstack
/
octavia
Code Issues Proposed changes

Change amphora statistics to use deltas 

Amphora statistics packets should report deltas instead of absolutes for
all relevant metrics.

Change-Id: I5cf6f1f20f2c6f1da39982b2d88e036eefe48b2f
Co-Authored-By: Anushka Singh <anushka.singh.2511@gmail.com>
Co-Authored-By: Stephanie Djajadi <stephanie.djajadi@gmail.com>
changes/15/740815/19
Adam Harwell 2020-07-13 14:16:52 -07:00
parent 51b93c0022
commit d15cccff2f
9 changed files with 699 additions and 267 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

162
octavia/amphorae/backends/health_daemon/health_daemon.py
View File

@ -17,10 +17,12 @@
import errno
import os
import queue
import stat
import time
from oslo_config import cfg
from oslo_log import log as logging
import simplejson
from octavia.amphorae.backends.agent.api_server import util
from octavia.amphorae.backends.health_daemon import health_sender
@ -38,8 +40,57 @@ SEQ = 0
#
# ver 1 - Adds UDP listener status when no pool or members are present
# ver 2 - Switch to all listeners in a single combined haproxy config
#
MSG_VER = 2
# ver 3 - Switch stats reporting to deltas
MSG_VER = 3
DELTA_METRICS = ('bin', 'bout', 'ereq', 'stot')
# Filesystem persistent counters for statistics deltas
COUNTERS = None
COUNTERS_FILE = None
def get_counters_file():
global COUNTERS_FILE
if COUNTERS_FILE is None:
stats_file_path = os.path.join(
CONF.haproxy_amphora.base_path, "stats_counters.json")
# Open for read+write and create if necessary
flags = os.O_RDWR | os.O_CREAT
# mode 00644
mode = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP
try:
COUNTERS_FILE = os.fdopen(
os.open(stats_file_path, flags, mode), 'r+')
except OSError:
LOG.info("Failed to open `%s`, ignoring...", stats_file_path)
COUNTERS_FILE.seek(0)
return COUNTERS_FILE
def get_counters():
global COUNTERS
if COUNTERS is None:
try:
COUNTERS = simplejson.load(get_counters_file()) or {}
except (simplejson.JSONDecodeError, AttributeError):
COUNTERS = {}
return COUNTERS
def persist_counters():
"""Attempt to persist the latest statistics values"""
if COUNTERS is None:
return
try:
stats = simplejson.dumps(COUNTERS)
counters_file = get_counters_file()
counters_file.truncate(0)
counters_file.write(stats)
counters_file.flush()
except (OSError, AttributeError):
LOG.warning("Couldn't persist statistics counter file!")
def list_sock_stat_files(hadir=None):
@ -83,11 +134,12 @@ def run_sender(cmd_queue):
LOG.error('Keepalived is configured but not running, '
'skipping health heartbeat.')
else:
LOG.error('Failed to check keepalived and haproxy status due '
'to exception %s, skipping health heartbeat.', e)
LOG.exception('Failed to check keepalived and haproxy status '
'due to exception %s, skipping health '
'heartbeat.', e)
except Exception as e:
LOG.error('Failed to check keepalived and haproxy status due to '
'exception %s, skipping health heartbeat.', e)
LOG.exception('Failed to check keepalived and haproxy status due '
'to exception %s, skipping health heartbeat.', e)
try:
cmd = cmd_queue.get_nowait()
@ -109,33 +161,59 @@ def get_stats(stat_sock_file):
return stats, pool_status
def calculate_stats_deltas(listener_id, row):
counters = get_counters()
listener_counters = counters.get(listener_id, {})
counters[listener_id] = listener_counters
delta_values = {}
for metric_key in DELTA_METRICS:
current_value = int(row[metric_key])
# Get existing counter for our metrics
last_value = listener_counters.get(metric_key, 0)
# Store the new absolute value
listener_counters[metric_key] = current_value
# Calculate a delta for each metric
delta = current_value - last_value
# Did HAProxy restart or reset counters?
if delta < 0:
delta = current_value # If so, reset ours.
delta_values[metric_key] = delta
return delta_values
def build_stats_message():
# Example version 2 message without UDP:
# {
# "id": "<amphora_id>",
# "seq": 67,
# "listeners": {
# "<listener_id>": {
# "status": "OPEN",
# "stats": {
# "tx": 0,
# "rx": 0,
# "conns": 0,
# "totconns": 0,
# "ereq": 0
# }
# }
# },
# "pools": {
# "<pool_id>:<listener_id>": {
# "status": "UP",
# "members": {
# "<member_id>": "no check"
# }
# }
# },
# "ver": 2
# }
"""Build a stats message based on retrieved listener statistics.
Example version 3 message without UDP (note that values are deltas,
not absolutes)::
{"id": "<amphora_id>",
"seq": 67,
"listeners": {
"<listener_id>": {
"status": "OPEN",
"stats": {
"tx": 0,
"rx": 0,
"conns": 0,
"totconns": 0,
"ereq": 0
}
}
},
"pools": {
"<pool_id>:<listener_id>": {
"status": "UP",
"members": {
"<member_id>": "no check"
}
}
},
"ver": 3
}
"""
global SEQ
msg = {'id': CONF.amphora_agent.amphora_id,
'seq': SEQ, 'listeners': {}, 'pools': {},
@ -149,13 +227,14 @@ def build_stats_message():
for row in stats:
if row['svname'] == 'FRONTEND':
listener_id = row['pxname']
delta_values = calculate_stats_deltas(listener_id, row)
msg['listeners'][listener_id] = {
'status': row['status'],
'stats': {'tx': int(row['bout']),
'rx': int(row['bin']),
'stats': {'tx': delta_values['bout'],
'rx': delta_values['bin'],
'conns': int(row['scur']),
'totconns': int(row['stot']),
'ereq': int(row['ereq'])}}
'totconns': delta_values['stot'],
'ereq': delta_values['ereq']}}
for pool_id, pool in pool_status.items():
msg['pools'][pool_id] = {"status": pool['status'],
"members": pool['members']}
@ -166,16 +245,18 @@ def build_stats_message():
listeners_stats = keepalivedlvs_query.get_udp_listeners_stats()
if listeners_stats:
for listener_id, listener_stats in listeners_stats.items():
delta_values = calculate_stats_deltas(
listener_id, listener_stats['stats'])
pool_status = keepalivedlvs_query.get_udp_listener_pool_status(
listener_id)
udp_listener_dict = dict()
udp_listener_dict['status'] = listener_stats['status']
udp_listener_dict['stats'] = {
'tx': listener_stats['stats']['bout'],
'rx': listener_stats['stats']['bin'],
'tx': delta_values['bout'],
'rx': delta_values['bin'],
'conns': listener_stats['stats']['scur'],
'totconns': listener_stats['stats']['stot'],
'ereq': listener_stats['stats']['ereq']
'totconns': delta_values['stot'],
'ereq': delta_values['ereq']
}
if pool_status:
pool_id = pool_status['lvs']['uuid']
@ -184,4 +265,5 @@ def build_stats_message():
"members": pool_status['lvs']['members']
}
msg['listeners'][listener_id] = udp_listener_dict
persist_counters()
return msg

4
octavia/common/data_models.py
View File

@ -208,6 +208,10 @@ class ListenerStatistics(BaseDataModel):
self.bytes_out += other.bytes_out
self.request_errors += other.request_errors
self.total_connections += other.total_connections
else:
raise TypeError( # noqa: O342
"unsupported operand type(s) for +=: '{0}' and '{1}'".format(
type(self), type(other)))
return self

58
octavia/controller/healthmanager/health_drivers/update_db.py
View File

@ -23,6 +23,7 @@ import sqlalchemy
from stevedore import driver as stevedore_driver
from octavia.common import constants
from octavia.common import data_models
from octavia.common import stats
from octavia.controller.healthmanager.health_drivers import update_base
from octavia.db import api as db_api
@ -461,10 +462,6 @@ class UpdateHealthDb(update_base.HealthUpdateBase):
class UpdateStatsDb(update_base.StatsUpdateBase, stats.StatsMixin):
def __init__(self):
super(UpdateStatsDb, self).__init__()
self.repo_listener = repo.ListenerRepository()
def update_stats(self, health_message, srcaddr):
# The executor will eat any exceptions from the update_stats code
# so we need to wrap it and log the unhandled exception
@ -533,7 +530,28 @@ class UpdateStatsDb(update_base.StatsUpdateBase, stats.StatsMixin):
"ver": 2
}
Example V3 message::
See V2 message, except values are deltas rather than absolutes.
"""
version = health_message.get("ver", 1)
if version <= 2:
self.version2(health_message)
elif version == 3:
self.version3(health_message)
else:
LOG.warning("Unknown message version: %s, ignoring...", version)
def version2(self, health_message):
"""Parse version 1 and 2 of the health message.
:param health_message: health message dictionary
:type health_message: dict
"""
session = db_api.get_session()
amphora_id = health_message['id']
@ -545,8 +563,36 @@ class UpdateStatsDb(update_base.StatsUpdateBase, stats.StatsMixin):
'active_connections': stats['conns'],
'total_connections': stats['totconns'],
'request_errors': stats['ereq']}
LOG.debug("Updating listener stats in db and sending event.")
LOG.debug("Listener %s / Amphora %s stats: %s",
LOG.debug("Updating listener stats in db."
"Listener %s / Amphora %s stats: %s",
listener_id, amphora_id, stats)
self.listener_stats_repo.replace(
session, listener_id, amphora_id, **stats)
def version3(self, health_message):
"""Parse version 3 of the health message.
:param health_message: health message dictionary
:type health_message: dict
"""
session = db_api.get_session()
amphora_id = health_message['id']
listeners = health_message['listeners']
for listener_id, listener in listeners.items():
delta_stats = listener.get('stats')
delta_stats_model = data_models.ListenerStatistics(
listener_id=listener_id,
amphora_id=amphora_id,
bytes_in=delta_stats['rx'],
bytes_out=delta_stats['tx'],
active_connections=delta_stats['conns'],
total_connections=delta_stats['totconns'],
request_errors=delta_stats['ereq']
)
LOG.debug("Updating listener stats in db."
"Listener %s / Amphora %s stats: %s",
listener_id, amphora_id, delta_stats_model.to_dict())
self.listener_stats_repo.increment(session, delta_stats_model)

15
octavia/db/models.py
View File

@ -175,6 +175,21 @@ class ListenerStatistics(base_models.BASE):
'Current value is %(value)d.') % data)
return value
def __iadd__(self, other):
if isinstance(other, (ListenerStatistics,
data_models.ListenerStatistics)):
self.bytes_in += other.bytes_in
self.bytes_out += other.bytes_out
self.request_errors += other.request_errors
self.total_connections += other.total_connections
else:
raise TypeError( # noqa: O342
"unsupported operand type(s) for +=: '{0}' and '{1}'".format(
type(self), type(other)))
return self
class Member(base_models.BASE, base_models.IdMixin, base_models.ProjectMixin,
models.TimestampMixin, base_models.NameMixin,

33
octavia/db/repositories.py
View File

@ -1222,8 +1222,39 @@ class ListenerStatisticsRepository(BaseRepository):
model_kwargs['amphora_id'] = amphora_id
self.create(session, **model_kwargs)
def increment(self, session, delta_stats):
"""Updates a listener's statistics, incrementing by the passed deltas.
:param session: A Sql Alchemy database session
:param delta_stats: Listener statistics deltas to add
:type delta_stats: octavia.common.data_models.ListenerStatistics
"""
with session.begin(subtransactions=True):
count = session.query(self.model_class).filter_by(
listener_id=delta_stats.listener_id,
amphora_id=delta_stats.amphora_id).count()
if count:
existing_stats = session.query(
self.model_class).with_for_update().filter_by(
listener_id=delta_stats.listener_id,
amphora_id=delta_stats.amphora_id).one()
existing_stats += delta_stats
existing_stats.active_connections = (
delta_stats.active_connections)
else:
self.create(session, **delta_stats.to_dict())
def update(self, session, listener_id, **model_kwargs):
"""Updates a listener's statistics by a listener's id."""
"""Updates a listener's statistics, overriding with the passed values.
:param session: A Sql Alchemy database session
:param listener_id: The UUID of the listener to update
:type listener_id: str
:param model_kwargs: Entity attributes that should be updated
"""
with session.begin(subtransactions=True):
session.query(self.model_class).filter_by(
listener_id=listener_id).update(model_kwargs)

369
octavia/tests/functional/db/test_repositories.py
View File

File diff suppressed because it is too large Load Diff

223
octavia/tests/unit/amphorae/backends/health_daemon/test_health_daemon.py
View File

@ -12,21 +12,26 @@
# License for the specific language governing permissions and limitations
# under the License.
#
import os
import queue
from unittest import mock
from oslo_config import cfg
from oslo_config import fixture as oslo_fixture
from oslo_utils import uuidutils
import simplejson
from octavia.amphorae.backends.health_daemon import health_daemon
from octavia.common import constants
from octavia.tests.common import utils as test_utils
import octavia.tests.unit.base as base
LB_ID1 = uuidutils.generate_uuid()
LISTENER_ID1 = uuidutils.generate_uuid()
LISTENER_ID2 = uuidutils.generate_uuid()
LISTENER_IDS = [LISTENER_ID1, LISTENER_ID2]
AMPHORA_ID = uuidutils.generate_uuid()
BASE_PATH = '/tmp/test'
SAMPLE_POOL_STATUS = {
'432fc8b3-d446-48d4-bb64-13beb90e22bc': {
@ -45,46 +50,46 @@ SAMPLE_BOGUS_POOL_STATUS = {LISTENER_ID1: {
'302e33d9-dee1-4de9-98d5-36329a06fb58':
'DOWN'}}}
SAMPLE_STATS = ({'': '', 'status': 'OPEN', 'lastchg': '',
'weight': '', 'slim': '2000', 'pid': '1', 'comp_byp': '0',
'lastsess': '', 'rate_lim': '0', 'check_duration': '',
'rate': '0', 'req_rate': '0', 'check_status': '',
'econ': '', 'comp_out': '0', 'wredis': '', 'dresp': '0',
'ereq': '0', 'tracked': '', 'comp_in': '0',
'pxname': LISTENER_ID1,
'dreq': '0', 'hrsp_5xx': '0', 'last_chk': '',
'check_code': '', 'sid': '0', 'bout': '0', 'hrsp_1xx': '0',
'qlimit': '', 'hrsp_other': '0', 'bin': '0', 'rtime': '',
'smax': '0', 'req_tot': '0', 'lbtot': '', 'stot': '0',
'wretr': '', 'req_rate_max': '0', 'ttime': '', 'iid': '2',
'hrsp_4xx': '0', 'chkfail': '', 'hanafail': '',
'downtime': '', 'qcur': '', 'eresp': '', 'comp_rsp': '0',
'cli_abrt': '', 'ctime': '', 'qtime': '', 'srv_abrt': '',
'throttle': '', 'last_agt': '', 'scur': '0', 'type': '0',
'bck': '', 'qmax': '', 'rate_max': '0', 'hrsp_2xx': '0',
'act': '', 'chkdown': '', 'svname': 'FRONTEND',
'hrsp_3xx': '0'},
{'': '', 'status': 'no check', 'lastchg': '', 'weight': '1',
'slim': '', 'pid': '1', 'comp_byp': '', 'lastsess': '-1',
'rate_lim': '', 'check_duration': '', 'rate': '0',
'req_rate': '', 'check_status': '', 'econ': '0',
'comp_out': '', 'wredis': '0', 'dresp': '0', 'ereq': '',
'tracked': '', 'comp_in': '',
'pxname': '432fc8b3-d446-48d4-bb64-13beb90e22bc',
'dreq': '', 'hrsp_5xx': '0', 'last_chk': '',
'check_code': '', 'sid': '1', 'bout': '0', 'hrsp_1xx': '0',
'qlimit': '', 'hrsp_other': '0', 'bin': '0', 'rtime': '0',
'smax': '0', 'req_tot': '', 'lbtot': '0', 'stot': '0',
'wretr': '0', 'req_rate_max': '', 'ttime': '0', 'iid': '3',
'hrsp_4xx': '0', 'chkfail': '', 'hanafail': '0',
'downtime': '', 'qcur': '0', 'eresp': '0', 'comp_rsp': '',
'cli_abrt': '0', 'ctime': '0', 'qtime': '0', 'srv_abrt': '0',
'throttle': '', 'last_agt': '', 'scur': '0', 'type': '2',
'bck': '0', 'qmax': '0', 'rate_max': '0', 'hrsp_2xx': '0',
'act': '1', 'chkdown': '',
'svname': '302e33d9-dee1-4de9-98d5-36329a06fb58',
'hrsp_3xx': '0'},
{'': '', 'status': 'UP', 'lastchg': '122', 'weight': '1',
FRONTEND_STATS = {'': '', 'status': 'OPEN', 'lastchg': '',
'weight': '', 'slim': '2000', 'pid': '1', 'comp_byp': '0',
'lastsess': '', 'rate_lim': '0', 'check_duration': '',
'rate': '0', 'req_rate': '0', 'check_status': '',
'econ': '', 'comp_out': '0', 'wredis': '', 'dresp': '0',
'ereq': '5', 'tracked': '', 'comp_in': '0',
'pxname': LISTENER_ID1,
'dreq': '0', 'hrsp_5xx': '0', 'last_chk': '',
'check_code': '', 'sid': '0', 'bout': '10', 'hrsp_1xx': '0',
'qlimit': '', 'hrsp_other': '0', 'bin': '5', 'rtime': '',
'smax': '0', 'req_tot': '0', 'lbtot': '', 'stot': '0',
'wretr': '', 'req_rate_max': '0', 'ttime': '', 'iid': '2',
'hrsp_4xx': '0', 'chkfail': '', 'hanafail': '',
'downtime': '', 'qcur': '', 'eresp': '', 'comp_rsp': '0',
'cli_abrt': '', 'ctime': '', 'qtime': '', 'srv_abrt': '',
'throttle': '', 'last_agt': '', 'scur': '0', 'type': '0',
'bck': '', 'qmax': '', 'rate_max': '0', 'hrsp_2xx': '0',
'act': '', 'chkdown': '', 'svname': 'FRONTEND',
'hrsp_3xx': '0'}
MEMBER_STATS = {'': '', 'status': 'no check', 'lastchg': '', 'weight': '1',
'slim': '', 'pid': '1', 'comp_byp': '', 'lastsess': '-1',
'rate_lim': '', 'check_duration': '', 'rate': '0',
'req_rate': '', 'check_status': '', 'econ': '0',
'comp_out': '', 'wredis': '0', 'dresp': '0', 'ereq': '',
'tracked': '', 'comp_in': '',
'pxname': '432fc8b3-d446-48d4-bb64-13beb90e22bc',
'dreq': '', 'hrsp_5xx': '0', 'last_chk': '',
'check_code': '', 'sid': '1', 'bout': '0', 'hrsp_1xx': '0',
'qlimit': '', 'hrsp_other': '0', 'bin': '0', 'rtime': '0',
'smax': '0', 'req_tot': '', 'lbtot': '0', 'stot': '0',
'wretr': '0', 'req_rate_max': '', 'ttime': '0', 'iid': '3',
'hrsp_4xx': '0', 'chkfail': '', 'hanafail': '0',
'downtime': '', 'qcur': '0', 'eresp': '0', 'comp_rsp': '',
'cli_abrt': '0', 'ctime': '0', 'qtime': '0', 'srv_abrt': '0',
'throttle': '', 'last_agt': '', 'scur': '0', 'type': '2',
'bck': '0', 'qmax': '0', 'rate_max': '0', 'hrsp_2xx': '0',
'act': '1', 'chkdown': '',
'svname': '302e33d9-dee1-4de9-98d5-36329a06fb58',
'hrsp_3xx': '0'}
BACKEND_STATS = {'': '', 'status': 'UP', 'lastchg': '122', 'weight': '1',
'slim': '200', 'pid': '1', 'comp_byp': '0', 'lastsess': '-1',
'rate_lim': '', 'check_duration': '', 'rate': '0',
'req_rate': '', 'check_status': '', 'econ': '0',
@ -101,14 +106,15 @@ SAMPLE_STATS = ({'': '', 'status': 'OPEN', 'lastchg': '',
'srv_abrt': '0', 'throttle': '', 'last_agt': '', 'scur': '0',
'type': '1', 'bck': '0', 'qmax': '0', 'rate_max': '0',
'hrsp_2xx': '0', 'act': '1', 'chkdown': '0',
'svname': 'BACKEND', 'hrsp_3xx': '0'})
'svname': 'BACKEND', 'hrsp_3xx': '0'}
SAMPLE_STATS = (FRONTEND_STATS, MEMBER_STATS, BACKEND_STATS)
SAMPLE_STATS_MSG = {
'listeners': {
LISTENER_ID1: {
'stats': {
'totconns': 0, 'conns': 0,
'tx': 0, 'rx': 0, 'ereq': 0},
'tx': 8, 'rx': 4, 'ereq': 5},
'status': 'OPEN'},
},
'pools': {
@ -119,8 +125,29 @@ SAMPLE_STATS_MSG = {
'members': {'e657f950-a6a2-4d28-bffa-0c8a8c05f815': 'DOWN'},
'status': 'UP'},
},
'id': None,
'seq': 0,
'id': AMPHORA_ID,
'seq': mock.ANY,
'ver': health_daemon.MSG_VER
}
SAMPLE_MSG_HAPROXY_RESTART = {
'listeners': {
LISTENER_ID1: {
'stats': {
'totconns': 0, 'conns': 0,
'tx': 10, 'rx': 5, 'ereq': 5},
'status': 'OPEN'},
},
'pools': {
'432fc8b3-d446-48d4-bb64-13beb90e22bc': {
'members': {'302e33d9-dee1-4de9-98d5-36329a06fb58': 'DOWN'},
'status': 'UP'},
'3661ed10-99db-4d2c-bffb-99b60eb876ff': {
'members': {'e657f950-a6a2-4d28-bffa-0c8a8c05f815': 'DOWN'},
'status': 'UP'},
},
'id': AMPHORA_ID,
'seq': mock.ANY,
'ver': health_daemon.MSG_VER
}
@ -131,6 +158,10 @@ class TestHealthDaemon(base.TestCase):
super(TestHealthDaemon, self).setUp()
conf = oslo_fixture.Config(cfg.CONF)
conf.config(group="haproxy_amphora", base_path=BASE_PATH)
conf.config(group="amphora_agent", amphora_id=AMPHORA_ID)
file_name = os.path.join(BASE_PATH, "stats_counters")
self.mock_open = self.useFixture(
test_utils.OpenFixture(file_name)).mock_open
@mock.patch('octavia.amphorae.backends.agent.'
'api_server.util.get_loadbalancers')
@ -296,18 +327,33 @@ class TestHealthDaemon(base.TestCase):
'health_daemon.list_sock_stat_files')
def test_build_stats_message(self, mock_list_files,
mock_get_stats, mock_is_running):
mock_list_files.return_value = {LISTENER_ID1: 'TEST',
LISTENER_ID2: 'TEST2'}
health_daemon.COUNTERS = None
health_daemon.COUNTERS_FILE = None
lb1_stats_socket = '/var/lib/octavia/{0}/haproxy.sock'.format(LB_ID1)
mock_list_files.return_value = {LB_ID1: lb1_stats_socket}
mock_is_running.return_value = True
mock_get_stats.return_value = SAMPLE_STATS, SAMPLE_POOL_STATUS
msg = health_daemon.build_stats_message()
with mock.patch('os.open'), mock.patch.object(
os, 'fdopen', self.mock_open) as mock_fdopen:
mock_fdopen().read.return_value = simplejson.dumps({
LISTENER_ID1: {'bin': 1, 'bout': 2},
})
msg = health_daemon.build_stats_message()
self.assertEqual(SAMPLE_STATS_MSG, msg)
mock_get_stats.assert_any_call('TEST')
mock_get_stats.assert_any_call('TEST2')
mock_get_stats.assert_any_call(lb1_stats_socket)
mock_fdopen().write.assert_called_once_with(simplejson.dumps({
LISTENER_ID1: {
'bin': int(FRONTEND_STATS['bin']),
'bout': int(FRONTEND_STATS['bout']),
'ereq': int(FRONTEND_STATS['ereq']),
'stot': int(FRONTEND_STATS['stot'])
}
}))
@mock.patch('octavia.amphorae.backends.agent.api_server.'
'util.is_lb_running')
@ -318,15 +364,19 @@ class TestHealthDaemon(base.TestCase):
def test_build_stats_message_no_listener(self, mock_list_files,
mock_get_stats,
mock_is_running):
mock_list_files.return_value = {LISTENER_ID1: 'TEST',
LISTENER_ID2: 'TEST2'}
health_daemon.COUNTERS = None
health_daemon.COUNTERS_FILE = None
lb1_stats_socket = '/var/lib/octavia/{0}/haproxy.sock'.format(LB_ID1)
mock_list_files.return_value = {LB_ID1: lb1_stats_socket}
mock_is_running.side_effect = [True, False]
mock_get_stats.return_value = SAMPLE_STATS, SAMPLE_POOL_STATUS
mock_is_running.return_value = False
health_daemon.build_stats_message()
with mock.patch('os.open'), mock.patch.object(
os, 'fdopen', self.mock_open) as mock_fdopen:
health_daemon.build_stats_message()
self.assertEqual(1, mock_get_stats.call_count)
self.assertEqual(0, mock_get_stats.call_count)
self.assertEqual(0, mock_fdopen().read.call_count)
@mock.patch("octavia.amphorae.backends.utils.keepalivedlvs_query."
"get_udp_listener_pool_status")
@ -335,8 +385,10 @@ class TestHealthDaemon(base.TestCase):
@mock.patch("octavia.amphorae.backends.agent.api_server.util."
"get_udp_listeners")
def test_build_stats_message_with_udp_listener(
self, mock_get_udp_listeners, mock_get_listener_stats,
mock_get_pool_status):
self, mock_get_udp_listeners,
mock_get_listener_stats, mock_get_pool_status):
health_daemon.COUNTERS = None
health_daemon.COUNTERS_FILE = None
udp_listener_id1 = uuidutils.generate_uuid()
udp_listener_id2 = uuidutils.generate_uuid()
udp_listener_id3 = uuidutils.generate_uuid()
@ -346,10 +398,11 @@ class TestHealthDaemon(base.TestCase):
mock_get_udp_listeners.return_value = [udp_listener_id1,
udp_listener_id2,
udp_listener_id3]
mock_get_listener_stats.return_value = {
udp_listener_id1: {
'status': constants.OPEN,
'stats': {'bin': 6387472, 'stot': 5, 'bout': 7490,
'stats': {'bin': 5, 'stot': 5, 'bout': 10,
'ereq': 0, 'scur': 0}},
udp_listener_id3: {
'status': constants.DOWN,
@ -373,7 +426,7 @@ class TestHealthDaemon(base.TestCase):
udp_listener_id1: {
'status': constants.OPEN,
'stats': {'conns': 0, 'totconns': 5, 'ereq': 0,
'rx': 6387472, 'tx': 7490}},
'rx': 4, 'tx': 8}},
udp_listener_id3: {
'status': constants.DOWN,
'stats': {'conns': 0, 'totconns': 0, 'ereq': 0,
@ -384,10 +437,58 @@ class TestHealthDaemon(base.TestCase):
'members': {
member_id1: constants.UP,
member_id2: constants.UP}}},
'id': None,
'id': AMPHORA_ID,
'seq': mock.ANY, 'ver': health_daemon.MSG_VER}
msg = health_daemon.build_stats_message()
with mock.patch('os.open'), mock.patch.object(
os, 'fdopen', self.mock_open) as mock_fdopen:
mock_fdopen().read.return_value = simplejson.dumps({
udp_listener_id1: {
'bin': 1, 'bout': 2, "ereq": 0, "stot": 0}
})
msg = health_daemon.build_stats_message()
self.assertEqual(expected, msg)
mock_fdopen().write.assert_called_once_with(simplejson.dumps({
udp_listener_id1: {'bin': 5, 'bout': 10, 'ereq': 0, 'stot': 5},
udp_listener_id3: {'bin': 0, 'bout': 0, 'ereq': 0, 'stot': 0},
}))
@mock.patch('octavia.amphorae.backends.agent.api_server.'
'util.is_lb_running')
@mock.patch('octavia.amphorae.backends.health_daemon.'
'health_daemon.get_stats')
@mock.patch('octavia.amphorae.backends.health_daemon.'
'health_daemon.list_sock_stat_files')
def test_haproxy_restart(self, mock_list_files,
mock_get_stats, mock_is_running):
health_daemon.COUNTERS = None
health_daemon.COUNTERS_FILE = None
lb1_stats_socket = '/var/lib/octavia/{0}/haproxy.sock'.format(LB_ID1)
mock_list_files.return_value = {LB_ID1: lb1_stats_socket}
mock_is_running.return_value = True
mock_get_stats.return_value = SAMPLE_STATS, SAMPLE_POOL_STATUS
with mock.patch('os.open'), mock.patch.object(
os, 'fdopen', self.mock_open) as mock_fdopen:
mock_fdopen().read.return_value = simplejson.dumps({
LISTENER_ID1: {'bin': 15, 'bout': 20},
})
msg = health_daemon.build_stats_message()
self.assertEqual(SAMPLE_MSG_HAPROXY_RESTART, msg)
mock_get_stats.assert_any_call(lb1_stats_socket)
mock_fdopen().write.assert_called_once_with(simplejson.dumps({
LISTENER_ID1: {
'bin': int(FRONTEND_STATS['bin']),
'bout': int(FRONTEND_STATS['bout']),
'ereq': int(FRONTEND_STATS['ereq']),
'stot': int(FRONTEND_STATS['stot'])
}
}))
class FileNotFoundError(IOError):

54
octavia/tests/unit/common/test_data_models.py
View File

@ -14,6 +14,7 @@
import copy
import datetime
import random
from oslo_utils import uuidutils
@ -27,6 +28,7 @@ class TestDataModels(base.TestCase):
def setUp(self):
self.LB_ID = uuidutils.generate_uuid()
self.LISTENER_ID = uuidutils.generate_uuid()
self.PROJECT_ID = uuidutils.generate_uuid()
self.SERVER_GROUP_ID = uuidutils.generate_uuid()
self.CREATED_AT = datetime.datetime.now()
@ -479,3 +481,55 @@ class TestDataModels(base.TestCase):
test_Quota_obj.update(update_dict)
self.assertEqual(reference_Quota_obj, test_Quota_obj)
def test_ListenerStatistics_iadd(self):
# test incrementing add function
bytes_in1 = random.randrange(1000000000)
bytes_out1 = random.randrange(1000000000)
active_conns1 = random.randrange(1000000000)
total_conns1 = random.randrange(1000000000)
request_errors1 = random.randrange(1000000000)
stats_1 = data_models.ListenerStatistics(
listener_id=self.LISTENER_ID,
amphora_id=self.AMP_ID,
bytes_in=bytes_in1,
bytes_out=bytes_out1,
active_connections=active_conns1,
total_connections=total_conns1,
request_errors=request_errors1
)
bytes_in2 = random.randrange(1000000000)
bytes_out2 = random.randrange(1000000000)
active_conns2 = random.randrange(1000000000)
total_conns2 = random.randrange(1000000000)
request_errors2 = random.randrange(1000000000)
stats_2 = data_models.ListenerStatistics(
listener_id="listener 2",
amphora_id="amphora 2",
bytes_in=bytes_in2,
bytes_out=bytes_out2,
active_connections=active_conns2,
total_connections=total_conns2,
request_errors=request_errors2
)
# test successful +=
stats_1 += stats_2
# not a delta, so it won't be incremented
self.assertEqual(stats_1.active_connections, active_conns1)
self.assertEqual(stats_1.listener_id, self.LISTENER_ID)
self.assertEqual(stats_1.amphora_id, self.AMP_ID)
# deltas will be incremented
self.assertEqual(stats_1.bytes_in, bytes_in1 + bytes_in2)
self.assertEqual(stats_1.bytes_out, bytes_out1 + bytes_out2)
self.assertEqual(stats_1.total_connections,
total_conns1 + total_conns2)
self.assertEqual(stats_1.request_errors,
request_errors1 + request_errors2)
# test incrementing an incompatible object
self.assertRaises(TypeError, stats_1.__iadd__, "boom")

48
octavia/tests/unit/controller/healthmanager/health_drivers/test_update_db.py
View File

@ -1385,7 +1385,7 @@ class TestUpdateStatsDb(base.TestCase):
self.loadbalancer_repo.get.return_value = self.loadbalancer
@mock.patch('octavia.db.api.get_session')
def test_update_stats(self, mock_session):
def test_update_stats_v1(self, mock_session):
health = {
"id": self.amphora_id,
@ -1422,11 +1422,53 @@ class TestUpdateStatsDb(base.TestCase):
total_connections=self.listener_stats.total_connections,
request_errors=self.listener_stats.request_errors)
# Test with missing DB listener
self.sm.repo_listener.get.return_value = None
# Test with update failure
mock_session.side_effect = Exception
self.sm.update_stats(health, '192.0.2.1')
@mock.patch('octavia.db.api.get_session')
def test_update_stats_v3(self, mock_session):
health = {
"id": self.amphora_id,
"seq": random.randint(0, 100),
"ver": 3,
"listeners": {
self.listener_id: {
"status": constants.OPEN,
"stats": {
"ereq": self.listener_stats.request_errors,
"conns": self.listener_stats.active_connections,
"totconns": self.listener_stats.total_connections,
"rx": self.listener_stats.bytes_in,
"tx": self.listener_stats.bytes_out,
},
"pools": {
"pool-id-1": {
"status": constants.UP,
"members": {"member-id-1": constants.ONLINE}
}
}
}
}
}
delta_stats_model = data_models.ListenerStatistics(
listener_id=self.listener_id,
amphora_id=self.amphora_id,
bytes_in=self.listener_stats.bytes_in,
bytes_out=self.listener_stats.bytes_out,
active_connections=self.listener_stats.active_connections,
total_connections=self.listener_stats.total_connections,
request_errors=self.listener_stats.request_errors
)
mock_session.return_value = 'blah'
self.sm.update_stats(health, '192.0.2.1')
self.listener_stats_repo.increment.assert_called_once_with(
'blah', delta_stats_model)
# Test with update failure
mock_session.side_effect = Exception
self.sm.update_stats(health, '192.0.2.1')