openstack
/
octavia
Code Issues Proposed changes
Browse Source

Change amphora statistics to use deltas 

Amphora statistics packets should report deltas instead of absolutes for
all relevant metrics.

Change-Id: I5cf6f1f20f2c6f1da39982b2d88e036eefe48b2f
Co-Authored-By: Anushka Singh <anushka.singh.2511@gmail.com>
Co-Authored-By: Stephanie Djajadi <stephanie.djajadi@gmail.com>
changes/15/740815/19
Adam Harwell 2 years ago
parent
51b93c0022
commit
d15cccff2f
9 changed files with 699 additions and 267 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 162
      octavia/amphorae/backends/health_daemon/health_daemon.py
  2. 4
      octavia/common/data_models.py
  3. 58
      octavia/controller/healthmanager/health_drivers/update_db.py
  4. 15
      octavia/db/models.py
  5. 33
      octavia/db/repositories.py
  6. 369
      octavia/tests/functional/db/test_repositories.py
  7. 223
      octavia/tests/unit/amphorae/backends/health_daemon/test_health_daemon.py
  8. 54
      octavia/tests/unit/common/test_data_models.py
  9. 48
      octavia/tests/unit/controller/healthmanager/health_drivers/test_update_db.py

162
octavia/amphorae/backends/health_daemon/health_daemon.py
Unescape View File

@ -17,10 +17,12 @@
import errno
import os
import queue
import stat
import time
from oslo_config import cfg
from oslo_log import log as logging
import simplejson
from octavia.amphorae.backends.agent.api_server import util
from octavia.amphorae.backends.health_daemon import health_sender
@ -38,8 +40,57 @@ SEQ = 0
#
# ver 1 - Adds UDP listener status when no pool or members are present
# ver 2 - Switch to all listeners in a single combined haproxy config
#
MSG_VER = 2
# ver 3 - Switch stats reporting to deltas
MSG_VER = 3
DELTA_METRICS = ('bin', 'bout', 'ereq', 'stot')
# Filesystem persistent counters for statistics deltas
COUNTERS = None
COUNTERS_FILE = None
def get_counters_file():
global COUNTERS_FILE
if COUNTERS_FILE is None:
stats_file_path = os.path.join(
CONF.haproxy_amphora.base_path, "stats_counters.json")
# Open for read+write and create if necessary
flags = os.O_RDWR | os.O_CREAT
# mode 00644
mode = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP
try:
COUNTERS_FILE = os.fdopen(
os.open(stats_file_path, flags, mode), 'r+')
except OSError:
LOG.info("Failed to open `%s`, ignoring...", stats_file_path)
COUNTERS_FILE.seek(0)
return COUNTERS_FILE
def get_counters():
global COUNTERS
if COUNTERS is None:
try:
COUNTERS = simplejson.load(get_counters_file()) or {}
except (simplejson.JSONDecodeError, AttributeError):
COUNTERS = {}
return COUNTERS
def persist_counters():
"""Attempt to persist the latest statistics values"""
if COUNTERS is None:
return
try:
stats = simplejson.dumps(COUNTERS)
counters_file = get_counters_file()
counters_file.truncate(0)
counters_file.write(stats)
counters_file.flush()
except (OSError, AttributeError):
LOG.warning("Couldn't persist statistics counter file!")
def list_sock_stat_files(hadir=None):
@ -83,11 +134,12 @@ def run_sender(cmd_queue):
LOG.error('Keepalived is configured but not running, '
'skipping health heartbeat.')
else:
LOG.error('Failed to check keepalived and haproxy status due '
'to exception %s, skipping health heartbeat.', e)
LOG.exception('Failed to check keepalived and haproxy status '
'due to exception %s, skipping health '
'heartbeat.', e)
except Exception as e:
LOG.error('Failed to check keepalived and haproxy status due to '
'exception %s, skipping health heartbeat.', e)
LOG.exception('Failed to check keepalived and haproxy status due '
'to exception %s, skipping health heartbeat.', e)
try:
cmd = cmd_queue.get_nowait()
@ -109,33 +161,59 @@ def get_stats(stat_sock_file):
return stats, pool_status
def calculate_stats_deltas(listener_id, row):
counters = get_counters()
listener_counters = counters.get(listener_id, {})
counters[listener_id] = listener_counters
delta_values = {}
for metric_key in DELTA_METRICS:
current_value = int(row[metric_key])
# Get existing counter for our metrics
last_value = listener_counters.get(metric_key, 0)
# Store the new absolute value
listener_counters[metric_key] = current_value
# Calculate a delta for each metric
delta = current_value - last_value
# Did HAProxy restart or reset counters?
if delta < 0:
delta = current_value # If so, reset ours.
delta_values[metric_key] = delta
return delta_values
def build_stats_message():
# Example version 2 message without UDP:
# {
# "id": "<amphora_id>",
# "seq": 67,
# "listeners": {
# "<listener_id>": {
# "status": "OPEN",
# "stats": {
# "tx": 0,
# "rx": 0,
# "conns": 0,
# "totconns": 0,
# "ereq": 0
# }
# }
# },
# "pools": {
# "<pool_id>:<listener_id>": {
# "status": "UP",
# "members": {
# "<member_id>": "no check"
# }
# }
# },
# "ver": 2
# }
"""Build a stats message based on retrieved listener statistics.
Example version 3 message without UDP (note that values are deltas,
not absolutes)::
{"id": "<amphora_id>",
"seq": 67,
"listeners": {
"<listener_id>": {
"status": "OPEN",
"stats": {
"tx": 0,
"rx": 0,
"conns": 0,
"totconns": 0,
"ereq": 0
}
}
},
"pools": {
"<pool_id>:<listener_id>": {
"status": "UP",
"members": {
"<member_id>": "no check"
}
}
},
"ver": 3
}
"""
global SEQ
msg = {'id': CONF.amphora_agent.amphora_id,
'seq': SEQ, 'listeners': {}, 'pools': {},
@ -149,13 +227,14 @@ def build_stats_message():
for row in stats:
if row['svname'] == 'FRONTEND':
listener_id = row['pxname']
delta_values = calculate_stats_deltas(listener_id, row)
msg['listeners'][listener_id] = {
'status': row['status'],
'stats': {'tx': int(row['bout']),
'rx': int(row['bin']),
'stats': {'tx': delta_values['bout'],
'rx': delta_values['bin'],
'conns': int(row['scur']),
'totconns': int(row['stot']),
'ereq': int(row['ereq'])}}
'totconns': delta_values['stot'],
'ereq': delta_values['ereq']}}
for pool_id, pool in pool_status.items():
msg['pools'][pool_id] = {"status": pool['status'],
"members": pool['members']}
@ -166,16 +245,18 @@ def build_stats_message():
listeners_stats = keepalivedlvs_query.get_udp_listeners_stats()
if listeners_stats:
for listener_id, listener_stats in listeners_stats.items():
delta_values = calculate_stats_deltas(
listener_id, listener_stats['stats'])
pool_status = keepalivedlvs_query.get_udp_listener_pool_status(
listener_id)
udp_listener_dict = dict()
udp_listener_dict['status'] = listener_stats['status']
udp_listener_dict['stats'] = {
'tx': listener_stats['stats']['bout'],
'rx': listener_stats['stats']['bin'],
'tx': delta_values['bout'],
'rx': delta_values['bin'],
'conns': listener_stats['stats']['scur'],
'totconns': listener_stats['stats']['stot'],
'ereq': listener_stats['stats']['ereq']
'totconns': delta_values['stot'],
'ereq': delta_values['ereq']
}
if pool_status:
pool_id = pool_status['lvs']['uuid']
@ -184,4 +265,5 @@ def build_stats_message():
"members": pool_status['lvs']['members']
}
msg['listeners'][listener_id] = udp_listener_dict
persist_counters()
return msg

4
octavia/common/data_models.py
Unescape View File

@ -208,6 +208,10 @@ class ListenerStatistics(BaseDataModel):
self.bytes_out += other.bytes_out
self.request_errors += other.request_errors
self.total_connections += other.total_connections
else:
raise TypeError( # noqa: O342
"unsupported operand type(s) for +=: '{0}' and '{1}'".format(
type(self), type(other)))
return self

58
octavia/controller/healthmanager/health_drivers/update_db.py
Unescape View File

@ -23,6 +23,7 @@ import sqlalchemy
from stevedore import driver as stevedore_driver
from octavia.common import constants
from octavia.common import data_models
from octavia.common import stats
from octavia.controller.healthmanager.health_drivers import update_base
from octavia.db import api as db_api
@ -461,10 +462,6 @@ class UpdateHealthDb(update_base.HealthUpdateBase):
class UpdateStatsDb(update_base.StatsUpdateBase, stats.StatsMixin):
def __init__(self):
super(UpdateStatsDb, self).__init__()
self.repo_listener = repo.ListenerRepository()
def update_stats(self, health_message, srcaddr):
# The executor will eat any exceptions from the update_stats code
# so we need to wrap it and log the unhandled exception
@ -533,7 +530,28 @@ class UpdateStatsDb(update_base.StatsUpdateBase, stats.StatsMixin):
"ver": 2
}
Example V3 message::
See V2 message, except values are deltas rather than absolutes.
"""
version = health_message.get("ver", 1)
if version <= 2:
self.version2(health_message)
elif version == 3:
self.version3(health_message)
else:
LOG.warning("Unknown message version: %s, ignoring...", version)
def version2(self, health_message):
"""Parse version 1 and 2 of the health message.
:param health_message: health message dictionary
:type health_message: dict
"""
session = db_api.get_session()
amphora_id = health_message['id']
@ -545,8 +563,36 @@ class UpdateStatsDb(update_base.StatsUpdateBase, stats.StatsMixin):
'active_connections': stats['conns'],
'total_connections': stats['totconns'],
'request_errors': stats['ereq']}
LOG.debug("Updating listener stats in db and sending event.")
LOG.debug("Listener %s / Amphora %s stats: %s",
LOG.debug("Updating listener stats in db."
"Listener %s / Amphora %s stats: %s",
listener_id, amphora_id, stats)
self.listener_stats_repo.replace(
session, listener_id, amphora_id, **stats)
def version3(self, health_message):
"""Parse version 3 of the health message.
:param health_message: health message dictionary
:type health_message: dict
"""
session = db_api.get_session()
amphora_id = health_message['id']
listeners = health_message['listeners']
for listener_id, listener in listeners.items():
delta_stats = listener.get('stats')
delta_stats_model = data_models.ListenerStatistics(
listener_id=listener_id,
amphora_id=amphora_id,
bytes_in=delta_stats['rx'],
bytes_out=delta_stats['tx'],
active_connections=delta_stats['conns'],
total_connections=delta_stats['totconns'],
request_errors=delta_stats['ereq']
)
LOG.debug("Updating listener stats in db."
"Listener %s / Amphora %s stats: %s",
listener_id, amphora_id, delta_stats_model.to_dict())
self.listener_stats_repo.increment(session, delta_stats_model)

15
octavia/db/models.py
Unescape View File

@ -175,6 +175,21 @@ class ListenerStatistics(base_models.BASE):
'Current value is %(value)d.') % data)
return value
def __iadd__(self, other):
if isinstance(other, (ListenerStatistics,
data_models.ListenerStatistics)):
self.bytes_in += other.bytes_in
self.bytes_out += other.bytes_out
self.request_errors += other.request_errors
self.total_connections += other.total_connections
else:
raise TypeError( # noqa: O342
"unsupported operand type(s) for +=: '{0}' and '{1}'".format(
type(self), type(other)))
return self
class Member(base_models.BASE, base_models.IdMixin, base_models.ProjectMixin,
models.TimestampMixin, base_models.NameMixin,

33
octavia/db/repositories.py
Unescape View File

@ -1222,8 +1222,39 @@ class ListenerStatisticsRepository(BaseRepository):
model_kwargs['amphora_id'] = amphora_id
self.create(session, **model_kwargs)
def increment(self, session, delta_stats):
"""Updates a listener's statistics, incrementing by the passed deltas.
:param session: A Sql Alchemy database session
:param delta_stats: Listener statistics deltas to add
:type delta_stats: octavia.common.data_models.ListenerStatistics
"""
with session.begin(subtransactions=True):
count = session.query(self.model_class).filter_by(
listener_id=delta_stats.listener_id,
amphora_id=delta_stats.amphora_id).count()
if count:
existing_stats = session.query(
self.model_class).with_for_update().filter_by(
listener_id=delta_stats.listener_id,
amphora_id=delta_stats.amphora_id).one()
existing_stats += delta_stats
existing_stats.active_connections = (
delta_stats.active_connections)
else:
self.create(session, **delta_stats.to_dict())
def update(self, session, listener_id, **model_kwargs):
"""Updates a listener's statistics by a listener's id."""
"""Updates a listener's statistics, overriding with the passed values.
:param session: A Sql Alchemy database session
:param listener_id: The UUID of the listener to update
:type listener_id: str
:param model_kwargs: Entity attributes that should be updated
"""
with session.begin(subtransactions=True):
session.query(self.model_class).filter_by(
listener_id=listener_id).update(model_kwargs)

369
octavia/tests/functional/db/test_repositories.py
View File

File diff suppressed because it is too large Load Diff

223
octavia/tests/unit/amphorae/backends/health_daemon/test_health_daemon.py
Unescape View File

@ -12,21 +12,26 @@
# License for the specific language governing permissions and limitations
# under the License.
#
import os
import queue
from unittest import mock
from oslo_config import cfg
from oslo_config import fixture as oslo_fixture
from oslo_utils import uuidutils
import simplejson
from octavia.amphorae.backends.health_daemon import health_daemon
from octavia.common import constants
from octavia.tests.common import utils as test_utils
import octavia.tests.unit.base as base
LB_ID1 = uuidutils.generate_uuid()
LISTENER_ID1 = uuidutils.generate_uuid()
LISTENER_ID2 = uuidutils.generate_uuid()
LISTENER_IDS = [LISTENER_ID1, LISTENER_ID2]
AMPHORA_ID = uuidutils.generate_uuid()
BASE_PATH = '/tmp/test'
SAMPLE_POOL_STATUS = {
'432fc8b3-d446-48d4-bb64-13beb90e22bc': {
@ -45,46 +50,46 @@ SAMPLE_BOGUS_POOL_STATUS = {LISTENER_ID1: {
'302e33d9-dee1-4de9-98d5-36329a06fb58':
'DOWN'}}}
SAMPLE_STATS = ({'': '', 'status': 'OPEN', 'lastchg': '',
'weight': '', 'slim': '2000', 'pid': '1', 'comp_byp': '0',
'lastsess': '', 'rate_lim': '0', 'check_duration': '',
'rate': '0', 'req_rate': '0', 'check_status': '',
'econ': '', 'comp_out': '0', 'wredis': '', 'dresp': '0',
'ereq': '0', 'tracked': '', 'comp_in': '0',
'pxname': LISTENER_ID1,
'dreq': '0', 'hrsp_5xx': '0', 'last_chk': '',
'check_code': '', 'sid': '0', 'bout': '0', 'hrsp_1xx': '0',
'qlimit': '', 'hrsp_other': '0', 'bin': '0', 'rtime': '',
'smax': '0', 'req_tot': '0', 'lbtot': '', 'stot': '0',
'wretr': '', 'req_rate_max': '0', 'ttime': '', 'iid': '2',
'hrsp_4xx': '0', 'chkfail': '', 'hanafail': '',
'downtime': '', 'qcur': '', 'eresp': '', 'comp_rsp': '0',
'cli_abrt': '', 'ctime': '', 'qtime': '', 'srv_abrt': '',
'throttle': '', 'last_agt': '', 'scur': '0', 'type': '0',
'bck': '', 'qmax': '', 'rate_max': '0', 'hrsp_2xx': '0',
'act': '', 'chkdown': '', 'svname': 'FRONTEND',
'hrsp_3xx': '0'},
{'': '', 'status': 'no check', 'lastchg': '', 'weight': '1',
'slim': '', 'pid': '1', 'comp_byp': '', 'lastsess': '-1',
'rate_lim': '', 'check_duration': '', 'rate': '0',
'req_rate': '', 'check_status': '', 'econ': '0',
'comp_out': '', 'wredis': '0', 'dresp': '0', 'ereq': '',
'tracked': '', 'comp_in': '',
'pxname': '432fc8b3-d446-48d4-bb64-13beb90e22bc',
'dreq': '', 'hrsp_5xx': '0', 'last_chk': '',
'check_code': '', 'sid': '1', 'bout': '0', 'hrsp_1xx': '0',
'qlimit': '', 'hrsp_other': '0', 'bin': '0', 'rtime': '0',
'smax': '0', 'req_tot': '', 'lbtot': '0', 'stot': '0',
'wretr': '0', 'req_rate_max': '', 'ttime': '0', 'iid': '3',
'hrsp_4xx': '0', 'chkfail': '', 'hanafail': '0',
'downtime': '', 'qcur': '0', 'eresp': '0', 'comp_rsp': '',
'cli_abrt': '0', 'ctime': '0', 'qtime': '0', 'srv_abrt': '0',
'throttle': '', 'last_agt': '', 'scur': '0', 'type': '2',
'bck': '0', 'qmax': '0', 'rate_max': '0', 'hrsp_2xx': '0',
'act': '1', 'chkdown': '',
'svname': '302e33d9-dee1-4de9-98d5-36329a06fb58',
'hrsp_3xx': '0'},
{'': '', 'status': 'UP', 'lastchg': '122', 'weight': '1',
FRONTEND_STATS = {'': '', 'status': 'OPEN', 'lastchg': '',
'weight': '', 'slim': '2000', 'pid': '1', 'comp_byp': '0',
'lastsess': '', 'rate_lim': '0', 'check_duration': '',
'rate': '0', 'req_rate': '0', 'check_status': '',
'econ': '', 'comp_out': '0', 'wredis': '', 'dresp': '0',
'ereq': '5', 'tracked': '', 'comp_in': '0',
'pxname': LISTENER_ID1,
'dreq': '0', 'hrsp_5xx': '0', 'last_chk': '',
'check_code': '', 'sid': '0', 'bout': '10', 'hrsp_1xx': '0',
'qlimit': '', 'hrsp_other': '0', 'bin': '5', 'rtime': '',
'smax': '0', 'req_tot': '0', 'lbtot': '', 'stot': '0',
'wretr': '', 'req_rate_max': '0', 'ttime': '', 'iid': '2',
'hrsp_4xx': '0', 'chkfail': '', 'hanafail': '',
'downtime': '', 'qcur': '', 'eresp': '', 'comp_rsp': '0',
'cli_abrt': '', 'ctime': '', 'qtime': '', 'srv_abrt': '',
'throttle': '', 'last_agt': '', 'scur': '0', 'type': '0',
'bck': '', 'qmax': '', 'rate_max': '0', 'hrsp_2xx': '0',
'act': '', 'chkdown': '', 'svname': 'FRONTEND',
'hrsp_3xx': '0'}
MEMBER_STATS = {'': '', 'status': 'no check', 'lastchg': '', 'weight': '1',
'slim': '', 'pid': '1', 'comp_byp': '', 'lastsess': '-1',
'rate_lim': '', 'check_duration': '', 'rate': '0',
'req_rate': '', 'check_status': '', 'econ': '0',
'comp_out': '', 'wredis': '0', 'dresp': '0', 'ereq': '',
'tracked': '', 'comp_in': '',
'pxname': '432fc8b3-d446-48d4-bb64-13beb90e22bc',
'dreq': '', 'hrsp_5xx': '0', 'last_chk': '',
'check_code': '', 'sid': '1', 'bout': '0', 'hrsp_1xx': '0',
'qlimit': '', 'hrsp_other': '0', 'bin': '0', 'rtime': '0',
'smax': '0', 'req_tot': '', 'lbtot': '0', 'stot': '0',
'wretr': '0', 'req_rate_max': '', 'ttime': '0', 'iid': '3',
'hrsp_4xx': '0', 'chkfail': '', 'hanafail': '0',
'downtime': '', 'qcur': '0', 'eresp': '0', 'comp_rsp': '',
'cli_abrt': '0', 'ctime': '0', 'qtime': '0', 'srv_abrt': '0',
'throttle': '', 'last_agt': '', 'scur': '0', 'type': '2',
'bck': '0', 'qmax': '0', 'rate_max': '0', 'hrsp_2xx': '0',
'act': '1', 'chkdown': '',
'svname': '302e33d9-dee1-4de9-98d5-36329a06fb58',
'hrsp_3xx': '0'}
BACKEND_STATS = {'': '', 'status': 'UP', 'lastchg': '122', 'weight': '1',
'slim': '200', 'pid': '1', 'comp_byp': '0', 'lastsess': '-1',
'rate_lim': '', 'check_duration': '', 'rate': '0',
'req_rate': '', 'check_status': '', 'econ': '0',
@ -101,14 +106,36 @@ SAMPLE_STATS = ({'': '', 'status': 'OPEN', 'lastchg': '',
'srv_abrt': '0', 'throttle': '', 'last_agt': '', 'scur': '0',
'type': '1', 'bck': '0', 'qmax': '0', 'rate_max': '0',
'hrsp_2xx': '0', 'act': '1', 'chkdown': '0',
'svname': 'BACKEND', 'hrsp_3xx': '0'})
'svname': 'BACKEND', 'hrsp_3xx': '0'}
SAMPLE_STATS = (FRONTEND_STATS, MEMBER_STATS, BACKEND_STATS)
SAMPLE_STATS_MSG = {
'listeners': {
LISTENER_ID1: {
'stats': {
'totconns': 0, 'conns': 0,
'tx': 0, 'rx': 0, 'ereq': 0},
'tx': 8, 'rx': 4, 'ereq': 5},
'status': 'OPEN'},
},
'pools': {
'432fc8b3-d446-48d4-bb64-13beb90e22bc': {
'members': {'302e33d9-dee1-4de9-98d5-36329a06fb58': 'DOWN'},
'status': 'UP'},
'3661ed10-99db-4d2c-bffb-99b60eb876ff': {
'members': {'e657f950-a6a2-4d28-bffa-0c8a8c05f815': 'DOWN'},
'status': 'UP'},
},
'id': AMPHORA_ID,
'seq': mock.ANY,
'ver': health_daemon.MSG_VER
}
SAMPLE_MSG_HAPROXY_RESTART = {
'listeners': {
LISTENER_ID1: {
'stats': {
'totconns': 0, 'conns': 0,
'tx': 10, 'rx': 5, 'ereq': 5},
'status': 'OPEN'},
},
'pools': {
@ -119,8 +146,8 @@ SAMPLE_STATS_MSG = {
'members': {'e657f950-a6a2-4d28-bffa-0c8a8c05f815': 'DOWN'},
'status': 'UP'},
},
'id': None,
'seq': 0,
'id': AMPHORA_ID,
'seq': mock.ANY,
'ver': health_daemon.MSG_VER
}
@ -131,6 +158,10 @@ class TestHealthDaemon(base.TestCase):
super(TestHealthDaemon, self).setUp()
conf = oslo_fixture.Config(cfg.CONF)
conf.config(group="haproxy_amphora", base_path=BASE_PATH)
conf.config(group="amphora_agent", amphora_id=AMPHORA_ID)
file_name = os.path.join(BASE_PATH, "stats_counters")
self.mock_open = self.useFixture(
test_utils.OpenFixture(file_name)).mock_open
@mock.patch('octavia.amphorae.backends.agent.'
'api_server.util.get_loadbalancers')
@ -296,18 +327,33 @@ class TestHealthDaemon(base.TestCase):
'health_daemon.list_sock_stat_files')
def test_build_stats_message(self, mock_list_files,
mock_get_stats, mock_is_running):
mock_list_files.return_value = {LISTENER_ID1: 'TEST',
LISTENER_ID2: 'TEST2'}
health_daemon.COUNTERS = None
health_daemon.COUNTERS_FILE = None
lb1_stats_socket = '/var/lib/octavia/{0}/haproxy.sock'.format(LB_ID1)
mock_list_files.return_value = {LB_ID1: lb1_stats_socket}
mock_is_running.return_value = True
mock_get_stats.return_value = SAMPLE_STATS, SAMPLE_POOL_STATUS
msg = health_daemon.build_stats_message()
with mock.patch('os.open'), mock.patch.object(
os, 'fdopen', self.mock_open) as mock_fdopen:
mock_fdopen().read.return_value = simplejson.dumps({
LISTENER_ID1: {'bin': 1, 'bout': 2},
})
msg = health_daemon.build_stats_message()
self.assertEqual(SAMPLE_STATS_MSG, msg)
mock_get_stats.assert_any_call('TEST')
mock_get_stats.assert_any_call('TEST2')
mock_get_stats.assert_any_call(lb1_stats_socket)
mock_fdopen().write.assert_called_once_with(simplejson.dumps({
LISTENER_ID1: {
'bin': int(FRONTEND_STATS['bin']),
'bout': int(FRONTEND_STATS['bout']),
'ereq': int(FRONTEND_STATS['ereq']),
'stot': int(FRONTEND_STATS['stot'])
}
}))
@mock.patch('octavia.amphorae.backends.agent.api_server.'
'util.is_lb_running')
@ -318,15 +364,19 @@ class TestHealthDaemon(base.TestCase):
def test_build_stats_message_no_listener(self, mock_list_files,
mock_get_stats,
mock_is_running):
mock_list_files.return_value = {LISTENER_ID1: 'TEST',
LISTENER_ID2: 'TEST2'}
health_daemon.COUNTERS = None
health_daemon.COUNTERS_FILE = None
lb1_stats_socket = '/var/lib/octavia/{0}/haproxy.sock'.format(LB_ID1)
mock_list_files.return_value = {LB_ID1: lb1_stats_socket}
mock_is_running.side_effect = [True, False]
mock_get_stats.return_value = SAMPLE_STATS, SAMPLE_POOL_STATUS
mock_is_running.return_value = False
health_daemon.build_stats_message()
with mock.patch('os.open'), mock.patch.object(
os, 'fdopen', self.mock_open) as mock_fdopen:
health_daemon.build_stats_message()
self.assertEqual(1, mock_get_stats.call_count)
self.assertEqual(0, mock_get_stats.call_count)
self.assertEqual(0, mock_fdopen().read.call_count)
@mock.patch("octavia.amphorae.backends.utils.keepalivedlvs_query."
"get_udp_listener_pool_status")
@ -335,8 +385,10 @@ class TestHealthDaemon(base.TestCase):
@mock.patch("octavia.amphorae.backends.agent.api_server.util."
"get_udp_listeners")
def test_build_stats_message_with_udp_listener(
self, mock_get_udp_listeners, mock_get_listener_stats,
mock_get_pool_status):
self, mock_get_udp_listeners,
mock_get_listener_stats, mock_get_pool_status):
health_daemon.COUNTERS = None
health_daemon.COUNTERS_FILE = None
udp_listener_id1 = uuidutils.generate_uuid()
udp_listener_id2 = uuidutils.generate_uuid()
udp_listener_id3 = uuidutils.generate_uuid()
@ -346,10 +398,11 @@ class TestHealthDaemon(base.TestCase):
mock_get_udp_listeners.return_value = [udp_listener_id1,
udp_listener_id2,
udp_listener_id3]
mock_get_listener_stats.return_value = {
udp_listener_id1: {
'status': constants.OPEN,
'stats': {'bin': 6387472, 'stot': 5, 'bout': 7490,
'stats': {'bin': 5, 'stot': 5, 'bout': 10,
'ereq': 0, 'scur': 0}},
udp_listener_id3: {
'status': constants.DOWN,
@ -373,7 +426,7 @@ class TestHealthDaemon(base.TestCase):
udp_listener_id1: {
'status': constants.OPEN,
'stats': {'conns': 0, 'totconns': 5, 'ereq': 0,
'rx': 6387472, 'tx': 7490}},
'rx': 4, 'tx': 8}},
udp_listener_id3: {
'status': constants.DOWN,
'stats': {'conns': 0, 'totconns': 0, 'ereq': 0,
@ -384,10 +437,58 @@ class TestHealthDaemon(base.TestCase):
'members': {
member_id1: constants.UP,
member_id2: constants.UP}}},
'id': None,
'id': AMPHORA_ID,
'seq': mock.ANY, 'ver': health_daemon.MSG_VER}
msg = health_daemon.build_stats_message()
with mock.patch('os.open'), mock.patch.object(
os, 'fdopen', self.mock_open) as mock_fdopen:
mock_fdopen().read.return_value = simplejson.dumps({
udp_listener_id1: {
'bin': 1, 'bout': 2, "ereq": 0, "stot": 0}
})
msg = health_daemon.build_stats_message()
self.assertEqual(expected, msg)
mock_fdopen().write.assert_called_once_with(simplejson.dumps({
udp_listener_id1: {'bin': 5, 'bout': 10, 'ereq': 0, 'stot': 5},
udp_listener_id3: {'bin': 0, 'bout': 0, 'ereq': 0, 'stot': 0},
}))
@mock.patch('octavia.amphorae.backends.agent.api_server.'
'util.is_lb_running')
@mock.patch('octavia.amphorae.backends.health_daemon.'
'health_daemon.get_stats')
@mock.patch('octavia.amphorae.backends.health_daemon.'
'health_daemon.list_sock_stat_files')
def test_haproxy_restart(self, mock_list_files,
mock_get_stats, mock_is_running):
health_daemon.COUNTERS = None
health_daemon.COUNTERS_FILE = None
lb1_stats_socket = '/var/lib/octavia/{0}/haproxy.sock'.format(LB_ID1)
mock_list_files.return_value = {LB_ID1: lb1_stats_socket}
mock_is_running.return_value = True
mock_get_stats.return_value = SAMPLE_STATS, SAMPLE_POOL_STATUS
with mock.patch('os.open'), mock.patch.object(
os, 'fdopen', self.mock_open) as mock_fdopen:
mock_fdopen().read.return_value = simplejson.dumps({
LISTENER_ID1: {'bin': 15, 'bout': 20},
})
msg = health_daemon.build_stats_message()
self.assertEqual(SAMPLE_MSG_HAPROXY_RESTART, msg)
mock_get_stats.assert_any_call(lb1_stats_socket)
mock_fdopen().write.assert_called_once_with(simplejson.dumps({
LISTENER_ID1: {
'bin': int(FRONTEND_STATS['bin']),
'bout': int(FRONTEND_STATS['bout']),
'ereq': int(FRONTEND_STATS['ereq']),
'stot': int(FRONTEND_STATS['stot'])
}
}))
class FileNotFoundError(IOError):

54
octavia/tests/unit/common/test_data_models.py
Unescape View File

@ -14,6 +14,7 @@
import copy
import datetime
import random
from oslo_utils import uuidutils
@ -27,6 +28,7 @@ class TestDataModels(base.TestCase):
def setUp(self):
self.LB_ID = uuidutils.generate_uuid()
self.LISTENER_ID = uuidutils.generate_uuid()
self.PROJECT_ID = uuidutils.generate_uuid()
self.SERVER_GROUP_ID = uuidutils.generate_uuid()
self.CREATED_AT = datetime.datetime.now()
@ -479,3 +481,55 @@ class TestDataModels(base.TestCase):
test_Quota_obj.update(update_dict)
self.assertEqual(reference_Quota_obj, test_Quota_obj)
def test_ListenerStatistics_iadd(self):
# test incrementing add function
bytes_in1 = random.randrange(1000000000)
bytes_out1 = random.randrange(1000000000)
active_conns1 = random.randrange(1000000000)
total_conns1 = random.randrange(1000000000)
request_errors1 = random.randrange(1000000000)
stats_1 = data_models.ListenerStatistics(
listener_id=self.LISTENER_ID,
amphora_id=self.AMP_ID,
bytes_in=bytes_in1,
bytes_out=bytes_out1,
active_connections=active_conns1,
total_connections=total_conns1,
request_errors=request_errors1
)
bytes_in2 = random.randrange(1000000000)
bytes_out2 = random.randrange(1000000000)
active_conns2 = random.randrange(1000000000)
total_conns2 = random.randrange(1000000000)
request_errors2 = random.randrange(1000000000)
stats_2 = data_models.ListenerStatistics(
listener_id="listener 2",
amphora_id="amphora 2",
bytes_in=bytes_in2,
bytes_out=bytes_out2,
active_connections=active_conns2,
total_connections=total_conns2,
request_errors=request_errors2
)
# test successful +=
stats_1 += stats_2
# not a delta, so it won't be incremented
self.assertEqual(stats_1.active_connections, active_conns1)
self.assertEqual(stats_1.listener_id, self.LISTENER_ID)
self.assertEqual(stats_1.amphora_id, self.AMP_ID)
# deltas will be incremented
self.assertEqual(stats_1.bytes_in, bytes_in1 + bytes_in2)
self.assertEqual(stats_1.bytes_out, bytes_out1 + bytes_out2)
self.assertEqual(stats_1.total_connections,
total_conns1 + total_conns2)
self.assertEqual(stats_1.request_errors,
request_errors1 + request_errors2)
# test incrementing an incompatible object
self.assertRaises(TypeError, stats_1.__iadd__, "boom")

48
octavia/tests/unit/controller/healthmanager/health_drivers/test_update_db.py
Unescape View File

@ -1385,7 +1385,7 @@ class TestUpdateStatsDb(base.TestCase):
self.loadbalancer_repo.get.return_value = self.loadbalancer
@mock.patch('octavia.db.api.get_session')
def test_update_stats(self, mock_session):
def test_update_stats_v1(self, mock_session):
health = {
"id": self.amphora_id,
@ -1422,11 +1422,53 @@ class TestUpdateStatsDb(base.TestCase):
total_connections=self.listener_stats.total_connections,
request_errors=self.listener_stats.request_errors)
# Test with missing DB listener
self.sm.repo_listener.get.return_value = None
# Test with update failure
mock_session.side_effect = Exception
self.sm.update_stats(health, '192.0.2.1')
@mock.patch('octavia.db.api.get_session')
def test_update_stats_v3(self, mock_session):
health = {
"id": self.amphora_id,
"seq": random.randint(0, 100),
"ver": 3,
"listeners": {
self.listener_id: {
"status": constants.OPEN,
"stats": {
"ereq": self.listener_stats.request_errors,
"conns": self.listener_stats.active_connections,
"totconns": self.listener_stats.total_connections,
"rx": self.listener_stats.bytes_in,
"tx": self.listener_stats.bytes_out,
},
"pools": {
"pool-id-1": {
"status": constants.UP,
"members": {"member-id-1": constants.ONLINE}
}
}
}
}
}
delta_stats_model = data_models.ListenerStatistics(
listener_id=self.listener_id,
amphora_id=self.amphora_id,
bytes_in=self.listener_stats.bytes_in,
bytes_out=self.listener_stats.bytes_out,
active_connections=self.listener_stats.active_connections,
total_connections=self.listener_stats.total_connections,
request_errors=self.listener_stats.request_errors
)
mock_session.return_value = 'blah'
self.sm.update_stats(health, '192.0.2.1')
self.listener_stats_repo.increment.assert_called_once_with(
'blah', delta_stats_model)
# Test with update failure
mock_session.side_effect = Exception
self.sm.update_stats(health, '192.0.2.1')

Write Preview
Loading…
Cancel
Save