Apply ServiceMonitorEvent to affected LBs
Every Health_Monitor over a LB creates Load_Balancer_Health_Check
NB entry, and it creates 1..n Service_Monitor OVN SB entries according
to the member to be monitored. When more than one LB exists on the
same network segment for the same member, Service_Monitor SB
entries are not duplicated, in order to not performing multiple
checks on the same member.
When a ServiceMonitorUpdateEvent is received with status information
from a member, the LBs that match the event information (protocol,
ip, port and logical_port) are searched. At this way we ensure that
network segments are separated where the same ip and port could match
but the logical_port would be different.
The current logic will update just the status of the first LB selected
as "related" to the member event received from the Service_Monitor,
but we could use this information to keep updated all those LBs, that
are in the same network segment and have a Health_Monitor associated
to.
This patch will fix this behaviour in order to provide status to
Octavia of all LBs affected by a ServiceMonitorUpdateEvent.
Partial-Bug: #1965772
Closes-Bug: #1965772
Change-Id: I7c75003516015863320e53f0175dec8fdf4e2cf7
(cherry picked from commit 7db9e23fd9)
This commit is contained in:
Fernando Royo
parent
6b2b7501a8
commit
b006d5273d
|
|
@ -2498,14 +2498,17 @@ class OvnProviderHelper():
|
|||
'update will have incomplete data', ovn_lb.name)
|
||||
return status
|
||||
|
||||
def _get_lb_on_hm_event(self, row):
|
||||
def _get_lbs_on_hm_event(self, row):
|
||||
"""Get the Load Balancer information on a health_monitor event
|
||||
|
||||
This function is called when the status of a member has
|
||||
been updated.
|
||||
been updated. As no duplicate entries are created on a same
|
||||
member for different LBs we will search all LBs affected by
|
||||
the member reported in the health check event
|
||||
|
||||
Input: Service Monitor row which is coming from
|
||||
ServiceMonitorUpdateEvent.
|
||||
Output: A row from load_balancer table table matching the member
|
||||
Output: Rows from load_balancer table table matching the member
|
||||
for which the event was generated.
|
||||
Exception: RowNotFound exception can be generated.
|
||||
"""
|
||||
|
|
@ -2525,11 +2528,11 @@ class OvnProviderHelper():
|
|||
lbs = self.ovn_nbdb_api.db_find_rows(
|
||||
'Load_Balancer', ('ip_port_mappings', '=', mappings),
|
||||
('protocol', '=', row.protocol[0])).execute()
|
||||
return lbs[0] if lbs else None
|
||||
return lbs if lbs else None
|
||||
|
||||
def hm_update_event_handler(self, row):
|
||||
try:
|
||||
ovn_lb = self._get_lb_on_hm_event(row)
|
||||
ovn_lb = self._get_lbs_on_hm_event(row)
|
||||
except idlutils.RowNotFound:
|
||||
LOG.debug("Load balancer information not found")
|
||||
return
|
||||
|
|
@ -2538,11 +2541,6 @@ class OvnProviderHelper():
|
|||
LOG.debug("Load balancer not found")
|
||||
return
|
||||
|
||||
if row.protocol != ovn_lb.protocol:
|
||||
LOG.debug('Row protocol (%s) does not match LB protocol (%s)',
|
||||
row.protocol, ovn_lb.protocol)
|
||||
return
|
||||
|
||||
request_info = {'ovn_lb': ovn_lb,
|
||||
'ip': row.ip,
|
||||
'port': str(row.port),
|
||||
|
|
@ -2661,37 +2659,50 @@ class OvnProviderHelper():
|
|||
return status
|
||||
|
||||
def hm_update_event(self, info):
|
||||
ovn_lb = info['ovn_lb']
|
||||
|
||||
ovn_lbs = info['ovn_lb']
|
||||
statuses = []
|
||||
# Lookup member
|
||||
member_id = None
|
||||
|
||||
for k, v in ovn_lb.external_ids.items():
|
||||
if ovn_const.LB_EXT_IDS_POOL_PREFIX not in k:
|
||||
continue
|
||||
for member_ip, member_port, subnet in self._extract_member_info(v):
|
||||
if info['ip'] != member_ip:
|
||||
for ovn_lb in ovn_lbs:
|
||||
for k, v in ovn_lb.external_ids.items():
|
||||
if ovn_const.LB_EXT_IDS_POOL_PREFIX not in k:
|
||||
continue
|
||||
if info['port'] != member_port:
|
||||
continue
|
||||
# match
|
||||
for (
|
||||
member_ip,
|
||||
member_port,
|
||||
subnet,
|
||||
) in self._extract_member_info(v):
|
||||
if info['ip'] != member_ip:
|
||||
continue
|
||||
if info['port'] != member_port:
|
||||
continue
|
||||
# match
|
||||
|
||||
member_id = [mb.split('_')[1] for mb in v.split(',')
|
||||
if member_ip in mb and member_port in mb][0]
|
||||
break
|
||||
member_id = [mb.split('_')[1] for mb in v.split(',')
|
||||
if member_ip in mb and member_port in mb][0]
|
||||
break
|
||||
|
||||
# found it in inner loop
|
||||
if member_id:
|
||||
break
|
||||
# found it in inner loop
|
||||
if member_id:
|
||||
break
|
||||
|
||||
if not member_id:
|
||||
LOG.warning('Member for event not found, info: %s', info)
|
||||
return
|
||||
if not member_id:
|
||||
LOG.warning('Member for event not found, info: %s', info)
|
||||
return
|
||||
|
||||
member_status = constants.ONLINE
|
||||
if info['status'] == ['offline']:
|
||||
member_status = constants.ERROR
|
||||
member_status = constants.ONLINE
|
||||
if info['status'] == ['offline']:
|
||||
member_status = constants.ERROR
|
||||
|
||||
self._update_member_status(ovn_lb, member_id, member_status)
|
||||
statuses.append(self._get_current_operating_statuses(ovn_lb))
|
||||
|
||||
status = {}
|
||||
|
||||
for status_lb in statuses:
|
||||
for k in status_lb.keys():
|
||||
if k not in status:
|
||||
status[k] = []
|
||||
status[k].extend(status_lb[k])
|
||||
|
||||
self._update_member_status(ovn_lb, member_id, member_status)
|
||||
status = self._get_current_operating_statuses(ovn_lb)
|
||||
return status
|
||||
|
|
|
|||
|
|
@ -3664,7 +3664,7 @@ class TestOvnProviderHelper(ovn_base.TestOvnOctaviaBase):
|
|||
self.hm_update_event.run('update', row, mock.ANY)
|
||||
expected = {
|
||||
'info':
|
||||
{'ovn_lb': self.ovn_hm_lb,
|
||||
{'ovn_lb': [self.ovn_hm_lb],
|
||||
'ip': self.member_address,
|
||||
'port': self.member_port,
|
||||
'status': ['offline']},
|
||||
|
|
@ -3706,21 +3706,6 @@ class TestOvnProviderHelper(ovn_base.TestOvnOctaviaBase):
|
|||
self.hm_update_event.run('update', row, mock.ANY)
|
||||
self.mock_add_request.assert_not_called()
|
||||
|
||||
def test_hm_update_event_lb_protocol_not_found(self):
|
||||
self.helper.ovn_nbdb_api.db_find_rows.\
|
||||
side_effect = [self.ovn_hm_lb, idlutils.RowNotFound]
|
||||
self.hm_update_event = ovn_event.ServiceMonitorUpdateEvent(
|
||||
self.helper)
|
||||
row = fakes.FakeOvsdbRow.create_one_ovsdb_row(
|
||||
attrs={'ip': self.member_address,
|
||||
'logical_port': 'a-logical-port',
|
||||
'src_ip': '10.22.33.4',
|
||||
'port': self.member_port,
|
||||
'protocol': 'unknown',
|
||||
'status': ['offline']})
|
||||
self.hm_update_event.run('update', row, mock.ANY)
|
||||
self.mock_add_request.assert_not_called()
|
||||
|
||||
def _test_hm_update_no_member(self, bad_ip, bad_port):
|
||||
fake_subnet = fakes.FakeSubnet.create_one_subnet()
|
||||
fake_port = fakes.FakePort.create_one_port(
|
||||
|
|
@ -3746,7 +3731,7 @@ class TestOvnProviderHelper(ovn_base.TestOvnOctaviaBase):
|
|||
if bad_port:
|
||||
port = 'bad-port'
|
||||
info = {
|
||||
'ovn_lb': self.ovn_hm_lb,
|
||||
'ovn_lb': [self.ovn_hm_lb],
|
||||
'ip': ip,
|
||||
'logical_port': 'a-logical-port',
|
||||
'src_ip': '10.22.33.4',
|
||||
|
|
@ -3765,7 +3750,7 @@ class TestOvnProviderHelper(ovn_base.TestOvnOctaviaBase):
|
|||
|
||||
def _test_hm_update_status(self, member_id, ip, port, member_status):
|
||||
info = {
|
||||
'ovn_lb': self.ovn_hm_lb,
|
||||
'ovn_lb': [self.ovn_hm_lb],
|
||||
'ip': ip,
|
||||
'logical_port': 'a-logical-port',
|
||||
'src_ip': '10.22.33.4',
|
||||
|
|
@ -3792,10 +3777,12 @@ class TestOvnProviderHelper(ovn_base.TestOvnOctaviaBase):
|
|||
ovn_const.OVN_MEMBER_STATUS_KEY] = jsonutils.dumps(
|
||||
member_statuses)
|
||||
|
||||
def _add_member(self, subnet, port):
|
||||
fake_port = fakes.FakePort.create_one_port(
|
||||
attrs={'allowed_address_pairs': ''})
|
||||
ip = fake_port['fixed_ips'][0]['ip_address']
|
||||
def _add_member(self, lb, subnet, port, ip=None):
|
||||
if not ip:
|
||||
fake_port = fakes.FakePort.create_one_port(
|
||||
attrs={'allowed_address_pairs': ''})
|
||||
ip = fake_port['fixed_ips'][0]['ip_address']
|
||||
|
||||
member = {'id': uuidutils.generate_uuid(),
|
||||
'address': ip,
|
||||
'protocol_port': port,
|
||||
|
|
@ -3809,8 +3796,8 @@ class TestOvnProviderHelper(ovn_base.TestOvnOctaviaBase):
|
|||
member['protocol_port'], member['subnet_id']))
|
||||
pool_key = 'pool_%s' % self.pool_id
|
||||
|
||||
existing_members = self.ovn_hm_lb.external_ids[pool_key]
|
||||
existing_member_status = self.ovn_hm_lb.external_ids[
|
||||
existing_members = lb.external_ids[pool_key]
|
||||
existing_member_status = lb.external_ids[
|
||||
ovn_const.OVN_MEMBER_STATUS_KEY]
|
||||
|
||||
try:
|
||||
|
|
@ -3820,22 +3807,22 @@ class TestOvnProviderHelper(ovn_base.TestOvnOctaviaBase):
|
|||
|
||||
if existing_members:
|
||||
existing_members = ','.join([existing_members, member_line])
|
||||
self.ovn_hm_lb.external_ids[pool_key] = existing_members
|
||||
lb.external_ids[pool_key] = existing_members
|
||||
member_statuses[member['id']] = constants.ONLINE
|
||||
self.ovn_hm_lb.external_ids[
|
||||
lb.external_ids[
|
||||
ovn_const.OVN_MEMBER_STATUS_KEY] = jsonutils.dumps(
|
||||
member_statuses)
|
||||
else:
|
||||
self.ovn_hm_lb.external_ids[pool_key] = member_line
|
||||
lb.external_ids[pool_key] = member_line
|
||||
member_status = '{"%s": "%s"}' % (member['id'],
|
||||
constants.ONLINE)
|
||||
self.ovn_hm_lb.external_ids[
|
||||
lb.external_ids[
|
||||
ovn_const.OVN_MEMBER_STATUS_KEY] = member_status
|
||||
return member
|
||||
|
||||
def test_hm_update_status_offline(self):
|
||||
fake_subnet = fakes.FakeSubnet.create_one_subnet()
|
||||
member = self._add_member(fake_subnet, 8080)
|
||||
member = self._add_member(self.ovn_hm_lb, fake_subnet, 8080)
|
||||
status = self._test_hm_update_status(
|
||||
member['id'], member['address'], '8080', 'offline')
|
||||
|
||||
|
|
@ -3852,9 +3839,54 @@ class TestOvnProviderHelper(ovn_base.TestOvnOctaviaBase):
|
|||
self.assertEqual(status['loadbalancers'][0]['operating_status'],
|
||||
constants.ERROR)
|
||||
|
||||
def test_hm_update_status_offline_two_lbs_affected(self):
|
||||
fake_subnet = fakes.FakeSubnet.create_one_subnet()
|
||||
ovn_hm_lb_2 = copy.deepcopy(self.ovn_hm_lb)
|
||||
ovn_hm_lb_2.uuid = uuidutils.generate_uuid()
|
||||
member = self._add_member(self.ovn_hm_lb, fake_subnet, 8080)
|
||||
member_2 = self._add_member(
|
||||
ovn_hm_lb_2, fake_subnet, 8080, ip=member['address'])
|
||||
|
||||
info = {
|
||||
'ovn_lb': [self.ovn_hm_lb, ovn_hm_lb_2],
|
||||
'ip': member['address'],
|
||||
'logical_port': 'a-logical-port',
|
||||
'src_ip': '10.22.33.4',
|
||||
'port': '8080',
|
||||
'protocol': self.ovn_hm_lb.protocol,
|
||||
'status': ['offline']}
|
||||
self._update_member_status(self.ovn_hm_lb, member['id'], 'offline')
|
||||
self._update_member_status(ovn_hm_lb_2, member_2['id'], 'offline')
|
||||
status = self.helper.hm_update_event(info)
|
||||
|
||||
self.assertEqual(status['pools'][0]['provisioning_status'],
|
||||
constants.ACTIVE)
|
||||
self.assertEqual(status['pools'][0]['operating_status'],
|
||||
constants.ERROR)
|
||||
self.assertEqual(status['members'][0]['provisioning_status'],
|
||||
constants.ACTIVE)
|
||||
self.assertEqual(status['members'][0]['operating_status'],
|
||||
constants.ERROR)
|
||||
self.assertEqual(status['loadbalancers'][0]['provisioning_status'],
|
||||
constants.ACTIVE)
|
||||
self.assertEqual(status['loadbalancers'][0]['operating_status'],
|
||||
constants.ERROR)
|
||||
self.assertEqual(status['pools'][1]['provisioning_status'],
|
||||
constants.ACTIVE)
|
||||
self.assertEqual(status['pools'][1]['operating_status'],
|
||||
constants.ERROR)
|
||||
self.assertEqual(status['members'][1]['provisioning_status'],
|
||||
constants.ACTIVE)
|
||||
self.assertEqual(status['members'][1]['operating_status'],
|
||||
constants.ERROR)
|
||||
self.assertEqual(status['loadbalancers'][1]['provisioning_status'],
|
||||
constants.ACTIVE)
|
||||
self.assertEqual(status['loadbalancers'][1]['operating_status'],
|
||||
constants.ERROR)
|
||||
|
||||
def test_hm_update_status_offline_lb_pool_offline(self):
|
||||
fake_subnet = fakes.FakeSubnet.create_one_subnet()
|
||||
member = self._add_member(fake_subnet, 8080)
|
||||
member = self._add_member(self.ovn_hm_lb, fake_subnet, 8080)
|
||||
status = self._test_hm_update_status(
|
||||
member['id'], member['address'], '8080', 'offline')
|
||||
|
||||
|
|
@ -3873,7 +3905,7 @@ class TestOvnProviderHelper(ovn_base.TestOvnOctaviaBase):
|
|||
|
||||
def test_hm_update_status_online(self):
|
||||
fake_subnet = fakes.FakeSubnet.create_one_subnet()
|
||||
member = self._add_member(fake_subnet, 8080)
|
||||
member = self._add_member(self.ovn_hm_lb, fake_subnet, 8080)
|
||||
status = self._test_hm_update_status(
|
||||
member['id'], member['address'], '8080', 'online')
|
||||
|
||||
|
|
@ -3892,7 +3924,7 @@ class TestOvnProviderHelper(ovn_base.TestOvnOctaviaBase):
|
|||
|
||||
def test_hm_update_status_online_lb_pool_offline(self):
|
||||
fake_subnet = fakes.FakeSubnet.create_one_subnet()
|
||||
member = self._add_member(fake_subnet, 8080)
|
||||
member = self._add_member(self.ovn_hm_lb, fake_subnet, 8080)
|
||||
status = self._test_hm_update_status(
|
||||
member['id'], member['address'], '8080', 'online')
|
||||
|
||||
|
|
@ -3911,9 +3943,9 @@ class TestOvnProviderHelper(ovn_base.TestOvnOctaviaBase):
|
|||
|
||||
def test_hm_update_status_offline_two_members(self):
|
||||
fake_subnet = fakes.FakeSubnet.create_one_subnet()
|
||||
member_1 = self._add_member(fake_subnet, 8080)
|
||||
member_1 = self._add_member(self.ovn_hm_lb, fake_subnet, 8080)
|
||||
ip_1 = member_1['address']
|
||||
member_2 = self._add_member(fake_subnet, 8081)
|
||||
member_2 = self._add_member(self.ovn_hm_lb, fake_subnet, 8081)
|
||||
ip_2 = member_2['address']
|
||||
# This is the Octavia API version
|
||||
fake_member = fakes.FakeMember(
|
||||
|
|
@ -3954,8 +3986,8 @@ class TestOvnProviderHelper(ovn_base.TestOvnOctaviaBase):
|
|||
|
||||
def test_hm_update_status_online_two_members(self):
|
||||
fake_subnet = fakes.FakeSubnet.create_one_subnet()
|
||||
member_1 = self._add_member(fake_subnet, 8080)
|
||||
member_2 = self._add_member(fake_subnet, 8081)
|
||||
member_1 = self._add_member(self.ovn_hm_lb, fake_subnet, 8080)
|
||||
member_2 = self._add_member(self.ovn_hm_lb, fake_subnet, 8081)
|
||||
ip_2 = member_2['address']
|
||||
# This is the Octavia API version
|
||||
fake_member = fakes.FakeMember(
|
||||
|
|
|
|||
Loading…
Reference in New Issue