Browse Source

[ovn] Stop monitoring the SB MAC_Binding table to reduce mem footprint

The MAC_Binding table in the SB database may grow indefinitely (due
to a lack of an aging mechanism of its entries) and eventually
lead to OOM killers for neutron-server which maintains an in-memory
copy of the database.

In order to stop monitoring this table, this patch is invoking
the ovsdb-client tool to remove the entries associated to
Floating IPs that have just been detached. The execution of this
tool is really fast as it will just invoke a JSON-RPC transact command
which doesn't require downloading the database contents.

In a scale test, the memory consumption of neutron-server dropped
from 75GB to 7GB with this patch.

Closes-Bug: #1946318

(Manually cherry picked from Neutron f6c35527698119ee6f73a6a3613c9beebb563840)

Note: The functional test has been changed for this stable branch
because the update_tables() method from ovsdbapp is not availabie for
this version of networking-ovn. So I changed the functional test to
add the MAC_Binding entry via ovsdb-client instead of using ovsdbapp
to do so.

Signed-off-by: Daniel Alvarez Sanchez <dalvarez@redhat.com>
Change-Id: Id84bf17953527c415d611bfc198038fb6f811de3
changes/78/814878/4
Lucas Alvares Gomes 7 months ago
parent
commit
4f8e03593d
  1. 17
      networking_ovn/ml2/mech_driver.py
  2. 1
      networking_ovn/ovsdb/ovsdb_monitor.py
  3. 66
      networking_ovn/tests/functional/test_ovsdb_monitor.py
  4. 25
      networking_ovn/tests/unit/ml2/test_mech_driver.py

17
networking_ovn/ml2/mech_driver.py

@ -37,6 +37,7 @@ from neutron_lib import context as n_context
from neutron_lib import exceptions as n_exc
from neutron_lib.plugins import directory
from neutron_lib.plugins.ml2 import api
from oslo_concurrency import processutils
from oslo_config import cfg
from oslo_db import exception as os_db_exc
from oslo_log import log
@ -958,10 +959,18 @@ class OVNMechanismDriver(api.MechanismDriver):
def delete_mac_binding_entries(self, external_ip):
"""Delete all MAC_Binding entries associated to this IP address"""
mac_binds = self._sb_ovn.db_find_rows(
'MAC_Binding', ('ip', '=', external_ip)).execute() or []
for entry in mac_binds:
self._sb_ovn.db_destroy('MAC_Binding', entry.uuid).execute()
cmd = ['ovsdb-client', 'transact', config.get_ovn_sb_connection()]
if config.get_ovn_sb_private_key():
cmd += ['-p', config.get_ovn_sb_private_key(), '-c',
config.get_ovn_sb_certificate(), '-C',
config.get_ovn_sb_ca_cert()]
cmd += ['["OVN_Southbound", {"op": "delete", "table": "MAC_Binding", '
'"where": [["ip", "==", "%s"]]}]' % external_ip]
return processutils.execute(*cmd,
log_errors=processutils.LOG_FINAL_ERROR)
def update_segment_host_mapping(self, host, phy_nets):
"""Update SegmentHostMapping in DB"""

1
networking_ovn/ovsdb/ovsdb_monitor.py

@ -537,7 +537,6 @@ class OvnSbIdl(OvnIdlDistributedLock):
helper.register_table('Encap')
helper.register_table('Port_Binding')
helper.register_table('Datapath_Binding')
helper.register_table('MAC_Binding')
return cls(driver, connection_string, helper)
def post_connect(self):

66
networking_ovn/tests/functional/test_ovsdb_monitor.py

@ -14,7 +14,9 @@
import mock
import fixtures
from oslo_concurrency import processutils
from oslo_config import cfg
from oslo_serialization import jsonutils
from oslo_utils import uuidutils
from networking_ovn.common import config as ovn_conf
@ -30,17 +32,6 @@ from neutron_lib.plugins import directory
from ovsdbapp.backend.ovs_idl import event
class WaitForMACBindingDeleteEvent(event.WaitEvent):
event_name = 'WaitForMACBindingDeleteEvent'
def __init__(self, entry):
table = 'MAC_Binding'
events = (self.ROW_DELETE,)
conditions = (('_uuid', '=', entry),)
super(WaitForMACBindingDeleteEvent, self).__init__(
events, table, conditions, timeout=15)
class DistributedLockTestEvent(event.WaitEvent):
ONETIME = False
COUNTER = 0
@ -114,6 +105,38 @@ class TestNBDbMonitor(base.TestOVNFunctionalBase):
'port_id': port['id']}})
return r1_f2
def _check_mac_binding_exists(self, macb_id):
cmd = ['ovsdb-client', 'transact', ovn_conf.get_ovn_sb_connection()]
if self._ovsdb_protocol == 'ssl':
cmd += ['-p', self.ovsdb_server_mgr.private_key, '-c',
self.ovsdb_server_mgr.certificate, '-C',
self.ovsdb_server_mgr.ca_cert]
cmd += ['["OVN_Southbound", {"op": "select", "table": "MAC_Binding", '
'"where": [["_uuid", "==", ["uuid", "%s"]]]}]' % macb_id]
out, _ = processutils.execute(*cmd,
log_errors=False)
return str(macb_id) in out
def _add_mac_binding_row(self, ip, datapath):
cmd = ['ovsdb-client', 'transact', ovn_conf.get_ovn_sb_connection()]
if self._ovsdb_protocol == 'ssl':
cmd += ['-p', self.ovsdb_server_mgr.private_key, '-c',
self.ovsdb_server_mgr.certificate, '-C',
self.ovsdb_server_mgr.ca_cert]
cmd += ['["OVN_Southbound", {"op": "insert", "table": "MAC_Binding", '
'"row": {"ip": "%s", "datapath": '
'["uuid", "%s"]}}]' % (ip, datapath)]
out, _ = processutils.execute(*cmd,
log_errors=False)
out_parsed = jsonutils.loads(out)
return out_parsed[0]['uuid'][1]
def test_floatingip_mac_bindings(self):
"""Check that MAC_Binding entries are cleared on FIP add/removal
@ -131,27 +154,26 @@ class TestNBDbMonitor(base.TestOVNFunctionalBase):
dp = self.sb_api.db_find(
'Datapath_Binding',
('external_ids', '=', {'name2': 'network1'})).execute()
macb_id = self.sb_api.db_create('MAC_Binding', datapath=dp[0]['_uuid'],
ip='100.0.0.21').execute()
macb_id = self._add_mac_binding_row(
ip='100.0.0.21', datapath=dp[0]['_uuid'])
port = self.create_port()
# Ensure that the MAC_Binding entry gets deleted after creating a FIP
row_event = WaitForMACBindingDeleteEvent(macb_id)
self.mech_driver._sb_ovn.idl.notify_handler.watch_event(row_event)
fip = self._create_fip(port, '100.0.0.21')
self.assertTrue(row_event.wait())
n_utils.wait_until_true(
lambda: not self._check_mac_binding_exists(macb_id),
timeout=15, sleep=1)
# Now that the FIP is created, add a new MAC_Binding entry with the
# same IP address
macb_id = self.sb_api.db_create('MAC_Binding', datapath=dp[0]['_uuid'],
ip='100.0.0.21').execute()
macb_id = self._add_mac_binding_row(
ip='100.0.0.21', datapath=dp[0]['_uuid'])
# Ensure that the MAC_Binding entry gets deleted after deleting the FIP
row_event = WaitForMACBindingDeleteEvent(macb_id)
self.mech_driver._sb_ovn.idl.notify_handler.watch_event(row_event)
self.l3_plugin.delete_floatingip(self.context, fip['id'])
self.assertTrue(row_event.wait())
n_utils.wait_until_true(
lambda: not self._check_mac_binding_exists(macb_id),
timeout=15, sleep=1)
def _test_port_binding_and_status(self, port_id, action, status):
# This function binds or unbinds port to chassis and

25
networking_ovn/tests/unit/ml2/test_mech_driver.py

@ -14,6 +14,7 @@
import copy
import datetime
import shlex
import uuid
import mock
@ -40,6 +41,7 @@ from neutron_lib import exceptions as n_exc
from neutron_lib.plugins import directory
from neutron_lib.tests import tools
from neutron_lib.utils import net as n_net
from oslo_concurrency import processutils
from oslo_config import cfg
from oslo_db import exception as os_db_exc
from oslo_serialization import jsonutils
@ -109,6 +111,29 @@ class TestOVNMechanismDriver(test_plugin.Ml2PluginV2TestCase):
p.start()
self.addCleanup(p.stop)
def test_delete_mac_binding_entries(self):
self.config(group='ovn', ovn_sb_private_key=None)
expected = ('ovsdb-client transact tcp:127.0.0.1:6642 '
'\'["OVN_Southbound", {"op": "delete", "table": '
'"MAC_Binding", "where": [["ip", "==", "1.1.1.1"]]}]\'')
with mock.patch.object(processutils, 'execute') as mock_execute:
self.mech_driver.delete_mac_binding_entries('1.1.1.1')
mock_execute.assert_called_once_with(*shlex.split(expected),
log_errors=processutils.LOG_FINAL_ERROR)
def test_delete_mac_binding_entries_ssl(self):
self.config(group='ovn', ovn_sb_private_key='pk')
self.config(group='ovn', ovn_sb_certificate='cert')
self.config(group='ovn', ovn_sb_ca_cert='ca')
expected = ('ovsdb-client transact tcp:127.0.0.1:6642 '
'-p pk -c cert -C ca '
'\'["OVN_Southbound", {"op": "delete", "table": '
'"MAC_Binding", "where": [["ip", "==", "1.1.1.1"]]}]\'')
with mock.patch.object(processutils, 'execute') as mock_execute:
self.mech_driver.delete_mac_binding_entries('1.1.1.1')
mock_execute.assert_called_once_with(*shlex.split(expected),
log_errors=processutils.LOG_FINAL_ERROR)
@mock.patch.object(db_rev, 'bump_revision')
def test__create_security_group(self, mock_bump):
self.mech_driver._create_security_group(

Loading…
Cancel
Save