[ovn] Stop monitoring the SB MAC_Binding table to reduce mem footprint
The MAC_Binding table in the SB database may grow indefinitely (due
to a lack of an aging mechanism of its entries) and eventually
lead to OOM killers for neutron-server which maintains an in-memory
copy of the database.
In order to stop monitoring this table, this patch is invoking
the ovsdb-client tool to remove the entries associated to
Floating IPs that have just been detached. The execution of this
tool is really fast as it will just invoke a JSON-RPC transact command
which doesn't require downloading the database contents.
In a scale test, the memory consumption of neutron-server dropped
from 75GB to 7GB with this patch.
Closes-Bug: #1946318
Conflicts:
neutron/plugins/ml2/drivers/ovn/mech_driver/mech_driver.py
neutron/tests/functional/plugins/ml2/drivers/ovn/mech_driver/ovsdb/test_ovsdb_monitor.py
neutron/tests/unit/plugins/ml2/drivers/ovn/mech_driver/test_mech_driver.py
Note: The functional test has been changed for this stable branch
because the update_tables() method from ovsdbapp is not availabie for
this version of Neutron. So I changed the functional test to add the
MAC_Binding entry via ovsdb-client instead of using ovsdbapp to do so.
Signed-off-by: Daniel Alvarez Sanchez <dalvarez@redhat.com>
Change-Id: Id84bf17953527c415d611bfc198038fb6f811de3
(cherry picked from commit f6c3552769
)
This commit is contained in:
parent
0c5a5ef921
commit
b767825d13
|
@ -32,6 +32,7 @@ from neutron_lib import context as n_context
|
||||||
from neutron_lib import exceptions as n_exc
|
from neutron_lib import exceptions as n_exc
|
||||||
from neutron_lib.plugins import directory
|
from neutron_lib.plugins import directory
|
||||||
from neutron_lib.plugins.ml2 import api
|
from neutron_lib.plugins.ml2 import api
|
||||||
|
from oslo_concurrency import processutils
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
from oslo_db import exception as os_db_exc
|
from oslo_db import exception as os_db_exc
|
||||||
from oslo_log import log
|
from oslo_log import log
|
||||||
|
@ -1054,10 +1055,18 @@ class OVNMechanismDriver(api.MechanismDriver):
|
||||||
|
|
||||||
def delete_mac_binding_entries(self, external_ip):
|
def delete_mac_binding_entries(self, external_ip):
|
||||||
"""Delete all MAC_Binding entries associated to this IP address"""
|
"""Delete all MAC_Binding entries associated to this IP address"""
|
||||||
mac_binds = self._sb_ovn.db_find_rows(
|
cmd = ['ovsdb-client', 'transact', ovn_conf.get_ovn_sb_connection()]
|
||||||
'MAC_Binding', ('ip', '=', external_ip)).execute() or []
|
|
||||||
for entry in mac_binds:
|
if ovn_conf.get_ovn_sb_private_key():
|
||||||
self._sb_ovn.db_destroy('MAC_Binding', entry.uuid).execute()
|
cmd += ['-p', ovn_conf.get_ovn_sb_private_key(), '-c',
|
||||||
|
ovn_conf.get_ovn_sb_certificate(), '-C',
|
||||||
|
ovn_conf.get_ovn_sb_ca_cert()]
|
||||||
|
|
||||||
|
cmd += ['["OVN_Southbound", {"op": "delete", "table": "MAC_Binding", '
|
||||||
|
'"where": [["ip", "==", "%s"]]}]' % external_ip]
|
||||||
|
|
||||||
|
return processutils.execute(*cmd,
|
||||||
|
log_errors=processutils.LOG_FINAL_ERROR)
|
||||||
|
|
||||||
def update_segment_host_mapping(self, host, phy_nets):
|
def update_segment_host_mapping(self, host, phy_nets):
|
||||||
"""Update SegmentHostMapping in DB"""
|
"""Update SegmentHostMapping in DB"""
|
||||||
|
|
|
@ -543,7 +543,6 @@ class OvnSbIdl(OvnIdlDistributedLock):
|
||||||
helper.register_table('Encap')
|
helper.register_table('Encap')
|
||||||
helper.register_table('Port_Binding')
|
helper.register_table('Port_Binding')
|
||||||
helper.register_table('Datapath_Binding')
|
helper.register_table('Datapath_Binding')
|
||||||
helper.register_table('MAC_Binding')
|
|
||||||
helper.register_table('Connection')
|
helper.register_table('Connection')
|
||||||
return cls(driver, connection_string, helper)
|
return cls(driver, connection_string, helper)
|
||||||
|
|
||||||
|
|
|
@ -15,7 +15,9 @@
|
||||||
import mock
|
import mock
|
||||||
|
|
||||||
import fixtures as og_fixtures
|
import fixtures as og_fixtures
|
||||||
|
from oslo_concurrency import processutils
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
|
from oslo_serialization import jsonutils
|
||||||
from oslo_utils import uuidutils
|
from oslo_utils import uuidutils
|
||||||
|
|
||||||
from neutron.common.ovn import constants as ovn_const
|
from neutron.common.ovn import constants as ovn_const
|
||||||
|
@ -32,17 +34,6 @@ from neutron_lib.plugins import directory
|
||||||
from ovsdbapp.backend.ovs_idl import event
|
from ovsdbapp.backend.ovs_idl import event
|
||||||
|
|
||||||
|
|
||||||
class WaitForMACBindingDeleteEvent(event.WaitEvent):
|
|
||||||
event_name = 'WaitForMACBindingDeleteEvent'
|
|
||||||
|
|
||||||
def __init__(self, entry):
|
|
||||||
table = 'MAC_Binding'
|
|
||||||
events = (self.ROW_DELETE,)
|
|
||||||
conditions = (('_uuid', '=', entry),)
|
|
||||||
super(WaitForMACBindingDeleteEvent, self).__init__(
|
|
||||||
events, table, conditions, timeout=15)
|
|
||||||
|
|
||||||
|
|
||||||
class WaitForDataPathBindingCreateEvent(event.WaitEvent):
|
class WaitForDataPathBindingCreateEvent(event.WaitEvent):
|
||||||
event_name = 'WaitForDataPathBindingCreateEvent'
|
event_name = 'WaitForDataPathBindingCreateEvent'
|
||||||
|
|
||||||
|
@ -127,6 +118,40 @@ class TestNBDbMonitor(base.TestOVNFunctionalBase):
|
||||||
'port_id': port['id']}})
|
'port_id': port['id']}})
|
||||||
return r1_f2
|
return r1_f2
|
||||||
|
|
||||||
|
def _check_mac_binding_exists(self, macb_id):
|
||||||
|
cmd = ['ovsdb-client', 'transact',
|
||||||
|
self.mech_driver.sb_ovn.connection_string]
|
||||||
|
|
||||||
|
if self._ovsdb_protocol == 'ssl':
|
||||||
|
cmd += ['-p', self.ovsdb_server_mgr.private_key, '-c',
|
||||||
|
self.ovsdb_server_mgr.certificate, '-C',
|
||||||
|
self.ovsdb_server_mgr.ca_cert]
|
||||||
|
|
||||||
|
cmd += ['["OVN_Southbound", {"op": "select", "table": "MAC_Binding", '
|
||||||
|
'"where": [["_uuid", "==", ["uuid", "%s"]]]}]' % macb_id]
|
||||||
|
|
||||||
|
out, _ = processutils.execute(*cmd,
|
||||||
|
log_errors=False)
|
||||||
|
return str(macb_id) in out
|
||||||
|
|
||||||
|
def _add_mac_binding_row(self, ip, datapath):
|
||||||
|
cmd = ['ovsdb-client', 'transact',
|
||||||
|
self.mech_driver.sb_ovn.connection_string]
|
||||||
|
|
||||||
|
if self._ovsdb_protocol == 'ssl':
|
||||||
|
cmd += ['-p', self.ovsdb_server_mgr.private_key, '-c',
|
||||||
|
self.ovsdb_server_mgr.certificate, '-C',
|
||||||
|
self.ovsdb_server_mgr.ca_cert]
|
||||||
|
|
||||||
|
cmd += ['["OVN_Southbound", {"op": "insert", "table": "MAC_Binding", '
|
||||||
|
'"row": {"ip": "%s", "datapath": '
|
||||||
|
'["uuid", "%s"]}}]' % (ip, datapath)]
|
||||||
|
|
||||||
|
out, _ = processutils.execute(*cmd,
|
||||||
|
log_errors=False)
|
||||||
|
out_parsed = jsonutils.loads(out)
|
||||||
|
return out_parsed[0]['uuid'][1]
|
||||||
|
|
||||||
def test_floatingip_mac_bindings(self):
|
def test_floatingip_mac_bindings(self):
|
||||||
"""Check that MAC_Binding entries are cleared on FIP add/removal
|
"""Check that MAC_Binding entries are cleared on FIP add/removal
|
||||||
|
|
||||||
|
@ -148,27 +173,26 @@ class TestNBDbMonitor(base.TestOVNFunctionalBase):
|
||||||
dp = self.sb_api.db_find(
|
dp = self.sb_api.db_find(
|
||||||
'Datapath_Binding',
|
'Datapath_Binding',
|
||||||
('external_ids', '=', {'name2': net_name})).execute()
|
('external_ids', '=', {'name2': net_name})).execute()
|
||||||
macb_id = self.sb_api.db_create('MAC_Binding', datapath=dp[0]['_uuid'],
|
macb_id = self._add_mac_binding_row(
|
||||||
ip='100.0.0.21').execute()
|
ip='100.0.0.21', datapath=dp[0]['_uuid'])
|
||||||
port = self.create_port()
|
port = self.create_port()
|
||||||
|
|
||||||
# Ensure that the MAC_Binding entry gets deleted after creating a FIP
|
# Ensure that the MAC_Binding entry gets deleted after creating a FIP
|
||||||
row_event = WaitForMACBindingDeleteEvent(macb_id)
|
|
||||||
self.mech_driver._sb_ovn.idl.notify_handler.watch_event(row_event)
|
|
||||||
fip = self._create_fip(port, '100.0.0.21')
|
fip = self._create_fip(port, '100.0.0.21')
|
||||||
self.assertTrue(row_event.wait())
|
n_utils.wait_until_true(
|
||||||
|
lambda: not self._check_mac_binding_exists(macb_id),
|
||||||
|
timeout=15, sleep=1)
|
||||||
|
|
||||||
# Now that the FIP is created, add a new MAC_Binding entry with the
|
# Now that the FIP is created, add a new MAC_Binding entry with the
|
||||||
# same IP address
|
# same IP address
|
||||||
|
macb_id = self._add_mac_binding_row(
|
||||||
macb_id = self.sb_api.db_create('MAC_Binding', datapath=dp[0]['_uuid'],
|
ip='100.0.0.21', datapath=dp[0]['_uuid'])
|
||||||
ip='100.0.0.21').execute()
|
|
||||||
|
|
||||||
# Ensure that the MAC_Binding entry gets deleted after deleting the FIP
|
# Ensure that the MAC_Binding entry gets deleted after deleting the FIP
|
||||||
row_event = WaitForMACBindingDeleteEvent(macb_id)
|
|
||||||
self.mech_driver._sb_ovn.idl.notify_handler.watch_event(row_event)
|
|
||||||
self.l3_plugin.delete_floatingip(self.context, fip['id'])
|
self.l3_plugin.delete_floatingip(self.context, fip['id'])
|
||||||
self.assertTrue(row_event.wait())
|
n_utils.wait_until_true(
|
||||||
|
lambda: not self._check_mac_binding_exists(macb_id),
|
||||||
|
timeout=15, sleep=1)
|
||||||
|
|
||||||
def _test_port_binding_and_status(self, port_id, action, status):
|
def _test_port_binding_and_status(self, port_id, action, status):
|
||||||
# This function binds or unbinds port to chassis and
|
# This function binds or unbinds port to chassis and
|
||||||
|
|
|
@ -14,6 +14,7 @@
|
||||||
|
|
||||||
import copy
|
import copy
|
||||||
import datetime
|
import datetime
|
||||||
|
import shlex
|
||||||
import uuid
|
import uuid
|
||||||
|
|
||||||
import mock
|
import mock
|
||||||
|
@ -30,6 +31,7 @@ from neutron_lib import exceptions as n_exc
|
||||||
from neutron_lib.plugins import directory
|
from neutron_lib.plugins import directory
|
||||||
from neutron_lib.tests import tools
|
from neutron_lib.tests import tools
|
||||||
from neutron_lib.utils import net as n_net
|
from neutron_lib.utils import net as n_net
|
||||||
|
from oslo_concurrency import processutils
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
from oslo_db import exception as os_db_exc
|
from oslo_db import exception as os_db_exc
|
||||||
from oslo_serialization import jsonutils
|
from oslo_serialization import jsonutils
|
||||||
|
@ -112,6 +114,29 @@ class TestOVNMechanismDriver(test_plugin.Ml2PluginV2TestCase):
|
||||||
p.start()
|
p.start()
|
||||||
self.addCleanup(p.stop)
|
self.addCleanup(p.stop)
|
||||||
|
|
||||||
|
def test_delete_mac_binding_entries(self):
|
||||||
|
self.config(group='ovn', ovn_sb_private_key=None)
|
||||||
|
expected = ('ovsdb-client transact tcp:127.0.0.1:6642 '
|
||||||
|
'\'["OVN_Southbound", {"op": "delete", "table": '
|
||||||
|
'"MAC_Binding", "where": [["ip", "==", "1.1.1.1"]]}]\'')
|
||||||
|
with mock.patch.object(processutils, 'execute') as mock_execute:
|
||||||
|
self.mech_driver.delete_mac_binding_entries('1.1.1.1')
|
||||||
|
mock_execute.assert_called_once_with(*shlex.split(expected),
|
||||||
|
log_errors=processutils.LOG_FINAL_ERROR)
|
||||||
|
|
||||||
|
def test_delete_mac_binding_entries_ssl(self):
|
||||||
|
self.config(group='ovn', ovn_sb_private_key='pk')
|
||||||
|
self.config(group='ovn', ovn_sb_certificate='cert')
|
||||||
|
self.config(group='ovn', ovn_sb_ca_cert='ca')
|
||||||
|
expected = ('ovsdb-client transact tcp:127.0.0.1:6642 '
|
||||||
|
'-p pk -c cert -C ca '
|
||||||
|
'\'["OVN_Southbound", {"op": "delete", "table": '
|
||||||
|
'"MAC_Binding", "where": [["ip", "==", "1.1.1.1"]]}]\'')
|
||||||
|
with mock.patch.object(processutils, 'execute') as mock_execute:
|
||||||
|
self.mech_driver.delete_mac_binding_entries('1.1.1.1')
|
||||||
|
mock_execute.assert_called_once_with(*shlex.split(expected),
|
||||||
|
log_errors=processutils.LOG_FINAL_ERROR)
|
||||||
|
|
||||||
@mock.patch.object(ovsdb_monitor.OvnInitPGNbIdl, 'from_server')
|
@mock.patch.object(ovsdb_monitor.OvnInitPGNbIdl, 'from_server')
|
||||||
@mock.patch.object(ovsdb_monitor, 'short_living_ovsdb_api')
|
@mock.patch.object(ovsdb_monitor, 'short_living_ovsdb_api')
|
||||||
def test__create_neutron_pg_drop_non_existing(
|
def test__create_neutron_pg_drop_non_existing(
|
||||||
|
|
Loading…
Reference in New Issue