From c5b76a8393a21adb87447c925da2ede4a75dd11a Mon Sep 17 00:00:00 2001 From: Rodolfo Alonso Hernandez Date: Thu, 7 Jul 2022 06:31:22 +0000 Subject: [PATCH] Script to remove duplicated port bindings A new script to remove the duplicated port bindings was added. This script will list all ``ml2_port_bindings`` records in the database, finding those ones with the same port ID. Then the script removes those ones with status=INACTIVE. This script is useful to remove those leftovers that remain in the database after a failed live migration. "dry_run" mode is possible if selected in "[cli_script] dry_run" boolean config option. The duplicated port bindings are printed in the shell but not deleted. Related-Bug: #1979072 Change-Id: I0de5fbb70eb852f82bd311616557985d1ce89bbf --- .../contributor/internals/live_migration.rst | 21 ++++++ .../cmd/remove_duplicated_port_bindings.py | 71 +++++++++++++++++++ neutron/common/config.py | 3 + neutron/conf/common.py | 13 ++++ neutron/objects/ports.py | 8 +++ neutron/tests/unit/objects/test_ports.py | 11 +++ ...icated-port-bindings-83b58060f3adb403.yaml | 10 +++ setup.cfg | 1 + 8 files changed, 138 insertions(+) create mode 100644 neutron/cmd/remove_duplicated_port_bindings.py create mode 100644 releasenotes/notes/remove-duplicated-port-bindings-83b58060f3adb403.yaml diff --git a/doc/source/contributor/internals/live_migration.rst b/doc/source/contributor/internals/live_migration.rst index 3ffd2eda867..cf7a8995eaf 100644 --- a/doc/source/contributor/internals/live_migration.rst +++ b/doc/source/contributor/internals/live_migration.rst @@ -172,6 +172,27 @@ after migration finished. During this time window, the instance might not be reachable via the network. This should be solved with bug https://bugs.launchpad.net/nova/+bug/1605016 +Error recovery +-------------- + +If the Live Migration fails, Nova will revert the operation. That implies +deleting any object created in the database or in the destination compute +node. However, in some cases have been reported the presence of `duplicated +port bindings per port `_. +In this state, the port cannot be migrated until the inactive port binding +(the failed destination host port binding) has been deleted. + +To this end, the script ``neutron-remove-duplicated-port-bindings`` has been +created. This script finds all duplicated port binding (that means, all port +bindings that point to the same port) and deletes the inactive one. + +.. note:: + + This script cannot be executed while a Live Migration or a cross cell Cold + Migration. The script will delete the inactive port binding and will break + the process. + + Flow Diagram ------------ diff --git a/neutron/cmd/remove_duplicated_port_bindings.py b/neutron/cmd/remove_duplicated_port_bindings.py new file mode 100644 index 00000000000..6dafc1531ab --- /dev/null +++ b/neutron/cmd/remove_duplicated_port_bindings.py @@ -0,0 +1,71 @@ +# Copyright (c) 2022 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from neutron_lib import constants +from neutron_lib import context +from neutron_lib.db import api as db_api +from oslo_config import cfg +from oslo_db import options as db_options +from oslo_log import log as logging + +from neutron.common import config as common_config +from neutron.objects import ports as ports_obj + + +LOG = logging.getLogger(__name__) + + +def setup_conf(conf): + common_config.register_common_config_options() + db_group, neutron_db_opts = db_options.list_opts()[0] + conf.register_cli_opts(neutron_db_opts, db_group) + conf() + + +def main(): + """Main method for removing the duplicated port binding registers. + + This script finds all ``PortBinding`` registers with the same ``port_id``. + That happens during the live-migration process. Once finished, the inactive + port binding register is deleted. However, it could happen that during the + live-migration, an error occurs and this deletion is not executed. The + related port cannot be migrated anymore. + + This script should not be executed during a live migration process. It will + remove the inactive port binding and will break the migration. + """ + conf = cfg.CONF + setup_conf(conf) + _dry_run = conf.cli_script.dry_run + admin_ctx = context.get_admin_context() + with db_api.CONTEXT_WRITER.using(admin_ctx): + dup_pbindings = ports_obj.PortBinding.get_duplicated_port_bindings( + admin_ctx) + + # Clean duplicated port bindings that are INACTIVE (if not in dry-run). + if not _dry_run: + for pbinding in dup_pbindings: + ports_obj.PortBinding.delete_objects( + admin_ctx, status=constants.INACTIVE, + port_id=pbinding.port_id) + + if dup_pbindings: + port_ids = [pbinding.port_id for pbinding in dup_pbindings] + action = 'can be' if _dry_run else 'have been' + LOG.info('The following duplicated PortBinding registers with ' + 'status=INACTIVE %s removed, port_ids: %s', + action, port_ids) + else: + LOG.info('No duplicated PortBinding registers has been found.') diff --git a/neutron/common/config.py b/neutron/common/config.py index bd2f0e037f6..84b8f5fbc79 100644 --- a/neutron/common/config.py +++ b/neutron/common/config.py @@ -87,6 +87,9 @@ def register_common_config_options(): common_config.IRONIC_CONF_SECTION) common_config.register_ironic_opts() + # Register the CLI script configuration options. + common_config.register_cli_script_opts() + _COMMON_OPTIONS_ALREADY_REGISTERED = True diff --git a/neutron/conf/common.py b/neutron/conf/common.py index 1209a9d0633..3af648bc7b4 100644 --- a/neutron/conf/common.py +++ b/neutron/conf/common.py @@ -216,3 +216,16 @@ ironic_opts = [ def register_ironic_opts(cfg=cfg.CONF): cfg.register_opts(ironic_opts, group=IRONIC_CONF_SECTION) + + +CLI_SCRIPT_SECTION = 'cli_script' + +cli_script_options = [ + cfg.BoolOpt('dry_run', default=False, + help=_('Dry-run execution of the CLI script. No change will ' + 'be performed on the system.')), +] + + +def register_cli_script_opts(cfg=cfg.CONF): + cfg.register_opts(cli_script_options, group=CLI_SCRIPT_SECTION) diff --git a/neutron/objects/ports.py b/neutron/objects/ports.py index b69dac17563..4185d690107 100644 --- a/neutron/objects/ports.py +++ b/neutron/objects/ports.py @@ -20,6 +20,7 @@ from neutron_lib.utils import net as net_utils from oslo_log import log as logging from oslo_utils import versionutils from oslo_versionedobjects import fields as obj_fields +import sqlalchemy from sqlalchemy import and_ from neutron.common import _constants @@ -102,6 +103,13 @@ class PortBinding(PortBindingBase): cls.db_model.status == status)) return query.all() + @classmethod + @db_api.CONTEXT_READER + def get_duplicated_port_bindings(cls, context): + return context.session.query( + cls.db_model).group_by( + cls.db_model.port_id).having(sqlalchemy.func.count() > 1).all() + @base.NeutronObjectRegistry.register class DistributedPortBinding(PortBindingBase): diff --git a/neutron/tests/unit/objects/test_ports.py b/neutron/tests/unit/objects/test_ports.py index 77cd1688a2b..6ca0c4b9ab4 100644 --- a/neutron/tests/unit/objects/test_ports.py +++ b/neutron/tests/unit/objects/test_ports.py @@ -54,6 +54,17 @@ class PortBindingDbObjectTestCase(obj_test_base.BaseDbObjectTestCase, BasePortBindingDbObjectTestCase): _test_class = ports.PortBinding + def test_get_duplicated_port_bindings(self): + port_id = self._create_test_port_id() + self.update_obj_fields({'port_id': port_id}, + objs=[self.objs[0], self.objs[1]]) + for i in range(3): + _obj = self._make_object(self.obj_fields[i]) + _obj.create() + dup_pb = ports.PortBinding.get_duplicated_port_bindings(self.context) + self.assertEqual(1, len(dup_pb)) + self.assertEqual(port_id, dup_pb[0].port_id) + class DistributedPortBindingIfaceObjTestCase( obj_test_base.BaseObjectIfaceTestCase): diff --git a/releasenotes/notes/remove-duplicated-port-bindings-83b58060f3adb403.yaml b/releasenotes/notes/remove-duplicated-port-bindings-83b58060f3adb403.yaml new file mode 100644 index 00000000000..34f101ac8a3 --- /dev/null +++ b/releasenotes/notes/remove-duplicated-port-bindings-83b58060f3adb403.yaml @@ -0,0 +1,10 @@ +--- +features: + - | + A new script to remove the duplicated port bindings was added. This script + will list all ``ml2_port_bindings`` records in the database, finding those + ones with the same port ID. Then the script removes those ones with + status=INACTIVE. This script is useful to remove those leftovers that + remain in the database after a failed live migration. It is important to + remark that this script should not be executed during any live migration + process. diff --git a/setup.cfg b/setup.cfg index 7f52a8138f1..d72492d1a2e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -62,6 +62,7 @@ console_scripts = neutron-ovn-db-sync-util = neutron.cmd.ovn.neutron_ovn_db_sync_util:main neutron-sanitize-port-binding-profile-allocation = neutron.cmd.sanitize_port_binding_profile_allocation:main neutron-sanitize-port-mac-addresses = neutron.cmd.sanitize_port_mac_addresses:main + neutron-remove-duplicated-port-bindings = neutron.cmd.remove_duplicated_port_bindings:main ml2ovn-trace = neutron.cmd.ovn.ml2ovn_trace:main neutron.core_plugins = ml2 = neutron.plugins.ml2.plugin:Ml2Plugin