From 827546a0fb936cba846e29dbefecd78d49e9f745 Mon Sep 17 00:00:00 2001 From: LIU Yulong Date: Mon, 24 Jun 2019 18:06:22 +0800 Subject: [PATCH] Optimize DVR related port DB query Save order by in port query when not require fixed_ips, and save some useless query for dvr subnet mac. Conflicts: neutron/db/db_base_plugin_common.py neutron/db/l3_dvr_db.py Closes-Bug: #1834308 Change-Id: I6836840edcaa5a21fd2ba9f65ffd24f7e5038fa3 (cherry picked from commit dd96f3775939dc0209396b4bd73038c1962e6a7e) (cherry picked from commit 2d0adf4a0554e7f8b06a2aaa37d1e772b301de95) --- neutron/api/rpc/handlers/l3_rpc.py | 4 +-- neutron/db/db_base_plugin_common.py | 11 +++++--- neutron/db/dvr_mac_db.py | 41 ++++++++++++++++++++++------- neutron/db/l3_dvr_db.py | 9 +++++++ 4 files changed, 49 insertions(+), 16 deletions(-) diff --git a/neutron/api/rpc/handlers/l3_rpc.py b/neutron/api/rpc/handlers/l3_rpc.py index 7cc8eb55d6d..974730ec584 100644 --- a/neutron/api/rpc/handlers/l3_rpc.py +++ b/neutron/api/rpc/handlers/l3_rpc.py @@ -293,8 +293,8 @@ class L3RpcCallback(object): """DVR: RPC called by dvr-agent to get all ports for subnet.""" subnet_id = kwargs.get('subnet_id') LOG.debug("DVR: subnet_id: %s", subnet_id) - filters = {'fixed_ips': {'subnet_id': [subnet_id]}} - return self.plugin.get_ports(context, filters=filters) + return self.l3plugin.get_ports_under_dvr_connected_subnet( + context, subnet_id) @db_api.retry_db_errors def get_agent_gateway_port(self, context, **kwargs): diff --git a/neutron/db/db_base_plugin_common.py b/neutron/db/db_base_plugin_common.py index 701a4bac3e5..cbdd18b07dd 100644 --- a/neutron/db/db_base_plugin_common.py +++ b/neutron/db/db_base_plugin_common.py @@ -202,7 +202,8 @@ class DbBasePluginCommon(common_db_mixin.CommonDbMixin): return db_utils.resource_fields(res, fields) def _make_port_dict(self, port, fields=None, - process_extensions=True): + process_extensions=True, + with_fixed_ips=True): res = {"id": port["id"], 'name': port['name'], "network_id": port["network_id"], @@ -210,11 +211,13 @@ class DbBasePluginCommon(common_db_mixin.CommonDbMixin): "mac_address": port["mac_address"], "admin_state_up": port["admin_state_up"], "status": port["status"], - "fixed_ips": [{'subnet_id': ip["subnet_id"], - 'ip_address': ip["ip_address"]} - for ip in port["fixed_ips"]], "device_id": port["device_id"], "device_owner": port["device_owner"]} + if with_fixed_ips: + res["fixed_ips"] = [ + {'subnet_id': ip["subnet_id"], + 'ip_address': str( + ip["ip_address"])} for ip in port["fixed_ips"]] # Call auxiliary extend functions, if any if process_extensions: resource_extend.apply_funcs(port_def.COLLECTION_NAME, res, port) diff --git a/neutron/db/dvr_mac_db.py b/neutron/db/dvr_mac_db.py index c21e68cd39b..86dbe4e5614 100644 --- a/neutron/db/dvr_mac_db.py +++ b/neutron/db/dvr_mac_db.py @@ -15,7 +15,6 @@ import netaddr -from neutron_lib.api.definitions import portbindings from neutron_lib.callbacks import events from neutron_lib.callbacks import registry from neutron_lib.callbacks import resources @@ -37,6 +36,7 @@ from neutron.db import api as db_api from neutron.db import models_v2 from neutron.extensions import dvr as ext_dvr from neutron.objects import router +from neutron.plugins.ml2 import models as ml2_models LOG = logging.getLogger(__name__) @@ -46,6 +46,18 @@ dvr_mac_db.register_db_dvr_mac_opts() l3_dvr_db.register_db_l3_dvr_opts() +def get_ports_query_by_subnet_and_ip(context, subnet, ip_addresses=None): + query = context.session.query(models_v2.Port) + query = query.join(models_v2.IPAllocation) + query = query.filter( + models_v2.Port.id == models_v2.IPAllocation.port_id, + models_v2.IPAllocation.subnet_id == subnet) + if ip_addresses: + query = query.filter( + models_v2.IPAllocation.ip_address.in_(ip_addresses)) + return query + + @registry.has_registry_receivers class DVRDbMixin(ext_dvr.DVRMacAddressPluginBase): """Mixin class to add dvr mac address to db_plugin_base_v2.""" @@ -147,18 +159,28 @@ class DVRDbMixin(ext_dvr.DVRMacAddressPluginBase): :param subnet: subnet id to match and extract ports of interest :returns: list -- Ports on the given subnet in the input host """ + host_dvr_for_dhcp = cfg.CONF.host_dvr_for_dhcp - filters = {'fixed_ips': {'subnet_id': [subnet]}, - portbindings.HOST_ID: [host]} - ports_query = self.plugin._get_ports_query(context, filters=filters) + + query = context.session.query(models_v2.Port) + query = query.join(ml2_models.PortBinding) + query = query.join(models_v2.IPAllocation) + query = query.filter( + models_v2.Port.id == ml2_models.PortBinding.port_id, + models_v2.Port.id == models_v2.IPAllocation.port_id, + ml2_models.PortBinding.host == host, + models_v2.IPAllocation.subnet_id == subnet) owner_filter = or_( models_v2.Port.device_owner.startswith( constants.DEVICE_OWNER_COMPUTE_PREFIX), models_v2.Port.device_owner.in_( utils.get_other_dvr_serviced_device_owners(host_dvr_for_dhcp))) - ports_query = ports_query.filter(owner_filter) + + ports_query = query.filter(owner_filter) + ports = [ - self.plugin._make_port_dict(port, process_extensions=False) + self.plugin._make_port_dict(port, process_extensions=False, + with_fixed_ips=False) for port in ports_query.all() ] LOG.debug("Returning list of dvr serviced ports on host %(host)s" @@ -186,11 +208,10 @@ class DVRDbMixin(ext_dvr.DVRMacAddressPluginBase): else: ip_address = subnet_info['gateway_ip'] - filter = {'fixed_ips': {'subnet_id': [subnet], - 'ip_address': [ip_address]}} + query = get_ports_query_by_subnet_and_ip( + context, subnet, [ip_address]) + internal_gateway_ports = query.all() - internal_gateway_ports = self.plugin.get_ports( - context, filters=filter) if not internal_gateway_ports: LOG.error("Could not retrieve gateway port " "for subnet %s", subnet_info) diff --git a/neutron/db/l3_dvr_db.py b/neutron/db/l3_dvr_db.py index 99533df61b4..a6fe655d968 100644 --- a/neutron/db/l3_dvr_db.py +++ b/neutron/db/l3_dvr_db.py @@ -42,6 +42,7 @@ from neutron.common import constants as l3_const from neutron.common import utils as n_utils from neutron.conf.db import l3_dvr_db from neutron.db import api as db_api +from neutron.db import dvr_mac_db from neutron.db import l3_attrs_db from neutron.db import l3_db from neutron.db.models import allowed_address_pair as aap_models @@ -1254,6 +1255,14 @@ class L3_NAT_with_dvr_db_mixin(_DVRAgentInterfaceMixin, self.get_router(context.elevated(), router_id)) return False + def get_ports_under_dvr_connected_subnet(self, context, subnet_id): + query = dvr_mac_db.get_ports_query_by_subnet_and_ip(context, subnet_id) + return [ + self.l3plugin._core_plugin._make_port_dict( + port, process_extensions=False) + for port in query.all() + ] + def is_distributed_router(router): """Return True if router to be handled is distributed."""