From 6e2fce563ab250d4bfe000dd2a2dbc00f094141b Mon Sep 17 00:00:00 2001 From: Eugene Nikanorov Date: Wed, 4 Jun 2014 14:40:13 +0400 Subject: [PATCH] Improve vxlan type driver initialization performance Vxlan type driver may take long time to initialize vxlan allocation table. Optimize db performance by issuing raw sql inserts coalesced into bulk statements. Also optimize deleting logic. Proposed patch gives ~2x performance gain in comparison with original code on Mysql and Postgesql backends Change-Id: I801d967e8e3c0260593f289097d17270ef0b391e Partial-Bug: #1324875 --- neutron/plugins/ml2/drivers/type_vxlan.py | 44 ++++++++++++++--------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/neutron/plugins/ml2/drivers/type_vxlan.py b/neutron/plugins/ml2/drivers/type_vxlan.py index 3e5d4756791..2d39d5ff32a 100644 --- a/neutron/plugins/ml2/drivers/type_vxlan.py +++ b/neutron/plugins/ml2/drivers/type_vxlan.py @@ -153,23 +153,33 @@ class VxlanTypeDriver(type_tunnel.TunnelTypeDriver): session = db_api.get_session() with session.begin(subtransactions=True): # remove from table unallocated tunnels not currently allocatable - allocs = session.query(VxlanAllocation).with_lockmode("update") - for alloc in allocs: - try: - # see if tunnel is allocatable - vxlan_vnis.remove(alloc.vxlan_vni) - except KeyError: - # it's not allocatable, so check if its allocated - if not alloc.allocated: - # it's not, so remove it from table - LOG.debug(_("Removing tunnel %s from pool"), - alloc.vxlan_vni) - session.delete(alloc) - - # add missing allocatable tunnels to table - for vxlan_vni in sorted(vxlan_vnis): - alloc = VxlanAllocation(vxlan_vni=vxlan_vni) - session.add(alloc) + # fetch results as list via all() because we'll be iterating + # through them twice + allocs = (session.query(VxlanAllocation). + with_lockmode("update").all()) + # collect all vnis present in db + existing_vnis = set(alloc.vxlan_vni for alloc in allocs) + # collect those vnis that needs to be deleted from db + vnis_to_remove = [alloc.vxlan_vni for alloc in allocs + if (alloc.vxlan_vni not in vxlan_vnis and + not alloc.allocated)] + # Immediately delete vnis in chunks. This leaves no work for + # flush at the end of transaction + bulk_size = 100 + chunked_vnis = (vnis_to_remove[i:i + bulk_size] for i in + range(0, len(vnis_to_remove), bulk_size)) + for vni_list in chunked_vnis: + session.query(VxlanAllocation).filter( + VxlanAllocation.vxlan_vni.in_(vni_list)).delete( + synchronize_session=False) + # collect vnis that need to be added + vnis = list(vxlan_vnis - existing_vnis) + chunked_vnis = (vnis[i:i + bulk_size] for i in + range(0, len(vnis), bulk_size)) + for vni_list in chunked_vnis: + bulk = [{'vxlan_vni': vni, 'allocated': False} + for vni in vni_list] + session.execute(VxlanAllocation.__table__.insert(), bulk) def get_vxlan_allocation(self, session, vxlan_vni): with session.begin(subtransactions=True):