Merge "Improve vxlan type driver initialization performance"

neutron/plugins/ml2/drivers/type_vxlan.py

@@ -153,23 +153,33 @@ class VxlanTypeDriver(type_tunnel.TunnelTypeDriver):
         session = db_api.get_session()
         with session.begin(subtransactions=True):
             # remove from table unallocated tunnels not currently allocatable
-            allocs = session.query(VxlanAllocation).with_lockmode("update")
-            for alloc in allocs:
-                try:
-                    # see if tunnel is allocatable
-                    vxlan_vnis.remove(alloc.vxlan_vni)
-                except KeyError:
-                    # it's not allocatable, so check if its allocated
-                    if not alloc.allocated:
-                        # it's not, so remove it from table
-                        LOG.debug(_("Removing tunnel %s from pool"),
-                                  alloc.vxlan_vni)
-                        session.delete(alloc)
-
-            # add missing allocatable tunnels to table
-            for vxlan_vni in sorted(vxlan_vnis):
-                alloc = VxlanAllocation(vxlan_vni=vxlan_vni)
-                session.add(alloc)
+            # fetch results as list via all() because we'll be iterating
+            # through them twice
+            allocs = (session.query(VxlanAllocation).
+                      with_lockmode("update").all())
+            # collect all vnis present in db
+            existing_vnis = set(alloc.vxlan_vni for alloc in allocs)
+            # collect those vnis that needs to be deleted from db
+            vnis_to_remove = [alloc.vxlan_vni for alloc in allocs
+                              if (alloc.vxlan_vni not in vxlan_vnis and
+                                  not alloc.allocated)]
+            # Immediately delete vnis in chunks. This leaves no work for
+            # flush at the end of transaction
+            bulk_size = 100
+            chunked_vnis = (vnis_to_remove[i:i + bulk_size] for i in
+                            range(0, len(vnis_to_remove), bulk_size))
+            for vni_list in chunked_vnis:
+                session.query(VxlanAllocation).filter(
+                    VxlanAllocation.vxlan_vni.in_(vni_list)).delete(
+                        synchronize_session=False)
+            # collect vnis that need to be added
+            vnis = list(vxlan_vnis - existing_vnis)
+            chunked_vnis = (vnis[i:i + bulk_size] for i in
+                            range(0, len(vnis), bulk_size))
+            for vni_list in chunked_vnis:
+                bulk = [{'vxlan_vni': vni, 'allocated': False}
+                        for vni in vni_list]
+                session.execute(VxlanAllocation.__table__.insert(), bulk)
 
     def get_vxlan_allocation(self, session, vxlan_vni):
         with session.begin(subtransactions=True):