Merge "Send VRF notifications"

gbpservice/neutron/plugins/ml2plus/drivers/apic_aim/mechanism_driver.py

@@ -1023,9 +1023,29 @@ class ApicMechanismDriver(api_plus.MechanismDriver,
             if subnets_size > 1:
                 raise exceptions.OnlyOneSubnetInSVINetwork()
 
+        if network_db.aim_mapping:
+            # Provide VRF notifications if creating subnets in
+            # unscoped networks.
+            # REVISIT: We may need to handle VRF notifications for
+            #          external networks as well.
+            vrf = self._get_network_vrf(network_db.aim_mapping)
+            if (vrf and (self._is_unrouted_vrf(vrf) or
+                         self._is_default_vrf(vrf))
+                    and not network_db.external):
+                vrfs_to_notify = self._add_vrf_notification(vrf)
+                self._notify_vrf_update(context._plugin_context,
+                                        vrfs_to_notify)
+
         # Neutron subnets in non-external networks are mapped to AIM
         # Subnets as they are added to routers as interfaces.
 
+    def _is_unrouted_vrf(self, vrf):
+        return (vrf.tenant_name == COMMON_TENANT_NAME and
+                vrf.name == self.apic_system_id + '_' + UNROUTED_VRF_NAME)
+
+    def _is_default_vrf(self, vrf):
+        return vrf.name == DEFAULT_VRF_NAME
+
     def update_subnet_precommit(self, context):
         current = context.current
         original = context.original
@@ -1097,6 +1117,19 @@ class ApicMechanismDriver(api_plus.MechanismDriver,
             ns.delete_subnet(aim_ctx, l3out,
                              self._subnet_to_gw_ip_mask(current))
 
+        if network_db.aim_mapping:
+            # Provide VRF notifications if deleting subnets from
+            # unscoped networks.
+            # REVISIT: We may need to handle VRF notifications for
+            #          external networks as well.
+            vrf = self._get_network_vrf(network_db.aim_mapping)
+            if (vrf and (self._is_unrouted_vrf(vrf) or
+                         self._is_default_vrf(vrf))
+                    and not network_db.external):
+                vrfs_to_notify = self._add_vrf_notification(vrf)
+                self._notify_vrf_update(context._plugin_context,
+                                        vrfs_to_notify)
+
         # Non-external neutron subnets are unmapped from AIM Subnets as
         # they are removed from routers.
 
@@ -1166,6 +1199,20 @@ class ApicMechanismDriver(api_plus.MechanismDriver,
 
         self.extend_subnet_dict_bulk(session, [(result, subnet_db)])
 
+    def _notify_vrf_for_scope(self, context):
+        session = context._plugin_context.session
+        scope_id = context.current['address_scope_id']
+        mapping = self._get_address_scope_mapping(session, scope_id)
+        if mapping:
+            vrf = self._get_address_scope_vrf(mapping)
+            if vrf:
+                vrfs_to_notify = self._add_vrf_notification(vrf)
+                self._notify_vrf_update(context._plugin_context,
+                                        vrfs_to_notify)
+
+    def create_subnetpool_precommit(self, context):
+        self._notify_vrf_for_scope(context)
+
     def update_subnetpool_precommit(self, context):
         current = context.current
         original = context.original
@@ -1216,6 +1263,14 @@ class ApicMechanismDriver(api_plus.MechanismDriver,
                 # just reject the update.
                 raise exceptions.ScopeUpdateNotSupported()
 
+        current_prefixes = set(current['prefixes'])
+        original_prefixes = set(original['prefixes'])
+        if current_scope_id and current_prefixes != original_prefixes:
+            self._notify_vrf_for_scope(context)
+
+    def delete_subnetpool_precommit(self, context):
+        self._notify_vrf_for_scope(context)
+
     def create_address_scope_precommit(self, context):
         current = context.current
         LOG.debug("APIC AIM MD creating address scope: %s", current)
@@ -1637,6 +1692,7 @@ class ApicMechanismDriver(api_plus.MechanismDriver,
                     raise exceptions.NonIsomorphicNetworkRoutingUnsupported()
 
         nets_to_notify = set()
+        vrfs_to_notify = set()
         ports_to_notify = set()
         router_topo_moved = False
 
@@ -1677,7 +1733,7 @@ class ApicMechanismDriver(api_plus.MechanismDriver,
                     vrf = self._ensure_default_vrf(aim_ctx, intf_vrf)
                     self._move_topology(
                         context, aim_ctx, router_topology, router_vrf, vrf,
-                        nets_to_notify)
+                        nets_to_notify, vrfs_to_notify)
                     router_topo_moved = True
                     self._cleanup_default_vrf(aim_ctx, router_vrf)
                 elif router_shared_net:
@@ -1688,7 +1744,7 @@ class ApicMechanismDriver(api_plus.MechanismDriver,
                     if net_intfs:
                         self._move_topology(
                             context, aim_ctx, intf_topology, intf_vrf, vrf,
-                            nets_to_notify)
+                            nets_to_notify, vrfs_to_notify)
                         self._cleanup_default_vrf(aim_ctx, intf_vrf)
                 else:
                     # This should never happen.
@@ -1708,10 +1764,12 @@ class ApicMechanismDriver(api_plus.MechanismDriver,
             # First interface for network.
             if network_db.aim_mapping.epg_name:
                 bd, epg = self._associate_network_with_vrf(
-                    context, aim_ctx, network_db, vrf, nets_to_notify)
+                    context, aim_ctx, network_db, vrf, nets_to_notify,
+                    scope_id, vrfs_to_notify)
             elif network_db.aim_mapping.l3out_name:
                 l3out, epg = self._associate_network_with_vrf(
-                    context, aim_ctx, network_db, vrf, nets_to_notify)
+                    context, aim_ctx, network_db, vrf, nets_to_notify,
+                    scope_id, vrfs_to_notify)
         else:
             # Network is already routed.
             #
@@ -1795,6 +1853,8 @@ class ApicMechanismDriver(api_plus.MechanismDriver,
             ports_to_notify.update(port_ids)
         if ports_to_notify:
             self._notify_port_update_bulk(context, ports_to_notify)
+        if vrfs_to_notify:
+            self._notify_vrf_update(context, vrfs_to_notify)
 
     def remove_router_interface(self, context, router_id, port, subnets):
         LOG.debug("APIC AIM MD removing subnets %(subnets)s from router "
@@ -1867,6 +1927,7 @@ class ApicMechanismDriver(api_plus.MechanismDriver,
                                   provided_contract_names=contracts)
 
         nets_to_notify = set()
+        vrfs_to_notify = set()
         ports_to_notify = set()
         router_topo_moved = False
 
@@ -1888,7 +1949,7 @@ class ApicMechanismDriver(api_plus.MechanismDriver,
                     intf_vrf = self._ensure_default_vrf(aim_ctx, intf_vrf)
                     self._move_topology(
                         context, aim_ctx, intf_topology, old_vrf, intf_vrf,
-                        nets_to_notify)
+                        nets_to_notify, vrfs_to_notify)
 
             # See if the router's topology must be moved.
             router_topology = self._router_topology(session, router_db.id)
@@ -1901,14 +1962,15 @@ class ApicMechanismDriver(api_plus.MechanismDriver,
                     router_vrf = self._ensure_default_vrf(aim_ctx, router_vrf)
                     self._move_topology(
                         context, aim_ctx, router_topology, old_vrf, router_vrf,
-                        nets_to_notify)
+                        nets_to_notify, vrfs_to_notify)
                     router_topo_moved = True
 
         # If network is no longer connected to any router, make the
         # network's BD unrouted.
         if not router_ids:
             self._dissassociate_network_from_vrf(
-                context, aim_ctx, network_db, old_vrf, nets_to_notify)
+                context, aim_ctx, network_db, old_vrf, nets_to_notify,
+                scope_id, vrfs_to_notify)
             if scope_id == NO_ADDR_SCOPE:
                 self._cleanup_default_vrf(aim_ctx, old_vrf)
 
@@ -1957,6 +2019,8 @@ class ApicMechanismDriver(api_plus.MechanismDriver,
             ports_to_notify.update(port_ids)
         if ports_to_notify:
             self._notify_port_update_bulk(context, ports_to_notify)
+        if vrfs_to_notify:
+            self._notify_vrf_update(context, vrfs_to_notify)
 
     def bind_port(self, context):
         port = context.current
@@ -2745,7 +2809,7 @@ class ApicMechanismDriver(api_plus.MechanismDriver,
         return rtr_dbs
 
     def _associate_network_with_vrf(self, ctx, aim_ctx, network_db, new_vrf,
-                                    nets_to_notify):
+                                    nets_to_notify, scope_id, vrfs_to_notify):
         LOG.debug("Associating previously unrouted network %(net_id)s named "
                   "'%(net_name)s' in project %(net_tenant)s with VRF %(vrf)s",
                   {'net_id': network_db.id, 'net_name': network_db.name,
@@ -2817,6 +2881,13 @@ class ApicMechanismDriver(api_plus.MechanismDriver,
         # Tenants have changed.
         nets_to_notify.add(network_db.id)
 
+        # Notify VRFs not associated with address_scopes that the
+        # subnet CIDRs within them have changed.
+        old_vrf = self._get_network_vrf(network_db.aim_mapping)
+        if self._is_default_vrf(new_vrf):
+            self._add_vrf_notification(new_vrf, vrfs_to_notify)
+        self._add_vrf_notification(old_vrf, vrfs_to_notify)
+
         if not self._is_svi_db(network_db):
             return bd, epg
         else:
@@ -2824,7 +2895,8 @@ class ApicMechanismDriver(api_plus.MechanismDriver,
             return l3out, ext_net
 
     def _dissassociate_network_from_vrf(self, ctx, aim_ctx, network_db,
-                                        old_vrf, nets_to_notify):
+                                        old_vrf, nets_to_notify, scope_id,
+                                        vrfs_to_notify):
         LOG.debug("Dissassociating network %(net_id)s named '%(net_name)s' in "
                   "project %(net_tenant)s from VRF %(vrf)s",
                   {'net_id': network_db.id, 'net_name': network_db.name,
@@ -2896,8 +2968,20 @@ class ApicMechanismDriver(api_plus.MechanismDriver,
         # Tenants have changed.
         nets_to_notify.add(network_db.id)
 
+        # Notify VRFs not associated with address_scopes that the
+        # subnet CIDRs within them have changed
+        if self._is_default_vrf(old_vrf):
+            self._add_vrf_notification(old_vrf, vrfs_to_notify)
+        self._add_vrf_notification(new_vrf, vrfs_to_notify)
+
+    def _add_vrf_notification(self, vrf, vrfs_to_notify=None):
+        vrfs_to_notify = set() if vrfs_to_notify is None else vrfs_to_notify
+        vrf_to_notify = '%s %s' % (vrf.tenant_name, vrf.name)
+        vrfs_to_notify.add(vrf_to_notify)
+        return vrfs_to_notify
+
     def _move_topology(self, ctx, aim_ctx, topology, old_vrf, new_vrf,
-                       nets_to_notify):
+                       nets_to_notify, vrfs_to_notify):
         LOG.info("Moving routed networks %(topology)s from VRF "
                  "%(old_vrf)s to VRF %(new_vrf)s",
                  {'topology': topology.keys(),
@@ -2976,6 +3060,9 @@ class ApicMechanismDriver(api_plus.MechanismDriver,
         # EPGs' Tenants have changed.
         nets_to_notify.update(topology.keys())
 
+        self._add_vrf_notification(old_vrf, vrfs_to_notify)
+        self._add_vrf_notification(new_vrf, vrfs_to_notify)
+
     def _router_topology(self, session, router_id):
         LOG.debug("Getting topology for router %s", router_id)
         visited_networks = {}
@@ -3480,6 +3567,7 @@ class ApicMechanismDriver(api_plus.MechanismDriver,
             portbindings.VIF_TYPE_BINDING_FAILED]
 
     def _notify_port_update(self, plugin_context, port_id):
+        # REVISIT: Avoid getting the port resource, if possible.
         port = self.plugin.get_port(plugin_context.elevated(), port_id)
         if self._is_port_bound(port):
             LOG.debug("Enqueing notify for port %s", port['id'])
@@ -3524,6 +3612,15 @@ class ApicMechanismDriver(api_plus.MechanismDriver,
         for p_id in port_ids:
             self._notify_port_update(plugin_context, p_id)
 
+    def _notify_vrf_update(self, plugin_context, vrfs_to_notify):
+        txn = local_api.get_outer_transaction(
+            plugin_context.session.transaction)
+        for vrf in vrfs_to_notify:
+            local_api.send_or_queue_notification(plugin_context.session,
+                                                 txn, self.notifier,
+                                                 'opflex_notify_vrf',
+                                                 [plugin_context, vrf])
+
     def get_or_allocate_snat_ip(self, plugin_context, host_or_vrf,
                                 ext_network):
         """Fetch or allocate SNAT IP on the external network.

gbpservice/neutron/tests/unit/plugins/ml2plus/test_apic_aim.py

@@ -1362,29 +1362,87 @@ class TestAimMapping(ApicAimTestCase):
         self._sg_rule_should_not_exist(sg_rule['id'])
 
     def test_subnet_lifecycle(self):
+        self._test_subnet_lifecycle()
+
+    def test_subnet_lifecycle_with_pool(self):
+        self._test_subnet_lifecycle(use_pool=True)
+
+    def test_subnet_lifecycle_with_pool_and_scope(self):
+        self._test_subnet_lifecycle(use_pool=True, use_scope=True)
+
+    def _test_subnet_lifecycle(self, use_pool=False, use_scope=False):
+        # When subnetpools are used with address scopes,
+        # we need to provide VRF change notifications to
+        # L2 agents so they request the changed prefixes
+        # under that VRF
+
+        scope_id = None
+        scope_vrf = None
+        if use_scope:
+            scope = self._make_address_scope(
+                self.fmt, 4, name='as1')['address_scope']
+            scope_id = scope['id']
+            scope_vrf = scope['apic:distinguished_names']['VRF']
+        unrouted_vrf = aim_resource.VRF(
+            tenant_name='common',
+            name = self.driver.apic_system_id + '_UnroutedVRF').dn
+        vrfs_to_notify = [unrouted_vrf]
+
+        with mock.patch.object(self.driver, 'notifier') as notify:
+            # Helper function
+            def check_vrf_notifies(mock_notif, vrf_list):
+                if not vrf_list:
+                    mock_notif.opflex_notify_vrf.assert_not_called()
+                else:
+                    calls = []
+                    for vrf in vrf_list:
+                        aim_vrf = aim_resource.VRF.from_dn(vrf)
+                        calls.append(mock.call.opflex_notify_vrf(mock.ANY,
+                            "%s %s" % (aim_vrf.tenant_name, aim_vrf.name)))
+                    mock_notif.assert_has_calls(calls, any_order=True)
+                self.assertEqual(len(vrf_list), len(mock_notif.mock_calls))
+                mock_notif.reset_mock()
+
+            if use_pool:
+                kwargs = {'name': 'sp1',
+                          'tenant_id': self._tenant_id,
+                          'default_prefixlen': 24}
+                if use_scope:
+                    kwargs.update({'address_scope_id': scope_id})
+
+                pool = self._make_subnetpool(self.fmt, ['10.0.0.0/8'],
+                                             **kwargs)['subnetpool']
+                check_vrf_notifies(notify, [scope_vrf] if use_scope else [])
+
             # Create network.
             net_resp = self._make_network(self.fmt, 'net1', True)
             net = net_resp['network']
 
             # Test create.
             gw_ip = '10.0.0.1'
+            kwargs = {'subnetpool_id': pool['id']} if use_pool else {}
             subnet = self._make_subnet(
-            self.fmt, net_resp, gw_ip, '10.0.0.0/24')['subnet']
+                self.fmt, net_resp, gw_ip,
+                '10.0.0.0/24', **kwargs)['subnet']
             subnet_id = subnet['id']
             self._check_subnet(subnet, net, [], [gw_ip])
+            check_vrf_notifies(notify, vrfs_to_notify)
 
             # Test show.
             subnet = self._show('subnets', subnet_id)['subnet']
             self._check_subnet(subnet, net, [], [gw_ip])
+            check_vrf_notifies(notify, [])
 
             # Test update.
             data = {'subnet': {'name': 'newnamefornet'}}
             subnet = self._update('subnets', subnet_id, data)['subnet']
             self._check_subnet(subnet, net, [], [gw_ip])
+            check_vrf_notifies(notify, [])
 
             # Test delete.
             self._delete('subnets', subnet_id)
             self._check_subnet_deleted(subnet)
+            check_vrf_notifies(notify, vrfs_to_notify)
 
     def test_address_scope_lifecycle(self):
         # Test create.
@@ -1406,6 +1464,70 @@ class TestAimMapping(ApicAimTestCase):
         self._delete('address-scopes', scope_id)
         self._check_address_scope_deleted(scope)
 
+    def _test_subnetpool_lifecycle(self, use_scopes=False):
+        # When subnetpools are used with address scopes,
+        # we need to provide VRF change notifications to
+        # L2 agents so they request the changed prefixes
+        # under that VRF
+
+        scope_id = None
+        scope_vrf = None
+        if use_scopes:
+            scope = self._make_address_scope(
+                self.fmt, 4, name='as1')['address_scope']
+            scope_id = scope['id']
+            scope_vrf = scope['apic:distinguished_names']['VRF']
+
+        with mock.patch.object(self.driver, 'notifier') as notify:
+            # Helper function
+            def check_vrf_notifies(mock_notif, vrf_list):
+                if not vrf_list:
+                    mock_notif.opflex_notify_vrf.assert_not_called()
+                else:
+                    calls = []
+                    for vrf in vrf_list:
+                        aim_vrf = aim_resource.VRF.from_dn(vrf)
+                        calls.append(mock.call.opflex_notify_vrf(mock.ANY,
+                            "%s %s" % (aim_vrf.tenant_name, aim_vrf.name)))
+                    mock_notif.assert_has_calls(calls, any_order=True)
+                self.assertEqual(len(vrf_list), len(mock_notif.mock_calls))
+                mock_notif.reset_mock()
+
+            # Make the subnetpool
+            kwargs = {'name': 'sp1',
+                      'tenant_id': self._tenant_id,
+                      'default_prefixlen': 24}
+            if use_scopes:
+                kwargs.update({'address_scope_id': scope_id})
+
+            pool = self._make_subnetpool(self.fmt, ['10.0.0.0/8'],
+                                         **kwargs)['subnetpool']
+            vrfs_to_notify = [scope_vrf] if use_scopes else []
+            check_vrf_notifies(notify, vrfs_to_notify)
+
+            # Change the name of the pool. When scopes are used, this
+            # should not trigger a notify
+            data = {'subnetpool': {'name': 'newname'}}
+            pool = self._update('subnetpools', pool['id'], data)['subnetpool']
+            check_vrf_notifies(notify, [])
+
+            # Add a prefix to the pool. When scopes are used, this should
+            # trigger a notify
+            data = {'subnetpool': {'prefixes': ['10.0.0.0/8', '20.0.0.0/8']}}
+            pool = self._update('subnetpools', pool['id'], data)['subnetpool']
+            check_vrf_notifies(notify, vrfs_to_notify)
+
+            # Delete the subnetpool. When scopes are used, this should
+            # trigger a notify
+            self._delete('subnetpools', pool['id'])
+            check_vrf_notifies(notify, vrfs_to_notify)
+
+    def test_subnetpool_lifecycle(self):
+        self._test_subnetpool_lifecycle()
+
+    def test_subnetpool_lifecycle_with_scopes(self):
+        self._test_subnetpool_lifecycle(use_scopes=True)
+
     def test_router_lifecycle(self):
         # Test create.
         router = self._make_router(
@@ -1971,14 +2093,32 @@ class TestAimMapping(ApicAimTestCase):
         # same network once they are supported. Also, test with shared
         # scopes?
 
+        # Test
+        with mock.patch.object(self.driver, 'notifier') as notify:
+            def check_vrf_notifies(mock_notif, vrf_list):
+                if not vrf_list:
+                    mock_notif.opflex_notify_vrf.assert_not_called()
+                else:
+                    calls = []
+                    for vrf in vrf_list:
+                        aim_vrf = aim_resource.VRF.from_dn(vrf)
+                        calls.append(mock.call.opflex_notify_vrf(mock.ANY,
+                            "%s %s" % (aim_vrf.tenant_name, aim_vrf.name)))
+                    mock_notif.assert_has_calls(calls, any_order=True)
+                self.assertEqual(len(vrf_list), len(mock_notif.mock_calls))
+                mock_notif.reset_mock()
+
             # Get default unscoped routed VRF DNs for main and sharing
-        # projects.
+            # projects, and the unrouted VRF
             tenant_aname = self.name_mapper.project(None, self._tenant_id)
             main_vrf = aim_resource.VRF(
                 tenant_name=tenant_aname, name='DefaultVRF').dn
             tenant_aname = self.name_mapper.project(None, 'tenant_2')
             shared_vrf = aim_resource.VRF(
                 tenant_name=tenant_aname, name='DefaultVRF').dn
+            unrouted_vrf = aim_resource.VRF(
+                tenant_name='common',
+                name = self.driver.apic_system_id + '_UnroutedVRF').dn
 
             # Create a v6 scope and pool.
             scope6 = self._make_address_scope(
@@ -1991,6 +2131,7 @@ class TestAimMapping(ApicAimTestCase):
                 tenant_id=self._tenant_id,
                 address_scope_id=scope6_id)['subnetpool']
             pool6_id = pool6['id']
+            check_vrf_notifies(notify, [scope46i_vrf])
 
             # Create isomorphic v4 scope and pool.
             scope4i = self._make_address_scope_for_vrf(
@@ -2001,9 +2142,11 @@ class TestAimMapping(ApicAimTestCase):
             self._scope_vrf_dnames[scope6_id] = 'as4i-as6'
             self._check_address_scope(scope4i)
             pool4i = self._make_subnetpool(
-            self.fmt, ['10.1.0.0/16'], name='sp4i', tenant_id=self._tenant_id,
+                self.fmt, ['10.1.0.0/16'], name='sp4i',
-            address_scope_id=scope4i_id, default_prefixlen=24)['subnetpool']
+                tenant_id=self._tenant_id, address_scope_id=scope4i_id,
+                default_prefixlen=24)['subnetpool']
             pool4i_id = pool4i['id']
+            check_vrf_notifies(notify, [scope46i_vrf])
 
             # Create non-isomorphic v4 scope and pool.
             scope4n = self._make_address_scope(
@@ -2012,20 +2155,25 @@ class TestAimMapping(ApicAimTestCase):
             self._check_address_scope(scope4n)
             scope4n_vrf = scope4n['apic:distinguished_names']['VRF']
             pool4n = self._make_subnetpool(
-            self.fmt, ['10.2.0.0/16'], name='sp4n', tenant_id=self._tenant_id,
+                self.fmt, ['10.2.0.0/16'], name='sp4n',
-            address_scope_id=scope4n_id, default_prefixlen=24)['subnetpool']
+                tenant_id=self._tenant_id, address_scope_id=scope4n_id,
+                default_prefixlen=24)['subnetpool']
             pool4n_id = pool4n['id']
+            check_vrf_notifies(notify, [scope4n_vrf])
 
             # Create unscoped pools if required.
             if use_unscoped_pools:
                 pool4u = self._make_subnetpool(
                     self.fmt, ['10.3.0.0/16', '10.4.0.0/16'], name='sp4u',
-                tenant_id=self._tenant_id, default_prefixlen=24)['subnetpool']
+                    tenant_id=self._tenant_id,
+                    default_prefixlen=24)['subnetpool']
                 pool4u_id = pool4u['id']
+                check_vrf_notifies(notify, [])
                 pool6u = self._make_subnetpool(
                     self.fmt, ['2001:db8:1::0/56'], name='sp6u',
                     tenant_id=self._tenant_id)['subnetpool']
                 pool6u_id = pool6u['id']
+                check_vrf_notifies(notify, [])
             else:
                 pool4u_id = None
                 pool6u_id = None
@@ -2039,11 +2187,13 @@ class TestAimMapping(ApicAimTestCase):
                 self.fmt, net_resp, gw4i1_ip, '10.1.1.0/24',
                 subnetpool_id=pool4i_id)['subnet']
             self._check_subnet(subnet4i1, net1, [], [gw4i1_ip])
+            check_vrf_notifies(notify, [unrouted_vrf])
             gw61_ip = '2001:db8:1:1::1'
             subnet61 = self._make_subnet(
                 self.fmt, net_resp, gw61_ip, '2001:db8:1:1::0/64',
                 ip_version=6, subnetpool_id=pool6_id)['subnet']
             self._check_subnet(subnet61, net1, [], [gw61_ip])
+            check_vrf_notifies(notify, [unrouted_vrf])
 
             # Create network with subnets using second v4 scope and v6 scope.
             net_resp = self._make_network(self.fmt, 'net2', True)
@@ -2054,11 +2204,13 @@ class TestAimMapping(ApicAimTestCase):
                 self.fmt, net_resp, gw4n2_ip, '10.2.1.0/24',
                 subnetpool_id=pool4n_id)['subnet']
             self._check_subnet(subnet4n2, net2, [], [gw4n2_ip])
+            check_vrf_notifies(notify, [unrouted_vrf])
             gw62_ip = '2001:db8:1:2::1'
             subnet62 = self._make_subnet(
                 self.fmt, net_resp, gw62_ip, '2001:db8:1:2::0/64',
                 ip_version=6, subnetpool_id=pool6_id)['subnet']
             self._check_subnet(subnet62, net2, [], [gw62_ip])
+            check_vrf_notifies(notify, [unrouted_vrf])
 
             # Create network with unscoped subnets.
             net_resp = self._make_network(self.fmt, 'net3', True)
@@ -2069,11 +2221,13 @@ class TestAimMapping(ApicAimTestCase):
                 self.fmt, net_resp, gw43_ip, '10.3.1.0/24',
                 subnetpool_id=pool4u_id)['subnet']
             self._check_subnet(subnet43, net3, [], [gw43_ip])
+            check_vrf_notifies(notify, [unrouted_vrf])
             gw63_ip = '2001:db8:1:3::1'
             subnet63 = self._make_subnet(
                 self.fmt, net_resp, gw63_ip, '2001:db8:1:3::0/64',
                 ip_version=6, subnetpool_id=pool6u_id)['subnet']
             self._check_subnet(subnet63, net3, [], [gw63_ip])
+            check_vrf_notifies(notify, [unrouted_vrf])
 
             # Create shared network with unscoped subnets.
             net_resp = self._make_network(
@@ -2085,11 +2239,13 @@ class TestAimMapping(ApicAimTestCase):
                 self.fmt, net_resp, gw44_ip, '10.4.1.0/24',
                 subnetpool_id=pool4u_id)['subnet']
             self._check_subnet(subnet44, net4, [], [gw44_ip])
+            check_vrf_notifies(notify, [unrouted_vrf])
             gw64_ip = '2001:db8:1:4::1'
             subnet64 = self._make_subnet(
                 self.fmt, net_resp, gw64_ip, '2001:db8:1:4::0/64',
                 ip_version=6, subnetpool_id=pool6u_id)['subnet']
             self._check_subnet(subnet64, net4, [], [gw64_ip])
+            check_vrf_notifies(notify, [unrouted_vrf])
 
             # Create two external networks with subnets.
             ext_net1 = self._make_ext_network(
@@ -2097,6 +2253,7 @@ class TestAimMapping(ApicAimTestCase):
             self._make_subnet(
                 self.fmt, {'network': ext_net1}, '100.100.100.1',
                 '100.100.100.0/24')
+            check_vrf_notifies(notify, [])
             ext_net2 = self._make_ext_network(
                 'ext-net2', dn=self.dn_t1_l2_n2)
             self._make_subnet(
@@ -2127,10 +2284,12 @@ class TestAimMapping(ApicAimTestCase):
             def check(nets, scopes, unscoped_project):
                 router = self._show('routers', router_id)['router']
                 expected_gw_ips = []
-            for net, routed_subnets, unrouted_subnets, scope, project in nets:
+                for (net, routed_subnets, unrouted_subnets,
+                        scope, project) in nets:
                     net = self._show('networks', net['id'])['network']
                     self._check_network(
-                    net, [router] if routed_subnets else [], scope, project)
+                        net, [router] if routed_subnets else [],
+                        scope, project)
                     for subnet in routed_subnets:
                         gw_ip = subnet['gateway_ip']
                         expected_gw_ips.append(gw_ip)
@@ -2160,11 +2319,14 @@ class TestAimMapping(ApicAimTestCase):
                     self.assertEqual(sorted(expected_vrf_dns), sorted(vrf_dns))
 
                 check_calls(
-                self.mock_ns.disconnect_vrf, disconnect_vrf_dns, from_net_dn)
+                    self.mock_ns.disconnect_vrf, disconnect_vrf_dns,
+                    from_net_dn)
                 check_calls(
                     self.mock_ns.connect_vrf, connect_vrf_dns, to_net_dn)
                 self.mock_ns.reset_mock()
 
+            check_vrf_notifies(notify, [])
+
             # Create router.
             router = self._make_router(
                 self.fmt, self._tenant_id, 'router1',
@@ -2186,6 +2348,7 @@ class TestAimMapping(ApicAimTestCase):
                    (net4, [], [subnet44, subnet64], None, None)],
                   [scope4i], None)
             check_ns([], None, [scope46i_vrf], self.dn_t1_l1_n1)
+            check_vrf_notifies(notify, [scope46i_vrf])
 
             # Add first scoped v6 subnet to router, which should not
             # effect external connectivity.
@@ -2196,6 +2359,7 @@ class TestAimMapping(ApicAimTestCase):
                    (net4, [], [subnet44, subnet64], None, None)],
                   [scope4i, scope6], None)
             check_ns([], None, [], None)
+            check_vrf_notifies(notify, [])
 
             # Add first unscoped v6 subnet to router, which should connect
             # the default VRF to ext_net1.
@@ -2206,6 +2370,7 @@ class TestAimMapping(ApicAimTestCase):
                    (net4, [], [subnet44, subnet64], None, None)],
                   [scope4i, scope6], self._tenant_id)
             check_ns([], None, [main_vrf], self.dn_t1_l1_n1)
+            check_vrf_notifies(notify, [main_vrf])
 
             # REVISIT: Enable when non-isomorphic network routing is
             # supported.
@@ -2218,6 +2383,7 @@ class TestAimMapping(ApicAimTestCase):
             #        (net3, [subnet63], [subnet43], None, None),
             #        (net4, [], [subnet44, subnet64], None, None)],
             #       [scope4i, scope6], self._tenant_id)
+            # check_vrf_notifies(....)
 
             # Add second scoped v4 subnet to router, which should connect
             # its VRF to ext_net1.
@@ -2228,6 +2394,7 @@ class TestAimMapping(ApicAimTestCase):
                    (net4, [], [subnet44, subnet64], None, None)],
                   [scope4i, scope4n, scope6], self._tenant_id)
             check_ns([], None, [scope4n_vrf], self.dn_t1_l1_n1)
+            check_vrf_notifies(notify, [scope4n_vrf])
 
             # Add first unscoped v4 subnet to router, which should not
             # effect external connectivity.
@@ -2238,6 +2405,9 @@ class TestAimMapping(ApicAimTestCase):
                    (net4, [], [subnet44, subnet64], None, None)],
                   [scope4i, scope4n, scope6], self._tenant_id)
             check_ns([], None, [], None)
+            # no notify here b/c attaching subnet63 above caused
+            # everything in the net/BD to get moved
+            check_vrf_notifies(notify, [])
 
             # Add shared unscoped v4 subnet to router, which should move
             # unscoped topology but not scoped topologies, and should
@@ -2249,7 +2419,9 @@ class TestAimMapping(ApicAimTestCase):
                    (net3, [subnet43, subnet63], [], None, 'tenant_2'),
                    (net4, [subnet44], [subnet64], None, 'tenant_2')],
                   [scope4i, scope4n, scope6], 'tenant_2')
-        check_ns([main_vrf], self.dn_t1_l1_n1, [shared_vrf], self.dn_t1_l1_n1)
+            check_ns([main_vrf], self.dn_t1_l1_n1,
+                     [shared_vrf], self.dn_t1_l1_n1)
+            check_vrf_notifies(notify, [main_vrf, shared_vrf])
 
             # Add shared unscoped v6 subnet to router, which should not
             # effect external connectivity.
@@ -2260,6 +2432,7 @@ class TestAimMapping(ApicAimTestCase):
                    (net4, [subnet44, subnet64], [], None, 'tenant_2')],
                   [scope4i, scope4n, scope6], 'tenant_2')
             check_ns([], None, [], None)
+            check_vrf_notifies(notify, [])
 
             # Update router with new gateway, which should disconnect all
             # VRFs from ext_net1 and connect them to ext_net2.
@@ -2273,6 +2446,7 @@ class TestAimMapping(ApicAimTestCase):
                   [scope4i, scope4n, scope6], 'tenant_2')
             check_ns([scope46i_vrf, scope4n_vrf, shared_vrf], self.dn_t1_l1_n1,
                      [scope46i_vrf, scope4n_vrf, shared_vrf], self.dn_t1_l2_n2)
+            check_vrf_notifies(notify, [])
 
             # Remove first scoped v4 subnet from router, which should not
             # effect external connectivity.
@@ -2283,6 +2457,7 @@ class TestAimMapping(ApicAimTestCase):
                    (net4, [subnet44, subnet64], [], None, 'tenant_2')],
                   [scope4n, scope6], 'tenant_2')
             check_ns([], None, [], None)
+            check_vrf_notifies(notify, [])
 
             # Remove first scoped v6 subnet from router, which should
             # disconnect isomorphic VRF from ext_net2.
@@ -2293,6 +2468,7 @@ class TestAimMapping(ApicAimTestCase):
                    (net4, [subnet44, subnet64], [], None, 'tenant_2')],
                   [scope4n], 'tenant_2')
             check_ns([scope46i_vrf], self.dn_t1_l2_n2, [], None)
+            check_vrf_notifies(notify, [unrouted_vrf])
 
             # Remove shared unscoped v4 subnet from router, which should
             # not effect external connecivity.
@@ -2303,6 +2479,7 @@ class TestAimMapping(ApicAimTestCase):
                    (net4, [subnet64], [subnet44], None, 'tenant_2')],
                   [scope4n], 'tenant_2')
             check_ns([], None, [], None)
+            check_vrf_notifies(notify, [])
 
             # Remove shared unscoped v6 subnet from router, which should
             # move remaining unscoped topology back to original tenant,
@@ -2314,7 +2491,9 @@ class TestAimMapping(ApicAimTestCase):
                    (net3, [subnet43, subnet63], [], None, None),
                    (net4, [], [subnet44, subnet64], None, None)],
                   [scope4n], self._tenant_id)
-        check_ns([shared_vrf], self.dn_t1_l2_n2, [main_vrf], self.dn_t1_l2_n2)
+            check_ns([shared_vrf], self.dn_t1_l2_n2,
+                     [main_vrf], self.dn_t1_l2_n2)
+            check_vrf_notifies(notify, [unrouted_vrf, shared_vrf, main_vrf])
 
             # Remove first unscoped v6 subnet from router, which should
             # not effect external connectivity.
@@ -2325,6 +2504,7 @@ class TestAimMapping(ApicAimTestCase):
                    (net4, [], [subnet44, subnet64], None, None)],
                   [scope4n], self._tenant_id)
             check_ns([], None, [], None)
+            check_vrf_notifies(notify, [])
 
             # Remove second scoped v4 subnet from router, which should
             # disconnect its VRF from ext_net2.
@@ -2335,6 +2515,7 @@ class TestAimMapping(ApicAimTestCase):
                    (net4, [], [subnet44, subnet64], None, None)],
                   [], self._tenant_id)
             check_ns([scope4n_vrf], self.dn_t1_l2_n2, [], None)
+            check_vrf_notifies(notify, [unrouted_vrf])
 
             # Remove second unscoped v4 subnet from router, which should
             # disconnect the default VRF from ext_net2.
@@ -2345,6 +2526,7 @@ class TestAimMapping(ApicAimTestCase):
                    (net4, [], [subnet44, subnet64], None, None)],
                   [], None)
             check_ns([main_vrf], self.dn_t1_l2_n2, [], None)
+            check_vrf_notifies(notify, [unrouted_vrf, main_vrf])
 
             # REVISIT: Enable when non-isomorphic network routing is
             # supported.
@@ -2357,6 +2539,7 @@ class TestAimMapping(ApicAimTestCase):
             #        (net4, [], [subnet44, subnet64], None, None)],
             #       [], None)
             # check_ns(...)
+            # check_vrf_notifies(...)
 
     def test_shared_address_scope(self):
         # Create shared scope as tenant_1.