[APIC mapping] Support per-tenant NAT EPGs

Adds support for option ('per_tenant_nat_epg') to
enable creating a NAT EPG per tenant. The default
(i.e. current) behavior is to lump together
floating-IPs mappend to endpoints from all tenants
into a single, common "NAT EPG" in APIC. When the new
option is enabled, instead each tenant gets its own
NAT EPG and floating-IPs associated with endpoints of
that tenant are placed in that tenant-specific NAT
EPG. SNAT endpoints are still put in the common
NAT EPG.

Changing this option does not affect existing NAT
EPG usage. That is, if a tenant was using the common
NAT EPG before this option was enabled, then it will
continue to use the common NAT EPG until the tenant
stops using the external-segment completely. This
ensures backwards compatibility during upgrade.

Closes-bug: 1595689

Change-Id: I31179f8b3b4a554fdfe85be9adbedb4d92220aca
Signed-off-by: Amit Bose <amitbose@gmail.com>

gbpservice/neutron/db/migration/alembic_migrations/versions/98862f2d814e_apic_ptne.py

@@ -0,0 +1,43 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+"""apic_per-tenant-nat-epg
+
+Revision ID: 98862f2d814e
+Revises: 12c1bc8d7026
+Create Date: 2016-06-22 17:36:28.386526
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '98862f2d814e'
+down_revision = '12c1bc8d7026'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.create_table(
+        'gp_apic_tenant_specific_nat_epg',
+        sa.Column('external_segment_id', sa.String(36), nullable=False),
+        sa.Column('tenant_id', sa.String(length=36), nullable=False),
+        sa.PrimaryKeyConstraint('external_segment_id', 'tenant_id'),
+        sa.ForeignKeyConstraint(['external_segment_id'],
+                                ['gp_external_segments.id'],
+                                ondelete='CASCADE',
+                                name='gp_apic_ptne_fk_es'))
+
+
+def downgrade():
+    op.drop_table('gp_apic_tenant_specific_nat_epg')

gbpservice/neutron/db/migration/alembic_migrations/versions/HEAD

@@ -1 +1 @@
-12c1bc8d7026
+98862f2d814e

gbpservice/neutron/services/grouppolicy/drivers/cisco/apic/apic_mapping.py

@@ -34,6 +34,7 @@ from neutron.common import rpc as n_rpc
 from neutron.common import topics
 from neutron import context as nctx
 from neutron.db import db_base_plugin_v2 as n_db
+from neutron.db import model_base
 from neutron.extensions import portbindings
 from neutron.extensions import providernet
 from neutron import manager
@@ -47,6 +48,7 @@ from oslo_concurrency import lockutils
 from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_serialization import jsonutils
+import sqlalchemy as sa
 
 from gbpservice.neutron.db.grouppolicy import group_policy_mapping_db as gpdb
 from gbpservice.neutron.extensions import driver_proxy_group as proxy_group
@@ -196,6 +198,16 @@ class ExplicitPortOverlap(gpexc.GroupPolicyBadRequest):
                 '%(ip)s has overlapping IP or MAC address with another port '
                 'in network %(net)s')
 
+
+class TenantSpecificNatEpg(model_base.BASEV2):
+    """Tenants that use a specific NAT EPG for an external segment."""
+    __tablename__ = 'gp_apic_tenant_specific_nat_epg'
+    external_segment_id = sa.Column(
+        sa.String(36), sa.ForeignKey('gp_external_segments.id',
+                                     ondelete='CASCADE'),
+        primary_key=True)
+    tenant_id = sa.Column(sa.String(36), primary_key=True)
+
 REVERSE_PREFIX = 'reverse-'
 SHADOW_PREFIX = 'Shd-'
 AUTO_PREFIX = 'Auto-'
@@ -269,6 +281,7 @@ class ApicMappingDriver(api.ResourceMappingDriver,
         self.enable_dhcp_opt = self.apic_manager.enable_optimized_dhcp
         self.enable_metadata_opt = self.apic_manager.enable_optimized_metadata
         self.nat_enabled = self.apic_manager.use_vmm
+        self.per_tenant_nat_epg = self.apic_manager.per_tenant_nat_epg
         self._gbp_plugin = None
         self.l3out_vlan_alloc = l3out_vlan_alloc.L3outVlanAlloc()
         self.l3out_vlan_alloc.sync_vlan_allocations(
@@ -596,9 +609,10 @@ class ApicMappingDriver(api.ResourceMappingDriver,
             ext_info = self.apic_manager.ext_net_dict.get(es['name'])
             if ext_info and self._is_edge_nat(ext_info):
                 continue
-            nat_epg_name = self._get_nat_epg_for_es(context, es)
+            nat_epg_tenant, nat_epg_name = self._determine_nat_epg_for_es(
+                context, es, l3_policy)
             nat_epg_tenant = self.apic_manager.apic.fvTenant.name(
-                self._tenant_by_sharing_policy(es))
+                nat_epg_tenant)
             fips_in_es = []
 
             if es['subnet_id']:
@@ -619,7 +633,10 @@ class ApicMappingDriver(api.ResourceMappingDriver,
             if not fips_in_es:
                 ipms.append({'external_segment_name': es['name'],
                              'nat_epg_name': nat_epg_name,
-                             'nat_epg_tenant': nat_epg_tenant})
+                             'nat_epg_tenant': nat_epg_tenant,
+                             'next_hop_ep_tenant': (
+                                self.apic_manager.apic.fvTenant.name(
+                                    self._tenant_by_sharing_policy(es)))})
             for f in fips_in_es:
                 f['nat_epg_name'] = nat_epg_name
                 f['nat_epg_tenant'] = nat_epg_tenant
@@ -1900,9 +1917,10 @@ class ApicMappingDriver(api.ResourceMappingDriver,
                     l3p_id=context.current['id'], es_id=es['id'])
             context.set_external_fixed_ips(es['id'], [ip])
 
+        is_edge_nat = self._is_edge_nat(ext_info)
         es_name = self.name_mapper.external_segment(context, es,
             prefix=self._get_shadow_prefix(context,
-                is_shadow, context.current, self._is_edge_nat(ext_info)))
+                is_shadow, context.current, is_edge_nat))
         es_name_pre = self.name_mapper.name_mapper.pre_existing(
             context._plugin_context, es['name'])
         es_tenant = self._get_tenant_for_shadow(is_shadow, context.current, es)
@@ -1917,8 +1935,7 @@ class ApicMappingDriver(api.ResourceMappingDriver,
             # don't need to explicitly create the shadow l3out in this case
             # because we are going to query APIC then use the pre-existing
             # l3out as a template then clone it accordingly
-            if (is_shadow and self._is_edge_nat(ext_info) and
+            if (is_shadow and is_edge_nat and self._is_pre_existing(es)):
-                self._is_pre_existing(es)):
                 is_l3out_creation_needed = False
 
             if is_l3out_creation_needed:
@@ -1941,7 +1958,7 @@ class ApicMappingDriver(api.ResourceMappingDriver,
 
             # if its edge nat then we have to flesh
             # out this shadow L3 out in APIC
-            if is_shadow and self._is_edge_nat(ext_info):
+            if is_shadow and is_edge_nat:
                 vlan_id = self.l3out_vlan_alloc.reserve_vlan(
                     es['name'], context.current['id'])
                 encap = 'vlan-' + str(vlan_id)
@@ -1978,13 +1995,18 @@ class ApicMappingDriver(api.ResourceMappingDriver,
                             self.name_mapper.l2_policy(context, l2p),
                             es_name)
 
-            if not is_shadow and nat_enabled:
+            if nat_enabled:
+                if not is_shadow:
                     # set L3-out for NAT-BD
                     self.apic_manager.set_l3out_for_bd(es_tenant,
                         self._get_nat_bd_for_es(context, es),
                         (self.name_mapper.name_mapper.pre_existing(
                             context, es['name']) if pre_existing else es_name),
                         transaction=trs)
+                elif not is_edge_nat:
+                    # create tenant-specific NAT EPG if required
+                    self._create_tenant_specific_nat_epg(context, es,
+                        context.current, transaction=trs)
         if not is_shadow:
             if nat_enabled:
                 # create shadow external-networks
@@ -2026,6 +2048,10 @@ class ApicMappingDriver(api.ResourceMappingDriver,
             if nat_enabled:
                 # remove shadow external-networks
                 self._unplug_l3p_from_es(context, es, True)
+                if not is_edge_nat:
+                    # remove tenant-specific NAT EPG if required
+                    self._remove_tenant_specific_nat_epg(context, es,
+                                                         context.current)
             else:
                 # Dissociate BDs of the VRF from L3-out
                 l2ps = self._get_l2_policies(context._plugin_context,
@@ -2147,13 +2173,14 @@ class ApicMappingDriver(api.ResourceMappingDriver,
                             l3policy_obj, transaction=trs)
                     if is_shadow:
                         if not self._is_edge_nat(ext_info):
+                            nat_epg_tenant, nat_epg_name = (
+                                self._determine_nat_epg_for_es(
+                                    context, es, l3policy_obj))
                             # set up link to NAT EPG
                             (self.apic_manager.
                              associate_external_epg_to_nat_epg(
                                  es_tenant, es_name, ep_name,
-                                 self._get_nat_epg_for_es(context, es),
+                                 nat_epg_name, target_owner=nat_epg_tenant,
-                                 target_owner=self._tenant_by_sharing_policy(
-                                     es),
                                  transaction=trs))
                     elif nat_enabled:
                         # 'real' external EPGs provide and consume
@@ -3673,3 +3700,98 @@ class ApicMappingDriver(api.ResourceMappingDriver,
         for ip in fixed_ips:
             ip.pop('subnet_id', None)
         return fixed_ips
+
+    def _determine_nat_epg_for_es(self, context, es, tenant_obj):
+        nat_epg_name = self._get_nat_epg_for_es(context, es)
+        nat_epg_tenant = None
+        if (es['shared'] and
+            self._tenant_uses_specific_nat_epg(context, es, tenant_obj)):
+                nat_epg_tenant = self.name_mapper.tenant(tenant_obj)
+        nat_epg_tenant = nat_epg_tenant or self._tenant_by_sharing_policy(es)
+        return nat_epg_tenant, nat_epg_name
+
+    def _tenant_uses_specific_nat_epg(self, context, es, tenant_obj):
+        session = context._plugin_context.session
+        cnt = session.query(TenantSpecificNatEpg).filter_by(
+            external_segment_id = es['id']).filter_by(
+                tenant_id = tenant_obj['tenant_id']).count()
+        return bool(cnt)
+
+    def _create_tenant_specific_nat_epg(self, context, es, l3_policy,
+                                        transaction=None):
+        if not es['shared']:
+            return
+
+        l3ps = self._get_l3_policies(context._plugin_context.elevated(),
+            filters={'id': [l3p for l3p in es['l3_policies']
+                            if l3p != l3_policy['id']],
+                     'tenant_id': [l3_policy['tenant_id']]})
+        if l3ps:
+            # there are other L3Ps from this tenant - don't explicitly create
+            # tenant-specific NAT EPG so that we continue using whatever
+            # those L3Ps were using
+            return
+
+        uses_specific_nat_epg = self._tenant_uses_specific_nat_epg(
+            context, es, l3_policy)
+        if uses_specific_nat_epg or not self.per_tenant_nat_epg:
+            return
+
+        nat_bd_name = self._get_nat_bd_for_es(context, es)
+        nat_epg_name = self._get_nat_epg_for_es(context, es)
+        nat_contract = self._get_nat_contract_for_es(context, es)
+        nat_epg_tenant = self.name_mapper.tenant(l3_policy)
+        es_tenant = self._tenant_by_sharing_policy(es)
+
+        with self.apic_manager.apic.transaction(transaction) as trs:
+            self.apic_manager.ensure_epg_created(
+                nat_epg_tenant, nat_epg_name, bd_name=nat_bd_name,
+                bd_owner=es_tenant, transaction=trs)
+            self.apic_manager.set_contract_for_epg(
+                nat_epg_tenant, nat_epg_name, nat_contract,
+                transaction=trs)
+            self.apic_manager.set_contract_for_epg(
+                nat_epg_tenant, nat_epg_name, nat_contract, provider=True,
+                transaction=trs)
+        session = context._plugin_context.session
+        with session.begin(subtransactions=True):
+            db_obj = TenantSpecificNatEpg(
+                external_segment_id=es['id'], tenant_id=l3_policy['tenant_id'])
+            session.add(db_obj)
+        LOG.debug('Created tenant-specific NAT EPG (%(tenant)s, %(epg)s) for '
+                  'external segment %(es)s',
+                  {'tenant': nat_epg_tenant, 'epg': nat_epg_name,
+                   'es': es['id']})
+
+    def _remove_tenant_specific_nat_epg(self, context, es, l3_policy,
+                                        transaction=None):
+        if not es['shared']:
+            return
+        uses_specific_nat_epg = self._tenant_uses_specific_nat_epg(
+            context, es, l3_policy)
+        if not uses_specific_nat_epg:
+            return
+
+        # remove NAT EPG if this is last L3P from this tenant
+        l3ps = self._get_l3_policies(context._plugin_context.elevated(),
+            filters={'id': [l3p for l3p in es['l3_policies']
+                            if l3p != l3_policy['id']],
+                     'tenant_id': [l3_policy['tenant_id']]})
+        if not l3ps:
+            session = context._plugin_context.session
+            with session.begin(subtransactions=True):
+                db_obj = session.query(TenantSpecificNatEpg).filter_by(
+                    external_segment_id = es['id']).filter_by(
+                        tenant_id = l3_policy['tenant_id']).first()
+                if db_obj:
+                    session.delete(db_obj)
+
+            nat_epg_name = self._get_nat_epg_for_es(context, es)
+            nat_epg_tenant = self.name_mapper.tenant(l3_policy)
+            self.apic_manager.delete_epg_for_network(
+                nat_epg_tenant, nat_epg_name, transaction=transaction)
+
+            LOG.debug('Removed tenant-specific NAT EPG (%(tenant)s, %(epg)s) '
+                      'for external segment %(es)s',
+                      {'tenant': nat_epg_tenant, 'epg': nat_epg_name,
+                       'es': es['id']})

gbpservice/neutron/tests/unit/services/grouppolicy/test_apic_mapping.py

@@ -103,7 +103,11 @@ class ApicMappingTestCase(
         config.cfg.CONF.set_override('enable_security_group', False,
                                      group='SECURITYGROUP')
         n_rpc.create_connection = mock.Mock()
-        amap.ApicMappingDriver.get_apic_manager = mock.MagicMock()
+        amap.ApicMappingDriver.get_apic_manager = mock.Mock(
+            return_value=mock.MagicMock(
+                name_mapper=mock.Mock(),
+                ext_net_dict={},
+                per_tenant_nat_epg=False))
         self.set_up_mocks()
         ml2_opts = ml2_options or {
             'mechanism_drivers': ['apic_gbp'],
@@ -143,8 +147,6 @@ class ApicMappingTestCase(
         self.driver.name_mapper.name_mapper.external_policy = echo
         self.driver.name_mapper.name_mapper.external_segment = echo
         self.driver.name_mapper.name_mapper.pre_existing = echo
-        self.driver.apic_manager = mock.Mock(name_mapper=mock.Mock(),
-                                             ext_net_dict={})
         self.driver.apic_manager.apic.transaction = self.fake_transaction
         self.driver.notifier = mock.Mock()
         self.driver.apic_manager.ext_net_dict = {}
@@ -393,13 +395,13 @@ class TestPolicyTarget(ApicMappingTestCase):
         self.assertEqual(l3p['tenant_id'], details['vrf_tenant'])
         self.assertEqual(l3p['id'], details['vrf_name'])
 
-    def test_get_gbp_details(self):
+    def _do_test_get_gbp_details(self):
         self._mock_external_dict([('supported', '192.168.0.2/24')])
         self.driver.apic_manager.ext_net_dict[
                 'supported']['host_pool_cidr'] = '192.168.200.1/24'
         es = self.create_external_segment(name='supported',
             cidr='192.168.0.2/24',
-            expected_res_status=201, shared=False)['external_segment']
+            expected_res_status=201, shared=True)['external_segment']
         self.create_nat_pool(external_segment_id=es['id'],
                              ip_pool='20.20.20.0/24')
         l3p = self.create_l3_policy(name='myl3',
@@ -436,7 +438,10 @@ class TestPolicyTarget(ApicMappingTestCase):
         fip = mapping['floating_ip'][0]
         self.assertEqual(pt1['port_id'], fip['port_id'])
         self.assertEqual("NAT-epg-%s" % es['id'], fip['nat_epg_name'])
-        self.assertEqual(es['tenant_id'], fip['nat_epg_tenant'])
+        self.assertEqual(
+            (es['tenant_id'] if self.driver.per_tenant_nat_epg
+                else self.common_tenant),
+            fip['nat_epg_tenant'])
 
         self.assertEqual(l3p['tenant_id'], mapping['vrf_tenant'])
         self.assertEqual(l3p['id'], mapping['vrf_name'])
@@ -476,6 +481,13 @@ class TestPolicyTarget(ApicMappingTestCase):
             mapping['host_snat_ips'][0]['host_snat_ip'])
         self.assertEqual(24, mapping['host_snat_ips'][0]['prefixlen'])
 
+    def test_get_gbp_details(self):
+        self._do_test_get_gbp_details()
+
+    def test_get_gbp_details_ptne(self):
+        self.driver.per_tenant_nat_epg = True
+        self._do_test_get_gbp_details()
+
     def test_get_snat_ip_for_vrf(self):
         TEST_VRF1 = 'testvrf1'
         TEST_VRF2 = 'testvrf2'
@@ -2176,8 +2188,12 @@ class TestL3Policy(ApicMappingTestCase):
                               'nexthop': '192.168.0.254'},
                              {'destination': '128.0.0.0/16',
                               'nexthop': None}])['external_segment']
+        owner = self.common_tenant if shared_es else es['tenant_id']
+
+        mgr = self.driver.apic_manager
+        mgr.ensure_epg_created.reset_mock()
+        mgr.set_contract_for_epg.reset_mock()
 
-        # Create with explicit address
         l3p = self.create_l3_policy(
             name='myl3p',
             shared=shared_l3p,
@@ -2188,15 +2204,37 @@ class TestL3Policy(ApicMappingTestCase):
         self.assertEqual(1, len(l3p['external_segments'][es['id']]))
         self.assertEqual('169.254.0.2', l3p['external_segments'][es['id']][0])
 
+        expected_epg_calls = []
+        expected_contract_calls = []
+        expected_nat_epg_tenant = owner
+        if self.nat_enabled and shared_es and self.driver.per_tenant_nat_epg:
+            expected_epg_calls.append(
+                mock.call(l3p['tenant_id'], "NAT-epg-%s" % es['id'],
+                    bd_name="NAT-bd-%s" % es['id'], bd_owner=owner,
+                    transaction=mock.ANY))
+            expected_contract_calls.extend([
+                mock.call(l3p['tenant_id'], "NAT-epg-%s" % es['id'],
+                    "NAT-allow-%s" % es['id'], transaction=mock.ANY),
+                mock.call(l3p['tenant_id'], "NAT-epg-%s" % es['id'],
+                    "NAT-allow-%s" % es['id'], provider=True,
+                    transaction=mock.ANY)])
+            expected_nat_epg_tenant = l3p['tenant_id']
+        self._check_call_list(expected_epg_calls,
+            mgr.ensure_epg_created.call_args_list)
+        self._check_call_list(expected_contract_calls,
+            mgr.set_contract_for_epg.call_args_list)
+        ctx = context.get_admin_context()
+        ctx._plugin_context = ctx
+        self.assertEqual((expected_nat_epg_tenant, "NAT-epg-%s" % es['id']),
+            self.driver._determine_nat_epg_for_es(ctx, es, l3p))
+
         l2ps = [self.create_l2_policy(name='myl2p-%s' % x,
                                       tenant_id=l3p['tenant_id'],
                                       shared=shared_l3p,
                                       l3_policy_id=l3p['id'])['l2_policy']
                 for x in range(0, 3)]
 
-        owner = self.common_tenant if shared_es else es['tenant_id']
         l3p_owner = self.common_tenant if shared_l3p else l3p['tenant_id']
-        mgr = self.driver.apic_manager
         call_name = mgr.ensure_external_routed_network_created
         l3out_str = "Shd-%s-%s"
         if is_edge_nat:
@@ -2333,6 +2371,16 @@ class TestL3Policy(ApicMappingTestCase):
                                                  shared_l3p=False,
                                                  is_edge_nat=True)
 
+    def test_l3p_plugged_to_es_at_creation_ptne_1(self):
+        self.driver.per_tenant_nat_epg = True
+        self._test_l3p_plugged_to_es_at_creation(shared_es=True,
+                                                 shared_l3p=False)
+
+    def test_l3p_plugged_to_es_at_creation_ptne_2(self):
+        self.driver.per_tenant_nat_epg = True
+        self._test_l3p_plugged_to_es_at_creation(shared_es=True,
+                                                 shared_l3p=True)
+
     def _test_l3p_plugged_to_es_at_update(self, shared_es,
                                           shared_l3p, is_edge_nat=False):
         # Verify L3P is correctly plugged to ES on APIC during update
@@ -2357,6 +2405,10 @@ class TestL3Policy(ApicMappingTestCase):
                                       l3_policy_id=l3p['id'])['l2_policy']
                 for x in range(0, 3)]
 
+        mgr = self.driver.apic_manager
+        mgr.ensure_epg_created.reset_mock()
+        mgr.set_contract_for_epg.reset_mock()
+
         # update L3P with ES
         l3p = self.update_l3_policy(l3p['id'], tenant_id=l3p['tenant_id'],
             external_segments={es['id']: []},
@@ -2364,7 +2416,6 @@ class TestL3Policy(ApicMappingTestCase):
         self.assertEqual(1, len(l3p['external_segments'][es['id']]))
         self.assertEqual('169.254.0.2', l3p['external_segments'][es['id']][0])
 
-        mgr = self.driver.apic_manager
         owner = self.common_tenant if shared_es else es['tenant_id']
         l3p_owner = self.common_tenant if shared_l3p else l3p['tenant_id']
         l3out_str = "Shd-%s-%s"
@@ -2475,6 +2526,30 @@ class TestL3Policy(ApicMappingTestCase):
         self._check_call_list(expected_set_l3out_for_bd_calls,
             mgr.set_l3out_for_bd.call_args_list)
 
+        expected_epg_calls = []
+        expected_contract_calls = []
+        expected_nat_epg_tenant = owner
+        if self.nat_enabled and shared_es and self.driver.per_tenant_nat_epg:
+            expected_epg_calls.append(
+                mock.call(l3p['tenant_id'], "NAT-epg-%s" % es['id'],
+                    bd_name="NAT-bd-%s" % es['id'], bd_owner=owner,
+                    transaction=mock.ANY))
+            expected_contract_calls.extend([
+                mock.call(l3p['tenant_id'], "NAT-epg-%s" % es['id'],
+                    "NAT-allow-%s" % es['id'], transaction=mock.ANY),
+                mock.call(l3p['tenant_id'], "NAT-epg-%s" % es['id'],
+                    "NAT-allow-%s" % es['id'], provider=True,
+                    transaction=mock.ANY)])
+            expected_nat_epg_tenant = l3p['tenant_id']
+        self._check_call_list(expected_epg_calls,
+            mgr.ensure_epg_created.call_args_list)
+        self._check_call_list(expected_contract_calls,
+            mgr.set_contract_for_epg.call_args_list)
+        ctx = context.get_admin_context()
+        ctx._plugin_context = ctx
+        self.assertEqual((expected_nat_epg_tenant, "NAT-epg-%s" % es['id']),
+            self.driver._determine_nat_epg_for_es(ctx, es, l3p))
+
     # Although the naming convention used here has been chosen poorly,
     # I'm separating the tests in order to get the mock re-set.
     def test_l3p_plugged_to_es_at_update_1(self):
@@ -2504,6 +2579,16 @@ class TestL3Policy(ApicMappingTestCase):
                                                shared_l3p=False,
                                                is_edge_nat=True)
 
+    def test_l3p_plugged_to_es_at_update_ptne_1(self):
+        self.driver.per_tenant_nat_epg = True
+        self._test_l3p_plugged_to_es_at_update(shared_es=True,
+                                               shared_l3p=False)
+
+    def test_l3p_plugged_to_es_at_update_ptne_2(self):
+        self.driver.per_tenant_nat_epg = True
+        self._test_l3p_plugged_to_es_at_update(shared_es=True,
+                                               shared_l3p=True)
+
     def _test_l3p_unplugged_from_es_on_delete(self, shared_es,
                                               shared_l3p, is_edge_nat=False):
         self._mock_external_dict([('supported1', '192.168.0.2/24'),
@@ -2564,6 +2649,18 @@ class TestL3Policy(ApicMappingTestCase):
         mgr.unset_l3out_for_bd.reset_mock()
         self.driver.l3out_vlan_alloc.release_vlan.reset_mock()
 
+        expected_epg_calls = []
+        if self.nat_enabled and shared_es and self.driver.per_tenant_nat_epg:
+            expected_epg_calls.append(
+                mock.call(l3p['tenant_id'], "NAT-epg-%s" % es1['id'],
+                    transaction=mock.ANY))
+        self._check_call_list(expected_epg_calls,
+            mgr.delete_epg_for_network.call_args_list)
+        ctx = context.get_admin_context()
+        ctx._plugin_context = ctx
+        self.assertEqual((owner, "NAT-epg-%s" % es1['id']),
+            self.driver._determine_nat_epg_for_es(ctx, es1, l3p))
+
         # Verify correct deletion for 2 ESs
         l3p = self.create_l3_policy(
             shared=shared_l3p,
@@ -2572,6 +2669,7 @@ class TestL3Policy(ApicMappingTestCase):
                                es2['id']: ['169.254.0.3']},
             expected_res_status=201)['l3_policy']
         mgr.set_context_for_external_routed_network.reset_mock()
+        mgr.delete_epg_for_network.reset_mock()
         req = self.new_delete_request('l3_policies', l3p['id'], self.fmt)
         res = req.get_response(self.ext_api)
         self.assertEqual(webob.exc.HTTPNoContent.code, res.status_int)
@@ -2620,6 +2718,15 @@ class TestL3Policy(ApicMappingTestCase):
                 expected_release_vlan_calls,
                 self.driver.l3out_vlan_alloc.release_vlan.call_args_list)
 
+        if self.nat_enabled and shared_es and self.driver.per_tenant_nat_epg:
+            expected_epg_calls.append(
+                mock.call(l3p['tenant_id'], "NAT-epg-%s" % es2['id'],
+                    transaction=mock.ANY))
+        self._check_call_list(expected_epg_calls,
+            mgr.delete_epg_for_network.call_args_list)
+        self.assertEqual((owner, "NAT-epg-%s" % es2['id']),
+            self.driver._determine_nat_epg_for_es(ctx, es2, l3p))
+
     # Although the naming convention used here has been chosen poorly,
     # I'm separating the tests in order to get the mock re-set.
     def test_l3p_unplugged_from_es_on_delete_1(self):
@@ -2649,6 +2756,16 @@ class TestL3Policy(ApicMappingTestCase):
                                                    shared_l3p=False,
                                                    is_edge_nat=True)
 
+    def test_l3p_unplugged_from_es_on_delete_ptne_1(self):
+        self.per_tenant_nat_epg = True
+        self._test_l3p_unplugged_from_es_on_delete(shared_es=True,
+                                                   shared_l3p=False)
+
+    def test_l3p_unplugged_from_es_on_delete_ptne_2(self):
+        self.per_tenant_nat_epg = True
+        self._test_l3p_unplugged_from_es_on_delete(shared_es=True,
+                                                   shared_l3p=True)
+
     def _test_l3p_unplugged_from_es_on_update(self, shared_es,
                                               shared_l3p, is_edge_nat=False):
         self._mock_external_dict([('supported1', '192.168.0.2/24'),
@@ -2818,6 +2935,18 @@ class TestL3Policy(ApicMappingTestCase):
         self._check_call_list(expected_set_l3out_for_bd_calls,
             mgr.set_l3out_for_bd.call_args_list)
 
+        expected_epg_calls = []
+        if self.nat_enabled and shared_es and self.driver.per_tenant_nat_epg:
+            expected_epg_calls.append(
+                mock.call(l3p['tenant_id'], "NAT-epg-%s" % es1['id'],
+                    transaction=mock.ANY))
+        self._check_call_list(expected_epg_calls,
+            mgr.delete_epg_for_network.call_args_list)
+        ctx = context.get_admin_context()
+        ctx._plugin_context = ctx
+        self.assertEqual((owner, "NAT-epg-%s" % es1['id']),
+            self.driver._determine_nat_epg_for_es(ctx, es1, l3p))
+
         self.driver.l3out_vlan_alloc.release_vlan.reset_mock()
         mgr.delete_external_routed_network.reset_mock()
         mgr.unset_l3out_for_bd.reset_mock()
@@ -2826,6 +2955,7 @@ class TestL3Policy(ApicMappingTestCase):
             external_segments={es1['id']: ['169.254.0.5'],
                                es2['id']: ['169.254.0.6']})
         mgr.set_context_for_external_routed_network.reset_mock()
+        mgr.delete_epg_for_network.reset_mock()
         self.update_l3_policy(
             l3p['id'], tenant_id=l3p['tenant_id'],
             expected_res_status=200, external_segments={})
@@ -2891,6 +3021,15 @@ class TestL3Policy(ApicMappingTestCase):
                 expected_release_vlan_calls,
                 self.driver.l3out_vlan_alloc.release_vlan.call_args_list)
 
+        if self.nat_enabled and shared_es and self.driver.per_tenant_nat_epg:
+            expected_epg_calls.append(
+                mock.call(l3p['tenant_id'], "NAT-epg-%s" % es2['id'],
+                    transaction=mock.ANY))
+        self._check_call_list(expected_epg_calls,
+            mgr.delete_epg_for_network.call_args_list)
+        self.assertEqual((owner, "NAT-epg-%s" % es2['id']),
+            self.driver._determine_nat_epg_for_es(ctx, es2, l3p))
+
     # Although the naming convention used here has been chosen poorly,
     # I'm separating the tests in order to get the mock re-set.
     def test_l3p_unplugged_from_es_on_update_1(self):
@@ -2920,6 +3059,16 @@ class TestL3Policy(ApicMappingTestCase):
                                                    shared_l3p=False,
                                                    is_edge_nat=True)
 
+    def test_l3p_unplugged_from_es_on_update_ptne_1(self):
+        self.driver.per_tenant_nat_epg = True
+        self._test_l3p_unplugged_from_es_on_update(shared_es=True,
+                                                   shared_l3p=False)
+
+    def test_l3p_unplugged_from_es_on_update_ptne_2(self):
+        self.driver.per_tenant_nat_epg = True
+        self._test_l3p_unplugged_from_es_on_update(shared_es=True,
+                                                   shared_l3p=True)
+
     def test_verify_unsupported_es_noop(self):
         # Verify L3P is correctly plugged to ES on APIC during update
         self._mock_external_dict([('supported', '192.168.0.2/24')])
@@ -2989,6 +3138,107 @@ class TestL3Policy(ApicMappingTestCase):
     def test_multi_es_with_ptg_2(self):
         self._test_multi_es_with_ptg(True)
 
+    def test_multi_l3p_ptne(self):
+        self.driver.per_tenant_nat_epg = True
+
+        self._mock_external_dict([('supported', '192.168.0.2/24')])
+        es = self.create_external_segment(
+            name='supported', shared=True)['external_segment']
+
+        mgr = self.driver.apic_manager
+        mgr.ensure_epg_created.reset_mock()
+
+        l3ps = []
+        for x in range(0, 3 if self.nat_enabled else 1):
+            l3ps.append(self.create_l3_policy(
+                            name='myl3p-%s' % x, tenant_id='another_tenant',
+                            external_segments={es['id']: []},
+                            expected_res_status=201)['l3_policy'])
+            if self.nat_enabled:
+                mgr.ensure_epg_created.assert_called_once_with(
+                    'another_tenant', "NAT-epg-%s" % es['id'],
+                    bd_name="NAT-bd-%s" % es['id'],
+                    bd_owner=self.common_tenant, transaction=mock.ANY)
+            else:
+                mgr.ensure_epg_created.assert_not_called()
+
+        for l3p in l3ps[:-1]:
+            self.delete_l3_policy(l3p['id'], tenant_id=l3p['tenant_id'])
+            mgr.delete_epg_for_network.assert_not_called()
+        self.delete_l3_policy(l3ps[-1]['id'], tenant_id=l3ps[-1]['tenant_id'])
+        if self.nat_enabled:
+            mgr.delete_epg_for_network.assert_called_once_with(
+                'another_tenant', "NAT-epg-%s" % es['id'],
+                transaction=mock.ANY)
+        else:
+            mgr.delete_epg_for_network.assert_not_called()
+
+    def test_ptne_upgrade(self):
+        # Simulate "upgrade" - tenants existing before upgrade should
+        # continue using non-specific NAT EPG where as new ones use
+        # specific NAT EPGs
+        self._mock_external_dict([('supported', '192.168.0.2/24')])
+        es = self.create_external_segment(
+            name='supported', shared=True)['external_segment']
+
+        mgr = self.driver.apic_manager
+        mgr.ensure_epg_created.reset_mock()
+        ctx = context.get_admin_context()
+        ctx._plugin_context = ctx
+
+        l3p_a_1 = self.create_l3_policy(
+            name='myl3p-a-1', tenant_id='tenant_a',
+            external_segments={es['id']: []})['l3_policy']
+        mgr.ensure_epg_created.assert_not_called()
+        self.assertEqual((self.common_tenant, "NAT-epg-%s" % es['id']),
+            self.driver._determine_nat_epg_for_es(ctx, es, l3p_a_1))
+
+        # "Upgrade" and change to per-tenant NAT EPG
+        self.driver.per_tenant_nat_epg = True
+
+        if self.nat_enabled:
+            l3p_a_2 = self.create_l3_policy(
+                name='myl3p-a-2', tenant_id='tenant_a',
+                external_segments={es['id']: []})['l3_policy']
+            mgr.ensure_epg_created.assert_not_called()
+            self.assertEqual((self.common_tenant, "NAT-epg-%s" % es['id']),
+                self.driver._determine_nat_epg_for_es(ctx, es, l3p_a_2))
+            self.delete_l3_policy(l3p_a_2['id'],
+                                  tenant_id=l3p_a_2['tenant_id'])
+
+        self.assertEqual((self.common_tenant, "NAT-epg-%s" % es['id']),
+            self.driver._determine_nat_epg_for_es(ctx, es, l3p_a_1))
+        self.delete_l3_policy(l3p_a_1['id'], tenant_id=l3p_a_1['tenant_id'])
+        mgr.delete_epg_for_network.assert_not_called()
+
+        l3p_a_3 = self.create_l3_policy(
+            name='myl3p-a-3', tenant_id='tenant_a',
+            external_segments={es['id']: []})['l3_policy']
+        if self.nat_enabled:
+            mgr.ensure_epg_created.assert_called_once_with(
+                'tenant_a', "NAT-epg-%s" % es['id'],
+                bd_name="NAT-bd-%s" % es['id'], bd_owner=self.common_tenant,
+                transaction=mock.ANY)
+            self.assertEqual(('tenant_a', "NAT-epg-%s" % es['id']),
+                self.driver._determine_nat_epg_for_es(ctx, es, l3p_a_3))
+        else:
+            mgr.ensure_epg_created.assert_not_called()
+        self.delete_l3_policy(l3p_a_3['id'], tenant_id=l3p_a_3['tenant_id'])
+        mgr.ensure_epg_created.reset_mock()
+
+        l3p_b_1 = self.create_l3_policy(
+            name='myl3p-b-1', tenant_id='tenant_b',
+            external_segments={es['id']: []})['l3_policy']
+        if self.nat_enabled:
+            mgr.ensure_epg_created.assert_called_once_with(
+                'tenant_b', "NAT-epg-%s" % es['id'],
+                bd_name="NAT-bd-%s" % es['id'], bd_owner=self.common_tenant,
+                transaction=mock.ANY)
+            self.assertEqual(('tenant_b', "NAT-epg-%s" % es['id']),
+                self.driver._determine_nat_epg_for_es(ctx, es, l3p_b_1))
+        else:
+            mgr.ensure_epg_created.assert_not_called()
+
 
 class TestL3PolicyNoNat(TestL3Policy):
     def setUp(self):
@@ -3903,7 +4153,7 @@ class TestExternalSegment(ApicMappingTestCase):
         self.assertEqual('169.254.0.2',
                          l3p['external_segments'][es['id']][0])
 
-    def test_plug_l3p_to_es_with_multi_ep(self):
+    def _do_test_plug_l3p_to_es_with_multi_ep(self):
         tenants = (['tenant_a', 'tenant_b', 'tenant_c']
                    if self.nat_enabled else ['tenant_a'])
 
@@ -3962,7 +4212,9 @@ class TestExternalSegment(ApicMappingTestCase):
                               "Shd-%s-%s" % (l3p['id'], es['id']),
                               "Shd-%s-%s" % (l3p['id'], ep['id']),
                               "NAT-epg-%s" % es['id'],
-                              target_owner=self.common_tenant,
+                              target_owner=(l3p['tenant_id']
+                                            if self.driver.per_tenant_nat_epg
+                                            else self.common_tenant),
                               transaction=mock.ANY))
                 l3out = es['name' if self.pre_l3out else 'id']
                 l3out_owner = (APIC_PRE_L3OUT_TENANT
@@ -3988,6 +4240,13 @@ class TestExternalSegment(ApicMappingTestCase):
         self._check_call_list(expected_contract_calls,
             mgr.set_contract_for_external_epg.call_args_list)
 
+    def test_plug_l3p_to_es_with_multi_ep(self):
+        self._do_test_plug_l3p_to_es_with_multi_ep()
+
+    def test_plug_l3p_to_es_with_multi_ep_ptne(self):
+        self.driver.per_tenant_nat_epg = True
+        self._do_test_plug_l3p_to_es_with_multi_ep()
+
 
 class TestExternalSegmentNoNat(TestExternalSegment):
     def setUp(self):