Merge "Support multi-gateway NS networking with EW enabled"

devstack/local.conf.node_1.sample

@@ -41,3 +41,13 @@ TRICIRCLE_START_SERVICES=True
 enable_plugin tricircle https://github.com/openstack/tricircle/
 
 disable_service horizon
+
+# Enable l2population for vxlan network
+[[post-config|/$Q_PLUGIN_CONF_FILE]]
+
+[ml2]
+mechanism_drivers = openvswitch,linuxbridge,l2population
+
+[agent]
+tunnel_types=vxlan
+l2_population=True

devstack/local.conf.node_2.sample

@@ -45,3 +45,13 @@ TRICIRCLE_START_SERVICES=False
 enable_plugin tricircle https://github.com/openstack/tricircle/
 
 disable_service horizon
+
+# Enable l2population for vxlan network
+[[post-config|/$Q_PLUGIN_CONF_FILE]]
+
+[ml2]
+mechanism_drivers = openvswitch,linuxbridge,l2population
+
+[agent]
+tunnel_types=vxlan
+l2_population=True

tricircle/common/constants.py

@@ -64,7 +64,8 @@ ns_router_name = 'ns_router_%s'
 
 shadow_port_name = 'shadow_port_%s'
 dhcp_port_name = 'dhcp_port_%s'  # subnet_id
-interface_port_name = 'interface_%s_%s'  # b_pod_id t_subnet_id
+snat_port_name = 'snat_port_%s'  # subnet_id
+interface_port_name = 'interface_%s_%s'  # b_region_name t_subnet_id
 interface_port_device_id = 'reserved_gateway_port'
 
 MAX_INT = 0x7FFFFFFF

tricircle/network/central_plugin.py

@@ -411,14 +411,7 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
             net[az_ext.AZ_HINTS] = self._convert_az2region(t_ctx, az_hints)
 
     def _convert_az2region(self, t_ctx, az_hints):
-        region_names = set()
-        for az_hint in az_hints:
-            pods = db_api.find_pods_by_az_or_region(t_ctx, az_hint)
-            if not pods:
-                continue
-            for pod in pods:
-                region_names.add(pod['region_name'])
-        return list(region_names)
+        return self.helper.convert_az2region(t_ctx, az_hints)
 
     def get_network(self, context, network_id, fields=None):
         net = super(TricirclePlugin, self).get_network(context, network_id,
@@ -444,21 +437,30 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
     def create_subnet(self, context, subnet):
         subnet_data = subnet['subnet']
         network = self.get_network(context, subnet_data['network_id'])
+        is_external = network.get(external_net.EXTERNAL)
         with context.session.begin(subtransactions=True):
             res = super(TricirclePlugin, self).create_subnet(context, subnet)
             # put inside a session so when bottom operations fails db can
             # rollback
-            if network.get(external_net.EXTERNAL):
+            if is_external:
                 self._create_bottom_external_subnet(
                     context, res, network, res['id'])
-        if res['enable_dhcp']:
-            try:
-                t_ctx = t_context.get_context_from_neutron_context(context)
+        snat_port_id = None
+        try:
+            t_ctx = t_context.get_context_from_neutron_context(context)
+            if not subnet_data['name'].startswith(
+                    t_constants.bridge_subnet_name[:-3]) and not is_external:
+                # do not reserve snat port for bridge and external subnet
+                snat_port_id = self.helper.prepare_top_snat_port(
+                    t_ctx, context, res['tenant_id'], network['id'], res['id'])
+            if res['enable_dhcp']:
                 self.helper.prepare_top_dhcp_port(
                     t_ctx, context, res['tenant_id'], network['id'], res['id'])
-            except Exception:
-                self.delete_subnet(context, res['id'])
-                raise
+        except Exception:
+            if snat_port_id:
+                super(TricirclePlugin, self).delete_port(context, snat_port_id)
+            self.delete_subnet(context, res['id'])
+            raise
         return res
 
     def _delete_pre_created_port(self, t_ctx, q_ctx, port_name):
@@ -495,6 +497,8 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
             raise
         dhcp_port_name = t_constants.dhcp_port_name % subnet_id
         self._delete_pre_created_port(t_ctx, context, dhcp_port_name)
+        snat_port_name = t_constants.snat_port_name % subnet_id
+        self._delete_pre_created_port(t_ctx, context, snat_port_name)
         super(TricirclePlugin, self).delete_subnet(context, subnet_id)
 
     def update_subnet(self, context, subnet_id, subnet):
@@ -635,9 +639,19 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
                       'following routers: %s, xjob triggered',
                       port_body['network_id'], router_ids)
             admin_context = t_context.get_admin_context()
-            self.xjob_handler.setup_bottom_router(
-                admin_context, port_body['network_id'],
-                router_ids[0], pod['pod_id'])
+            t_ctx = t_context.get_context_from_neutron_context(context)
+
+            for router_id in router_ids:
+                router = self.get_router(context, router_id)
+                if not self.helper.is_local_router(t_ctx, router):
+                    # for local router, job will be triggered after router
+                    # interface attachment.
+                    self.xjob_handler.setup_bottom_router(
+                        admin_context, port_body['network_id'],
+                        router_id, pod['pod_id'])
+                    # network will be attached to only one non-local router,
+                    # so we break here
+                    break
         else:
             LOG.debug('Update port: no interfaces found, xjob not'
                       'triggered')
@@ -1345,6 +1359,15 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
             net_id = subnet['network_id']
         return net_id
 
+    def _get_subnet_id_by_interface_info(self, q_ctx, add_by_port,
+                                         interface_info):
+        if add_by_port:
+            port = self.get_port(q_ctx, interface_info['port_id'])
+            # here we assume the port has an IP
+            return port['fixed_ips'][0]['subnet_id']
+        else:
+            return interface_info['subnet_id']
+
     def _get_net_pods_by_interface_info(self, t_ctx, q_ctx, add_by_port,
                                         interface_info):
         net_id = self._get_net_id_by_interface_info(q_ctx, add_by_port,
@@ -1537,6 +1560,7 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
         return ret
 
     def validate_router_net_location_match(self, t_ctx, router, net):
+        is_local_router = self.helper.is_local_router(t_ctx, router)
         router_az_hints = self.helper.get_router_az_hints(router)
         if not router_az_hints:
             return
@@ -1545,6 +1569,12 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
 
         net_az_hints = net.get(az_ext.AZ_HINTS)
         if not net_az_hints:
+            if is_local_router:
+                # network az hints parameter is not specified, meaning that
+                # this network can be located in any pod, such network is
+                # allowed to be attached to a local router, for supporting
+                # multi-gateway l3 mode
+                return
             raise t_exceptions.RouterNetworkLocationMismatch(
                 router_az_hints=router_region_names,
                 net_az_hints=['All Region'])
@@ -1555,6 +1585,17 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
         else:
             net_region_names = self._convert_az2region(t_ctx, net_az_hints)
         net_region_set = set(net_region_names)
+
+        if is_local_router:
+            if router_region_set <= net_region_set:
+                # pods that this network can be located include the pod of the
+                # local router, this attachment is allowed, for supporting
+                # multi-gateway l3 mode
+                return
+            raise t_exceptions.RouterNetworkLocationMismatch(
+                router_az_hints=router_region_names,
+                net_az_hints=net_region_names)
+
         diff = net_region_set - router_region_set
         if diff:
             raise t_exceptions.RouterNetworkLocationMismatch(
@@ -1569,10 +1610,34 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
         add_by_port, _ = self._validate_interface_info(interface_info)
         net_id = self._get_net_id_by_interface_info(
             context, add_by_port, interface_info)
+        subnet_id = self._get_subnet_id_by_interface_info(
+            context, add_by_port, interface_info)
         net = self.get_network(context, net_id)
+        subnet = self.get_subnet(context, subnet_id)
         self.validate_router_net_location_match(t_ctx, router, net)
         is_local_router = self.helper.is_local_router(t_ctx, router)
 
+        if is_local_router:
+            other_infs = super(TricirclePlugin, self).get_ports(
+                context, filters={
+                    'network_id': [net_id],
+                    'device_owner': [constants.DEVICE_OWNER_ROUTER_INTF]})
+            for other_inf in other_infs:
+                if not other_inf['device_id']:
+                    continue
+                other_ip = other_inf['fixed_ips'][0]['ip_address']
+                other_router = super(
+                    TricirclePlugin, self).get_router(context,
+                                                      other_inf['device_id'])
+                if not self.helper.is_local_router(t_ctx, other_router) and (
+                        other_ip == subnet['gateway_ip']):
+                    # this network has already been attached to a non-local
+                    # router and the gateway port is on that router, in this
+                    # case, we don't allow this network to be attached other
+                    # local routers
+                    raise t_network_exc.NetAttachedToNonLocalRouter(
+                        network_id=net_id, router_id=other_inf['device_id'])
+
         t_pod = db_api.get_top_pod(t_ctx)
         assert t_pod
 
@@ -1584,8 +1649,24 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
             self._get_bridge_network_subnet(
                 t_ctx, context, project_id, t_pod, pool_id)
 
+        if is_local_router:
+            if self.helper.is_local_network(t_ctx, net):
+                router_region = self.helper.get_router_az_hints(router)[0]
+                b_client = self._get_client(router_region)
+                b_pod = db_api.get_pod_by_name(t_ctx, router_region)
+                # get bottom network will create bottom network and subnet
+                b_client.get_networks(t_ctx, net_id)
+                # create resource mapping so job will be triggered
+                db_api.create_resource_mapping(
+                    t_ctx, net_id, net_id, b_pod['pod_id'], net['project_id'],
+                    t_constants.RT_NETWORK)
+                db_api.create_resource_mapping(
+                    t_ctx, subnet_id, subnet_id, b_pod['pod_id'],
+                    subnet['project_id'], t_constants.RT_SUBNET)
+
         return_info = super(TricirclePlugin, self).add_router_interface(
             context, router_id, interface_info)
+
         _, b_pods = self._get_net_pods_by_interface_info(
             t_ctx, context, add_by_port, interface_info)
         if not b_pods:

tricircle/network/exceptions.py

@@ -38,3 +38,12 @@ class DhcpPortNotFound(exceptions.NotFound):
 class GatewayPortNotFound(exceptions.NotFound):
     message = _('Gateway port for subnet %(subnet_id)s and region %(region)s '
                 'not found')
+
+
+class CentralizedSNATPortNotFound(exceptions.NotFound):
+    message = _('Centralized snat port for subnet %(subnet_id)s not found')
+
+
+class NetAttachedToNonLocalRouter(exceptions.Conflict):
+    message = _('Network %(network_id)s has already been attached to non '
+                'local router %(router_id)s')

tricircle/network/helper.py

@@ -366,6 +366,8 @@ class NetworkHelper(object):
         t_gateway_ip = t_subnet['gateway_ip']
         new_pools = NetworkHelper._split_pools_by_bottom_gateway_ip(
             pools, b_gateway_ip)
+        if t_gateway_ip == b_gateway_ip:
+            return new_pools
         return NetworkHelper._merge_pools_by_top_gateway_ip(new_pools,
                                                             t_gateway_ip)
 
@@ -595,6 +597,42 @@ class NetworkHelper(object):
         }
         return body
 
+    def prepare_top_snat_port(self, t_ctx, q_ctx, project_id, t_net_id,
+                              t_subnet_id):
+        """Create top centralized snat port
+
+        :param t_ctx: tricircle context
+        :param q_ctx: neutron context
+        :param project_id: project id
+        :param t_net_id: top network id
+        :param t_subnet_id: top subnet id
+        :return: top centralized snat port
+        """
+        t_snat_name = t_constants.snat_port_name % t_subnet_id
+        t_snat_port_body = {
+            'port': {
+                'tenant_id': project_id,
+                'admin_state_up': True,
+                'network_id': t_net_id,
+                'name': t_snat_name,
+                'binding:profile': {},
+                'device_id': '',
+                'device_owner': constants.DEVICE_OWNER_ROUTER_SNAT,
+            }
+        }
+        if self.call_obj:
+            t_snat_port_body['port'].update(
+                {'mac_address': constants.ATTR_NOT_SPECIFIED,
+                 'fixed_ips': constants.ATTR_NOT_SPECIFIED})
+
+        # NOTE(zhiyuan) for one subnet in different pods, we just create one
+        # centralized snat port. though snat port in different pods will have
+        # the same IP, VM packets will only got to the local router namespace
+        _, t_snat_port_id = self.prepare_top_element(
+            t_ctx, q_ctx, project_id, db_api.get_top_pod(t_ctx),
+            {'id': t_snat_name}, t_constants.RT_PORT, t_snat_port_body)
+        return t_snat_port_id
+
     def prepare_top_dhcp_port(self, t_ctx, q_ctx, project_id, t_net_id,
                               t_subnet_id):
         """Create top dhcp port
@@ -706,6 +744,17 @@ class NetworkHelper(object):
             'port_range_min': rule.get('port_range_min'),
             'security_group_id': sg_id}}
 
+    @staticmethod
+    def convert_az2region(t_ctx, az_hints):
+        region_names = set()
+        for az_hint in az_hints:
+            pods = db_api.find_pods_by_az_or_region(t_ctx, az_hint)
+            if not pods:
+                continue
+            for pod in pods:
+                region_names.add(pod['region_name'])
+        return list(region_names)
+
     @staticmethod
     def get_router_az_hints(router):
         # when called by api, availability_zone_hints included in
@@ -731,6 +780,18 @@ class NetworkHelper(object):
         router_az_hint = router_az_hints[0]
         return bool(db_api.get_pod_by_name(t_ctx, router_az_hint))
 
+    @staticmethod
+    def is_local_network(t_ctx, net):
+        if net[provider_net.NETWORK_TYPE] == t_constants.NT_LOCAL:
+            return True
+        net_az_hints = net.get(AZ_HINTS)
+        if not net_az_hints:
+            return False
+        if len(net_az_hints) > 1:
+            return False
+        net_az_hint = net_az_hints[0]
+        return bool(db_api.get_pod_by_name(t_ctx, net_az_hint))
+
     @staticmethod
     def get_agent_type_by_vif(vif_type):
         return VIF_AGENT_TYPE_MAP.get(vif_type)

tricircle/network/local_plugin.py

@@ -402,7 +402,7 @@ class TricirclePlugin(plugin.Ml2Plugin):
         t_ctx = t_context.get_context_from_neutron_context(context)
         b_subnet = self.core_plugin.get_subnet(context, _id)
         origin_enable_dhcp = b_subnet['enable_dhcp']
-        req_enable_dhcp = subnet['subnet']['enable_dhcp']
+        req_enable_dhcp = subnet['subnet'].get('enable_dhcp')
         # when request enable dhcp, and origin dhcp is disabled,
         # ensure subnet dhcp port is created
         if req_enable_dhcp and not origin_enable_dhcp:
@@ -424,7 +424,15 @@ class TricirclePlugin(plugin.Ml2Plugin):
             return
         subnet_id = port['fixed_ips'][0]['subnet_id']
         t_subnet = self.neutron_handle.handle_get(t_ctx, 'subnet', subnet_id)
-        port['fixed_ips'][0]['ip_address'] = t_subnet['gateway_ip']
+        snat_port_name = t_constants.snat_port_name % t_subnet['id']
+        raw_client = self.neutron_handle._get_client(t_ctx)
+        params = {'name': snat_port_name}
+        t_ports = raw_client.list_ports(**params)['ports']
+        if not t_ports:
+            raise t_exceptions.CentralizedSNATPortNotFound(
+                subnet_id=t_subnet['id'])
+        port['fixed_ips'][0][
+            'ip_address'] = t_ports[0]['fixed_ips'][0]['ip_address']
 
     def create_port_bulk(self, context, ports):
         # NOTE(zhiyuan) currently this bulk operation is only for shadow port

tricircle/tests/unit/network/test_central_plugin.py

@@ -1042,6 +1042,23 @@ class FakeBaseXManager(xmanager.XManager):
     def _get_client(self, region_name=None):
         return FakeClient(region_name)
 
+    def setup_bottom_router(self, ctx, payload):
+        (b_pod_id,
+         t_router_id, t_net_id) = payload[constants.JT_ROUTER_SETUP].split('#')
+
+        if b_pod_id == constants.POD_NOT_SPECIFIED:
+            mappings = db_api.get_bottom_mappings_by_top_id(
+                ctx, t_net_id, constants.RT_NETWORK)
+            b_pods = [mapping[0] for mapping in mappings]
+            for b_pod in b_pods:
+                resource_id = '%s#%s#%s' % (b_pod['pod_id'],
+                                            t_router_id, t_net_id)
+                _payload = {constants.JT_ROUTER_SETUP: resource_id}
+                super(FakeBaseXManager,
+                      self).setup_bottom_router(ctx, _payload)
+        else:
+            super(FakeBaseXManager, self).setup_bottom_router(ctx, payload)
+
 
 class FakeXManager(FakeBaseXManager):
     def __init__(self, fake_plugin):
@@ -1919,6 +1936,7 @@ class PluginTest(unittest.TestCase,
                 'id': t_net_id,
                 'name': 'top_net_%d' % index,
                 'tenant_id': project_id,
+                'project_id': project_id,
                 'description': 'description',
                 'admin_state_up': False,
                 'shared': False,
@@ -1937,6 +1955,7 @@ class PluginTest(unittest.TestCase,
                 'ipv6_address_mode': '',
                 'ipv6_ra_mode': '',
                 'tenant_id': project_id,
+                'project_id': project_id,
                 'description': 'description',
                 'host_routes': [],
                 'dns_nameservers': []
@@ -1950,8 +1969,6 @@ class PluginTest(unittest.TestCase,
                                                      'comparator': 'eq',
                                                      'value': t_subnet_name}])
             t_subnet_id = t_subnets[0]['id']
-            # top and bottom ids are the same
-            return t_net_id, t_subnet_id, t_net_id, t_subnet_id
 
         b_net_id = t_net_id
         b_subnet_id = t_subnet_id
@@ -1959,6 +1976,7 @@ class PluginTest(unittest.TestCase,
             'id': b_net_id,
             'name': t_net_id,
             'tenant_id': project_id,
+            'project_id': project_id,
             'description': 'description',
             'admin_state_up': False,
             'shared': False,
@@ -1976,6 +1994,7 @@ class PluginTest(unittest.TestCase,
             'ipv6_address_mode': '',
             'ipv6_ra_mode': '',
             'tenant_id': project_id,
+            'project_id': project_id,
             'description': 'description',
             'host_routes': [],
             'dns_nameservers': []
@@ -2057,6 +2076,21 @@ class PluginTest(unittest.TestCase,
                               'resource_type': constants.RT_PORT})
         return t_port_id, b_port_id
 
+    def _prepare_router(self, project_id, router_az_hints=None):
+        t_router_id = uuidutils.generate_uuid()
+        t_router = {
+            'id': t_router_id,
+            'name': 'top_router',
+            'distributed': False,
+            'tenant_id': project_id,
+            'attached_ports': DotList(),
+            'extra_attributes': {
+                'availability_zone_hints': router_az_hints
+            }
+        }
+        TOP_ROUTERS.append(DotDict(t_router))
+        return t_router_id
+
     def _prepare_router_test(self, tenant_id, ctx, region_name, index,
                              router_az_hints=None, net_az_hints=None,
                              create_new_router=False,
@@ -2065,19 +2099,8 @@ class PluginTest(unittest.TestCase,
          b_subnet_id) = self._prepare_network_subnet(
             tenant_id, ctx, region_name, index, az_hints=net_az_hints,
             network_type=network_type)
-        t_router_id = uuidutils.generate_uuid()
-        t_router = {
-            'id': t_router_id,
-            'name': 'top_router',
-            'distributed': False,
-            'tenant_id': tenant_id,
-            'attached_ports': DotList(),
-            'extra_attributes': {
-                'availability_zone_hints': router_az_hints
-            }
-        }
         if create_new_router or len(TOP_ROUTERS) == 0:
-            TOP_ROUTERS.append(DotDict(t_router))
+            t_router_id = self._prepare_router(tenant_id, router_az_hints)
         else:
             t_router_id = TOP_ROUTERS[0]['id']
 
@@ -2565,9 +2588,12 @@ class PluginTest(unittest.TestCase,
             tenant_id, t_ctx, 'pod_1', 2, router_az_hints, net_az_hints, True)
         router = fake_plugin._get_router(q_ctx, t_router_id)
         net = fake_plugin.get_network(q_ctx, t_net_id)
-        self.assertRaises(t_exceptions.RouterNetworkLocationMismatch,
-                          fake_plugin.validate_router_net_location_match,
-                          t_ctx, router, net)
+        is_local_router = helper.NetworkHelper.is_local_router(t_ctx, router)
+        fake_plugin.validate_router_net_location_match(t_ctx, router, net)
+        # for supporting multi-gateway l3 mode, we allow attaching a network
+        # to a local router if the regions of the network include the region
+        # of the router
+        self.assertEqual(True, is_local_router)
 
         router_az_hints = '["az_name_1"]'
         net_az_hints = '["az_name_1", "pod_2"]'
@@ -2587,9 +2613,12 @@ class PluginTest(unittest.TestCase,
             tenant_id, t_ctx, 'pod_1', 4, router_az_hints, net_az_hints, True)
         router = fake_plugin._get_router(q_ctx, t_router_id)
         net = fake_plugin.get_network(q_ctx, t_net_id)
-        self.assertRaises(t_exceptions.RouterNetworkLocationMismatch,
-                          fake_plugin.validate_router_net_location_match,
-                          t_ctx, router, net)
+        is_local_router = helper.NetworkHelper.is_local_router(t_ctx, router)
+        fake_plugin.validate_router_net_location_match(t_ctx, router, net)
+        # for supporting multi-gateway l3 mode, we allow attaching a network
+        # to a local router if the regions of the network include the region
+        # of the router
+        self.assertEqual(True, is_local_router)
 
         router_az_hints = None
         net_az_hints = '["pod_1", "az_name_2"]'
@@ -2855,6 +2884,156 @@ class PluginTest(unittest.TestCase,
             t_ctx, b_router_id, {'port_id': b_interface_id})
         mock_rpc.assert_called_with(t_ctx, t_router_id)
 
+    def _prepare_interface_port(self, t_ctx, t_subnet_id, ip_suffix):
+        t_client = FakeClient()
+        t_subnet = t_client.get_subnets(t_ctx, t_subnet_id)
+        t_net = t_client.get_networks(t_ctx, t_subnet['network_id'])
+        t_port_id = uuidutils.generate_uuid()
+        t_port = {
+            'id': t_port_id,
+            'network_id': t_net['id'],
+            'device_id': '',
+            'device_owner': '',
+            'fixed_ips': [{'subnet_id': t_subnet['id'],
+                           'ip_address': '%s%d' % (
+                               t_subnet['cidr'][:-4], ip_suffix)}],
+            'mac_address': 'fa:16:3e:d4:%02x:%02x' % (
+                int(t_subnet['cidr'].split('.')[2]), ip_suffix),
+            'security_groups': [],
+            'tenant_id': t_subnet['tenant_id']
+        }
+        TOP_PORTS.append(DotDict(t_port))
+        return t_port_id
+
+    @patch.object(directory, 'get_plugin', new=fake_get_plugin)
+    @patch.object(driver.Pool, 'get_instance', new=fake_get_instance)
+    @patch.object(ipam_pluggable_backend.IpamPluggableBackend,
+                  '_allocate_ips_for_port', new=fake_allocate_ips_for_port)
+    @patch.object(db_base_plugin_common.DbBasePluginCommon,
+                  '_make_subnet_dict', new=fake_make_subnet_dict)
+    @patch.object(l3_db.L3_NAT_dbonly_mixin, '_make_router_dict',
+                  new=fake_make_router_dict)
+    @patch.object(FakeClient, 'add_gateway_routers')
+    @patch.object(FakeBaseRPCAPI, 'configure_extra_routes')
+    @patch.object(context, 'get_context_from_neutron_context')
+    def test_east_west_gw_router(self, mock_context, mock_rpc, mock_action):
+        self._basic_pod_route_setup()
+
+        fake_plugin = FakePlugin()
+        q_ctx = FakeNeutronContext()
+        t_ctx = context.get_db_context()
+        mock_context.return_value = t_ctx
+        tenant_id = TEST_TENANT_ID
+
+        # prepare three networks, net1 in pod1, net2 in pod2, net3 is in both
+        # pod1 and pod2
+        (t_net1_id, t_subnet1_id, b_net1_id,
+         b_subnet1_id) = self._prepare_network_subnet(
+            tenant_id, t_ctx, 'pod_1', 1)
+        (t_net2_id, t_subnet2_id, b_net2_id,
+         b_subnet2_id) = self._prepare_network_subnet(
+            tenant_id, t_ctx, 'pod_2', 2)
+        (t_net3_id, t_subnet3_id, b_net3_id,
+         b_subnet3_id) = self._prepare_network_subnet(
+            tenant_id, t_ctx, 'pod_1', 3)
+        self._prepare_network_subnet(tenant_id, t_ctx, 'pod_2', 3)
+        t_subnet_ids = [t_subnet1_id, t_subnet2_id, t_subnet3_id]
+        b_net_ids = [b_net1_id, b_net2_id, b_net3_id]
+
+        # prepare three routers, router1 and router2 are local routers
+        t_router1_id = self._prepare_router(tenant_id, ['pod_1'])
+        t_router2_id = self._prepare_router(tenant_id, ['pod_2'])
+        t_router3_id = self._prepare_router(tenant_id)
+        t_router_ids = [t_router1_id, t_router2_id, t_router3_id]
+
+        inf1_id = self._prepare_interface_port(t_ctx, t_subnet1_id, 5)
+        inf2_id = self._prepare_interface_port(t_ctx, t_subnet2_id, 5)
+        inf3_1_id = self._prepare_interface_port(t_ctx, t_subnet3_id, 5)
+        inf3_2_id = self._prepare_interface_port(t_ctx, t_subnet3_id, 6)
+
+        # attach router interface, net1 is attached to router1 and router3,
+        # default gateway is on router1; net2 is attached to router2 and
+        # router3, default gateway is on router2; net3 is attached to router1,
+        # router2 and router3
+        fake_plugin.add_router_interface(
+            q_ctx, t_router1_id, {'subnet_id': t_subnet1_id})['port_id']
+        fake_plugin.add_router_interface(
+            q_ctx, t_router3_id, {'port_id': inf1_id})
+        fake_plugin.add_router_interface(
+            q_ctx, t_router2_id, {'subnet_id': t_subnet2_id})['port_id']
+        fake_plugin.add_router_interface(
+            q_ctx, t_router3_id, {'port_id': inf2_id})
+        fake_plugin.add_router_interface(
+            q_ctx, t_router1_id, {'subnet_id': t_subnet3_id})['port_id']
+        fake_plugin.add_router_interface(
+            q_ctx, t_router2_id, {'port_id': inf3_1_id})
+        fake_plugin.add_router_interface(
+            q_ctx, t_router3_id, {'port_id': inf3_2_id})
+
+        b_router_id_map = {}
+        for pod_idx, router_idx in [(1, 1), (2, 2), (1, 3), (2, 3)]:
+            b_router_id = db_api.get_bottom_id_by_top_id_region_name(
+                t_ctx, t_router_ids[router_idx - 1], 'pod_%d' % pod_idx,
+                constants.RT_ROUTER)
+            b_router_id_map[(pod_idx, router_idx)] = b_router_id
+
+        actual_ips_map = {}
+        for pod_idx, net_idx, router_idx in [
+                (1, 1, 1), (1, 1, 3), (1, 3, 1), (1, 3, 3),
+                (2, 2, 2), (2, 2, 3), (2, 3, 2), (2, 3, 3)]:
+            b_router_id = b_router_id_map[(pod_idx, router_idx)]
+            b_ports = BOTTOM1_PORTS if pod_idx == 1 else BOTTOM2_PORTS
+            b_infs = [
+                e for e in b_ports if e['device_id'] == b_router_id and (
+                    e['network_id'] == b_net_ids[net_idx - 1])]
+            inf_ip = b_infs[0]['fixed_ips'][0]['ip_address']
+            actual_ips_map[(pod_idx, net_idx, router_idx)] = inf_ip
+
+        t_infs_ip_map = {}
+        for pod_idx, subnet_idx in [(1, 1), (1, 3), (2, 2), (2, 3)]:
+            inf_name = 'interface_pod_%d_%s' % (pod_idx,
+                                                t_subnet_ids[subnet_idx - 1])
+            infs = [e for e in TOP_PORTS if e.get('name') == inf_name]
+            t_infs_ip_map[(pod_idx, subnet_idx)] = infs[0][
+                'fixed_ips'][0]['ip_address']
+
+        t_client = FakeClient()
+        t_subnet1 = t_client.get_subnets(t_ctx, t_subnet1_id)
+        t_subnet2 = t_client.get_subnets(t_ctx, t_subnet2_id)
+        t_subnet3 = t_client.get_subnets(t_ctx, t_subnet3_id)
+        inf3_1 = t_client.get_ports(t_ctx, inf3_1_id)
+
+        # tuple means (pod_idx, net_idx, router_idx)
+        # (1, 1, 1) net1 attached to router1, subnet gateway is used as the
+        #           interface ip
+        # (1, 1, 3) net1 attached to router3, reserved gateway for non-local
+        #           router is used as the interface ip
+        # (1, 3, 1) net3 attached to router1, top and bottom interface ips
+        #           are the same
+        # (1, 3, 3) net3 attached to router3 in pod1, reserved gateway for
+        #           non-local router is used as the interface ip
+        # (2, 2, 2) net2 attached to router2, subnet gateway is used as the
+        #           interface ip
+        # (2, 2, 3) net2 attached to router3, reserved gateway for non-local
+        #           router is used as the interface ip
+        # (2, 3, 2) net3 attached to router2, top and bottom interface ips
+        #           are the same
+        # (2, 3, 3) net3 attached to router3 in pod2, reserved gateway for
+        #           non-local router is used as the interface ip
+        expect_ips_map = {
+            (1, 1, 1): t_subnet1['gateway_ip'],
+            (1, 1, 3): t_infs_ip_map[(1, 1)],
+            (1, 3, 1): t_subnet3['gateway_ip'],
+            (1, 3, 3): t_infs_ip_map[(1, 3)],
+            (2, 2, 2): t_subnet2['gateway_ip'],
+            (2, 2, 3): t_infs_ip_map[(2, 2)],
+            (2, 3, 2): inf3_1['fixed_ips'][0]['ip_address'],
+            (2, 3, 3): t_infs_ip_map[(2, 3)]
+        }
+
+        for key in actual_ips_map:
+            self.assertEqual(expect_ips_map[key], actual_ips_map[key])
+
     @patch.object(directory, 'get_plugin', new=fake_get_plugin)
     @patch.object(context, 'get_context_from_neutron_context')
     def test_create_external_network_no_az_pod(self, mock_context):

tricircle/tests/unit/xjob/test_xmanager.py

@@ -158,6 +158,9 @@ class FakeClient(object):
             'subnet', cxt,
             [{'key': 'id', 'comparator': 'eq', 'value': subnet_id}])[0]
 
+    def update_subnets(self, cxt, subnet_id, body):
+        pass
+
     def get_networks(self, cxt, net_id):
         return self.list_resources(
             'network', cxt,
@@ -432,6 +435,144 @@ class XManagerTest(unittest.TestCase):
             calls.append(call)
         self._check_extra_routes_calls(calls, mock_update.call_args_list)
 
+    @patch.object(FakeClient, 'update_subnets')
+    @patch.object(FakeClient, 'update_routers')
+    def test_configure_extra_routes_ew_gw(self, router_update, subnet_update):
+        for i in (1, 2):
+            pod_dict = {'pod_id': 'pod_id_%d' % i,
+                        'region_name': 'pod_%d' % i,
+                        'az_name': 'az_name_%d' % i}
+            db_api.create_pod(self.context, pod_dict)
+        for i in (1, 2, 3):
+            router = {'id': 'top_router_%d_id' % i}
+            TOP_ROUTER.append(router)
+
+        # gateway in podX is attached to routerX
+        gw_map = {'net1_pod1_gw': '10.0.1.1',
+                  'net2_pod2_gw': '10.0.2.1',
+                  'net3_pod1_gw': '10.0.3.3',
+                  'net3_pod2_gw': '10.0.3.4'}
+        # interfaces are all attached to router3
+        inf_map = {'net1_pod1_inf': '10.0.1.3',
+                   'net2_pod2_inf': '10.0.2.3',
+                   'net3_pod1_inf': '10.0.3.5',
+                   'net3_pod2_inf': '10.0.3.6'}
+        get_gw_map = lambda n_idx, p_idx: gw_map[
+            'net%d_pod%d_gw' % (n_idx, p_idx)]
+        get_inf_map = lambda n_idx, p_idx: inf_map[
+            'net%d_pod%d_inf' % (n_idx, p_idx)]
+        bridge_infos = []
+
+        for net_idx, router_idx, pod_idx in [(1, 1, 1), (3, 1, 1), (1, 3, 1),
+                                             (3, 3, 1), (2, 2, 2), (3, 2, 2),
+                                             (2, 3, 2), (3, 3, 2)]:
+            region_name = 'pod_%d' % pod_idx
+            pod_id = 'pod_id_%d' % pod_idx
+            top_router_id = 'top_router_%d_id' % router_idx
+
+            network = {'id': 'network_%d_id' % net_idx}
+            router = {'id': 'router_%d_%d_id' % (pod_idx, router_idx)}
+            subnet = {'id': 'subnet_%d_id' % net_idx,
+                      'network_id': network['id'],
+                      'cidr': '10.0.%d.0/24' % net_idx,
+                      'gateway_ip': get_gw_map(net_idx, pod_idx)}
+            port = {'network_id': network['id'],
+                    'device_id': router['id'],
+                    'device_owner': 'network:router_interface',
+                    'fixed_ips': [{'subnet_id': subnet['id']}]}
+            if router_idx == 3:
+                port['fixed_ips'][0][
+                    'ip_address'] = get_inf_map(net_idx, pod_idx)
+            else:
+                port['fixed_ips'][0][
+                    'ip_address'] = get_gw_map(net_idx, pod_idx)
+
+            if net_idx == pod_idx and router_idx == 3:
+                vm_idx = net_idx * 2 + pod_idx + 10
+                vm_ip = '10.0.%d.%d' % (net_idx, vm_idx)
+                vm_port = {'id': 'vm_port_%d_id' % vm_idx,
+                           'network_id': network['id'],
+                           'device_id': 'vm%d_id' % vm_idx,
+                           'device_owner': 'compute:None',
+                           'fixed_ips': [{'subnet_id': subnet['id'],
+                                          'ip_address': vm_ip}]}
+                bridge_network = {'id': 'bridge_network_%d_id' % net_idx}
+                bridge_subnet = {'id': 'bridge_subnet_%d_id' % net_idx,
+                                 'network_id': bridge_network['id'],
+                                 'cidr': '100.0.1.0/24',
+                                 'gateway_ip': '100.0.1.1'}
+                bridge_cidr = bridge_subnet['cidr']
+                bridge_port_ip = '%s.%d' % (
+                    bridge_cidr[:bridge_cidr.rindex('.')], 2 + pod_idx)
+                bridge_infos.append({'router_id': router['id'],
+                                     'bridge_ip': bridge_port_ip,
+                                     'vm_ip': vm_ip})
+                bridge_port = {
+                    'network_id': bridge_network['id'],
+                    'device_id': router['id'],
+                    'device_owner': 'network:router_gateway',
+                    'fixed_ips': [{'subnet_id': bridge_subnet['id'],
+                                   'ip_address': bridge_port_ip}]
+                }
+                RES_MAP[region_name]['port'].append(vm_port)
+                RES_MAP[region_name]['network'].append(bridge_network)
+                RES_MAP[region_name]['subnet'].append(bridge_subnet)
+                RES_MAP[region_name]['port'].append(bridge_port)
+
+            RES_MAP[region_name]['network'].append(network)
+            RES_MAP[region_name]['subnet'].append(subnet)
+            RES_MAP[region_name]['port'].append(port)
+            RES_MAP[region_name]['router'].append(router)
+
+            db_api.create_resource_mapping(self.context, top_router_id,
+                                           router['id'], pod_id, 'project_id',
+                                           constants.RT_ROUTER)
+        # the above codes create this topology
+        # pod1: net1 is attached to R1, default gateway is set on R1
+        #       net1 is attached to R3
+        #       net3 is attached to R1, default gateway is set on R1
+        #       net3 is attached to R3
+        # pod2: net2 is attached to R2, default gateway is set on R2
+        #       net2 is attached to R3
+        #       net3 is attached to R2, default gateway is set on R2
+        #       net3 is attached to R3
+
+        target_router_id = 'top_router_3_id'
+        db_api.new_job(self.context, constants.JT_ROUTER, target_router_id)
+        self.xmanager.configure_extra_routes(
+            self.context, payload={constants.JT_ROUTER: target_router_id})
+
+        # for the following paths, packets will go to R3 via the interface
+        # which is attached to R3
+        # net1 in pod1 -> net2 in pod2
+        # net2 in pod2 -> net1 in pod1
+        # net3 in pod1 -> net2 in pod2
+        # net3 in pod2 -> net1 in pod1
+        expect_calls = [
+            mock.call(self.context, 'subnet_1_id', {'subnet': {
+                'host_routes': [{'nexthop': get_inf_map(1, 1),
+                                 'destination': '10.0.2.0/24'}]}}),
+            mock.call(self.context, 'subnet_2_id', {'subnet': {
+                'host_routes': [{'nexthop': get_inf_map(2, 2),
+                                 'destination': '10.0.1.0/24'}]}}),
+            mock.call(self.context, 'subnet_3_id', {'subnet': {
+                'host_routes': [{'nexthop': get_inf_map(3, 1),
+                                 'destination': '10.0.2.0/24'}]}}),
+            mock.call(self.context, 'subnet_3_id', {'subnet': {
+                'host_routes': [{'nexthop': get_inf_map(3, 2),
+                                 'destination': '10.0.1.0/24'}]}})]
+        subnet_update.assert_has_calls(expect_calls, any_order=True)
+        expect_calls = []
+        for i in (0, 1):
+            bridge_info = bridge_infos[i]
+            expect_call = mock.call(
+                self.context, bridge_infos[1 - i]['router_id'],
+                {'router': {'routes': [
+                    {'nexthop': bridge_info['bridge_ip'],
+                     'destination': bridge_info['vm_ip'] + '/32'}]}})
+            expect_calls.append(expect_call)
+        router_update.assert_has_calls(expect_calls, any_order=True)
+
     @patch.object(FakeClient, 'delete_security_group_rules')
     @patch.object(FakeClient, 'create_security_group_rules')
     def test_configure_security_group_rules(self, mock_create, mock_delete):

tricircle/xjob/xmanager.py

@@ -305,13 +305,23 @@ class XManager(PeriodicTasks):
         router_body = {'router': {'name': t_router['id'],
                                   'distributed': is_distributed}}
         project_id = t_router['tenant_id']
+        q_ctx = None  # no need to pass neutron context when using client
+        is_local_router = self.helper.is_local_router(ctx, t_router)
+
+        if is_local_router:
+            # for local router, it's safe for us to get the first element as
+            # pod name
+            pod_name = self.helper.get_router_az_hints(t_router)[0]
+            if pod_name != b_pod['region_name']:
+                # now we allow to attach a cross-pod network to a local router,
+                # so if the pod of the local router is different from the pod
+                # of the bottom network, we do nothing.
+                return
 
         # create bottom router in target bottom pod
         _, b_router_id = self.helper.prepare_bottom_element(
             ctx, project_id, b_pod, t_router, constants.RT_ROUTER, router_body)
 
-        q_ctx = None  # no need to pass neutron context when using client
-        is_local_router = self.helper.is_local_router(ctx, t_router)
         if not is_local_router:
             # create top bridge port
             t_bridge_port_id = self.helper.get_bridge_interface(
@@ -360,7 +370,9 @@ class XManager(PeriodicTasks):
 
             # only consider ipv4 address currently
             t_subnet_id = t_port['fixed_ips'][0]['subnet_id']
+            t_port_ip = t_port['fixed_ips'][0]['ip_address']
             t_subnet = t_client.get_subnets(ctx, t_subnet_id)
+            is_default_gw = t_port_ip == t_subnet['gateway_ip']
 
             if CONF.enable_api_gateway:
                 (b_net_id,
@@ -370,12 +382,49 @@ class XManager(PeriodicTasks):
                 (b_net_id,
                  subnet_map) = (t_net['id'], {t_subnet['id']: t_subnet['id']})
 
-            # the gateway ip of bottom subnet is set to the ip of t_port, so
-            # we just attach the bottom subnet to the bottom router and neutron
-            # server in the bottom pod will create the interface for us, using
-            # the gateway ip.
-            b_client.action_routers(ctx, 'add_interface', b_router_id,
-                                    {'subnet_id': subnet_map[t_subnet_id]})
+            if is_local_router:
+                # if the attaching router is local router, we update the bottom
+                # subnet gateway ip to the interface ip
+                new_pools = self.helper.get_bottom_subnet_pools(t_subnet,
+                                                                t_port_ip)
+                b_client.update_subnets(ctx, t_subnet_id,
+                                        {'subnet': {
+                                            'gateway_ip': t_port_ip,
+                                            'allocation_pools': new_pools}})
+                b_client.action_routers(
+                    ctx, 'add_interface', b_router_id,
+                    {'subnet_id': subnet_map[t_subnet_id]})
+            else:
+                # the attaching router is not local router
+                if is_default_gw:
+                    # if top interface ip is equal to gateway ip of top subnet,
+                    # bottom subnet gateway is set to the ip of the reservered
+                    # gateway port, so we just attach the bottom subnet to the
+                    # bottom router and local neutron server will create the
+                    # interface for us, using the gateway ip.
+                    b_client.action_routers(
+                        ctx, 'add_interface', b_router_id,
+                        {'subnet_id': subnet_map[t_subnet_id]})
+                else:
+                    # if top interface ip is different from gateway ip of top
+                    # subnet, meaning that this interface is explicitly created
+                    # by users, then the subnet may be already attached to a
+                    # local router and its gateway ip is changed, so we need to
+                    # query the reservered gateway port to get its ip.
+                    gateway_port_name = constants.interface_port_name % (
+                        b_pod['region_name'], t_subnet['id'])
+                    gateway_port = t_client.list_ports(
+                        ctx, filters=[{'key': 'name',
+                                       'comparator': 'eq',
+                                       'value': gateway_port_name}])[0]
+                    b_port_body = self.helper.get_create_port_body(
+                        gateway_port['project_id'], gateway_port,
+                        {t_subnet_id: t_subnet_id}, b_net_id)
+                    b_port_body['port'][
+                        'device_owner'] = q_constants.DEVICE_OWNER_ROUTER_INTF
+                    b_port = b_client.create_ports(ctx, b_port_body)
+                    b_client.action_routers(ctx, 'add_interface', b_router_id,
+                                            {'port_id': b_port['id']})
 
         if not t_router['external_gateway_info']:
             return
@@ -560,6 +609,10 @@ class XManager(PeriodicTasks):
         router_ew_bridge_ip_map = {}
         router_ns_bridge_ip_map = {}
         router_ips_map = {}
+
+        pod_subnet_nexthop_map = {}  # {pod_name: {subnet_id: nexthop}
+        subnet_cidr_map = {}  # {subnet_id: cidr}
+
         for i, b_pod in enumerate(b_pods):
             is_ns_router = b_router_ids[i] == b_ns_router_id
             bottom_client = self._get_client(b_pod['region_name'])
@@ -575,6 +628,8 @@ class XManager(PeriodicTasks):
                                'comparator': 'eq',
                                'value': device_owner_filter}])
             router_ips_map[b_router_ids[i]] = {}
+            pod_subnet_nexthop_map[b_pod['region_name']] = {}
+
             for b_interface in b_interfaces:
                 ip = b_interface['fixed_ips'][0]['ip_address']
                 bridge_cidr = CONF.client.bridge_cidr
@@ -588,8 +643,18 @@ class XManager(PeriodicTasks):
                         router_ew_bridge_ip_map[b_router_ids[i]] = ip
                     continue
                 b_net_id = b_interface['network_id']
-                b_subnet = bottom_client.get_subnets(
-                    ctx, b_interface['fixed_ips'][0]['subnet_id'])
+                b_subnet_id = b_interface['fixed_ips'][0]['subnet_id']
+
+                b_subnet = bottom_client.get_subnets(ctx, b_subnet_id)
+                if b_subnet['gateway_ip'] != ip:
+                    # ip of the interface attached to the non local router is
+                    # different from the gateway ip, meaning that the interface
+                    # is for east-west traffic purpose, so we save necessary
+                    # information for next process
+                    pod_subnet_nexthop_map[
+                        b_pod['region_name']][b_subnet_id] = ip
+                    subnet_cidr_map[b_subnet_id] = b_subnet['cidr']
+
                 b_ports = bottom_client.list_ports(
                     ctx, filters=[{'key': 'network_id',
                                    'comparator': 'eq',
@@ -631,6 +696,20 @@ class XManager(PeriodicTasks):
             bottom_client.update_routers(
                 ctx, b_router_id, {'router': {'routes': extra_routes}})
 
+        # configure host routes for local network attached to local router
+        for (pod_name,
+             subnet_nexthop_map) in pod_subnet_nexthop_map.items():
+            for subnet_id, nexthop in subnet_nexthop_map.items():
+                host_routes = []
+                for _subnet_id, cidr in subnet_cidr_map.items():
+                    if _subnet_id in subnet_nexthop_map:
+                        continue
+                    host_routes.append({'destination': cidr,
+                                        'nexthop': nexthop})
+                bottom_client = self._get_client(pod_name)
+                bottom_client.update_subnets(
+                    ctx, subnet_id, {'subnet': {'host_routes': host_routes}})
+
         if not b_ns_router_id:
             # router for north-south networking not exist, skip extra routes
             # configuration for north-south router