Support Dual-Stack Gateway Ports on Neutron Routers

(Patch set #2 for multiple-ipv6-prefixes blueprint)

This patchset adds support for dual-stack gateway ports on Neutron
routers. Some background on the changes included in this patchset:

- The L3 driver's init_l3() method has been changed to accept a list
  of gateway IPs, rather than a single gateway IP.
- The Neutron port dictionary's singular 'subnet' entry has been
  replaced with a 'subnets' list, since ports can now be associated
  with multiple subnets.
- The Neutron port dictionary no longer has a (singular) 'ip_cidr'
  entry, since a port can now be associated with multiple IP CIDRs
  (e.g. up to one IP CIDR per IP family on gateway ports).
  Instead, a 'prefixlen' entry has been added to the Neutron
  fixed_ips dictionary, so that the port's (multiple) IP CIDRs can
  be derived from the matching 'ip_address' and 'prefixlen' pairs
  in the port's fixed_ips.

Change-Id: I150da5938e79eeef0c947ddb1a4282e37d0515ee
Partially-implements: blueprint multiple-ipv6-prefixes

neutron/agent/l3/dvr_fip_ns.py

@@ -14,7 +14,6 @@
 
 import os
 
-import netaddr
 from oslo_log import log as logging
 
 from neutron.agent.l3 import link_local_allocator as lla
@@ -100,20 +99,21 @@ class FipNamespace(namespaces.Namespace):
                              namespace=ns_name,
                              prefix=FIP_EXT_DEV_PREFIX)
 
-        self.driver.init_l3(interface_name,
-                            [ex_gw_port['ip_cidr']],
-                            namespace=ns_name)
+        ip_cidrs = common_utils.fixed_ip_cidrs(ex_gw_port['fixed_ips'])
+        self.driver.init_l3(interface_name, ip_cidrs, namespace=ns_name)
 
-        ip_address = str(netaddr.IPNetwork(ex_gw_port['ip_cidr']).ip)
-        ip_lib.send_gratuitous_arp(ns_name,
-                                   interface_name,
-                                   ip_address,
-                                   self.agent_conf.send_arp_for_ha)
+        for fixed_ip in ex_gw_port['fixed_ips']:
+            ip_lib.send_gratuitous_arp(ns_name,
+                                       interface_name,
+                                       fixed_ip['ip_address'],
+                                       self.agent_conf.send_arp_for_ha)
 
-        gw_ip = ex_gw_port['subnet']['gateway_ip']
-        if gw_ip:
-            ipd = ip_lib.IPDevice(interface_name, namespace=ns_name)
-            ipd.route.add_gateway(gw_ip)
+        for subnet in ex_gw_port['subnets']:
+            gw_ip = subnet.get('gateway_ip')
+            if gw_ip:
+                ipd = ip_lib.IPDevice(interface_name,
+                                      namespace=ns_name)
+                ipd.route.add_gateway(gw_ip)
 
         cmd = ['sysctl', '-w', 'net.ipv4.conf.%s.proxy_arp=1' % interface_name]
         # TODO(Carl) mlavelle's work has self.ip_wrapper

neutron/agent/l3/dvr_router.py

@@ -199,13 +199,8 @@ class DvrRouter(router.RouterInfo):
             with excutils.save_and_reraise_exception():
                 LOG.exception(_LE("DVR: Failed updating arp entry"))
 
-    def _set_subnet_arp_info(self, port):
+    def _set_subnet_arp_info(self, subnet_id):
         """Set ARP info retrieved from Plugin for existing ports."""
-        if 'id' not in port['subnet']:
-            return
-
-        subnet_id = port['subnet']['id']
-
         # TODO(Carl) Can we eliminate the need to make this RPC while
         # processing a router.
         subnet_ports = self.agent.get_ports_by_subnet(subnet_id)
@@ -251,32 +246,50 @@ class DvrRouter(router.RouterInfo):
             snat_idx = net.value
         return snat_idx
 
-    def _snat_redirect_add(self, gateway, sn_port, sn_int):
-        """Adds rules and routes for SNAT redirection."""
+    def _snat_redirect_modify(self, gateway, sn_port, sn_int, is_add):
+        """Adds or removes rules and routes for SNAT redirection."""
         try:
-            ip_cidr = sn_port['ip_cidr']
-            snat_idx = self._get_snat_idx(ip_cidr)
             ns_ipr = ip_lib.IPRule(namespace=self.ns_name)
             ns_ipd = ip_lib.IPDevice(sn_int, namespace=self.ns_name)
-            ns_ipwrapr = ip_lib.IPWrapper(namespace=self.ns_name)
-            ns_ipd.route.add_gateway(gateway, table=snat_idx)
-            ns_ipr.rule.add(ip_cidr, snat_idx, snat_idx)
-            ns_ipwrapr.netns.execute(['sysctl', '-w', 'net.ipv4.conf.%s.'
-                                     'send_redirects=0' % sn_int])
+            if is_add:
+                ns_ipwrapr = ip_lib.IPWrapper(namespace=self.ns_name)
+            for port_fixed_ip in sn_port['fixed_ips']:
+                # Find the first gateway IP address matching this IP version
+                port_ip_addr = port_fixed_ip['ip_address']
+                port_ip_vers = netaddr.IPAddress(port_ip_addr).version
+                for gw_fixed_ip in gateway['fixed_ips']:
+                    gw_ip_addr = gw_fixed_ip['ip_address']
+                    if netaddr.IPAddress(gw_ip_addr).version == port_ip_vers:
+                        sn_port_cidr = common_utils.ip_to_cidr(
+                            port_ip_addr, port_fixed_ip['prefixlen'])
+                        snat_idx = self._get_snat_idx(sn_port_cidr)
+                        if is_add:
+                            ns_ipd.route.add_gateway(gw_ip_addr,
+                                                     table=snat_idx)
+                            ns_ipr.rule.add(sn_port_cidr, snat_idx, snat_idx)
+                            ns_ipwrapr.netns.execute(
+                                ['sysctl', '-w',
+                                 'net.ipv4.conf.%s.send_redirects=0' % sn_int])
+                        else:
+                            ns_ipd.route.delete_gateway(gw_ip_addr,
+                                                        table=snat_idx)
+                            ns_ipr.rule.delete(sn_port_cidr, snat_idx,
+                                               snat_idx)
+                        break
         except Exception:
-            LOG.exception(_LE('DVR: error adding redirection logic'))
+            if is_add:
+                exc = _LE('DVR: error adding redirection logic')
+            else:
+                exc = _LE('DVR: removed snat failed')
+            LOG.exception(exc)
+
+    def _snat_redirect_add(self, gateway, sn_port, sn_int):
+        """Adds rules and routes for SNAT redirection."""
+        self._snat_redirect_modify(gateway, sn_port, sn_int, is_add=True)
 
     def _snat_redirect_remove(self, gateway, sn_port, sn_int):
         """Removes rules and routes for SNAT redirection."""
-        try:
-            ip_cidr = sn_port['ip_cidr']
-            snat_idx = self._get_snat_idx(ip_cidr)
-            ns_ipr = ip_lib.IPRule(namespace=self.ns_name)
-            ns_ipd = ip_lib.IPDevice(sn_int, namespace=self.ns_name)
-            ns_ipd.route.delete_gateway(gateway, table=snat_idx)
-            ns_ipr.rule.delete(ip_cidr, snat_idx, snat_idx)
-        except Exception:
-            LOG.exception(_LE('DVR: removed snat failed'))
+        self._snat_redirect_modify(gateway, sn_port, sn_int, is_add=False)
 
     def get_gw_port_host(self):
         host = self.router.get('gw_port_host')
@@ -304,26 +317,24 @@ class DvrRouter(router.RouterInfo):
             return
 
         interface_name = self.get_internal_device_name(port['id'])
-        self._snat_redirect_add(sn_port['fixed_ips'][0]['ip_address'],
-                                port,
-                                interface_name)
+        self._snat_redirect_add(sn_port, port, interface_name)
 
         if not self._is_this_snat_host():
             return
 
         ns_name = dvr_snat_ns.SnatNamespace.get_snat_ns_name(self.router['id'])
-        self._set_subnet_info(sn_port)
         interface_name = self.get_snat_int_device_name(sn_port['id'])
         self._internal_network_added(
             ns_name,
             sn_port['network_id'],
             sn_port['id'],
-            sn_port['ip_cidr'],
+            sn_port['fixed_ips'],
             sn_port['mac_address'],
             interface_name,
             dvr_snat_ns.SNAT_INT_DEV_PREFIX)
 
-        self._set_subnet_arp_info(port)
+        for subnet in port['subnets']:
+            self._set_subnet_arp_info(subnet['id'])
 
     def _dvr_internal_network_removed(self, port):
         if not self.ex_gw_port:
@@ -335,9 +346,7 @@ class DvrRouter(router.RouterInfo):
 
         # DVR handling code for SNAT
         interface_name = self.get_internal_device_name(port['id'])
-        self._snat_redirect_remove(sn_port['fixed_ips'][0]['ip_address'],
-                                   port,
-                                   interface_name)
+        self._snat_redirect_remove(sn_port, port, interface_name)
 
         mode = self.agent_conf.agent_mode
         is_this_snat_host = (mode == l3_constants.L3_AGENT_MODE_DVR_SNAT
@@ -375,10 +384,9 @@ class DvrRouter(router.RouterInfo):
         # connect snat_ports to br_int from SNAT namespace
         for port in snat_ports:
             # create interface_name
-            self._set_subnet_info(port)
             interface_name = self.get_snat_int_device_name(port['id'])
             self._internal_network_added(snat_ns.name, port['network_id'],
-                                         port['id'], port['ip_cidr'],
+                                         port['id'], port['fixed_ips'],
                                          port['mac_address'], interface_name,
                                          dvr_snat_ns.SNAT_INT_DEV_PREFIX)
         self._external_gateway_added(ex_gw_port, gw_interface_name,
@@ -401,8 +409,7 @@ class DvrRouter(router.RouterInfo):
             gateway = self._map_internal_interfaces(p, snat_ports)
             id_name = self.get_internal_device_name(p['id'])
             if gateway:
-                self._snat_redirect_add(
-                    gateway['fixed_ips'][0]['ip_address'], p, id_name)
+                self._snat_redirect_add(gateway, p, id_name)
 
         if self._is_this_snat_host():
             self._create_dvr_gateway(ex_gw_port, interface_name, snat_ports)
@@ -436,9 +443,7 @@ class DvrRouter(router.RouterInfo):
         for p in self.internal_ports:
             gateway = self._map_internal_interfaces(p, snat_ports)
             internal_interface = self.get_internal_device_name(p['id'])
-            self._snat_redirect_remove(gateway['fixed_ips'][0]['ip_address'],
-                                       p,
-                                       internal_interface)
+            self._snat_redirect_remove(gateway, p, internal_interface)
 
         if not self._is_this_snat_host():
             # no centralized SNAT gateway for this node/agent
@@ -491,10 +496,9 @@ class DvrRouter(router.RouterInfo):
         if floating_ips:
             is_first = self.fip_ns.subscribe(self.router_id)
             if is_first and fip_agent_port:
-                if 'subnet' not in fip_agent_port:
+                if 'subnets' not in fip_agent_port:
                     LOG.error(_LE('Missing subnet/agent_gateway_port'))
                 else:
-                    self._set_subnet_info(fip_agent_port)
                     self.fip_ns.create_gateway_port(fip_agent_port)
 
         if self.fip_ns.agent_gateway_port and floating_ips:

neutron/agent/l3/ha_router.py

@@ -80,7 +80,6 @@ class HaRouter(router.RouterInfo):
                       self.router_id)
             return
 
-        self._set_subnet_info(ha_port)
         self.ha_port = ha_port
         self._init_keepalived_manager(process_monitor)
         self.ha_network_added()
@@ -103,12 +102,13 @@ class HaRouter(router.RouterInfo):
         config = self.keepalived_manager.config
 
         interface_name = self.get_ha_device_name()
-        ha_port_cidr = self.ha_port['subnet']['cidr']
+        subnets = self.ha_port.get('subnets', [])
+        ha_port_cidrs = [subnet['cidr'] for subnet in subnets]
         instance = keepalived.KeepalivedInstance(
             'BACKUP',
             interface_name,
             self.ha_vr_id,
-            ha_port_cidr,
+            ha_port_cidrs,
             nopreempt=True,
             advert_int=self.agent_conf.ha_vrrp_advert_int,
             priority=self.ha_priority)
@@ -148,8 +148,8 @@ class HaRouter(router.RouterInfo):
                          self.ha_port['mac_address'],
                          namespace=self.ns_name,
                          prefix=HA_DEV_PREFIX)
-        self.driver.init_l3(interface_name,
-                            [self.ha_port['ip_cidr']],
+        ip_cidrs = common_utils.fixed_ip_cidrs(self.ha_port['fixed_ips'])
+        self.driver.init_l3(interface_name, ip_cidrs,
                             namespace=self.ns_name,
                             preserve_ips=[self._get_primary_vip()])
 
@@ -195,19 +195,22 @@ class HaRouter(router.RouterInfo):
         self.routes = new_routes
 
     def _add_default_gw_virtual_route(self, ex_gw_port, interface_name):
-        gw_ip = ex_gw_port['subnet']['gateway_ip']
-        if gw_ip:
-            # TODO(Carl) This is repeated everywhere.  A method would be nice.
-            default_gw = (n_consts.IPv4_ANY if
-                          netaddr.IPAddress(gw_ip).version == 4 else
-                          n_consts.IPv6_ANY)
-            instance = self._get_keepalived_instance()
-            instance.virtual_routes = (
-                [route for route in instance.virtual_routes
-                 if route.destination != default_gw])
-            instance.virtual_routes.append(
-                keepalived.KeepalivedVirtualRoute(
-                    default_gw, gw_ip, interface_name))
+        subnets = ex_gw_port.get('subnets', [])
+        for subnet in subnets:
+            gw_ip = subnet['gateway_ip']
+            if gw_ip:
+                # TODO(Carl) This is repeated everywhere.  A method would
+                # be nice.
+                default_gw = (n_consts.IPv4_ANY if
+                              netaddr.IPAddress(gw_ip).version == 4 else
+                              n_consts.IPv6_ANY)
+                instance = self._get_keepalived_instance()
+                instance.virtual_routes = (
+                    [route for route in instance.virtual_routes
+                     if route.destination != default_gw])
+                instance.virtual_routes.append(
+                    keepalived.KeepalivedVirtualRoute(
+                        default_gw, gw_ip, interface_name))
 
     def _should_delete_ipv6_lladdr(self, ipv6_lladdr):
         """Only the master should have any IP addresses configured.
@@ -238,7 +241,8 @@ class HaRouter(router.RouterInfo):
         self._add_vip(ipv6_lladdr, interface_name, scope='link')
 
     def _add_gateway_vip(self, ex_gw_port, interface_name):
-        self._add_vip(ex_gw_port['ip_cidr'], interface_name)
+        for ip_cidr in common_utils.fixed_ip_cidrs(ex_gw_port['fixed_ips']):
+            self._add_vip(ip_cidr, interface_name)
         self._add_default_gw_virtual_route(ex_gw_port, interface_name)
 
     def add_floating_ip(self, fip, interface_name, device):
@@ -264,7 +268,8 @@ class HaRouter(router.RouterInfo):
                              prefix=router.INTERNAL_DEV_PREFIX)
 
         self._disable_ipv6_addressing_on_interface(interface_name)
-        self._add_vip(port['ip_cidr'], interface_name)
+        for ip_cidr in common_utils.fixed_ip_cidrs(port['fixed_ips']):
+            self._add_vip(ip_cidr, interface_name)
 
     def internal_network_removed(self, port):
         super(HaRouter, self).internal_network_removed(port)
@@ -329,8 +334,9 @@ class HaRouter(router.RouterInfo):
 
     def external_gateway_updated(self, ex_gw_port, interface_name):
         self._plug_external_gateway(ex_gw_port, interface_name, self.ns_name)
-        old_gateway_cidr = self.ex_gw_port['ip_cidr']
-        self._remove_vip(old_gateway_cidr)
+        ip_cidrs = common_utils.fixed_ip_cidrs(self.ex_gw_port['fixed_ips'])
+        for old_gateway_cidr in ip_cidrs:
+            self._remove_vip(old_gateway_cidr)
         self._add_gateway_vip(ex_gw_port, interface_name)
 
     def external_gateway_removed(self, ex_gw_port, interface_name):

neutron/agent/l3/router_info.py

@@ -22,7 +22,7 @@ from neutron.agent.linux import iptables_manager
 from neutron.common import constants as l3_constants
 from neutron.common import exceptions as n_exc
 from neutron.common import utils as common_utils
-from neutron.i18n import _LE, _LW
+from neutron.i18n import _LW
 
 LOG = logging.getLogger(__name__)
 INTERNAL_DEV_PREFIX = namespaces.INTERNAL_DEV_PREFIX
@@ -95,16 +95,6 @@ class RouterInfo(object):
     def get_external_device_interface_name(self, ex_gw_port):
         return self.get_external_device_name(ex_gw_port['id'])
 
-    def _set_subnet_info(self, port):
-        ips = port['fixed_ips']
-        if not ips:
-            raise Exception(_("Router port %s has no IP address") % port['id'])
-        if len(ips) > 1:
-            LOG.error(_LE("Ignoring multiple IPs on router port %s"),
-                      port['id'])
-        prefixlen = netaddr.IPNetwork(port['subnet']['cidr']).prefixlen
-        port['ip_cidr'] = "%s/%s" % (ips[0]['ip_address'], prefixlen)
-
     def perform_snat_action(self, snat_callback, *args):
         # Process SNAT rules for attached subnets
         if self._snat_action:
@@ -267,7 +257,7 @@ class RouterInfo(object):
             self.router_namespace.delete()
 
     def _internal_network_added(self, ns_name, network_id, port_id,
-                                internal_cidr, mac_address,
+                                fixed_ips, mac_address,
                                 interface_name, prefix):
         if not ip_lib.device_exists(interface_name,
                                     namespace=ns_name):
@@ -275,18 +265,18 @@ class RouterInfo(object):
                              namespace=ns_name,
                              prefix=prefix)
 
-        self.driver.init_l3(interface_name, [internal_cidr],
-                            namespace=ns_name)
-        ip_address = internal_cidr.split('/')[0]
-        ip_lib.send_gratuitous_arp(ns_name,
-                                   interface_name,
-                                   ip_address,
-                                   self.agent_conf.send_arp_for_ha)
+        ip_cidrs = common_utils.fixed_ip_cidrs(fixed_ips)
+        self.driver.init_l3(interface_name, ip_cidrs, namespace=ns_name)
+        for fixed_ip in fixed_ips:
+            ip_lib.send_gratuitous_arp(ns_name,
+                                       interface_name,
+                                       fixed_ip['ip_address'],
+                                       self.agent_conf.send_arp_for_ha)
 
     def internal_network_added(self, port):
         network_id = port['network_id']
         port_id = port['id']
-        internal_cidr = port['ip_cidr']
+        fixed_ips = port['fixed_ips']
         mac_address = port['mac_address']
 
         interface_name = self.get_internal_device_name(port_id)
@@ -294,7 +284,7 @@ class RouterInfo(object):
         self._internal_network_added(self.ns_name,
                                      network_id,
                                      port_id,
-                                     internal_cidr,
+                                     fixed_ips,
                                      mac_address,
                                      interface_name,
                                      INTERNAL_DEV_PREFIX)
@@ -326,19 +316,22 @@ class RouterInfo(object):
         new_ipv6_port = False
         old_ipv6_port = False
         for p in new_ports:
-            self._set_subnet_info(p)
             self.internal_network_added(p)
             self.internal_ports.append(p)
-            if (not new_ipv6_port and
-                    netaddr.IPNetwork(p['subnet']['cidr']).version == 6):
-                new_ipv6_port = True
+            if not new_ipv6_port:
+                for subnet in p['subnets']:
+                    if netaddr.IPNetwork(subnet['cidr']).version == 6:
+                        new_ipv6_port = True
+                        break
 
         for p in old_ports:
             self.internal_network_removed(p)
             self.internal_ports.remove(p)
-            if (not old_ipv6_port and
-                    netaddr.IPNetwork(p['subnet']['cidr']).version == 6):
-                old_ipv6_port = True
+            if not old_ipv6_port:
+                for subnet in p['subnets']:
+                    if netaddr.IPNetwork(subnet['cidr']).version == 6:
+                        old_ipv6_port = True
+                        break
 
         # Enable RA
         if new_ipv6_port or old_ipv6_port:
@@ -379,17 +372,23 @@ class RouterInfo(object):
                                 ns_name, preserve_ips):
         self._plug_external_gateway(ex_gw_port, interface_name, ns_name)
 
+        # Build up the interface and gateway IP addresses that
+        # will be added to the interface.
+        ip_cidrs = common_utils.fixed_ip_cidrs(ex_gw_port['fixed_ips'])
+        gateway_ips = [subnet['gateway_ip']
+                       for subnet in ex_gw_port['subnets']
+                       if subnet['gateway_ip']]
         self.driver.init_l3(interface_name,
-                            [ex_gw_port['ip_cidr']],
+                            ip_cidrs,
                             namespace=ns_name,
-                            gateway=ex_gw_port['subnet'].get('gateway_ip'),
+                            gateway_ips=gateway_ips,
                             extra_subnets=ex_gw_port.get('extra_subnets', []),
                             preserve_ips=preserve_ips)
-        ip_address = ex_gw_port['ip_cidr'].split('/')[0]
-        ip_lib.send_gratuitous_arp(ns_name,
-                                   interface_name,
-                                   ip_address,
-                                   self.agent_conf.send_arp_for_ha)
+        for fixed_ip in ex_gw_port['fixed_ips']:
+            ip_lib.send_gratuitous_arp(ns_name,
+                                       interface_name,
+                                       fixed_ip['ip_address'],
+                                       self.agent_conf.send_arp_for_ha)
 
     def external_gateway_added(self, ex_gw_port, interface_name):
         preserve_ips = self._list_floating_ip_cidrs()
@@ -426,7 +425,6 @@ class RouterInfo(object):
                 port2_filtered = _get_filtered_dict(port2, keys_to_ignore)
                 return port1_filtered == port2_filtered
 
-            self._set_subnet_info(ex_gw_port)
             if not self.ex_gw_port:
                 self.external_gateway_added(ex_gw_port, interface_name)
             elif not _gateway_ports_equal(ex_gw_port, self.ex_gw_port):

neutron/agent/linux/interface.py

@@ -78,11 +78,12 @@ class LinuxInterfaceDriver(object):
         self.conf = conf
 
     def init_l3(self, device_name, ip_cidrs, namespace=None,
-                preserve_ips=[], gateway=None, extra_subnets=[]):
+                preserve_ips=[], gateway_ips=None, extra_subnets=[]):
         """Set the L3 settings for the interface using data from the port.
 
         ip_cidrs: list of 'X.X.X.X/YY' strings
         preserve_ips: list of ip cidrs that should not be removed from device
+        gateway_ips: For gateway ports, list of external gateway ip addresses
         """
         device = ip_lib.IPDevice(device_name, namespace=namespace)
 
@@ -110,8 +111,8 @@ class LinuxInterfaceDriver(object):
                 device.addr.delete(ip_cidr)
                 self.delete_conntrack_state(namespace=namespace, ip=ip_cidr)
 
-        if gateway:
-            device.route.add_gateway(gateway)
+        for gateway_ip in gateway_ips or []:
+            device.route.add_gateway(gateway_ip)
 
         new_onlink_routes = set(s['cidr'] for s in extra_subnets)
         existing_onlink_routes = set(

neutron/agent/linux/ip_lib.py

@@ -595,16 +595,18 @@ def device_exists(device_name, namespace=None):
     return bool(address)
 
 
-def device_exists_with_ip_mac(device_name, ip_cidr, mac, namespace=None):
-    """Return True if the device with the given IP and MAC addresses
+def device_exists_with_ips_and_mac(device_name, ip_cidrs, mac, namespace=None):
+    """Return True if the device with the given IP addresses and MAC address
     exists in the namespace.
     """
     try:
         device = IPDevice(device_name, namespace=namespace)
         if mac != device.link.address:
             return False
-        if ip_cidr not in (ip['cidr'] for ip in device.addr.list()):
-            return False
+        device_ip_cidrs = [ip['cidr'] for ip in device.addr.list()]
+        for ip_cidr in ip_cidrs:
+            if ip_cidr not in device_ip_cidrs:
+                return False
     except RuntimeError:
         return False
     else:

neutron/agent/linux/keepalived.py

@@ -110,7 +110,7 @@ class KeepalivedVirtualRoute(object):
 class KeepalivedInstance(object):
     """Instance section of a keepalived configuration."""
 
-    def __init__(self, state, interface, vrouter_id, ha_cidr,
+    def __init__(self, state, interface, vrouter_id, ha_cidrs,
                  priority=HA_DEFAULT_PRIORITY, advert_int=None,
                  mcast_src_ip=None, nopreempt=False):
         self.name = 'VR_%s' % vrouter_id
@@ -132,9 +132,7 @@ class KeepalivedInstance(object):
         metadata_cidr = '169.254.169.254/32'
         self.primary_vip_range = get_free_range(
             parent_range='169.254.0.0/16',
-            excluded_ranges=[metadata_cidr,
-                             FIP_LL_SUBNET,
-                             ha_cidr],
+            excluded_ranges=[metadata_cidr, FIP_LL_SUBNET] + ha_cidrs,
             size=PRIMARY_VIP_RANGE_SIZE)
 
     def set_authentication(self, auth_type, password):

neutron/agent/linux/ra.py

@@ -78,15 +78,17 @@ class DaemonMonitor(object):
                                               True)
         buf = six.StringIO()
         for p in router_ports:
-            prefix = p['subnet']['cidr']
-            if netaddr.IPNetwork(prefix).version == 6:
-                interface_name = self._dev_name_helper(p['id'])
-                ra_mode = p['subnet']['ipv6_ra_mode']
-                buf.write('%s' % CONFIG_TEMPLATE.render(
-                    ra_mode=ra_mode,
-                    interface_name=interface_name,
-                    prefix=prefix,
-                    constants=constants))
+            subnets = p.get('subnets', [])
+            for subnet in subnets:
+                prefix = subnet['cidr']
+                if netaddr.IPNetwork(prefix).version == 6:
+                    interface_name = self._dev_name_helper(p['id'])
+                    ra_mode = subnet['ipv6_ra_mode']
+                    buf.write('%s' % CONFIG_TEMPLATE.render(
+                        ra_mode=ra_mode,
+                        interface_name=interface_name,
+                        prefix=prefix,
+                        constants=constants))
 
         utils.replace_file(radvd_conf, buf.getvalue())
         return radvd_conf
@@ -121,16 +123,15 @@ class DaemonMonitor(object):
 
     def enable(self, router_ports):
         for p in router_ports:
-            if netaddr.IPNetwork(p['subnet']['cidr']).version == 6:
-                break
-        else:
-            # Kill the daemon if it's running
-            self.disable()
-            return
+            for subnet in p['subnets']:
+                if netaddr.IPNetwork(subnet['cidr']).version == 6:
+                    LOG.debug("Enable IPv6 RA for router %s", self._router_id)
+                    radvd_conf = self._generate_radvd_conf(router_ports)
+                    self._spawn_radvd(radvd_conf)
+                    return
 
-        LOG.debug("Enable IPv6 RA for router %s", self._router_id)
-        radvd_conf = self._generate_radvd_conf(router_ports)
-        self._spawn_radvd(radvd_conf)
+        # Kill the daemon if it's running
+        self.disable()
 
     def disable(self):
         self._process_monitor.unregister(uuid=self._router_id,

neutron/common/utils.py

@@ -387,6 +387,15 @@ def ip_to_cidr(ip, prefix=None):
     return str(net)
 
 
+def fixed_ip_cidrs(fixed_ips):
+    """Create a list of a port's fixed IPs in cidr notation.
+
+    :param fixed_ips: A neutron port's fixed_ips dictionary
+    """
+    return [ip_to_cidr(fixed_ip['ip_address'], fixed_ip.get('prefixlen'))
+            for fixed_ip in fixed_ips]
+
+
 def is_cidr_host(cidr):
     """Determines if the cidr passed in represents a single host network
 

neutron/db/l3_db.py

@@ -1074,7 +1074,7 @@ class L3_NAT_dbonly_mixin(l3.RouterPluginBase):
         filters = {'id': gw_port_ids}
         gw_ports = self._core_plugin.get_ports(context, filters)
         if gw_ports:
-            self._populate_subnet_for_ports(context, gw_ports)
+            self._populate_subnets_for_ports(context, gw_ports)
         return gw_ports
 
     def get_sync_interfaces(self, context, router_ids, device_owners=None):
@@ -1093,34 +1093,31 @@ class L3_NAT_dbonly_mixin(l3.RouterPluginBase):
         ports = [rp.port.id for rp in qry]
         interfaces = self._core_plugin.get_ports(context, {'id': ports})
         if interfaces:
-            self._populate_subnet_for_ports(context, interfaces)
+            self._populate_subnets_for_ports(context, interfaces)
         return interfaces
 
-    def _populate_subnet_for_ports(self, context, ports):
-        """Populate ports with subnet.
+    def _populate_subnets_for_ports(self, context, ports):
+        """Populate ports with subnets.
 
         These ports already have fixed_ips populated.
         """
         if not ports:
             return
 
-        def each_port_with_ip():
+        def each_port_having_fixed_ips():
             for port in ports:
                 fixed_ips = port.get('fixed_ips', [])
-                if len(fixed_ips) > 1:
-                    LOG.info(_LI("Ignoring multiple IPs on router port %s"),
-                             port['id'])
-                    continue
-                elif not fixed_ips:
+                if not fixed_ips:
                     # Skip ports without IPs, which can occur if a subnet
                     # attached to a router is deleted
                     LOG.info(_LI("Skipping port %s as no IP is configure on "
                                  "it"),
                              port['id'])
                     continue
-                yield (port, fixed_ips[0])
+                yield port
 
-        network_ids = set(p['network_id'] for p, _ in each_port_with_ip())
+        network_ids = set(p['network_id']
+                          for p in each_port_having_fixed_ips())
         filters = {'network_id': [id for id in network_ids]}
         fields = ['id', 'cidr', 'gateway_ip',
                   'network_id', 'ipv6_ra_mode']
@@ -1129,17 +1126,28 @@ class L3_NAT_dbonly_mixin(l3.RouterPluginBase):
         for subnet in self._core_plugin.get_subnets(context, filters, fields):
             subnets_by_network[subnet['network_id']].append(subnet)
 
-        for port, fixed_ip in each_port_with_ip():
+        for port in each_port_having_fixed_ips():
+
+            port['subnets'] = []
             port['extra_subnets'] = []
             for subnet in subnets_by_network[port['network_id']]:
+                # If this subnet is used by the port (has a matching entry
+                # in the port's fixed_ips), then add this subnet to the
+                # port's subnets list, and populate the fixed_ips entry
+                # entry with the subnet's prefix length.
                 subnet_info = {'id': subnet['id'],
                                'cidr': subnet['cidr'],
                                'gateway_ip': subnet['gateway_ip'],
                                'ipv6_ra_mode': subnet['ipv6_ra_mode']}
-
-                if subnet['id'] == fixed_ip['subnet_id']:
-                    port['subnet'] = subnet_info
+                for fixed_ip in port['fixed_ips']:
+                    if fixed_ip['subnet_id'] == subnet['id']:
+                        port['subnets'].append(subnet_info)
+                        prefixlen = netaddr.IPNetwork(
+                            subnet['cidr']).prefixlen
+                        fixed_ip['prefixlen'] = prefixlen
+                        break
                 else:
+                    # This subnet is not used by the port.
                     port['extra_subnets'].append(subnet_info)
 
     def _process_floating_ips(self, context, routers_dict, floating_ips):

neutron/db/l3_dvr_db.py

@@ -372,7 +372,7 @@ class L3_NAT_with_dvr_db_mixin(l3_db.L3_NAT_db_mixin,
         interfaces = self._core_plugin.get_ports(context, {'id': ports})
         LOG.debug("Return the SNAT ports: %s", interfaces)
         if interfaces:
-            self._populate_subnet_for_ports(context, interfaces)
+            self._populate_subnets_for_ports(context, interfaces)
         return interfaces
 
     def _build_routers_list(self, context, routers, gw_ports):
@@ -450,7 +450,7 @@ class L3_NAT_with_dvr_db_mixin(l3_db.L3_NAT_db_mixin,
         interfaces = self._core_plugin.get_ports(context.elevated(), filters)
         LOG.debug("Return the FIP ports: %s ", interfaces)
         if interfaces:
-            self._populate_subnet_for_ports(context, interfaces)
+            self._populate_subnets_for_ports(context, interfaces)
         return interfaces
 
     def get_sync_data(self, context, router_ids=None, active=None):
@@ -553,12 +553,12 @@ class L3_NAT_with_dvr_db_mixin(l3_db.L3_NAT_db_mixin,
                               'admin_state_up': True,
                               'name': ''}})
                 if agent_port:
-                    self._populate_subnet_for_ports(context, [agent_port])
+                    self._populate_subnets_for_ports(context, [agent_port])
                     return agent_port
                 msg = _("Unable to create the Agent Gateway Port")
                 raise n_exc.BadRequest(resource='router', msg=msg)
             else:
-                self._populate_subnet_for_ports(context, [f_port])
+                self._populate_subnets_for_ports(context, [f_port])
                 return f_port
 
     def get_snat_interface_ports_for_router(self, context, router_id):
@@ -600,7 +600,7 @@ class L3_NAT_with_dvr_db_mixin(l3_db.L3_NAT_db_mixin,
             context.session.add(router_port)
 
         if do_pop:
-            return self._populate_subnet_for_ports(context, [snat_port])
+            return self._populate_subnets_for_ports(context, [snat_port])
         return snat_port
 
     def create_snat_intf_ports_if_not_exists(self, context, router):
@@ -613,7 +613,7 @@ class L3_NAT_with_dvr_db_mixin(l3_db.L3_NAT_db_mixin,
         port_list = self.get_snat_interface_ports_for_router(
             context, router.id)
         if port_list:
-            self._populate_subnet_for_ports(context, port_list)
+            self._populate_subnets_for_ports(context, port_list)
             return port_list
         port_list = []
 
@@ -635,7 +635,7 @@ class L3_NAT_with_dvr_db_mixin(l3_db.L3_NAT_db_mixin,
                     intf['fixed_ips'][0]['subnet_id'], do_pop=False)
                 port_list.append(snat_port)
         if port_list:
-            self._populate_subnet_for_ports(context, port_list)
+            self._populate_subnets_for_ports(context, port_list)
         return port_list
 
     def dvr_vmarp_table_update(self, context, port_dict, action):

neutron/db/l3_hamode_db.py

@@ -458,7 +458,7 @@ class L3_HA_NAT_db_mixin(l3_dvr_db.L3_NAT_with_dvr_db_mixin):
         for router in routers_dict.values():
             interface = router.get(constants.HA_INTERFACE_KEY)
             if interface:
-                self._populate_subnet_for_ports(context, [interface])
+                self._populate_subnets_for_ports(context, [interface])
 
         return routers_dict.values()
 

neutron/tests/functional/agent/linux/test_ip_lib.py

@@ -27,7 +27,8 @@ from neutron.common import utils
 from neutron.tests.functional.agent.linux import base
 
 LOG = logging.getLogger(__name__)
-Device = collections.namedtuple('Device', 'name ip_cidr mac_address namespace')
+Device = collections.namedtuple('Device',
+                                'name ip_cidrs mac_address namespace')
 
 
 class IpLibTestFramework(base.BaseLinuxTestCase):
@@ -45,10 +46,10 @@ class IpLibTestFramework(base.BaseLinuxTestCase):
         self.driver = importutils.import_object(cfg.CONF.interface_driver,
                                                 cfg.CONF)
 
-    def generate_device_details(self, name=None, ip_cidr=None,
+    def generate_device_details(self, name=None, ip_cidrs=None,
                                 mac_address=None, namespace=None):
         return Device(name or base.get_rand_name(),
-                      ip_cidr or '240.0.0.1/24',
+                      ip_cidrs or ['240.0.0.1/24'],
                       mac_address or
                       utils.get_random_mac('fa:16:3e:00:00:00'.split(':')),
                       namespace or base.get_rand_name())
@@ -73,7 +74,7 @@ class IpLibTestFramework(base.BaseLinuxTestCase):
         tap_device = ip.add_tuntap(attr.name)
         self.addCleanup(self._safe_delete_device, tap_device)
         tap_device.link.set_address(attr.mac_address)
-        self.driver.init_l3(attr.name, [attr.ip_cidr],
+        self.driver.init_l3(attr.name, attr.ip_cidrs,
                             namespace=attr.namespace)
         tap_device.link.set_up()
         return tap_device
@@ -96,34 +97,34 @@ class IpLibTestCase(IpLibTestFramework):
         self.assertFalse(
             ip_lib.device_exists(attr.name, namespace=attr.namespace))
 
-    def test_device_exists_with_ip_mac(self):
+    def test_device_exists_with_ips_and_mac(self):
         attr = self.generate_device_details()
         device = self.manage_device(attr)
         self.assertTrue(
-            ip_lib.device_exists_with_ip_mac(*attr))
+            ip_lib.device_exists_with_ips_and_mac(*attr))
 
         wrong_ip_cidr = '10.0.0.1/8'
         wrong_mac_address = 'aa:aa:aa:aa:aa:aa'
 
         attr = self.generate_device_details(name='wrong_name')
         self.assertFalse(
-            ip_lib.device_exists_with_ip_mac(*attr))
+            ip_lib.device_exists_with_ips_and_mac(*attr))
 
-        attr = self.generate_device_details(ip_cidr=wrong_ip_cidr)
-        self.assertFalse(ip_lib.device_exists_with_ip_mac(*attr))
+        attr = self.generate_device_details(ip_cidrs=[wrong_ip_cidr])
+        self.assertFalse(ip_lib.device_exists_with_ips_and_mac(*attr))
 
         attr = self.generate_device_details(mac_address=wrong_mac_address)
-        self.assertFalse(ip_lib.device_exists_with_ip_mac(*attr))
+        self.assertFalse(ip_lib.device_exists_with_ips_and_mac(*attr))
 
         attr = self.generate_device_details(namespace='wrong_namespace')
-        self.assertFalse(ip_lib.device_exists_with_ip_mac(*attr))
+        self.assertFalse(ip_lib.device_exists_with_ips_and_mac(*attr))
 
         device.link.delete()
 
     def test_get_routing_table(self):
         attr = self.generate_device_details()
         device = self.manage_device(attr)
-        device_ip = attr.ip_cidr.split('/')[0]
+        device_ip = attr.ip_cidrs[0].split('/')[0]
         destination = '8.8.8.0/24'
         device.route.add_route(destination, device_ip)
 
@@ -133,7 +134,7 @@ class IpLibTestCase(IpLibTestFramework):
                            {'nexthop': None,
                             'device': attr.name,
                             'destination': str(
-                                netaddr.IPNetwork(attr.ip_cidr).cidr)}]
+                                netaddr.IPNetwork(attr.ip_cidrs[0]).cidr)}]
 
         routes = ip_lib.get_routing_table(namespace=attr.namespace)
         self.assertEqual(expected_routes, routes)

neutron/tests/functional/agent/test_l3_agent.py

@@ -101,8 +101,9 @@ class L3AgentTestFramework(base.BaseOVSLinuxTestCase):
         return agent
 
     def generate_router_info(self, enable_ha, ip_version=4, extra_routes=True,
-                             enable_fip=True, enable_snat=True):
-        if ip_version == 6:
+                             enable_fip=True, enable_snat=True,
+                             dual_stack=False):
+        if ip_version == 6 and not dual_stack:
             enable_snat = False
             enable_fip = False
             extra_routes = False
@@ -111,7 +112,8 @@ class L3AgentTestFramework(base.BaseOVSLinuxTestCase):
                                                  enable_snat=enable_snat,
                                                  enable_floating_ip=enable_fip,
                                                  enable_ha=enable_ha,
-                                                 extra_routes=extra_routes)
+                                                 extra_routes=extra_routes,
+                                                 dual_stack=dual_stack)
 
     def manage_router(self, agent, router):
         self.addCleanup(self._delete_router, agent, router['id'])
@@ -145,12 +147,19 @@ class L3AgentTestFramework(base.BaseOVSLinuxTestCase):
             router.ns_name)
         return pm.active
 
-    def device_exists_with_ip_mac(self, expected_device, name_getter,
-                                  namespace):
-        return ip_lib.device_exists_with_ip_mac(
-            name_getter(expected_device['id']), expected_device['ip_cidr'],
+    def device_exists_with_ips_and_mac(self, expected_device, name_getter,
+                                       namespace):
+        ip_cidrs = common_utils.fixed_ip_cidrs(expected_device['fixed_ips'])
+        return ip_lib.device_exists_with_ips_and_mac(
+            name_getter(expected_device['id']), ip_cidrs,
             expected_device['mac_address'], namespace)
 
+    @staticmethod
+    def _port_first_ip_cidr(port):
+        fixed_ip = port['fixed_ips'][0]
+        return common_utils.ip_to_cidr(fixed_ip['ip_address'],
+                                       fixed_ip['prefixlen'])
+
     def get_device_mtu(self, target_device, name_getter, namespace):
         device = ip_lib.IPDevice(name_getter(target_device), namespace)
         return device.link.mtu
@@ -158,20 +167,20 @@ class L3AgentTestFramework(base.BaseOVSLinuxTestCase):
     def get_expected_keepalive_configuration(self, router):
         router_id = router.router_id
         ha_device_name = router.get_ha_device_name()
-        ha_device_cidr = router.ha_port['ip_cidr']
+        ha_device_cidr = self._port_first_ip_cidr(router.ha_port)
         external_port = router.get_ex_gw_port()
         ex_port_ipv6 = ip_lib.get_ipv6_lladdr(external_port['mac_address'])
         external_device_name = router.get_external_device_name(
             external_port['id'])
-        external_device_cidr = external_port['ip_cidr']
+        external_device_cidr = self._port_first_ip_cidr(external_port)
         internal_port = router.router[l3_constants.INTERFACE_KEY][0]
         int_port_ipv6 = ip_lib.get_ipv6_lladdr(internal_port['mac_address'])
         internal_device_name = router.get_internal_device_name(
             internal_port['id'])
-        internal_device_cidr = internal_port['ip_cidr']
+        internal_device_cidr = self._port_first_ip_cidr(internal_port)
         floating_ip_cidr = common_utils.ip_to_cidr(
             router.get_floating_ips()[0]['floating_ip_address'])
-        default_gateway_ip = external_port['subnet'].get('gateway_ip')
+        default_gateway_ip = external_port['subnets'][0].get('gateway_ip')
 
         return """vrrp_instance VR_1 {
     state BACKUP
@@ -250,7 +259,7 @@ class L3AgentTestFramework(base.BaseOVSLinuxTestCase):
         internal_devices = router.router[l3_constants.INTERFACE_KEY]
         self.assertTrue(len(internal_devices))
         for device in internal_devices:
-            self.assertTrue(self.device_exists_with_ip_mac(
+            self.assertTrue(self.device_exists_with_ips_and_mac(
                 device, router.get_internal_device_name, router.ns_name))
 
     def _assert_extra_routes(self, router):
@@ -272,11 +281,12 @@ class L3AgentTestFramework(base.BaseOVSLinuxTestCase):
     def floating_ips_configured(self, router):
         floating_ips = router.router[l3_constants.FLOATINGIP_KEY]
         external_port = router.get_ex_gw_port()
-        return len(floating_ips) and all(ip_lib.device_exists_with_ip_mac(
-            router.get_external_device_name(external_port['id']),
-            '%s/32' % fip['floating_ip_address'],
-            external_port['mac_address'],
-            namespace=router.ns_name) for fip in floating_ips)
+        return len(floating_ips) and all(
+            ip_lib.device_exists_with_ips_and_mac(
+                router.get_external_device_name(external_port['id']),
+                ['%s/32' % fip['floating_ip_address']],
+                external_port['mac_address'],
+                namespace=router.ns_name) for fip in floating_ips)
 
     def fail_ha_router(self, router):
         device_name = router.get_ha_device_name()
@@ -352,7 +362,7 @@ class L3AgentTestCase(L3AgentTestFramework):
                 calls)
 
     def test_legacy_router_lifecycle(self):
-        self._router_lifecycle(enable_ha=False)
+        self._router_lifecycle(enable_ha=False, dual_stack=True)
 
     def test_ha_router_lifecycle(self):
         self._router_lifecycle(enable_ha=True)
@@ -418,8 +428,15 @@ class L3AgentTestCase(L3AgentTestFramework):
         existing_fip = '19.4.4.2'
         new_fip = '19.4.4.3'
         self._add_fip(router, new_fip)
-        router.router['gw_port']['subnet']['gateway_ip'] = '19.4.4.5'
-        router.router['gw_port']['fixed_ips'][0]['ip_address'] = '19.4.4.10'
+        subnet_id = _uuid()
+        fixed_ips = [{'ip_address': '19.4.4.10',
+                      'prefixlen': 24,
+                      'subnet_id': subnet_id}]
+        subnets = [{'id': subnet_id,
+                    'cidr': '19.4.4.0/24',
+                    'gateway_ip': '19.4.4.5'}]
+        router.router['gw_port']['subnets'] = subnets
+        router.router['gw_port']['fixed_ips'] = fixed_ips
 
         self.agent.process_router(router)
 
@@ -444,8 +461,9 @@ class L3AgentTestCase(L3AgentTestFramework):
                       (new_external_device_ip, external_device_name),
                       new_config)
 
-    def _router_lifecycle(self, enable_ha, ip_version=4):
-        router_info = self.generate_router_info(enable_ha, ip_version)
+    def _router_lifecycle(self, enable_ha, ip_version=4, dual_stack=False):
+        router_info = self.generate_router_info(enable_ha, ip_version,
+                                                dual_stack=dual_stack)
         router = self.manage_router(self.agent, router_info)
 
         if enable_ha:
@@ -461,7 +479,7 @@ class L3AgentTestCase(L3AgentTestFramework):
             # device has an IP address.
             device = router.router[l3_constants.INTERFACE_KEY][-1]
             device_exists = functools.partial(
-                self.device_exists_with_ip_mac,
+                self.device_exists_with_ips_and_mac,
                 device,
                 router.get_internal_device_name,
                 router.ns_name)
@@ -472,7 +490,7 @@ class L3AgentTestCase(L3AgentTestFramework):
             lambda: self._metadata_proxy_exists(self.agent.conf, router))
         self._assert_internal_devices(router)
         self._assert_external_device(router)
-        if ip_version == 4:
+        if not (enable_ha and (ip_version == 6 or dual_stack)):
             # Note(SridharG): enable the assert_gateway for IPv6 once
             # keepalived on Ubuntu14.04 (i.e., check-neutron-dsvm-functional
             # platform) is updated to 1.2.10 (or above).
@@ -497,7 +515,7 @@ class L3AgentTestCase(L3AgentTestFramework):
 
     def _assert_external_device(self, router):
         external_port = router.get_ex_gw_port()
-        self.assertTrue(self.device_exists_with_ip_mac(
+        self.assertTrue(self.device_exists_with_ips_and_mac(
             external_port, router.get_external_device_name,
             router.ns_name))
 
@@ -507,16 +525,19 @@ class L3AgentTestCase(L3AgentTestFramework):
             external_port['id'])
         external_device = ip_lib.IPDevice(external_device_name,
                                           namespace=router.ns_name)
-        existing_gateway = (
-            external_device.route.get_gateway().get('gateway'))
-        expected_gateway = external_port['subnet']['gateway_ip']
-        self.assertEqual(expected_gateway, existing_gateway)
+        for subnet in external_port['subnets']:
+            expected_gateway = subnet['gateway_ip']
+            ip_vers = netaddr.IPAddress(expected_gateway).version
+            existing_gateway = (external_device.route.get_gateway(
+                ip_version=ip_vers).get('gateway'))
+            self.assertEqual(expected_gateway, existing_gateway)
 
     def _assert_ha_device(self, router):
-        device = router.router[l3_constants.HA_INTERFACE_KEY]
-        self.assertTrue(ip_lib.device_exists_with_ip_mac(
-            router.get_ha_device_name(), device['ip_cidr'],
-            device['mac_address'], router.ns_name))
+        def ha_router_dev_name_getter(not_used):
+            return router.get_ha_device_name()
+        self.assertTrue(self.device_exists_with_ips_and_mac(
+            router.router[l3_constants.HA_INTERFACE_KEY],
+            ha_router_dev_name_getter, router.ns_name))
 
     @classmethod
     def _get_addresses_on_device(cls, namespace, interface):
@@ -631,7 +652,7 @@ class MetadataL3AgentTestCase(L3AgentTestFramework):
 
         # Create and configure client namespace
         client_ns = self._create_namespace()
-        router_ip_cidr = router.internal_ports[0]['ip_cidr']
+        router_ip_cidr = self._port_first_ip_cidr(router.internal_ports[0])
         ip_cidr = net_helpers.increment_ip_cidr(router_ip_cidr)
         br_int = get_ovs_bridge(self.agent.conf.ovs_integration_bridge)
         port = self.bind_namespace_to_cidr(client_ns, br_int, ip_cidr)
@@ -746,24 +767,26 @@ class TestDvrRouter(L3AgentTestFramework):
         if not fip_gw_port_list and external_gw_port:
             # Get values from external gateway port
             fixed_ip = external_gw_port['fixed_ips'][0]
-            float_subnet = external_gw_port['subnet']
+            float_subnet = external_gw_port['subnets'][0]
             port_ip = fixed_ip['ip_address']
             # Pick an ip address which is not the same as port_ip
             fip_gw_port_ip = str(netaddr.IPAddress(port_ip) + 5)
             # Add floatingip agent gateway port info to router
+            prefixlen = netaddr.IPNetwork(float_subnet['cidr']).prefixlen
             router[l3_constants.FLOATINGIP_AGENT_INTF_KEY] = [
-                {'subnet':
+                {'subnets': [
                     {'cidr': float_subnet['cidr'],
-                        'gateway_ip': float_subnet['gateway_ip'],
-                        'id': fixed_ip['subnet_id']},
-                    'network_id': external_gw_port['network_id'],
-                    'device_owner': 'network:floatingip_agent_gateway',
-                    'mac_address': 'fa:16:3e:80:8d:89',
-                    'binding:host_id': self.agent.conf.host,
-                    'fixed_ips': [{'subnet_id': fixed_ip['subnet_id'],
-                                    'ip_address': fip_gw_port_ip}],
-                    'id': _uuid(),
-                    'device_id': _uuid()}
+                     'gateway_ip': float_subnet['gateway_ip'],
+                     'id': fixed_ip['subnet_id']}],
+                 'network_id': external_gw_port['network_id'],
+                 'device_owner': 'network:floatingip_agent_gateway',
+                 'mac_address': 'fa:16:3e:80:8d:89',
+                 'binding:host_id': self.agent.conf.host,
+                 'fixed_ips': [{'subnet_id': fixed_ip['subnet_id'],
+                                'ip_address': fip_gw_port_ip,
+                                'prefixlen': prefixlen}],
+                 'id': _uuid(),
+                 'device_id': _uuid()}
             ]
 
     def _add_snat_port_info_to_router(self, router, internal_ports):
@@ -773,24 +796,26 @@ class TestDvrRouter(L3AgentTestFramework):
             # Get values from internal port
             port = internal_ports[0]
             fixed_ip = port['fixed_ips'][0]
-            snat_subnet = port['subnet']
+            snat_subnet = port['subnets'][0]
             port_ip = fixed_ip['ip_address']
             # Pick an ip address which is not the same as port_ip
             snat_ip = str(netaddr.IPAddress(port_ip) + 5)
             # Add the info to router as the first snat port
             # in the list of snat ports
+            prefixlen = netaddr.IPNetwork(snat_subnet['cidr']).prefixlen
             router[l3_constants.SNAT_ROUTER_INTF_KEY] = [
-                {'subnet':
+                {'subnets': [
                     {'cidr': snat_subnet['cidr'],
-                        'gateway_ip': snat_subnet['gateway_ip'],
-                        'id': fixed_ip['subnet_id']},
-                    'network_id': port['network_id'],
-                    'device_owner': 'network:router_centralized_snat',
-                    'mac_address': 'fa:16:3e:80:8d:89',
-                    'fixed_ips': [{'subnet_id': fixed_ip['subnet_id'],
-                                    'ip_address': snat_ip}],
-                    'id': _uuid(),
-                    'device_id': _uuid()}
+                     'gateway_ip': snat_subnet['gateway_ip'],
+                     'id': fixed_ip['subnet_id']}],
+                 'network_id': port['network_id'],
+                 'device_owner': 'network:router_centralized_snat',
+                 'mac_address': 'fa:16:3e:80:8d:89',
+                 'fixed_ips': [{'subnet_id': fixed_ip['subnet_id'],
+                                'ip_address': snat_ip,
+                                'prefixlen': prefixlen}],
+                 'id': _uuid(),
+                 'device_id': _uuid()}
             ]
 
     def _assert_dvr_external_device(self, router):
@@ -802,7 +827,7 @@ class TestDvrRouter(L3AgentTestFramework):
         # that the correct ports and ip addresses exist in the
         # snat_ns_name namespace
         if self.agent.conf.agent_mode == 'dvr_snat':
-            self.assertTrue(self.device_exists_with_ip_mac(
+            self.assertTrue(self.device_exists_with_ips_and_mac(
                 external_port, router.get_external_device_name,
                 snat_ns_name))
         # if the agent is in dvr mode then the snat_ns_name namespace
@@ -841,7 +866,7 @@ class TestDvrRouter(L3AgentTestFramework):
                                           namespace=namespace)
         existing_gateway = (
             external_device.route.get_gateway().get('gateway'))
-        expected_gateway = external_port['subnet']['gateway_ip']
+        expected_gateway = external_port['subnets'][0]['gateway_ip']
         self.assertEqual(expected_gateway, existing_gateway)
 
     def _assert_snat_namespace_does_not_exist(self, router):
@@ -864,9 +889,9 @@ class TestDvrRouter(L3AgentTestFramework):
         external_gw_port = floating_agent_gw_port[0]
         fip_ns = self.agent.get_fip_ns(floating_ips[0]['floating_network_id'])
         fip_ns_name = fip_ns.get_name()
-        fg_port_created_successfully = ip_lib.device_exists_with_ip_mac(
+        fg_port_created_successfully = ip_lib.device_exists_with_ips_and_mac(
             fip_ns.get_ext_device_name(external_gw_port['id']),
-            external_gw_port['ip_cidr'],
+            [self._port_first_ip_cidr(external_gw_port)],
             external_gw_port['mac_address'],
             namespace=fip_ns_name)
         self.assertTrue(fg_port_created_successfully)

neutron/tests/unit/agent/l3/test_dvr_router.py

@@ -63,13 +63,16 @@ class TestDvrRouterOperations(base.BaseTestCase):
         router = mock.MagicMock()
         ri = self._create_router(router)
         ext_net_id = _uuid()
+        subnet_id = _uuid()
         agent_gw_port = {'fixed_ips': [{'ip_address': '20.0.0.30',
-                                        'subnet_id': _uuid()}],
-                         'subnet': {'gateway_ip': '20.0.0.1'},
+                                        'prefixlen': 24,
+                                        'subnet_id': subnet_id}],
+                         'subnets': [{'id': subnet_id,
+                                      'cidr': '20.0.0.0/24',
+                                      'gateway_ip': '20.0.0.1'}],
                          'id': _uuid(),
                          'network_id': ext_net_id,
-                         'mac_address': 'ca:fe:de:ad:be:ef',
-                         'ip_cidr': '20.0.0.30/24'}
+                         'mac_address': 'ca:fe:de:ad:be:ef'}
 
         fip = {'id': _uuid(),
                'host': HOSTNAME,
@@ -95,13 +98,16 @@ class TestDvrRouterOperations(base.BaseTestCase):
         router = mock.MagicMock()
         ri = self._create_router(router)
 
+        subnet_id = _uuid()
         agent_gw_port = {'fixed_ips': [{'ip_address': '20.0.0.30',
-                                        'subnet_id': _uuid()}],
-                         'subnet': {'gateway_ip': '20.0.0.1'},
+                                        'prefixlen': 24,
+                                        'subnet_id': subnet_id}],
+                         'subnets': [{'id': subnet_id,
+                                      'cidr': '20.0.0.0/24',
+                                      'gateway_ip': '20.0.0.1'}],
                          'id': _uuid(),
                          'network_id': _uuid(),
-                         'mac_address': 'ca:fe:de:ad:be:ef',
-                         'ip_cidr': '20.0.0.30/24'}
+                         'mac_address': 'ca:fe:de:ad:be:ef'}
         fip_cidr = '11.22.33.44/24'
 
         ri.dist_fip_count = 2

neutron/tests/unit/agent/linux/test_keepalived.py

@@ -62,7 +62,7 @@ class KeepalivedConfBaseMixin(object):
         config = keepalived.KeepalivedConf()
 
         instance1 = keepalived.KeepalivedInstance('MASTER', 'eth0', 1,
-                                                  '169.254.192.0/18',
+                                                  ['169.254.192.0/18'],
                                                   advert_int=5)
         instance1.set_authentication('AH', 'pass123')
         instance1.track_interfaces.append("eth0")
@@ -90,7 +90,7 @@ class KeepalivedConfBaseMixin(object):
         instance1.virtual_routes.append(virtual_route)
 
         instance2 = keepalived.KeepalivedInstance('MASTER', 'eth4', 2,
-                                                  '169.254.192.0/18',
+                                                  ['169.254.192.0/18'],
                                                   mcast_src_ip='224.0.0.1')
         instance2.track_interfaces.append("eth4")
 
@@ -178,11 +178,12 @@ class KeepalivedStateExceptionTestCase(base.BaseTestCase):
         invalid_vrrp_state = 'a seal walks'
         self.assertRaises(keepalived.InvalidInstanceStateException,
                           keepalived.KeepalivedInstance,
-                          invalid_vrrp_state, 'eth0', 33, '169.254.192.0/18')
+                          invalid_vrrp_state, 'eth0', 33,
+                          ['169.254.192.0/18'])
 
         invalid_auth_type = 'into a club'
         instance = keepalived.KeepalivedInstance('MASTER', 'eth0', 1,
-                                                 '169.254.192.0/18')
+                                                 ['169.254.192.0/18'])
         self.assertRaises(keepalived.InvalidAuthenticationTypeException,
                           instance.set_authentication,
                           invalid_auth_type, 'some_password')
@@ -192,7 +193,7 @@ class KeepalivedInstanceTestCase(base.BaseTestCase,
                                  KeepalivedConfBaseMixin):
     def test_get_primary_vip(self):
         instance = keepalived.KeepalivedInstance('MASTER', 'ha0', 42,
-                                                 '169.254.192.0/18')
+                                                 ['169.254.192.0/18'])
         self.assertEqual('169.254.0.42/24', instance.get_primary_vip())
 
     def test_remove_adresses_by_interface(self):
@@ -256,7 +257,7 @@ vrrp_instance VR_2 {
     }
 }"""
         instance = keepalived.KeepalivedInstance(
-            'MASTER', 'eth0', 1, '169.254.192.0/18')
+            'MASTER', 'eth0', 1, ['169.254.192.0/18'])
         self.assertEqual(expected, '\n'.join(instance.build_config()))
 
 

neutron/tests/unit/agent/test_dvr_fip_ns.py

@@ -70,13 +70,16 @@ class TestDvrFipNs(base.BaseTestCase):
     @mock.patch.object(ip_lib, 'send_gratuitous_arp')
     @mock.patch.object(ip_lib, 'device_exists')
     def test_gateway_added(self, device_exists, send_arp, IPDevice, IPWrapper):
+        subnet_id = _uuid()
         agent_gw_port = {'fixed_ips': [{'ip_address': '20.0.0.30',
-                                        'subnet_id': _uuid()}],
-                         'subnet': {'gateway_ip': '20.0.0.1'},
+                                        'prefixlen': 24,
+                                        'subnet_id': subnet_id}],
+                         'subnets': [{'id': subnet_id,
+                                      'cidr': '20.0.0.0/24',
+                                      'gateway_ip': '20.0.0.1'}],
                          'id': _uuid(),
                          'network_id': self.net_id,
-                         'mac_address': 'ca:fe:de:ad:be:ef',
-                         'ip_cidr': '20.0.0.30/24'}
+                         'mac_address': 'ca:fe:de:ad:be:ef'}
 
         device_exists.return_value = False
         self.fip_ns._gateway_added(agent_gw_port,

neutron/tests/unit/db/test_l3_ha_db.py

@@ -225,7 +225,10 @@ class L3HATestCase(L3HATestFramework):
 
         self.assertEqual(constants.DEVICE_OWNER_ROUTER_HA_INTF,
                          interface['device_owner'])
-        self.assertEqual(cfg.CONF.l3_ha_net_cidr, interface['subnet']['cidr'])
+
+        subnets = interface['subnets']
+        self.assertEqual(1, len(subnets))
+        self.assertEqual(cfg.CONF.l3_ha_net_cidr, subnets[0]['cidr'])
 
     def test_unique_ha_network_per_tenant(self):
         tenant1 = _uuid()

neutron/tests/unit/test_l3_agent.py

@@ -59,66 +59,90 @@ class FakeDev(object):
 
 
 def router_append_interface(router, count=1, ip_version=4, ra_mode=None,
-                            addr_mode=None):
-    if ip_version == 4:
-        ip_pool = '35.4.%i.4'
-        cidr_pool = '35.4.%i.0/24'
-        gw_pool = '35.4.%i.1'
-    elif ip_version == 6:
-        ip_pool = 'fd01:%x:1::6'
-        cidr_pool = 'fd01:%x:1::/64'
-        gw_pool = 'fd01:%x:1::1'
-    else:
-        raise ValueError("Invalid ip_version: %s" % ip_version)
-
+                            addr_mode=None, dual_stack=False):
     interfaces = router[l3_constants.INTERFACE_KEY]
     current = sum(
-        [netaddr.IPNetwork(p['subnet']['cidr']).version == ip_version
-         for p in interfaces])
+        [netaddr.IPNetwork(subnet['cidr']).version == ip_version
+         for p in interfaces for subnet in p['subnets']])
 
     mac_address = netaddr.EUI('ca:fe:de:ad:be:ef')
     mac_address.dialect = netaddr.mac_unix
     for i in range(current, current + count):
+        fixed_ips = []
+        subnets = []
+        for loop_version in (4, 6):
+            if loop_version == 4 and (ip_version == 4 or dual_stack):
+                ip_pool = '35.4.%i.4'
+                cidr_pool = '35.4.%i.0/24'
+                prefixlen = 24
+                gw_pool = '35.4.%i.1'
+            elif loop_version == 6 and (ip_version == 6 or dual_stack):
+                ip_pool = 'fd01:%x:1::6'
+                cidr_pool = 'fd01:%x:1::/64'
+                prefixlen = 64
+                gw_pool = 'fd01:%x:1::1'
+            else:
+                continue
+            subnet_id = _uuid()
+            fixed_ips.append({'ip_address': ip_pool % i,
+                              'subnet_id': subnet_id,
+                              'prefixlen': prefixlen})
+            subnets.append({'id': subnet_id,
+                            'cidr': cidr_pool % i,
+                            'gateway_ip': gw_pool % i,
+                            'ipv6_ra_mode': ra_mode,
+                            'ipv6_address_mode': addr_mode})
+        if not fixed_ips:
+            raise ValueError("Invalid ip_version: %s" % ip_version)
+
         interfaces.append(
             {'id': _uuid(),
              'network_id': _uuid(),
              'admin_state_up': True,
-             'fixed_ips': [{'ip_address': ip_pool % i,
-                            'subnet_id': _uuid()}],
+             'fixed_ips': fixed_ips,
              'mac_address': str(mac_address),
-             'subnet': {'cidr': cidr_pool % i,
-                        'gateway_ip': gw_pool % i,
-                        'ipv6_ra_mode': ra_mode,
-                        'ipv6_address_mode': addr_mode}})
+             'subnets': subnets})
         mac_address.value += 1
 
 
 def prepare_router_data(ip_version=4, enable_snat=None, num_internal_ports=1,
                         enable_floating_ip=False, enable_ha=False,
-                        extra_routes=False):
-    if ip_version == 4:
-        ip_addr = '19.4.4.4'
-        cidr = '19.4.4.0/24'
-        gateway_ip = '19.4.4.1'
-    elif ip_version == 6:
-        ip_addr = 'fd00::4'
-        cidr = 'fd00::/64'
-        gateway_ip = 'fd00::1'
-    else:
+                        extra_routes=False, dual_stack=False):
+    fixed_ips = []
+    subnets = []
+    for loop_version in (4, 6):
+        if loop_version == 4 and (ip_version == 4 or dual_stack):
+            ip_address = '19.4.4.4'
+            prefixlen = 24
+            subnet_cidr = '19.4.4.0/24'
+            gateway_ip = '19.4.4.1'
+        elif loop_version == 6 and (ip_version == 6 or dual_stack):
+            ip_address = 'fd00::4'
+            prefixlen = 64
+            subnet_cidr = 'fd00::/64'
+            gateway_ip = 'fd00::1'
+        else:
+            continue
+        subnet_id = _uuid()
+        fixed_ips.append({'ip_address': ip_address,
+                          'subnet_id': subnet_id,
+                          'prefixlen': prefixlen})
+        subnets.append({'id': subnet_id,
+                        'cidr': subnet_cidr,
+                        'gateway_ip': gateway_ip})
+    if not fixed_ips:
         raise ValueError("Invalid ip_version: %s" % ip_version)
 
     router_id = _uuid()
     ex_gw_port = {'id': _uuid(),
                   'mac_address': 'ca:fe:de:ad:be:ee',
                   'network_id': _uuid(),
-                  'fixed_ips': [{'ip_address': ip_addr,
-                                 'subnet_id': _uuid()}],
-                  'subnet': {'cidr': cidr,
-                             'gateway_ip': gateway_ip}}
+                  'fixed_ips': fixed_ips,
+                  'subnets': subnets}
 
     routes = []
     if extra_routes:
-        routes = [{'destination': '8.8.8.0/24', 'nexthop': ip_addr}]
+        routes = [{'destination': '8.8.8.0/24', 'nexthop': '19.4.4.4'}]
 
     router = {
         'id': router_id,
@@ -135,7 +159,7 @@ def prepare_router_data(ip_version=4, enable_snat=None, num_internal_ports=1,
             'fixed_ip_address': '10.0.0.1'}]
 
     router_append_interface(router, count=num_internal_ports,
-                            ip_version=ip_version)
+                            ip_version=ip_version, dual_stack=dual_stack)
     if enable_ha:
         router['ha'] = True
         router['ha_vr_id'] = 1
@@ -154,19 +178,21 @@ def _get_subnet_id(port):
 # and the functional tests, and should be moved elsewhere (probably
 # neutron/tests/common/).
 def get_ha_interface(ip='169.254.192.1', mac='12:34:56:78:2b:5d'):
+    subnet_id = _uuid()
     return {'admin_state_up': True,
             'device_id': _uuid(),
             'device_owner': 'network:router_ha_interface',
             'fixed_ips': [{'ip_address': ip,
-                           'subnet_id': _uuid()}],
+                           'prefixlen': 18,
+                           'subnet_id': subnet_id}],
             'id': _uuid(),
             'mac_address': mac,
             'name': u'L3 HA Admin port 0',
             'network_id': _uuid(),
             'status': u'ACTIVE',
-            'subnet': {'cidr': '169.254.192.0/18',
-                       'gateway_ip': '169.254.255.254',
-                       'id': _uuid()},
+            'subnets': [{'cidr': '169.254.192.0/18',
+                         'gateway_ip': '169.254.255.254',
+                         'id': subnet_id}],
             'tenant_id': '',
             'agent_id': _uuid(),
             'agent_host': 'aaa',
@@ -256,25 +282,27 @@ class BasicRouterOperationsFramework(base.BaseTestCase):
             'neutron.openstack.common.loopingcall.FixedIntervalLoopingCall')
         self.looping_call_p.start()
 
-        self.snat_ports = [{'subnet': {'cidr': '152.2.0.0/16',
-                                       'gateway_ip': '152.2.0.1',
-                                       'id': _uuid()},
+        subnet_id_1 = _uuid()
+        subnet_id_2 = _uuid()
+        self.snat_ports = [{'subnets': [{'cidr': '152.2.0.0/16',
+                                         'gateway_ip': '152.2.0.1',
+                                         'id': subnet_id_1}],
                            'network_id': _uuid(),
                            'device_owner': 'network:router_centralized_snat',
-                           'ip_cidr': '152.2.0.13/16',
                            'mac_address': 'fa:16:3e:80:8d:80',
-                           'fixed_ips': [{'subnet_id': _uuid(),
-                                          'ip_address': '152.2.0.13'}],
+                           'fixed_ips': [{'subnet_id': subnet_id_1,
+                                          'ip_address': '152.2.0.13',
+                                          'prefixlen': 16}],
                            'id': _uuid(), 'device_id': _uuid()},
-                          {'subnet': {'cidr': '152.10.0.0/16',
-                                      'gateway_ip': '152.10.0.1',
-                                      'id': _uuid()},
+                          {'subnets': [{'cidr': '152.10.0.0/16',
+                                        'gateway_ip': '152.10.0.1',
+                                        'id': subnet_id_2}],
                            'network_id': _uuid(),
                            'device_owner': 'network:router_centralized_snat',
-                           'ip_cidr': '152.10.0.13/16',
                            'mac_address': 'fa:16:3e:80:8d:80',
-                           'fixed_ips': [{'subnet_id': _uuid(),
-                                         'ip_address': '152.10.0.13'}],
+                           'fixed_ips': [{'subnet_id': subnet_id_2,
+                                         'ip_address': '152.10.0.13',
+                                         'prefixlen': 16}],
                            'id': _uuid(), 'device_id': _uuid()}]
 
         self.ri_kwargs = {'agent_conf': self.conf,
@@ -325,20 +353,23 @@ class TestBasicRouterOperations(BasicRouterOperationsFramework):
         self.assertTrue(ri.ns_name.endswith(id))
 
     def test_router_info_create_with_router(self):
-        id = _uuid()
+        ns_id = _uuid()
+        subnet_id = _uuid()
         ex_gw_port = {'id': _uuid(),
                       'network_id': _uuid(),
                       'fixed_ips': [{'ip_address': '19.4.4.4',
-                                     'subnet_id': _uuid()}],
-                      'subnet': {'cidr': '19.4.4.0/24',
-                                 'gateway_ip': '19.4.4.1'}}
+                                     'prefixlen': 24,
+                                     'subnet_id': subnet_id}],
+                      'subnets': [{'id': subnet_id,
+                                   'cidr': '19.4.4.0/24',
+                                   'gateway_ip': '19.4.4.1'}]}
         router = {
             'id': _uuid(),
             'enable_snat': True,
             'routes': [],
             'gw_port': ex_gw_port}
-        ri = l3router.RouterInfo(id, router, **self.ri_kwargs)
-        self.assertTrue(ri.ns_name.endswith(id))
+        ri = l3router.RouterInfo(ns_id, router, **self.ri_kwargs)
+        self.assertTrue(ri.ns_name.endswith(ns_id))
         self.assertEqual(ri.router, router)
 
     def test_agent_create(self):
@@ -350,8 +381,10 @@ class TestBasicRouterOperations(BasicRouterOperationsFramework):
         ri = l3router.RouterInfo(router_id, router, **self.ri_kwargs)
         port = {'network_id': _uuid(),
                 'id': _uuid(),
-                'ip_cidr': '99.0.1.9/24',
-                'mac_address': 'ca:fe:de:ad:be:ef'}
+                'mac_address': 'ca:fe:de:ad:be:ef',
+                'fixed_ips': [{'subnet_id': _uuid(),
+                               'ip_address': '99.0.1.9',
+                               'prefixlen': 24}]}
 
         interface_name = ri.get_internal_device_name(port['id'])
 
@@ -369,37 +402,44 @@ class TestBasicRouterOperations(BasicRouterOperationsFramework):
         else:
             raise Exception("Invalid action %s" % action)
 
+    @staticmethod
+    def _fixed_ip_cidr(fixed_ip):
+        return '%s/%s' % (fixed_ip['ip_address'], fixed_ip['prefixlen'])
+
     def _test_internal_network_action_dist(self, action):
         router = prepare_router_data(num_internal_ports=2)
         router_id = router['id']
         agent = l3_agent.L3NATAgent(HOSTNAME, self.conf)
         ri = dvr_router.DvrRouter(
             agent, HOSTNAME, router_id, router, **self.ri_kwargs)
+        subnet_id = _uuid()
         port = {'network_id': _uuid(),
                 'id': _uuid(),
-                'ip_cidr': '99.0.1.9/24',
-                'mac_address': 'ca:fe:de:ad:be:ef'}
+                'mac_address': 'ca:fe:de:ad:be:ef',
+                'fixed_ips': [{'subnet_id': subnet_id,
+                               'ip_address': '99.0.1.9',
+                               'prefixlen': 24}],
+                'subnets': [{'id': subnet_id}]}
 
         ri.router['gw_port_host'] = HOSTNAME
         agent.host = HOSTNAME
         agent.conf.agent_mode = 'dvr_snat'
         sn_port = {'fixed_ips': [{'ip_address': '20.0.0.31',
                                  'subnet_id': _uuid()}],
-                  'subnet': {'gateway_ip': '20.0.0.1'},
+                  'subnets': [{'gateway_ip': '20.0.0.1'}],
                   'extra_subnets': [{'cidr': '172.16.0.0/24'}],
                   'id': _uuid(),
                   'network_id': _uuid(),
-                  'mac_address': 'ca:fe:de:ad:be:ef',
-                  'ip_cidr': '20.0.0.31/24'}
+                  'mac_address': 'ca:fe:de:ad:be:ef'}
         ex_gw_port = {'fixed_ips': [{'ip_address': '20.0.0.30',
+                                     'prefixlen': 24,
                                      'subnet_id': _uuid()}],
-                      'subnet': {'gateway_ip': '20.0.0.1'},
+                      'subnets': [{'gateway_ip': '20.0.0.1'}],
                       'extra_subnets': [{'cidr': '172.16.0.0/24'}],
                       'id': _uuid(),
                       'binding:host_id': HOSTNAME,
                       'network_id': _uuid(),
-                      'mac_address': 'ca:fe:de:ad:be:ef',
-                      'ip_cidr': '20.0.0.30/24'}
+                      'mac_address': 'ca:fe:de:ad:be:ef'}
         ri.snat_ports = sn_port
         ri.ex_gw_port = ex_gw_port
         ri.snat_namespace = mock.Mock()
@@ -409,32 +449,30 @@ class TestBasicRouterOperations(BasicRouterOperationsFramework):
 
             ri._map_internal_interfaces = mock.Mock(return_value=sn_port)
             ri._snat_redirect_add = mock.Mock()
-            ri._set_subnet_info = mock.Mock()
             ri._set_subnet_arp_info = mock.Mock()
             ri._internal_network_added = mock.Mock()
             ri._set_subnet_arp_info = mock.Mock()
             ri.internal_network_added(port)
             self.assertEqual(ri._snat_redirect_add.call_count, 1)
-            self.assertEqual(ri._set_subnet_info.call_count, 1)
             self.assertEqual(ri._internal_network_added.call_count, 2)
-            ri._set_subnet_arp_info.assert_called_once_with(port)
+            ri._set_subnet_arp_info.assert_called_once_with(subnet_id)
             ri._internal_network_added.assert_called_with(
                 dvr_snat_ns.SnatNamespace.get_snat_ns_name(ri.router['id']),
                 sn_port['network_id'],
                 sn_port['id'],
-                sn_port['ip_cidr'],
+                sn_port['fixed_ips'],
                 sn_port['mac_address'],
                 ri.get_snat_int_device_name(sn_port['id']),
                 dvr_snat_ns.SNAT_INT_DEV_PREFIX)
         elif action == 'remove':
             self.device_exists.return_value = False
             ri._map_internal_interfaces = mock.Mock(return_value=sn_port)
-            ri._snat_redirect_remove = mock.Mock()
+            ri._snat_redirect_modify = mock.Mock()
             ri.internal_network_removed(port)
-            ri._snat_redirect_remove.assert_called_with(
-                sn_port['fixed_ips'][0]['ip_address'],
-                port,
-                ri.get_internal_device_name(port['id']))
+            ri._snat_redirect_modify.assert_called_with(
+                sn_port, port,
+                ri.get_internal_device_name(port['id']),
+                is_add=False)
 
     def test_agent_add_internal_network(self):
         self._test_internal_network_action('add')
@@ -448,7 +486,7 @@ class TestBasicRouterOperations(BasicRouterOperationsFramework):
     def test_agent_remove_internal_network_dist(self):
         self._test_internal_network_action_dist('remove')
 
-    def _test_external_gateway_action(self, action, router):
+    def _test_external_gateway_action(self, action, router, dual_stack=False):
         agent = l3_agent.L3NATAgent(HOSTNAME, self.conf)
         ex_net_id = _uuid()
         sn_port = self.snat_ports[1]
@@ -471,14 +509,27 @@ class TestBasicRouterOperations(BasicRouterOperationsFramework):
                 router['id'], router,
                 **self.ri_kwargs)
 
-        ex_gw_port = {'fixed_ips': [{'ip_address': '20.0.0.30',
-                                     'subnet_id': _uuid()}],
-                      'subnet': {'gateway_ip': '20.0.0.1'},
+        subnet_id = _uuid()
+        fixed_ips = [{'subnet_id': subnet_id,
+                      'ip_address': '20.0.0.30',
+                      'prefixlen': 24}]
+        subnets = [{'id': subnet_id,
+                    'cidr': '20.0.0.0/24',
+                    'gateway_ip': '20.0.0.1'}]
+        if dual_stack:
+            subnet_id_v6 = _uuid()
+            fixed_ips.append({'subnet_id': subnet_id_v6,
+                              'ip_address': '2001:192:168:100::2',
+                              'prefixlen': 64})
+            subnets.append({'id': subnet_id_v6,
+                            'cidr': '2001:192:168:100::/64',
+                            'gateway_ip': '2001:192:168:100::1'})
+        ex_gw_port = {'fixed_ips': fixed_ips,
+                      'subnets': subnets,
                       'extra_subnets': [{'cidr': '172.16.0.0/24'}],
                       'id': _uuid(),
                       'network_id': ex_net_id,
-                      'mac_address': 'ca:fe:de:ad:be:ef',
-                      'ip_cidr': '20.0.0.30/24'}
+                      'mac_address': 'ca:fe:de:ad:be:ef'}
         interface_name = ri.get_external_device_name(ex_gw_port['id'])
 
         if action == 'add':
@@ -492,15 +543,24 @@ class TestBasicRouterOperations(BasicRouterOperationsFramework):
             if not router.get('distributed'):
                 self.assertEqual(self.mock_driver.plug.call_count, 1)
                 self.assertEqual(self.mock_driver.init_l3.call_count, 1)
-                self.send_arp.assert_called_once_with(ri.ns_name,
-                                                      interface_name,
-                                                      '20.0.0.30', mock.ANY)
+                exp_arp_calls = [mock.call(ri.ns_name, interface_name,
+                                           '20.0.0.30', mock.ANY)]
+                if dual_stack:
+                    exp_arp_calls += [mock.call(ri.ns_name, interface_name,
+                                                '2001:192:168:100::2',
+                                                mock.ANY)]
+                self.send_arp.assert_has_calls(exp_arp_calls)
+                ip_cidrs = ['20.0.0.30/24']
+                gateway_ips = ['20.0.0.1']
+                if dual_stack:
+                    ip_cidrs.append('2001:192:168:100::2/64')
+                    gateway_ips.append('2001:192:168:100::1')
                 kwargs = {'preserve_ips': ['192.168.1.34/32'],
+                          'gateway_ips': gateway_ips,
                           'namespace': 'qrouter-' + router['id'],
-                          'gateway': '20.0.0.1',
                           'extra_subnets': [{'cidr': '172.16.0.0/24'}]}
                 self.mock_driver.init_l3.assert_called_with(interface_name,
-                                                            ['20.0.0.30/24'],
+                                                            ip_cidrs,
                                                             **kwargs)
             else:
                 ri._create_dvr_gateway.assert_called_once_with(
@@ -520,31 +580,44 @@ class TestBasicRouterOperations(BasicRouterOperationsFramework):
                     prefix=mock.ANY)
             else:
                 ri._snat_redirect_remove.assert_called_with(
-                    sn_port['fixed_ips'][0]['ip_address'],
-                    sn_port,
+                    sn_port, sn_port,
                     ri.get_internal_device_name(sn_port['id']))
         else:
             raise Exception("Invalid action %s" % action)
 
-    def _prepare_ext_gw_test(self, ri):
-        ex_gw_port = {'fixed_ips': [{'ip_address': '20.0.0.30',
-                                     'subnet_id': _uuid()}],
-                      'subnet': {'gateway_ip': '20.0.0.1'},
+    def _prepare_ext_gw_test(self, ri, dual_stack=False):
+        subnet_id = _uuid()
+        fixed_ips = [{'subnet_id': subnet_id,
+                      'ip_address': '20.0.0.30',
+                      'prefixlen': 24}]
+        subnets = [{'id': subnet_id,
+                    'cidr': '20.0.0.0/24',
+                    'gateway_ip': '20.0.0.1'}]
+        if dual_stack:
+            subnet_id_v6 = _uuid()
+            fixed_ips.append({'subnet_id': subnet_id_v6,
+                              'ip_address': '2001:192:168:100::2',
+                              'prefixlen': 64})
+            subnets.append({'id': subnet_id_v6,
+                            'cidr': '2001:192:168:100::/64',
+                            'gateway_ip': '2001:192:168:100::1'})
+        ex_gw_port = {'fixed_ips': fixed_ips,
+                      'subnets': subnets,
                       'extra_subnets': [{'cidr': '172.16.0.0/24'}],
                       'id': _uuid(),
                       'network_id': _uuid(),
-                      'mac_address': 'ca:fe:de:ad:be:ef',
-                      'ip_cidr': '20.0.0.30/24'}
+                      'mac_address': 'ca:fe:de:ad:be:ef'}
         interface_name = ri.get_external_device_name(ex_gw_port['id'])
 
         self.device_exists.return_value = True
 
         return interface_name, ex_gw_port
 
-    def test_external_gateway_updated(self):
+    def _test_external_gateway_updated(self, dual_stack=False):
         router = prepare_router_data(num_internal_ports=2)
         ri = l3router.RouterInfo(router['id'], router, **self.ri_kwargs)
-        interface_name, ex_gw_port = self._prepare_ext_gw_test(ri)
+        interface_name, ex_gw_port = self._prepare_ext_gw_test(
+            ri, dual_stack=dual_stack)
 
         fake_fip = {'floatingips': [{'id': _uuid(),
                                      'floating_ip_address': '192.168.1.34',
@@ -554,16 +627,31 @@ class TestBasicRouterOperations(BasicRouterOperationsFramework):
         ri.external_gateway_updated(ex_gw_port, interface_name)
         self.assertEqual(self.mock_driver.plug.call_count, 0)
         self.assertEqual(self.mock_driver.init_l3.call_count, 1)
-        self.send_arp.assert_called_once_with(ri.ns_name, interface_name,
-                                              '20.0.0.30', mock.ANY)
+        exp_arp_calls = [mock.call(ri.ns_name, interface_name,
+                                   '20.0.0.30', mock.ANY)]
+        if dual_stack:
+            exp_arp_calls += [mock.call(ri.ns_name, interface_name,
+                                        '2001:192:168:100::2', mock.ANY)]
+        self.send_arp.assert_has_calls(exp_arp_calls)
+        ip_cidrs = ['20.0.0.30/24']
+        gateway_ips = ['20.0.0.1']
+        if dual_stack:
+            ip_cidrs.append('2001:192:168:100::2/64')
+            gateway_ips.append('2001:192:168:100::1')
         kwargs = {'preserve_ips': ['192.168.1.34/32'],
+                  'gateway_ips': gateway_ips,
                   'namespace': 'qrouter-' + router['id'],
-                  'gateway': '20.0.0.1',
                   'extra_subnets': [{'cidr': '172.16.0.0/24'}]}
         self.mock_driver.init_l3.assert_called_with(interface_name,
-                                                    ['20.0.0.30/24'],
+                                                    ip_cidrs,
                                                     **kwargs)
 
+    def test_external_gateway_updated(self):
+        self._test_external_gateway_updated()
+
+    def test_external_gateway_updated_dual_stack(self):
+        self._test_external_gateway_updated(dual_stack=True)
+
     def _test_ext_gw_updated_dvr_agent_mode(self, host,
                                             agent_mode, expected_call_count):
         router = prepare_router_data(num_internal_ports=2)
@@ -603,30 +691,51 @@ class TestBasicRouterOperations(BasicRouterOperationsFramework):
         router = prepare_router_data(num_internal_ports=2)
         self._test_external_gateway_action('add', router)
 
+    def test_agent_add_external_gateway_dual_stack(self):
+        router = prepare_router_data(num_internal_ports=2)
+        self._test_external_gateway_action('add', router, dual_stack=True)
+
     def test_agent_add_external_gateway_dist(self):
         router = prepare_router_data(num_internal_ports=2)
         router['distributed'] = True
         router['gw_port_host'] = HOSTNAME
         self._test_external_gateway_action('add', router)
 
+    def test_agent_add_external_gateway_dist_dual_stack(self):
+        router = prepare_router_data(num_internal_ports=2)
+        router['distributed'] = True
+        router['gw_port_host'] = HOSTNAME
+        self._test_external_gateway_action('add', router, dual_stack=True)
+
     def test_agent_remove_external_gateway(self):
         router = prepare_router_data(num_internal_ports=2)
         self._test_external_gateway_action('remove', router)
 
+    def test_agent_remove_external_gateway_dual_stack(self):
+        router = prepare_router_data(num_internal_ports=2)
+        self._test_external_gateway_action('remove', router, dual_stack=True)
+
     def test_agent_remove_external_gateway_dist(self):
         router = prepare_router_data(num_internal_ports=2)
         router['distributed'] = True
         router['gw_port_host'] = HOSTNAME
         self._test_external_gateway_action('remove', router)
 
+    def test_agent_remove_external_gateway_dist_dual_stack(self):
+        router = prepare_router_data(num_internal_ports=2)
+        router['distributed'] = True
+        router['gw_port_host'] = HOSTNAME
+        self._test_external_gateway_action('remove', router, dual_stack=True)
+
     def _verify_snat_rules(self, rules, router, negate=False):
         interfaces = router[l3_constants.INTERFACE_KEY]
         source_cidrs = []
         for iface in interfaces:
-            prefix = iface['subnet']['cidr'].split('/')[1]
-            source_cidr = "%s/%s" % (iface['fixed_ips'][0]['ip_address'],
-                                     prefix)
-            source_cidrs.append(source_cidr)
+            for subnet in iface['subnets']:
+                prefix = subnet['cidr'].split('/')[1]
+                source_cidr = "%s/%s" % (iface['fixed_ips'][0]['ip_address'],
+                                         prefix)
+                source_cidrs.append(source_cidr)
         source_nat_ip = router['gw_port']['fixed_ips'][0]['ip_address']
         interface_name = ('qg-%s' % router['gw_port']['id'])[:14]
         expected_rules = [
@@ -668,22 +777,25 @@ class TestBasicRouterOperations(BasicRouterOperationsFramework):
         ri = dvr_router.DvrRouter(
             agent, HOSTNAME, router['id'], router, **self.ri_kwargs)
         ports = ri.router.get(l3_constants.INTERFACE_KEY, [])
+        subnet_id = _get_subnet_id(ports[0])
         test_ports = [{'mac_address': '00:11:22:33:44:55',
                       'device_owner': 'network:dhcp',
-                      'subnet_id': _get_subnet_id(ports[0]),
-                      'fixed_ips': [{'ip_address': '1.2.3.4'}]}]
+                      'fixed_ips': [{'ip_address': '1.2.3.4',
+                                     'prefixlen': 24,
+                                     'subnet_id': subnet_id}]}]
 
         self.plugin_api.get_ports_by_subnet.return_value = test_ports
 
         # Test basic case
-        ports[0]['subnet']['id'] = _get_subnet_id(ports[0])
-        ri._set_subnet_arp_info(ports[0])
+        ports[0]['subnets'] = [{'id': subnet_id,
+                                'cidr': '1.2.3.0/24'}]
+        ri._set_subnet_arp_info(subnet_id)
         self.mock_ip_dev.neigh.add.assert_called_once_with(
             '1.2.3.4', '00:11:22:33:44:55')
 
         # Test negative case
         router['distributed'] = False
-        ri._set_subnet_arp_info(ports[0])
+        ri._set_subnet_arp_info(subnet_id)
         self.mock_ip_dev.neigh.add.never_called()
 
     def test_add_arp_entry(self):
@@ -860,16 +972,19 @@ class TestBasicRouterOperations(BasicRouterOperationsFramework):
 
     def test_get_floating_agent_gw_interfaces(self):
         fake_network_id = _uuid()
+        subnet_id = _uuid()
         agent_gateway_port = (
             [{'fixed_ips': [{'ip_address': '20.0.0.30',
-             'subnet_id': _uuid()}],
-             'subnet': {'gateway_ip': '20.0.0.1'},
-             'id': _uuid(),
-             'binding:host_id': 'myhost',
-             'device_owner': 'network:floatingip_agent_gateway',
-             'network_id': fake_network_id,
-             'mac_address': 'ca:fe:de:ad:be:ef',
-             'ip_cidr': '20.0.0.30/24'}]
+                             'prefixlen': 24,
+                             'subnet_id': subnet_id}],
+              'subnets': [{'id': subnet_id,
+                           'cidr': '20.0.0.0/24',
+                           'gateway_ip': '20.0.0.1'}],
+              'id': _uuid(),
+              'binding:host_id': 'myhost',
+              'device_owner': 'network:floatingip_agent_gateway',
+              'network_id': fake_network_id,
+              'mac_address': 'ca:fe:de:ad:be:ef'}]
         )
 
         router = prepare_router_data(enable_snat=True)
@@ -885,6 +1000,7 @@ class TestBasicRouterOperations(BasicRouterOperationsFramework):
     @mock.patch.object(lla.LinkLocalAllocator, '_write')
     def test_create_dvr_fip_interfaces(self, lla_write):
         fake_network_id = _uuid()
+        subnet_id = _uuid()
         fake_floatingips = {'floatingips': [
             {'id': _uuid(),
              'floating_ip_address': '20.0.0.3',
@@ -893,13 +1009,16 @@ class TestBasicRouterOperations(BasicRouterOperationsFramework):
              'port_id': _uuid(),
              'host': HOSTNAME}]}
         agent_gateway_port = (
-            [{'fixed_ips': [{'ip_address': '20.0.0.30',
-             'subnet_id': _uuid()}],
-             'subnet': {'gateway_ip': '20.0.0.1'},
+            [{'fixed_ips': [
+                {'ip_address': '20.0.0.30',
+                 'prefixlen': 24,
+                 'subnet_id': subnet_id}],
+             'subnets': [
+                 {'id': subnet_id,
+                  'gateway_ip': '20.0.0.1'}],
              'id': _uuid(),
              'network_id': fake_network_id,
-             'mac_address': 'ca:fe:de:ad:be:ef',
-             'ip_cidr': '20.0.0.30/24'}]
+             'mac_address': 'ca:fe:de:ad:be:ef'}]
         )
 
         router = prepare_router_data(enable_snat=True)
@@ -918,12 +1037,9 @@ class TestBasicRouterOperations(BasicRouterOperationsFramework):
         with contextlib.nested(mock.patch.object(ri,
                                                  'get_floating_ips'),
                                mock.patch.object(
-                                   ri, 'get_floating_agent_gw_interface'),
-                               mock.patch.object(
-                                   ri, '_set_subnet_info')
+                                   ri, 'get_floating_agent_gw_interface')
                                ) as (fips,
-                                     fip_gw_port,
-                                     sub_info):
+                                     fip_gw_port):
             fips.return_value = fake_floatingips
             fip_gw_port.return_value = agent_gateway_port[0]
             ri.create_dvr_fip_interfaces(ext_gw_port)
@@ -976,14 +1092,16 @@ class TestBasicRouterOperations(BasicRouterOperationsFramework):
         ri.iptables_manager.ipv4['nat'] = mock.MagicMock()
         ri.dist_fip_count = 0
         fip_ns = agent.get_fip_ns(mock.sentinel.ext_net_id)
+        subnet_id = _uuid()
         fip_ns.agent_gateway_port = (
             {'fixed_ips': [{'ip_address': '20.0.0.30',
-             'subnet_id': _uuid()}],
-             'subnet': {'gateway_ip': '20.0.0.1'},
+                            'subnet_id': subnet_id}],
+             'subnets': [{'id': subnet_id,
+                          'cidr': '20.0.0.0/24',
+                          'gateway_ip': '20.0.0.1'}],
              'id': _uuid(),
              'network_id': _uuid(),
-             'mac_address': 'ca:fe:de:ad:be:ef',
-             'ip_cidr': '20.0.0.30/24'}
+             'mac_address': 'ca:fe:de:ad:be:ef'}
         )
 
     def test_process_router_snat_disabled(self):
@@ -1043,9 +1161,9 @@ class TestBasicRouterOperations(BasicRouterOperationsFramework):
         # send_arp is called both times process_router is called
         self.assertEqual(self.send_arp.call_count, 2)
 
-    def test_process_ipv6_only_gw(self):
+    def _test_process_ipv6_only_or_dual_stack_gw(self, dual_stack=False):
         agent = l3_agent.L3NATAgent(HOSTNAME, self.conf)
-        router = prepare_router_data(ip_version=6)
+        router = prepare_router_data(ip_version=6, dual_stack=dual_stack)
         # Get NAT rules without the gw_port
         gw_port = router['gw_port']
         router['gw_port'] = None
@@ -1057,15 +1175,27 @@ class TestBasicRouterOperations(BasicRouterOperationsFramework):
         # Get NAT rules with the gw_port
         router['gw_port'] = gw_port
         ri = l3router.RouterInfo(router['id'], router, **self.ri_kwargs)
+        orig_ext_gw_nat_rules = ri.external_gateway_nat_rules
         with mock.patch.object(
                 ri,
                 'external_gateway_nat_rules') as external_gateway_nat_rules:
+            external_gateway_nat_rules.side_effect = orig_ext_gw_nat_rules
             self._process_router_instance_for_agent(agent, ri, router)
             new_nat_rules = ri.iptables_manager.ipv4['nat'].rules[:]
 
-            # There should be no change with the NAT rules
-            self.assertFalse(external_gateway_nat_rules.called)
-            self.assertEqual(orig_nat_rules, new_nat_rules)
+            # NAT rules should only change for dual_stack operation
+            if dual_stack:
+                self.assertTrue(external_gateway_nat_rules.called)
+                self.assertNotEqual(orig_nat_rules, new_nat_rules)
+            else:
+                self.assertFalse(external_gateway_nat_rules.called)
+                self.assertEqual(orig_nat_rules, new_nat_rules)
+
+    def test_process_ipv6_only_gw(self):
+        self._test_process_ipv6_only_or_dual_stack_gw()
+
+    def test_process_dual_stack_gw(self):
+        self._test_process_ipv6_only_or_dual_stack_gw(dual_stack=True)
 
     def _process_router_ipv6_interface_added(
             self, router, ra_mode=None, addr_mode=None):
@@ -1751,13 +1881,16 @@ class TestBasicRouterOperations(BasicRouterOperationsFramework):
                                   **self.ri_kwargs)
 
         port_id = _uuid()
+        subnet_id = _uuid()
         dvr_gw_port = {'fixed_ips': [{'ip_address': '20.0.0.30',
-                                     'subnet_id': _uuid()}],
-                       'subnet': {'gateway_ip': '20.0.0.1'},
+                                      'prefixlen': 24,
+                                      'subnet_id': subnet_id}],
+                       'subnets': [{'id': subnet_id,
+                                    'cidr': '20.0.0.0/24',
+                                    'gateway_ip': '20.0.0.1'}],
                        'id': port_id,
                        'network_id': _uuid(),
-                       'mac_address': 'ca:fe:de:ad:be:ef',
-                       'ip_cidr': '20.0.0.30/24'}
+                       'mac_address': 'ca:fe:de:ad:be:ef'}
 
         interface_name = ri.get_snat_int_device_name(port_id)
         self.device_exists.return_value = False
@@ -1826,15 +1959,19 @@ class TestBasicRouterOperations(BasicRouterOperationsFramework):
         nat.add_rule = mock.Mock()
         if fip_ns:
             ri.fip_ns = agent.get_fip_ns(external_net_id)
+            subnet_id = _uuid()
             ri.fip_ns.agent_gateway_port = {
                 'fixed_ips': [{
-                               'ip_address': '20.0.0.30', 'subnet_id': _uuid()
+                               'ip_address': '20.0.0.30',
+                               'prefixlen': 24,
+                               'subnet_id': subnet_id
                             }],
-                'subnet': {'gateway_ip': '20.0.0.1'},
+                'subnets': [{'id': subnet_id,
+                             'cidr': '20.0.0.0/24',
+                            'gateway_ip': '20.0.0.1'}],
                 'id': _uuid(),
                 'network_id': external_net_id,
-                'mac_address': 'ca:fe:de:ad:be:ef',
-                'ip_cidr': '20.0.0.30/24'}
+                'mac_address': 'ca:fe:de:ad:be:ef'}
 
             vm_floating_ip = '19.4.4.2'
             ri.floating_ips_dict[vm_floating_ip] = FIP_PRI

neutron/tests/unit/test_l3_plugin.py

@@ -2234,7 +2234,9 @@ class L3AgentDbTestCaseBase(L3NatTestCaseMixin):
                 self.assertEqual(1, len(routers))
                 interfaces = routers[0][l3_constants.INTERFACE_KEY]
                 self.assertEqual(1, len(interfaces))
-                subnet_id = interfaces[0]['subnet']['id']
+                subnets = interfaces[0]['subnets']
+                self.assertEqual(1, len(subnets))
+                subnet_id = subnets[0]['id']
                 wanted_subnetid = p['port']['fixed_ips'][0]['subnet_id']
                 self.assertEqual(wanted_subnetid, subnet_id)
                 # clean-up
@@ -2280,7 +2282,9 @@ class L3AgentDbTestCaseBase(L3NatTestCaseMixin):
                     context.get_admin_context(), [r['router']['id']])
                 self.assertEqual(1, len(routers))
                 gw_port = routers[0]['gw_port']
-                self.assertEqual(s['subnet']['id'], gw_port['subnet']['id'])
+                subnets = gw_port.get('subnets')
+                self.assertEqual(1, len(subnets))
+                self.assertEqual(s['subnet']['id'], subnets[0]['id'])
                 self._remove_external_gateway_from_router(
                     r['router']['id'],
                     s['subnet']['network_id'])

neutron/tests/unit/test_linux_interface.py

@@ -123,7 +123,7 @@ class TestABCDriver(TestBase):
              mock.call().addr.add('192.168.1.2/24')])
         self.assertFalse(self.ip_dev().addr.delete.called)
 
-    def test_l3_init_with_ipv6(self):
+    def _test_l3_init_with_ipv6(self, include_gw_ip):
         addresses = [dict(scope='global',
                           dynamic=False,
                           cidr='2001:db8:a::123/64')]
@@ -132,16 +132,54 @@ class TestABCDriver(TestBase):
 
         bc = BaseChild(self.conf)
         ns = '12345678-1234-5678-90ab-ba0987654321'
-        bc.init_l3('tap0', ['2001:db8:a::124/64'], namespace=ns,
-                   extra_subnets=[{'cidr': '2001:db8:b::/64'}])
-        self.ip_dev.assert_has_calls(
+        new_cidr = '2001:db8:a::124/64'
+        kwargs = {'namespace': ns,
+                  'extra_subnets': [{'cidr': '2001:db8:b::/64'}]}
+        if include_gw_ip:
+            kwargs['gateway_ips'] = ['2001:db8:a::1']
+        bc.init_l3('tap0', [new_cidr], **kwargs)
+        expected_calls = (
             [mock.call('tap0', namespace=ns),
              mock.call().addr.list(scope='global', filters=['permanent']),
              mock.call().addr.add('2001:db8:a::124/64'),
+             mock.call().addr.delete('2001:db8:a::123/64')])
+        if include_gw_ip:
+            expected_calls += (
+                [mock.call().route.add_gateway('2001:db8:a::1')])
+        expected_calls += (
+             [mock.call().route.list_onlink_routes(constants.IP_VERSION_4),
+              mock.call().route.list_onlink_routes(constants.IP_VERSION_6),
+              mock.call().route.add_onlink_route('2001:db8:b::/64')])
+        self.ip_dev.assert_has_calls(expected_calls)
+
+    def test_l3_init_ipv6_with_gw_ip(self):
+        self._test_l3_init_with_ipv6(include_gw_ip=True)
+
+    def test_l3_init_ipv6_without_gw_ip(self):
+        self._test_l3_init_with_ipv6(include_gw_ip=False)
+
+    def test_l3_init_ext_gw_with_dual_stack(self):
+        old_addrs = [dict(ip_version=4, scope='global',
+                          dynamic=False, cidr='172.16.77.240/24'),
+                     dict(ip_version=6, scope='global',
+                          dynamic=False, cidr='2001:db8:a::123/64')]
+        self.ip_dev().addr.list = mock.Mock(return_value=old_addrs)
+        self.ip_dev().route.list_onlink_routes.return_value = []
+        bc = BaseChild(self.conf)
+        ns = '12345678-1234-5678-90ab-ba0987654321'
+        new_cidrs = ['192.168.1.2/24', '2001:db8:a::124/64']
+        bc.init_l3('tap0', new_cidrs, namespace=ns,
+                   extra_subnets=[{'cidr': '172.20.0.0/24'}])
+        self.ip_dev.assert_has_calls(
+            [mock.call('tap0', namespace=ns),
+             mock.call().addr.list(scope='global', filters=['permanent']),
+             mock.call().addr.add('192.168.1.2/24'),
+             mock.call().addr.add('2001:db8:a::124/64'),
+             mock.call().addr.delete('172.16.77.240/24'),
              mock.call().addr.delete('2001:db8:a::123/64'),
              mock.call().route.list_onlink_routes(constants.IP_VERSION_4),
              mock.call().route.list_onlink_routes(constants.IP_VERSION_6),
-             mock.call().route.add_onlink_route('2001:db8:b::/64')])
+             mock.call().route.add_onlink_route('172.20.0.0/24')])
 
     def test_l3_init_with_ipv6_delete_onlink_routes(self):
         addresses = [dict(scope='global',