Add L3 VRRP HA base classes

Add L3 HA base classes on the plugin side. A new admin-only ha
attribute is added to the API router resource. Conversion from
or to HA router is possible. Each tenant gets a single network
used for HA traffic. The tenant_id for that network is set to
'' so that it isn't visible via the CLI or GUI. A new table
is added to map a tenant to its HA network. Specific HA
attributes are added to the extra router attributes table.
Finally, each HA router gets a port on the HA network, per
l3 agent it is scheduled on. A new table is added to track
these bindings. A new table is added in order to track
VRID allocations.

DVR integration is not expected to work. Any issues will
be reported as bugs and handled after the feature merges.
Migrating a router to HA or from HA works server side
but is not expected to work (Yet) agent side. This will be
dealt with as a bug in the future.

DocImpact
Partially-implements: blueprint l3-high-availability
Change-Id: I9d935cf5e0c231e8cb7af5f61b9a9fc552c3521e
Co-Authored-By: Assaf Muller <amuller@redhat.com>

etc/neutron.conf

@@ -172,6 +172,22 @@ lock_path = $state_path/lock
 
 # ===========  end of items for agent scheduler extension =====
 
+# =========== items for l3 extension ==============
+# Enable high availability for virtual routers.
+# l3_ha = False
+#
+# Maximum number of l3 agents which a HA router will be scheduled on. If it
+# is set to 0 the router will be scheduled on every agent.
+# max_l3_agents_per_router = 3
+#
+# Minimum number of l3 agents which a HA router will be scheduled on. The
+# default value is 2.
+# min_l3_agents_per_router = 2
+#
+# CIDR of the administrative network if HA mode is enabled
+# l3_ha_net_cidr = 169.254.192.0/18
+# =========== end of items for l3 extension =======
+
 # =========== WSGI parameters related to the API server ==============
 # Number of separate worker processes to spawn.  The default, 0, runs the
 # worker thread in the current process.  Greater than 0 launches that number of

etc/policy.json

@@ -58,13 +58,16 @@
     "update_port:mac_learning_enabled": "rule:admin_or_network_owner",
     "delete_port": "rule:admin_or_owner",
 
+    "get_router:ha": "rule:admin_only",
     "create_router": "rule:regular_user",
     "create_router:external_gateway_info:enable_snat": "rule:admin_only",
     "create_router:distributed": "rule:admin_only",
+    "create_router:ha": "rule:admin_only",
     "get_router": "rule:admin_or_owner",
     "get_router:distributed": "rule:admin_only",
     "update_router:external_gateway_info:enable_snat": "rule:admin_only",
     "update_router:distributed": "rule:admin_only",
+    "update_router:ha": "rule:admin_only",
     "delete_router": "rule:admin_or_owner",
 
     "add_router_interface": "rule:admin_or_owner",

neutron/api/rpc/handlers/l3_rpc.py

@@ -38,7 +38,8 @@ class L3RpcCallback(n_rpc.RpcCallback):
     # 1.1  Support update_floatingip_statuses
     # 1.2 Added methods for DVR support
     # 1.3 Added a method that returns the list of activated services
-    RPC_API_VERSION = '1.3'
+    # 1.4 Added L3 HA update_router_state
+    RPC_API_VERSION = '1.4'
 
     @property
     def plugin(self):
@@ -104,6 +105,10 @@ class L3RpcCallback(n_rpc.RpcCallback):
             for interface in router.get(constants.INTERFACE_KEY, []):
                 self._ensure_host_set_on_port(context, host,
                                               interface, router['id'])
+            interface = router.get(constants.HA_INTERFACE_KEY)
+            if interface:
+                self._ensure_host_set_on_port(context, host, interface,
+                                              router['id'])
 
     def _ensure_host_set_on_port(self, context, host, port, router_id=None):
         if (port and
@@ -224,3 +229,11 @@ class L3RpcCallback(n_rpc.RpcCallback):
                   'and on host %(host)s', {'snat_port_list': snat_port_list,
                   'host': host})
         return snat_port_list
+
+    def update_router_state(self, context, **kwargs):
+        router_id = kwargs.get('router_id')
+        state = kwargs.get('state')
+        host = kwargs.get('host')
+
+        return self.l3plugin.update_router_state(context, router_id, state,
+                                                 host=host)

neutron/common/constants.py

@@ -29,6 +29,7 @@ FLOATINGIP_STATUS_ACTIVE = 'ACTIVE'
 FLOATINGIP_STATUS_DOWN = 'DOWN'
 FLOATINGIP_STATUS_ERROR = 'ERROR'
 
+DEVICE_OWNER_ROUTER_HA_INTF = "network:router_ha_interface"
 DEVICE_OWNER_ROUTER_INTF = "network:router_interface"
 DEVICE_OWNER_ROUTER_GW = "network:router_gateway"
 DEVICE_OWNER_FLOATINGIP = "network:floatingip"
@@ -42,10 +43,17 @@ DEVICE_ID_RESERVED_DHCP_PORT = "reserved_dhcp_port"
 
 FLOATINGIP_KEY = '_floatingips'
 INTERFACE_KEY = '_interfaces'
+HA_INTERFACE_KEY = '_ha_interface'
+HA_ROUTER_STATE_KEY = '_ha_state'
 METERING_LABEL_KEY = '_metering_labels'
 FLOATINGIP_AGENT_INTF_KEY = '_floatingip_agent_interfaces'
 SNAT_ROUTER_INTF_KEY = '_snat_router_interfaces'
 
+HA_NETWORK_NAME = 'HA network tenant %s'
+HA_SUBNET_NAME = 'HA subnet tenant %s'
+HA_PORT_NAME = 'HA port tenant %s'
+MINIMUM_AGENTS_FOR_HA = 2
+
 IPv4 = 'IPv4'
 IPv6 = 'IPv6'
 
@@ -101,6 +109,7 @@ L3_AGENT_SCHEDULER_EXT_ALIAS = 'l3_agent_scheduler'
 DHCP_AGENT_SCHEDULER_EXT_ALIAS = 'dhcp_agent_scheduler'
 LBAAS_AGENT_SCHEDULER_EXT_ALIAS = 'lbaas_agent_scheduler'
 L3_DISTRIBUTED_EXT_ALIAS = 'dvr'
+L3_HA_MODE_EXT_ALIAS = 'l3-ha'
 
 # Protocol names and numbers for Security Groups/Firewalls
 PROTO_NAME_TCP = 'tcp'

neutron/db/l3_agentschedulers_db.py

@@ -284,8 +284,14 @@ class L3AgentSchedulerDbMixin(l3agentscheduler.L3AgentSchedulerPluginBase,
                 RouterL3AgentBinding.router_id.in_(router_ids))
         router_ids = [item[0] for item in query]
         if router_ids:
-            return self.get_sync_data(context, router_ids=router_ids,
-                                      active=True)
+            if n_utils.is_extension_supported(self,
+                                              constants.L3_HA_MODE_EXT_ALIAS):
+                return self.get_ha_sync_data_for_host(context, host,
+                                                      router_ids=router_ids,
+                                                      active=True)
+            else:
+                return self.get_sync_data(context, router_ids=router_ids,
+                                          active=True)
         else:
             return []
 

neutron/db/l3_attrs_db.py

@@ -40,6 +40,11 @@ class RouterExtraAttributes(model_base.BASEV2):
     service_router = sa.Column(sa.Boolean, default=False,
                                server_default=sa.sql.false(),
                                nullable=False)
+    ha = sa.Column(sa.Boolean, default=False,
+                   server_default=sa.sql.false(),
+                   nullable=False)
+    ha_vr_id = sa.Column(sa.Integer())
+
     router = orm.relationship(
         l3_db.Router,
         backref=orm.backref("extra_attributes", lazy='joined',

neutron/db/l3_dvr_db.py

@@ -61,7 +61,7 @@ class L3_NAT_with_dvr_db_mixin(l3_db.L3_NAT_db_mixin,
 
     def _create_router_db(self, context, router, tenant_id):
         """Create a router db object with dvr additions."""
-        router['distributed'] = _is_distributed_router(router)
+        router['distributed'] = is_distributed_router(router)
         with context.session.begin(subtransactions=True):
             router_db = super(
                 L3_NAT_with_dvr_db_mixin, self)._create_router_db(
@@ -128,7 +128,7 @@ class L3_NAT_with_dvr_db_mixin(l3_db.L3_NAT_db_mixin,
         router_is_uuid = isinstance(router, basestring)
         if router_is_uuid:
             router = self._get_router(context, router)
-        if _is_distributed_router(router):
+        if is_distributed_router(router):
             return DEVICE_OWNER_DVR_INTERFACE
         return super(L3_NAT_with_dvr_db_mixin,
                      self)._get_device_owner(context, router)
@@ -534,7 +534,7 @@ class L3_NAT_with_dvr_db_mixin(l3_db.L3_NAT_db_mixin,
                                                   l3_port_check=False)
 
 
-def _is_distributed_router(router):
+def is_distributed_router(router):
     """Return True if router to be handled is distributed."""
     try:
         # See if router is a DB object first

neutron/db/l3_hamode_db.py

@@ -0,0 +1,459 @@
+# Copyright (C) 2014 eNovance SAS <licensing@enovance.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+import netaddr
+from oslo.config import cfg
+from oslo.db import exception as db_exc
+import sqlalchemy as sa
+from sqlalchemy import orm
+
+from neutron.api.v2 import attributes
+from neutron.common import constants
+from neutron.db import agents_db
+from neutron.db import l3_dvr_db
+from neutron.db import model_base
+from neutron.db import models_v2
+from neutron.extensions import l3_ext_ha_mode as l3_ha
+from neutron.openstack.common import excutils
+from neutron.openstack.common.gettextutils import _LI
+from neutron.openstack.common.gettextutils import _LW
+from neutron.openstack.common import log as logging
+
+VR_ID_RANGE = set(range(1, 255))
+MAX_ALLOCATION_TRIES = 10
+
+LOG = logging.getLogger(__name__)
+
+L3_HA_OPTS = [
+    cfg.BoolOpt('l3_ha',
+                default=False,
+                help=_('Enable HA mode for virtual routers.')),
+    cfg.IntOpt('max_l3_agents_per_router',
+               default=3,
+               help=_('Maximum number of agents on which a router will be '
+                      'scheduled.')),
+    cfg.IntOpt('min_l3_agents_per_router',
+               default=constants.MINIMUM_AGENTS_FOR_HA,
+               help=_('Minimum number of agents on which a router will be '
+                      'scheduled.')),
+    cfg.StrOpt('l3_ha_net_cidr',
+               default='169.254.192.0/18',
+               help=_('Subnet used for the l3 HA admin network.')),
+]
+cfg.CONF.register_opts(L3_HA_OPTS)
+
+
+class L3HARouterAgentPortBinding(model_base.BASEV2):
+    """Represent agent binding state of a HA router port.
+
+    A HA Router has one HA port per agent on which it is spawned.
+    This binding table stores which port is used for a HA router by a
+    L3 agent.
+    """
+
+    __tablename__ = 'ha_router_agent_port_bindings'
+
+    port_id = sa.Column(sa.String(36), sa.ForeignKey('ports.id',
+                                                     ondelete='CASCADE'),
+                        nullable=False, primary_key=True)
+    port = orm.relationship(models_v2.Port)
+
+    router_id = sa.Column(sa.String(36), sa.ForeignKey('routers.id',
+                                                       ondelete='CASCADE'),
+                          nullable=False)
+
+    l3_agent_id = sa.Column(sa.String(36),
+                            sa.ForeignKey("agents.id",
+                                          ondelete='CASCADE'))
+    agent = orm.relationship(agents_db.Agent)
+
+    state = sa.Column(sa.Enum('active', 'standby', name='l3_ha_states'),
+                      default='standby',
+                      server_default='standby')
+
+
+class L3HARouterNetwork(model_base.BASEV2):
+    """Host HA network for a tenant.
+
+    One HA Network is used per tenant, all HA router ports are created
+    on this network.
+    """
+
+    __tablename__ = 'ha_router_networks'
+
+    tenant_id = sa.Column(sa.String(255), primary_key=True,
+                          nullable=False)
+    network_id = sa.Column(sa.String(36),
+                           sa.ForeignKey('networks.id', ondelete="CASCADE"),
+                           nullable=False, primary_key=True)
+    network = orm.relationship(models_v2.Network)
+
+
+class L3HARouterVRIdAllocation(model_base.BASEV2):
+    """VRID allocation per HA network.
+
+    Keep a track of the VRID allocations per HA network.
+    """
+
+    __tablename__ = 'ha_router_vrid_allocations'
+
+    network_id = sa.Column(sa.String(36),
+                           sa.ForeignKey('networks.id', ondelete="CASCADE"),
+                           nullable=False, primary_key=True)
+    vr_id = sa.Column(sa.Integer(), nullable=False, primary_key=True)
+
+
+class L3_HA_NAT_db_mixin(l3_dvr_db.L3_NAT_with_dvr_db_mixin):
+    """Mixin class to add high availability capability to routers."""
+
+    extra_attributes = (
+        l3_dvr_db.L3_NAT_with_dvr_db_mixin.extra_attributes + [
+            {'name': 'ha', 'default': cfg.CONF.l3_ha},
+            {'name': 'ha_vr_id', 'default': 0}])
+
+    def _verify_configuration(self):
+        self.ha_cidr = cfg.CONF.l3_ha_net_cidr
+        try:
+            net = netaddr.IPNetwork(self.ha_cidr)
+        except netaddr.AddrFormatError:
+            raise l3_ha.HANetworkCIDRNotValid(cidr=self.ha_cidr)
+        if ('/' not in self.ha_cidr or net.network != net.ip):
+            raise l3_ha.HANetworkCIDRNotValid(cidr=self.ha_cidr)
+
+        if cfg.CONF.min_l3_agents_per_router < constants.MINIMUM_AGENTS_FOR_HA:
+            raise l3_ha.HAMinimumAgentsNumberNotValid()
+
+    def __init__(self):
+        self._verify_configuration()
+        super(L3_HA_NAT_db_mixin, self).__init__()
+
+    def get_ha_network(self, context, tenant_id):
+        return (context.session.query(L3HARouterNetwork).
+                filter(L3HARouterNetwork.tenant_id == tenant_id).
+                first())
+
+    def _get_allocated_vr_id(self, context, network_id):
+        with context.session.begin(subtransactions=True):
+            query = (context.session.query(L3HARouterVRIdAllocation).
+                     filter(L3HARouterVRIdAllocation.network_id == network_id))
+
+            allocated_vr_ids = set(a.vr_id for a in query) - set([0])
+
+        return allocated_vr_ids
+
+    def _allocate_vr_id(self, context, network_id, router_id):
+        for count in range(MAX_ALLOCATION_TRIES):
+            try:
+                with context.session.begin(subtransactions=True):
+                    allocated_vr_ids = self._get_allocated_vr_id(context,
+                                                                 network_id)
+                    available_vr_ids = VR_ID_RANGE - allocated_vr_ids
+
+                    if not available_vr_ids:
+                        raise l3_ha.NoVRIDAvailable(router_id=router_id)
+
+                    allocation = L3HARouterVRIdAllocation()
+                    allocation.network_id = network_id
+                    allocation.vr_id = available_vr_ids.pop()
+
+                    context.session.add(allocation)
+
+                    return allocation.vr_id
+
+            except db_exc.DBDuplicateEntry:
+                LOG.info(_LI("Attempt %(count)s to allocate a VRID in the "
+                             "network %(network)s for the router %(router)s"),
+                         {'count': count, 'network': network_id,
+                          'router': router_id})
+
+        raise l3_ha.MaxVRIDAllocationTriesReached(
+            network_id=network_id, router_id=router_id,
+            max_tries=MAX_ALLOCATION_TRIES)
+
+    def _delete_vr_id_allocation(self, context, ha_network, vr_id):
+        with context.session.begin(subtransactions=True):
+            context.session.query(L3HARouterVRIdAllocation).filter_by(
+                network_id=ha_network.network_id,
+                vr_id=vr_id).delete()
+
+    def _set_vr_id(self, context, router, ha_network):
+        with context.session.begin(subtransactions=True):
+            router.extra_attributes.ha_vr_id = self._allocate_vr_id(
+                context, ha_network.network_id, router.id)
+
+    def _create_ha_subnet(self, context, network_id, tenant_id):
+        args = {'subnet':
+                {'network_id': network_id,
+                 'tenant_id': '',
+                 'name': constants.HA_SUBNET_NAME % tenant_id,
+                 'ip_version': 4,
+                 'cidr': cfg.CONF.l3_ha_net_cidr,
+                 'enable_dhcp': False,
+                 'host_routes': attributes.ATTR_NOT_SPECIFIED,
+                 'dns_nameservers': attributes.ATTR_NOT_SPECIFIED,
+                 'allocation_pools': attributes.ATTR_NOT_SPECIFIED,
+                 'gateway_ip': None}}
+        return self._core_plugin.create_subnet(context, args)
+
+    def _create_ha_network_tenant_binding(self, context, tenant_id,
+                                          network_id):
+        with context.session.begin(subtransactions=True):
+            ha_network = L3HARouterNetwork(tenant_id=tenant_id,
+                                           network_id=network_id)
+            context.session.add(ha_network)
+        return ha_network
+
+    def _create_ha_network(self, context, tenant_id):
+        admin_ctx = context.elevated()
+
+        args = {'network':
+                {'name': constants.HA_NETWORK_NAME % tenant_id,
+                 'tenant_id': '',
+                 'shared': False,
+                 'admin_state_up': True,
+                 'status': constants.NET_STATUS_ACTIVE}}
+        network = self._core_plugin.create_network(context, args)
+        try:
+            ha_network = self._create_ha_network_tenant_binding(admin_ctx,
+                                                                tenant_id,
+                                                                network['id'])
+        except Exception:
+            with excutils.save_and_reraise_exception():
+                self._core_plugin.delete_network(admin_ctx, network['id'])
+
+        try:
+            self._create_ha_subnet(admin_ctx, network['id'], tenant_id)
+        except Exception:
+            with excutils.save_and_reraise_exception():
+                self._core_plugin.delete_network(admin_ctx, network['id'])
+
+        return ha_network
+
+    def get_number_of_agents_for_scheduling(self, context):
+        """Return the number of agents on which the router will be scheduled.
+
+        Raises an exception if there are not enough agents available to honor
+        the min_agents config parameter. If the max_agents parameter is set to
+        0 all the agents will be used.
+        """
+
+        min_agents = cfg.CONF.min_l3_agents_per_router
+        num_agents = len(self.get_l3_agents(context))
+        max_agents = cfg.CONF.max_l3_agents_per_router
+        if max_agents:
+            if max_agents > num_agents:
+                LOG.info(_LI("Number of available agents lower than "
+                             "max_l3_agents_per_router. L3 agents "
+                             "available: %s"), num_agents)
+            else:
+                num_agents = max_agents
+
+        if num_agents < min_agents:
+            raise l3_ha.HANotEnoughAvailableAgents(min_agents=min_agents,
+                                                   num_agents=num_agents)
+
+        return num_agents
+
+    def _create_ha_port_binding(self, context, port_id, router_id):
+        with context.session.begin(subtransactions=True):
+            portbinding = L3HARouterAgentPortBinding(port_id=port_id,
+                                                     router_id=router_id)
+            context.session.add(portbinding)
+
+        return portbinding
+
+    def add_ha_port(self, context, router_id, network_id, tenant_id):
+        port = self._core_plugin.create_port(context, {
+            'port':
+            {'tenant_id': '',
+             'network_id': network_id,
+             'fixed_ips': attributes.ATTR_NOT_SPECIFIED,
+             'mac_address': attributes.ATTR_NOT_SPECIFIED,
+             'admin_state_up': True,
+             'device_id': router_id,
+             'device_owner': constants.DEVICE_OWNER_ROUTER_HA_INTF,
+             'name': constants.HA_PORT_NAME % tenant_id}})
+
+        try:
+            return self._create_ha_port_binding(context, port['id'], router_id)
+        except Exception:
+            with excutils.save_and_reraise_exception():
+                self._core_plugin.delete_port(context, port['id'],
+                                              l3_port_check=False)
+
+    def _create_ha_interfaces(self, context, router, ha_network):
+        admin_ctx = context.elevated()
+
+        num_agents = self.get_number_of_agents_for_scheduling(context)
+
+        port_ids = []
+        try:
+            for index in range(num_agents):
+                binding = self.add_ha_port(admin_ctx, router.id,
+                                           ha_network.network['id'],
+                                           router.tenant_id)
+                port_ids.append(binding.port_id)
+        except Exception:
+            with excutils.save_and_reraise_exception():
+                for port_id in port_ids:
+                    self._core_plugin.delete_port(admin_ctx, port_id,
+                                                  l3_port_check=False)
+
+    def _delete_ha_interfaces(self, context, router_id):
+        admin_ctx = context.elevated()
+        device_filter = {'device_id': [router_id],
+                         'device_owner':
+                         [constants.DEVICE_OWNER_ROUTER_HA_INTF]}
+        ports = self._core_plugin.get_ports(admin_ctx, filters=device_filter)
+
+        for port in ports:
+            self._core_plugin.delete_port(admin_ctx, port['id'],
+                                          l3_port_check=False)
+
+    def _notify_ha_interfaces_updated(self, context, router_id):
+        self.l3_rpc_notifier.routers_updated(context, [router_id])
+
+    @classmethod
+    def _is_ha(cls, router):
+        ha = router.get('ha')
+        if not attributes.is_attr_set(ha):
+            ha = cfg.CONF.l3_ha
+        return ha
+
+    def _create_router_db(self, context, router, tenant_id):
+        router['ha'] = self._is_ha(router)
+
+        if router['ha'] and l3_dvr_db.is_distributed_router(router):
+            raise l3_ha.DistributedHARouterNotSupported()
+
+        with context.session.begin(subtransactions=True):
+            router_db = super(L3_HA_NAT_db_mixin, self)._create_router_db(
+                context, router, tenant_id)
+
+        if router['ha']:
+            try:
+                ha_network = self.get_ha_network(context,
+                                                 router_db.tenant_id)
+                if not ha_network:
+                    ha_network = self._create_ha_network(context,
+                                                         router_db.tenant_id)
+
+                self._set_vr_id(context, router_db, ha_network)
+                self._create_ha_interfaces(context, router_db, ha_network)
+                self._notify_ha_interfaces_updated(context, router_db.id)
+            except Exception:
+                with excutils.save_and_reraise_exception():
+                    self.delete_router(context, router_db.id)
+
+        return router_db
+
+    def _update_router_db(self, context, router_id, data, gw_info):
+        ha = data.pop('ha', None)
+
+        if ha and data.get('distributed'):
+            raise l3_ha.DistributedHARouterNotSupported()
+
+        with context.session.begin(subtransactions=True):
+            router_db = super(L3_HA_NAT_db_mixin, self)._update_router_db(
+                context, router_id, data, gw_info)
+
+            ha_not_changed = ha is None or ha == router_db.extra_attributes.ha
+            if ha_not_changed:
+                return router_db
+
+            ha_network = self.get_ha_network(context,
+                                             router_db.tenant_id)
+            router_db.extra_attributes.ha = ha
+            if not ha:
+                self._delete_vr_id_allocation(
+                    context, ha_network, router_db.extra_attributes.ha_vr_id)
+                router_db.extra_attributes.ha_vr_id = None
+
+        if ha:
+            if not ha_network:
+                ha_network = self._create_ha_network(context,
+                                                     router_db.tenant_id)
+
+            self._set_vr_id(context, router_db, ha_network)
+            self._create_ha_interfaces(context, router_db, ha_network)
+            self._notify_ha_interfaces_updated(context, router_db.id)
+        else:
+            self._delete_ha_interfaces(context, router_db.id)
+            self._notify_ha_interfaces_updated(context, router_db.id)
+
+        return router_db
+
+    def update_router_state(self, context, router_id, state, host):
+        with context.session.begin(subtransactions=True):
+            bindings = self.get_ha_router_port_bindings(context, [router_id],
+                                                        host)
+            if bindings:
+                if len(bindings) > 1:
+                    LOG.warn(_LW("The router %(router_id)s is bound multiple "
+                                 "times on the agent %(host)s"),
+                             {'router_id': router_id, 'host': host})
+
+                bindings[0].update({'state': state})
+
+    def delete_router(self, context, id):
+        router_db = self._get_router(context, id)
+        if router_db.extra_attributes.ha:
+            ha_network = self.get_ha_network(context,
+                                             router_db.tenant_id)
+            if ha_network:
+                self._delete_vr_id_allocation(
+                    context, ha_network, router_db.extra_attributes.ha_vr_id)
+                self._delete_ha_interfaces(context, router_db.id)
+
+        return super(L3_HA_NAT_db_mixin, self).delete_router(context, id)
+
+    def get_ha_router_port_bindings(self, context, router_ids, host=None):
+        query = context.session.query(L3HARouterAgentPortBinding)
+
+        if host:
+            query = query.join(agents_db.Agent).filter(
+                agents_db.Agent.host == host)
+
+        query = query.filter(
+            L3HARouterAgentPortBinding.router_id.in_(router_ids))
+
+        return query.all()
+
+    def _process_sync_ha_data(self, context, routers, host):
+        routers_dict = dict((router['id'], router) for router in routers)
+
+        bindings = self.get_ha_router_port_bindings(context,
+                                                    routers_dict.keys(),
+                                                    host)
+        for binding in bindings:
+            port_dict = self._core_plugin._make_port_dict(binding.port)
+
+            router = routers_dict.get(binding.router_id)
+            router[constants.HA_INTERFACE_KEY] = port_dict
+            router[constants.HA_ROUTER_STATE_KEY] = binding.state
+
+        for router in routers_dict.values():
+            interface = router.get(constants.HA_INTERFACE_KEY)
+            if interface:
+                self._populate_subnet_for_ports(context, [interface])
+
+        return routers_dict.values()
+
+    def get_ha_sync_data_for_host(self, context, host=None, router_ids=None,
+                                  active=None):
+        sync_data = super(L3_HA_NAT_db_mixin, self).get_sync_data(context,
+                                                                  router_ids,
+                                                                  active)
+        return self._process_sync_ha_data(context, sync_data, host)

neutron/db/migration/alembic_migrations/versions/16a27a58e093_ext_l3_ha_mode.py

@@ -0,0 +1,86 @@
+# Copyright 2014 OpenStack Foundation
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+"""ext_l3_ha_mode
+
+Revision ID: 16a27a58e093
+Revises: 86d6d9776e2b
+Create Date: 2014-02-01 10:24:12.412733
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '16a27a58e093'
+down_revision = '86d6d9776e2b'
+
+
+from alembic import op
+import sqlalchemy as sa
+
+l3_ha_states = sa.Enum('active', 'standby', name='l3_ha_states')
+
+
+def upgrade(active_plugins=None, options=None):
+    op.add_column('router_extra_attributes',
+                  sa.Column('ha', sa.Boolean(),
+                            nullable=False,
+                            server_default=sa.sql.false()))
+    op.add_column('router_extra_attributes',
+                  sa.Column('ha_vr_id', sa.Integer()))
+
+    op.create_table('ha_router_agent_port_bindings',
+                    sa.Column('port_id', sa.String(length=36),
+                              nullable=False),
+                    sa.Column('router_id', sa.String(length=36),
+                              nullable=False),
+                    sa.Column('l3_agent_id', sa.String(length=36),
+                              nullable=True),
+                    sa.Column('state', l3_ha_states,
+                              server_default='standby'),
+                    sa.PrimaryKeyConstraint('port_id'),
+                    sa.ForeignKeyConstraint(['port_id'], ['ports.id'],
+                                            ondelete='CASCADE'),
+                    sa.ForeignKeyConstraint(['router_id'], ['routers.id'],
+                                            ondelete='CASCADE'),
+                    sa.ForeignKeyConstraint(['l3_agent_id'], ['agents.id'],
+                                            ondelete='CASCADE'))
+
+    op.create_table('ha_router_networks',
+                    sa.Column('tenant_id', sa.String(length=255),
+                              nullable=False, primary_key=True),
+                    sa.Column('network_id', sa.String(length=36),
+                              nullable=False,
+                              primary_key=True),
+                    sa.ForeignKeyConstraint(['network_id'], ['networks.id'],
+                                            ondelete='CASCADE'))
+
+    op.create_table('ha_router_vrid_allocations',
+                    sa.Column('network_id', sa.String(length=36),
+                              nullable=False,
+                              primary_key=True),
+                    sa.Column('vr_id', sa.Integer(),
+                              nullable=False,
+                              primary_key=True),
+                    sa.ForeignKeyConstraint(['network_id'], ['networks.id'],
+                                            ondelete='CASCADE'))
+
+
+def downgrade(active_plugins=None, options=None):
+    op.drop_table('ha_router_vrid_allocations')
+    op.drop_table('ha_router_networks')
+    op.drop_table('ha_router_agent_port_bindings')
+    l3_ha_states.drop(op.get_bind(), checkfirst=False)
+    op.drop_column('router_extra_attributes', 'ha_vr_id')
+    op.drop_column('router_extra_attributes', 'ha')

neutron/db/migration/alembic_migrations/versions/HEAD

@@ -1 +1 @@
-86d6d9776e2b
+16a27a58e093

neutron/db/migration/models/head.py

@@ -34,6 +34,7 @@ from neutron.db import l3_attrs_db  # noqa
 from neutron.db import l3_db  # noqa
 from neutron.db import l3_dvrscheduler_db  # noqa
 from neutron.db import l3_gwmode_db  # noqa
+from neutron.db import l3_hamode_db  # noqa
 from neutron.db.loadbalancer import loadbalancer_db  # noqa
 from neutron.db.metering import metering_db  # noqa
 from neutron.db import model_base

neutron/extensions/l3_ext_ha_mode.py

@@ -0,0 +1,91 @@
+# Copyright (C) 2014 eNovance SAS <licensing@enovance.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+from neutron.api import extensions
+from neutron.api.v2 import attributes
+from neutron.common import constants
+from neutron.common import exceptions
+
+HA_INFO = 'ha'
+EXTENDED_ATTRIBUTES_2_0 = {
+    'routers': {
+        HA_INFO: {'allow_post': True, 'allow_put': True,
+                  'default': attributes.ATTR_NOT_SPECIFIED, 'is_visible': True,
+                  'enforce_policy': True,
+                  'convert_to': attributes.convert_to_boolean_if_not_none}
+    }
+}
+
+
+class DistributedHARouterNotSupported(NotImplementedError):
+    message = _("Currenly distributed HA routers are "
+                "not supported.")
+
+
+class MaxVRIDAllocationTriesReached(exceptions.NeutronException):
+    message = _("Failed to allocate a VRID in the network %(network_id)s "
+                "for the router %(router_id)s after %(max_tries)s tries.")
+
+
+class NoVRIDAvailable(exceptions.Conflict):
+    message = _("No more Virtual Router Identifier (VRID) available when "
+                "creating router %(router_id)s. The limit of number "
+                "of HA Routers per tenant is 254.")
+
+
+class HANetworkCIDRNotValid(exceptions.NeutronException):
+    message = _("The HA Network CIDR specified in the configuration file "
+                "isn't valid; %(cidr)s.")
+
+
+class HANotEnoughAvailableAgents(exceptions.NeutronException):
+    message = _("Not enough l3 agents available to ensure HA. Minimum "
+                "required %(min_agents)s, available %(num_agents)s.")
+
+
+class HAMinimumAgentsNumberNotValid(exceptions.NeutronException):
+    message = (_("min_l3_agents_per_router config parameter is not valid. "
+                 "It has to be equal to or more than %s for HA.") %
+               constants.MINIMUM_AGENTS_FOR_HA)
+
+
+class L3_ext_ha_mode(extensions.ExtensionDescriptor):
+    """Extension class supporting virtual router in HA mode."""
+
+    @classmethod
+    def get_name(cls):
+        return "HA Router extension"
+
+    @classmethod
+    def get_alias(cls):
+        return constants.L3_HA_MODE_EXT_ALIAS
+
+    @classmethod
+    def get_description(cls):
+        return "Add HA capability to routers."
+
+    @classmethod
+    def get_namespace(cls):
+        return ""
+
+    @classmethod
+    def get_updated(cls):
+        return "2014-04-26T00:00:00-00:00"
+
+    def get_extended_resources(self, version):
+        if version == "2.0":
+            return EXTENDED_ATTRIBUTES_2_0
+        else:
+            return {}

neutron/services/l3_router/l3_router_plugin.py

@@ -24,18 +24,18 @@ from neutron.common import rpc as n_rpc
 from neutron.common import topics
 from neutron.db import common_db_mixin
 from neutron.db import extraroute_db
-from neutron.db import l3_dvr_db
 from neutron.db import l3_dvrscheduler_db
 from neutron.db import l3_gwmode_db
+from neutron.db import l3_hamode_db
 from neutron.openstack.common import importutils
 from neutron.plugins.common import constants
 
 
 class L3RouterPlugin(common_db_mixin.CommonDbMixin,
                      extraroute_db.ExtraRoute_db_mixin,
-                     l3_dvr_db.L3_NAT_with_dvr_db_mixin,
                      l3_gwmode_db.L3_NAT_db_mixin,
-                     l3_dvrscheduler_db.L3_DVRsch_db_mixin):
+                     l3_dvrscheduler_db.L3_DVRsch_db_mixin,
+                     l3_hamode_db.L3_HA_NAT_db_mixin):
 
     """Implementation of the Neutron L3 Router Service Plugin.
 
@@ -43,17 +43,19 @@ class L3RouterPlugin(common_db_mixin.CommonDbMixin,
     router and floatingip resources and manages associated
     request/response.
     All DB related work is implemented in classes
-    l3_db.L3_NAT_db_mixin, l3_dvr_db.L3_NAT_with_dvr_db_mixin, and
-    extraroute_db.ExtraRoute_db_mixin.
+    l3_db.L3_NAT_db_mixin, l3_hamode_db.L3_HA_NAT_db_mixin,
+    l3_dvr_db.L3_NAT_with_dvr_db_mixin, and extraroute_db.ExtraRoute_db_mixin.
     """
     supported_extension_aliases = ["dvr", "router", "ext-gw-mode",
-                                   "extraroute", "l3_agent_scheduler"]
+                                   "extraroute", "l3_agent_scheduler",
+                                   "l3-ha"]
 
     def __init__(self):
         self.setup_rpc()
         self.router_scheduler = importutils.import_object(
             cfg.CONF.router_scheduler_driver)
         self.start_periodic_agent_status_check()
+        super(L3RouterPlugin, self).__init__()
 
     def setup_rpc(self):
         # RPC support

neutron/tests/unit/db/test_l3_dvr_db.py

@@ -118,7 +118,7 @@ class L3DvrTestCase(testlib_api.SqlTestCase):
             pass_router_id=False)
 
     def _test__is_distributed_router(self, router, expected):
-        result = l3_dvr_db._is_distributed_router(router)
+        result = l3_dvr_db.is_distributed_router(router)
         self.assertEqual(expected, result)
 
     def test__is_distributed_router_by_db_object(self):

neutron/tests/unit/db/test_l3_ha_db.py

@@ -0,0 +1,390 @@
+# Copyright (C) 2014 eNovance SAS <licensing@enovance.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import mock
+from oslo.config import cfg
+
+from neutron.common import constants
+from neutron import context
+from neutron.db import agents_db
+from neutron.db import common_db_mixin
+from neutron.db import l3_hamode_db
+from neutron.extensions import l3_ext_ha_mode
+from neutron import manager
+from neutron.openstack.common import uuidutils
+from neutron.tests.unit import testlib_api
+from neutron.tests.unit import testlib_plugin
+
+_uuid = uuidutils.generate_uuid
+
+
+class FakeL3Plugin(common_db_mixin.CommonDbMixin,
+                   l3_hamode_db.L3_HA_NAT_db_mixin):
+    pass
+
+
+class FakeL3PluginWithAgents(FakeL3Plugin,
+                             agents_db.AgentDbMixin):
+    pass
+
+
+class L3HATestFramework(testlib_api.SqlTestCase,
+                        testlib_plugin.PluginSetupHelper):
+    def setUp(self):
+        super(L3HATestFramework, self).setUp()
+
+        self.admin_ctx = context.get_admin_context()
+        self.setup_coreplugin('neutron.plugins.ml2.plugin.Ml2Plugin')
+        self.core_plugin = manager.NeutronManager.get_plugin()
+        mock.patch.object(l3_hamode_db.L3_HA_NAT_db_mixin, 'get_l3_agents',
+                          create=True, return_value=[1, 2]).start()
+        notif_p = mock.patch.object(l3_hamode_db.L3_HA_NAT_db_mixin,
+                                    '_notify_ha_interfaces_updated')
+        self.notif_m = notif_p.start()
+        cfg.CONF.set_override('allow_overlapping_ips', True)
+
+    def _create_router(self, ha=True, tenant_id='tenant1', distributed=None):
+        router = {'name': 'router1', 'admin_state_up': True}
+        if ha is not None:
+            router['ha'] = ha
+        if distributed is not None:
+            router['distributed'] = distributed
+        return self.plugin._create_router_db(self.admin_ctx, router, tenant_id)
+
+    def _update_router(self, router_id, ha=True, distributed=None):
+        data = {'ha': ha} if ha is not None else {}
+        if distributed is not None:
+            data['distributed'] = distributed
+        return self.plugin._update_router_db(self.admin_ctx, router_id,
+                                             data, None)
+
+
+class L3HAGetSyncDataTestCase(L3HATestFramework):
+
+    def setUp(self):
+        super(L3HAGetSyncDataTestCase, self).setUp()
+        self.plugin = FakeL3PluginWithAgents()
+        self._register_agents()
+
+    def _register_agents(self):
+        agent_status = {
+            'agent_type': constants.AGENT_TYPE_L3,
+            'binary': 'neutron-l3-agent',
+            'host': 'l3host',
+            'topic': 'N/A'
+        }
+        self.plugin.create_or_update_agent(self.admin_ctx, agent_status)
+        agent_status['host'] = 'l3host_2'
+        self.plugin.create_or_update_agent(self.admin_ctx, agent_status)
+        self.agent1, self.agent2 = self.plugin.get_agents(self.admin_ctx)
+
+    def _bind_router(self, router_id):
+        with self.admin_ctx.session.begin(subtransactions=True):
+            bindings = self.plugin.get_ha_router_port_bindings(self.admin_ctx,
+                                                               [router_id])
+
+            for agent_id, binding in zip(
+                    [self.agent1['id'], self.agent2['id']], bindings):
+                binding.l3_agent_id = agent_id
+
+    def test_l3_agent_routers_query_interface(self):
+        router = self._create_router()
+        self._bind_router(router.id)
+        routers = self.plugin.get_ha_sync_data_for_host(self.admin_ctx,
+                                                        self.agent1['host'])
+        self.assertEqual(1, len(routers))
+        router = routers[0]
+
+        self.assertIsNotNone(router.get('ha'))
+
+        interface = router.get(constants.HA_INTERFACE_KEY)
+        self.assertIsNotNone(interface)
+
+        self.assertEqual(constants.DEVICE_OWNER_ROUTER_HA_INTF,
+                         interface['device_owner'])
+        self.assertEqual(cfg.CONF.l3_ha_net_cidr, interface['subnet']['cidr'])
+
+    def test_update_state(self):
+        router = self._create_router()
+        self._bind_router(router.id)
+        routers = self.plugin.get_ha_sync_data_for_host(self.admin_ctx,
+                                                        self.agent1['host'])
+        state = routers[0].get(constants.HA_ROUTER_STATE_KEY)
+        self.assertEqual('standby', state)
+
+        self.plugin.update_router_state(self.admin_ctx, router.id, 'active',
+                                        self.agent1['host'])
+
+        routers = self.plugin.get_ha_sync_data_for_host(self.admin_ctx,
+                                                        self.agent1['host'])
+
+        state = routers[0].get(constants.HA_ROUTER_STATE_KEY)
+        self.assertEqual('active', state)
+
+
+class L3HATestCase(L3HATestFramework):
+
+    def setUp(self):
+        super(L3HATestCase, self).setUp()
+        self.plugin = FakeL3Plugin()
+
+    def test_verify_configuration_succeed(self):
+        # Default configuration should pass
+        self.plugin._verify_configuration()
+
+    def test_verify_configuration_l3_ha_net_cidr_is_not_a_cidr(self):
+        cfg.CONF.set_override('l3_ha_net_cidr', 'not a cidr')
+        self.assertRaises(
+            l3_ext_ha_mode.HANetworkCIDRNotValid,
+            self.plugin._verify_configuration)
+
+    def test_verify_configuration_l3_ha_net_cidr_is_not_a_subnet(self):
+        cfg.CONF.set_override('l3_ha_net_cidr', '10.0.0.1/8')
+        self.assertRaises(
+            l3_ext_ha_mode.HANetworkCIDRNotValid,
+            self.plugin._verify_configuration)
+
+    def test_verify_conifguration_min_l3_agents_per_router_below_minimum(self):
+        cfg.CONF.set_override('min_l3_agents_per_router', 0)
+        self.assertRaises(
+            l3_ext_ha_mode.HAMinimumAgentsNumberNotValid,
+            self.plugin._verify_configuration)
+
+    def test_ha_router_create(self):
+        router = self._create_router()
+        self.assertTrue(router.extra_attributes['ha'])
+
+    def test_ha_router_create_with_distributed(self):
+        self.assertRaises(l3_ext_ha_mode.DistributedHARouterNotSupported,
+                          self._create_router,
+                          distributed=True)
+
+    def test_no_ha_router_create(self):
+        router = self._create_router(ha=False)
+        self.assertFalse(router.extra_attributes['ha'])
+
+    def test_router_create_with_ha_conf_enabled(self):
+        cfg.CONF.set_override('l3_ha', True)
+
+        router = self._create_router(ha=None)
+        self.assertTrue(router.extra_attributes['ha'])
+
+    def test_migration_from_ha(self):
+        router = self._create_router()
+        self.assertTrue(router.extra_attributes['ha'])
+
+        router = self._update_router(router.id, ha=False)
+        self.assertFalse(router.extra_attributes['ha'])
+        self.assertIsNone(router.extra_attributes['ha_vr_id'])
+
+    def test_migration_to_ha(self):
+        router = self._create_router(ha=False)
+        self.assertFalse(router.extra_attributes['ha'])
+
+        router = self._update_router(router.id, ha=True)
+        self.assertTrue(router.extra_attributes['ha'])
+        self.assertIsNotNone(router.extra_attributes['ha_vr_id'])
+
+    def test_migrate_ha_router_to_distributed(self):
+        router = self._create_router()
+        self.assertTrue(router.extra_attributes['ha'])
+
+        self.assertRaises(l3_ext_ha_mode.DistributedHARouterNotSupported,
+                          self._update_router,
+                          router.id,
+                          distributed=True)
+
+    def test_unique_ha_network_per_tenant(self):
+        tenant1 = _uuid()
+        tenant2 = _uuid()
+        self._create_router(tenant_id=tenant1)
+        self._create_router(tenant_id=tenant2)
+        ha_network1 = self.plugin.get_ha_network(self.admin_ctx, tenant1)
+        ha_network2 = self.plugin.get_ha_network(self.admin_ctx, tenant2)
+        self.assertNotEqual(
+            ha_network1['network_id'], ha_network2['network_id'])
+
+    def _deployed_router_change_ha_flag(self, to_ha):
+        self._create_router(ha=not to_ha)
+        routers = self.plugin.get_ha_sync_data_for_host(self.admin_ctx)
+        router = routers[0]
+        interface = router.get(constants.HA_INTERFACE_KEY)
+        if to_ha:
+            self.assertIsNone(interface)
+        else:
+            self.assertIsNotNone(interface)
+
+        self._update_router(router['id'], to_ha)
+        routers = self.plugin.get_ha_sync_data_for_host(self.admin_ctx)
+        router = routers[0]
+        interface = router.get(constants.HA_INTERFACE_KEY)
+        if to_ha:
+            self.assertIsNotNone(interface)
+        else:
+            self.assertIsNone(interface)
+
+    def test_deployed_router_can_have_ha_enabled(self):
+        self._deployed_router_change_ha_flag(to_ha=True)
+
+    def test_deployed_router_can_have_ha_disabled(self):
+        self._deployed_router_change_ha_flag(to_ha=False)
+
+    def test_create_ha_router_notifies_agent(self):
+        self._create_router()
+        self.assertTrue(self.notif_m.called)
+
+    def test_update_router_to_ha_notifies_agent(self):
+        router = self._create_router(ha=False)
+        self.notif_m.reset_mock()
+        self._update_router(router.id, ha=True)
+        self.assertTrue(self.notif_m.called)
+
+    def test_unique_vr_id_between_routers(self):
+        self._create_router()
+        self._create_router()
+        routers = self.plugin.get_ha_sync_data_for_host(self.admin_ctx)
+        self.assertEqual(2, len(routers))
+        self.assertNotEqual(routers[0]['ha_vr_id'], routers[1]['ha_vr_id'])
+
+    @mock.patch('neutron.db.l3_hamode_db.VR_ID_RANGE', new=set(range(1, 1)))
+    def test_vr_id_depleted(self):
+        self.assertRaises(l3_ext_ha_mode.NoVRIDAvailable, self._create_router)
+
+    @mock.patch('neutron.db.l3_hamode_db.VR_ID_RANGE', new=set(range(1, 2)))
+    def test_vr_id_unique_range_per_tenant(self):
+        self._create_router()
+        self._create_router(tenant_id=_uuid())
+        routers = self.plugin.get_ha_sync_data_for_host(self.admin_ctx)
+        self.assertEqual(2, len(routers))
+        self.assertEqual(routers[0]['ha_vr_id'], routers[1]['ha_vr_id'])
+
+    @mock.patch('neutron.db.l3_hamode_db.MAX_ALLOCATION_TRIES', new=2)
+    def test_vr_id_allocation_contraint_conflict(self):
+        router = self._create_router()
+        network = self.plugin.get_ha_network(self.admin_ctx, router.tenant_id)
+
+        with mock.patch.object(self.plugin, '_get_allocated_vr_id',
+                               return_value=set()) as alloc:
+            self.assertRaises(l3_ext_ha_mode.MaxVRIDAllocationTriesReached,
+                              self.plugin._allocate_vr_id, self.admin_ctx,
+                              network.network_id, router.id)
+            self.assertEqual(2, len(alloc.mock_calls))
+
+    def test_vr_id_allocation_delete_router(self):
+        router = self._create_router()
+        network = self.plugin.get_ha_network(self.admin_ctx, router.tenant_id)
+
+        allocs_before = self.plugin._get_allocated_vr_id(self.admin_ctx,
+                                                         network.network_id)
+        router = self._create_router()
+        allocs_current = self.plugin._get_allocated_vr_id(self.admin_ctx,
+                                                          network.network_id)
+        self.assertNotEqual(allocs_before, allocs_current)
+
+        self.plugin.delete_router(self.admin_ctx, router.id)
+        allocs_after = self.plugin._get_allocated_vr_id(self.admin_ctx,
+                                                        network.network_id)
+        self.assertEqual(allocs_before, allocs_after)
+
+    def test_vr_id_allocation_router_migration(self):
+        router = self._create_router()
+        network = self.plugin.get_ha_network(self.admin_ctx, router.tenant_id)
+
+        allocs_before = self.plugin._get_allocated_vr_id(self.admin_ctx,
+                                                         network.network_id)
+        router = self._create_router()
+        self._update_router(router.id, ha=False)
+        allocs_after = self.plugin._get_allocated_vr_id(self.admin_ctx,
+                                                        network.network_id)
+        self.assertEqual(allocs_before, allocs_after)
+
+    def test_one_ha_router_one_not(self):
+        self._create_router(ha=False)
+        self._create_router()
+        routers = self.plugin.get_ha_sync_data_for_host(self.admin_ctx)
+
+        ha0 = routers[0]['ha']
+        ha1 = routers[1]['ha']
+
+        self.assertNotEqual(ha0, ha1)
+
+    def test_add_ha_port_binding_failure_rolls_back_port(self):
+        router = self._create_router()
+        device_filter = {'device_id': [router.id]}
+        ports_before = self.core_plugin.get_ports(
+            self.admin_ctx, filters=device_filter)
+        network = self.plugin.get_ha_network(self.admin_ctx, router.tenant_id)
+
+        with mock.patch.object(self.plugin, '_create_ha_port_binding',
+                               side_effect=ValueError):
+            self.assertRaises(ValueError, self.plugin.add_ha_port,
+                              self.admin_ctx, router.id, network.network_id,
+                              router.tenant_id)
+
+        ports_after = self.core_plugin.get_ports(
+            self.admin_ctx, filters=device_filter)
+
+        self.assertEqual(ports_before, ports_after)
+
+    def test_create_ha_network_binding_failure_rolls_back_network(self):
+        networks_before = self.core_plugin.get_networks(self.admin_ctx)
+
+        with mock.patch.object(self.plugin,
+                               '_create_ha_network_tenant_binding',
+                               side_effect=ValueError):
+            self.assertRaises(ValueError, self.plugin._create_ha_network,
+                              self.admin_ctx, _uuid())
+
+        networks_after = self.core_plugin.get_networks(self.admin_ctx)
+        self.assertEqual(networks_before, networks_after)
+
+    def test_create_ha_network_subnet_failure_rolls_back_network(self):
+        networks_before = self.core_plugin.get_networks(self.admin_ctx)
+
+        with mock.patch.object(self.plugin, '_create_ha_subnet',
+                               side_effect=ValueError):
+            self.assertRaises(ValueError, self.plugin._create_ha_network,
+                              self.admin_ctx, _uuid())
+
+        networks_after = self.core_plugin.get_networks(self.admin_ctx)
+        self.assertEqual(networks_before, networks_after)
+
+    def test_create_ha_interfaces_binding_failure_rolls_back_ports(self):
+        router = self._create_router()
+        network = self.plugin.get_ha_network(self.admin_ctx, router.tenant_id)
+        device_filter = {'device_id': [router.id]}
+        ports_before = self.core_plugin.get_ports(
+            self.admin_ctx, filters=device_filter)
+
+        with mock.patch.object(self.plugin, '_create_ha_port_binding',
+                               side_effect=ValueError):
+            self.assertRaises(ValueError, self.plugin._create_ha_interfaces,
+                              self.admin_ctx, router, network)
+
+        ports_after = self.core_plugin.get_ports(
+            self.admin_ctx, filters=device_filter)
+        self.assertEqual(ports_before, ports_after)
+
+    def test_create_router_db_ha_attribute_failure_rolls_back_router(self):
+        routers_before = self.plugin.get_routers(self.admin_ctx)
+
+        for method in ('_set_vr_id',
+                       '_create_ha_interfaces',
+                       '_notify_ha_interfaces_updated'):
+            with mock.patch.object(self.plugin, method,
+                                   side_effect=ValueError):
+                self.assertRaises(ValueError, self._create_router)
+
+        routers_after = self.plugin.get_routers(self.admin_ctx)
+        self.assertEqual(routers_before, routers_after)