openstack/neutron-fwaas
Code Issues Proposed changes

Add L3 VRRP HA base classes

Browse Source 
Add L3 HA base classes on the plugin side. A new admin-only ha
attribute is added to the API router resource. Conversion from
or to HA router is possible. Each tenant gets a single network
used for HA traffic. The tenant_id for that network is set to
'' so that it isn't visible via the CLI or GUI. A new table
is added to map a tenant to its HA network. Specific HA
attributes are added to the extra router attributes table.
Finally, each HA router gets a port on the HA network, per
l3 agent it is scheduled on. A new table is added to track
these bindings. A new table is added in order to track
VRID allocations.

DVR integration is not expected to work. Any issues will
be reported as bugs and handled after the feature merges.
Migrating a router to HA or from HA works server side
but is not expected to work (Yet) agent side. This will be
dealt with as a bug in the future.

DocImpact
Partially-implements: blueprint l3-high-availability
Change-Id: I9d935cf5e0c231e8cb7af5f61b9a9fc552c3521e
Co-Authored-By: Assaf Muller <amuller@redhat.com>
This commit is contained in:
Sylvain Afchain 2014-05-26 14:28:06 +02:00
parent 0507f6a58c
commit 5730d87c26
15 changed files with 1095 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
etc/neutron.conf
View File

@ -172,6 +172,22 @@ lock_path = $state_path/lock
# =========== end of items for agent scheduler extension =====
# =========== items for l3 extension ==============
# Enable high availability for virtual routers.
# l3_ha = False
#
# Maximum number of l3 agents which a HA router will be scheduled on. If it
# is set to 0 the router will be scheduled on every agent.
# max_l3_agents_per_router = 3
#
# Minimum number of l3 agents which a HA router will be scheduled on. The
# default value is 2.
# min_l3_agents_per_router = 2
#
# CIDR of the administrative network if HA mode is enabled
# l3_ha_net_cidr = 169.254.192.0/18
# =========== end of items for l3 extension =======
# =========== WSGI parameters related to the API server ==============
# Number of separate worker processes to spawn. The default, 0, runs the
# worker thread in the current process. Greater than 0 launches that number of

3
etc/policy.json
View File

@ -58,13 +58,16 @@
"update_port:mac_learning_enabled": "rule:admin_or_network_owner",
"delete_port": "rule:admin_or_owner",
"get_router:ha": "rule:admin_only",
"create_router": "rule:regular_user",
"create_router:external_gateway_info:enable_snat": "rule:admin_only",
"create_router:distributed": "rule:admin_only",
"create_router:ha": "rule:admin_only",
"get_router": "rule:admin_or_owner",
"get_router:distributed": "rule:admin_only",
"update_router:external_gateway_info:enable_snat": "rule:admin_only",
"update_router:distributed": "rule:admin_only",
"update_router:ha": "rule:admin_only",
"delete_router": "rule:admin_or_owner",
"add_router_interface": "rule:admin_or_owner",

15
neutron/api/rpc/handlers/l3_rpc.py
View File

@ -38,7 +38,8 @@ class L3RpcCallback(n_rpc.RpcCallback):
# 1.1 Support update_floatingip_statuses
# 1.2 Added methods for DVR support
# 1.3 Added a method that returns the list of activated services
RPC_API_VERSION = '1.3'
# 1.4 Added L3 HA update_router_state
RPC_API_VERSION = '1.4'
@property
def plugin(self):
@ -104,6 +105,10 @@ class L3RpcCallback(n_rpc.RpcCallback):
for interface in router.get(constants.INTERFACE_KEY, []):
self._ensure_host_set_on_port(context, host,
interface, router['id'])
interface = router.get(constants.HA_INTERFACE_KEY)
if interface:
self._ensure_host_set_on_port(context, host, interface,
router['id'])
def _ensure_host_set_on_port(self, context, host, port, router_id=None):
if (port and
@ -224,3 +229,11 @@ class L3RpcCallback(n_rpc.RpcCallback):
'and on host %(host)s', {'snat_port_list': snat_port_list,
'host': host})
return snat_port_list
def update_router_state(self, context, **kwargs):
router_id = kwargs.get('router_id')
state = kwargs.get('state')
host = kwargs.get('host')
return self.l3plugin.update_router_state(context, router_id, state,
host=host)

9
neutron/common/constants.py
View File

@ -29,6 +29,7 @@ FLOATINGIP_STATUS_ACTIVE = 'ACTIVE'
FLOATINGIP_STATUS_DOWN = 'DOWN'
FLOATINGIP_STATUS_ERROR = 'ERROR'
DEVICE_OWNER_ROUTER_HA_INTF = "network:router_ha_interface"
DEVICE_OWNER_ROUTER_INTF = "network:router_interface"
DEVICE_OWNER_ROUTER_GW = "network:router_gateway"
DEVICE_OWNER_FLOATINGIP = "network:floatingip"
@ -42,10 +43,17 @@ DEVICE_ID_RESERVED_DHCP_PORT = "reserved_dhcp_port"
FLOATINGIP_KEY = '_floatingips'
INTERFACE_KEY = '_interfaces'
HA_INTERFACE_KEY = '_ha_interface'
HA_ROUTER_STATE_KEY = '_ha_state'
METERING_LABEL_KEY = '_metering_labels'
FLOATINGIP_AGENT_INTF_KEY = '_floatingip_agent_interfaces'
SNAT_ROUTER_INTF_KEY = '_snat_router_interfaces'
HA_NETWORK_NAME = 'HA network tenant %s'
HA_SUBNET_NAME = 'HA subnet tenant %s'
HA_PORT_NAME = 'HA port tenant %s'
MINIMUM_AGENTS_FOR_HA = 2
IPv4 = 'IPv4'
IPv6 = 'IPv6'
@ -101,6 +109,7 @@ L3_AGENT_SCHEDULER_EXT_ALIAS = 'l3_agent_scheduler'
DHCP_AGENT_SCHEDULER_EXT_ALIAS = 'dhcp_agent_scheduler'
LBAAS_AGENT_SCHEDULER_EXT_ALIAS = 'lbaas_agent_scheduler'
L3_DISTRIBUTED_EXT_ALIAS = 'dvr'
L3_HA_MODE_EXT_ALIAS = 'l3-ha'
# Protocol names and numbers for Security Groups/Firewalls
PROTO_NAME_TCP = 'tcp'

10
neutron/db/l3_agentschedulers_db.py
View File

@ -284,8 +284,14 @@ class L3AgentSchedulerDbMixin(l3agentscheduler.L3AgentSchedulerPluginBase,
RouterL3AgentBinding.router_id.in_(router_ids))
router_ids = [item[0] for item in query]
if router_ids:
return self.get_sync_data(context, router_ids=router_ids,
active=True)
if n_utils.is_extension_supported(self,
constants.L3_HA_MODE_EXT_ALIAS):
return self.get_ha_sync_data_for_host(context, host,
router_ids=router_ids,
active=True)
else:
return self.get_sync_data(context, router_ids=router_ids,
active=True)
else:
return []

5
neutron/db/l3_attrs_db.py
View File

@ -40,6 +40,11 @@ class RouterExtraAttributes(model_base.BASEV2):
service_router = sa.Column(sa.Boolean, default=False,
server_default=sa.sql.false(),
nullable=False)
ha = sa.Column(sa.Boolean, default=False,
server_default=sa.sql.false(),
nullable=False)
ha_vr_id = sa.Column(sa.Integer())
router = orm.relationship(
l3_db.Router,
backref=orm.backref("extra_attributes", lazy='joined',

6
neutron/db/l3_dvr_db.py
View File

@ -61,7 +61,7 @@ class L3_NAT_with_dvr_db_mixin(l3_db.L3_NAT_db_mixin,
def _create_router_db(self, context, router, tenant_id):
"""Create a router db object with dvr additions."""
router['distributed'] = _is_distributed_router(router)
router['distributed'] = is_distributed_router(router)
with context.session.begin(subtransactions=True):
router_db = super(
L3_NAT_with_dvr_db_mixin, self)._create_router_db(
@ -128,7 +128,7 @@ class L3_NAT_with_dvr_db_mixin(l3_db.L3_NAT_db_mixin,
router_is_uuid = isinstance(router, basestring)
if router_is_uuid:
router = self._get_router(context, router)
if _is_distributed_router(router):
if is_distributed_router(router):
return DEVICE_OWNER_DVR_INTERFACE
return super(L3_NAT_with_dvr_db_mixin,
self)._get_device_owner(context, router)
@ -534,7 +534,7 @@ class L3_NAT_with_dvr_db_mixin(l3_db.L3_NAT_db_mixin,
l3_port_check=False)
def _is_distributed_router(router):
def is_distributed_router(router):
"""Return True if router to be handled is distributed."""
try:
# See if router is a DB object first

459
neutron/db/l3_hamode_db.py Normal file
View File

@ -0,0 +1,459 @@
# Copyright (C) 2014 eNovance SAS <licensing@enovance.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import netaddr
from oslo.config import cfg
from oslo.db import exception as db_exc
import sqlalchemy as sa
from sqlalchemy import orm
from neutron.api.v2 import attributes
from neutron.common import constants
from neutron.db import agents_db
from neutron.db import l3_dvr_db
from neutron.db import model_base
from neutron.db import models_v2
from neutron.extensions import l3_ext_ha_mode as l3_ha
from neutron.openstack.common import excutils
from neutron.openstack.common.gettextutils import _LI
from neutron.openstack.common.gettextutils import _LW
from neutron.openstack.common import log as logging
VR_ID_RANGE = set(range(1, 255))
MAX_ALLOCATION_TRIES = 10
LOG = logging.getLogger(__name__)
L3_HA_OPTS = [
cfg.BoolOpt('l3_ha',
default=False,
help=_('Enable HA mode for virtual routers.')),
cfg.IntOpt('max_l3_agents_per_router',
default=3,
help=_('Maximum number of agents on which a router will be '
'scheduled.')),
cfg.IntOpt('min_l3_agents_per_router',
default=constants.MINIMUM_AGENTS_FOR_HA,
help=_('Minimum number of agents on which a router will be '
'scheduled.')),
cfg.StrOpt('l3_ha_net_cidr',
default='169.254.192.0/18',
help=_('Subnet used for the l3 HA admin network.')),
]
cfg.CONF.register_opts(L3_HA_OPTS)
class L3HARouterAgentPortBinding(model_base.BASEV2):
"""Represent agent binding state of a HA router port.
A HA Router has one HA port per agent on which it is spawned.
This binding table stores which port is used for a HA router by a
L3 agent.
"""
__tablename__ = 'ha_router_agent_port_bindings'
port_id = sa.Column(sa.String(36), sa.ForeignKey('ports.id',
ondelete='CASCADE'),
nullable=False, primary_key=True)
port = orm.relationship(models_v2.Port)
router_id = sa.Column(sa.String(36), sa.ForeignKey('routers.id',
ondelete='CASCADE'),
nullable=False)
l3_agent_id = sa.Column(sa.String(36),
sa.ForeignKey("agents.id",
ondelete='CASCADE'))
agent = orm.relationship(agents_db.Agent)
state = sa.Column(sa.Enum('active', 'standby', name='l3_ha_states'),
default='standby',
server_default='standby')
class L3HARouterNetwork(model_base.BASEV2):
"""Host HA network for a tenant.
One HA Network is used per tenant, all HA router ports are created
on this network.
"""
__tablename__ = 'ha_router_networks'
tenant_id = sa.Column(sa.String(255), primary_key=True,
nullable=False)
network_id = sa.Column(sa.String(36),
sa.ForeignKey('networks.id', ondelete="CASCADE"),
nullable=False, primary_key=True)
network = orm.relationship(models_v2.Network)
class L3HARouterVRIdAllocation(model_base.BASEV2):
"""VRID allocation per HA network.
Keep a track of the VRID allocations per HA network.
"""
__tablename__ = 'ha_router_vrid_allocations'
network_id = sa.Column(sa.String(36),
sa.ForeignKey('networks.id', ondelete="CASCADE"),
nullable=False, primary_key=True)
vr_id = sa.Column(sa.Integer(), nullable=False, primary_key=True)
class L3_HA_NAT_db_mixin(l3_dvr_db.L3_NAT_with_dvr_db_mixin):
"""Mixin class to add high availability capability to routers."""
extra_attributes = (
l3_dvr_db.L3_NAT_with_dvr_db_mixin.extra_attributes + [
{'name': 'ha', 'default': cfg.CONF.l3_ha},
{'name': 'ha_vr_id', 'default': 0}])
def _verify_configuration(self):
self.ha_cidr = cfg.CONF.l3_ha_net_cidr
try:
net = netaddr.IPNetwork(self.ha_cidr)
except netaddr.AddrFormatError:
raise l3_ha.HANetworkCIDRNotValid(cidr=self.ha_cidr)
if ('/' not in self.ha_cidr or net.network != net.ip):
raise l3_ha.HANetworkCIDRNotValid(cidr=self.ha_cidr)
if cfg.CONF.min_l3_agents_per_router < constants.MINIMUM_AGENTS_FOR_HA:
raise l3_ha.HAMinimumAgentsNumberNotValid()
def __init__(self):
self._verify_configuration()
super(L3_HA_NAT_db_mixin, self).__init__()
def get_ha_network(self, context, tenant_id):
return (context.session.query(L3HARouterNetwork).
filter(L3HARouterNetwork.tenant_id == tenant_id).
first())
def _get_allocated_vr_id(self, context, network_id):
with context.session.begin(subtransactions=True):
query = (context.session.query(L3HARouterVRIdAllocation).
filter(L3HARouterVRIdAllocation.network_id == network_id))
allocated_vr_ids = set(a.vr_id for a in query) - set([0])
return allocated_vr_ids
def _allocate_vr_id(self, context, network_id, router_id):
for count in range(MAX_ALLOCATION_TRIES):
try:
with context.session.begin(subtransactions=True):
allocated_vr_ids = self._get_allocated_vr_id(context,
network_id)
available_vr_ids = VR_ID_RANGE - allocated_vr_ids
if not available_vr_ids:
raise l3_ha.NoVRIDAvailable(router_id=router_id)
allocation = L3HARouterVRIdAllocation()
allocation.network_id = network_id
allocation.vr_id = available_vr_ids.pop()
context.session.add(allocation)
return allocation.vr_id
except db_exc.DBDuplicateEntry:
LOG.info(_LI("Attempt %(count)s to allocate a VRID in the "
"network %(network)s for the router %(router)s"),
{'count': count, 'network': network_id,
'router': router_id})
raise l3_ha.MaxVRIDAllocationTriesReached(
network_id=network_id, router_id=router_id,
max_tries=MAX_ALLOCATION_TRIES)
def _delete_vr_id_allocation(self, context, ha_network, vr_id):
with context.session.begin(subtransactions=True):
context.session.query(L3HARouterVRIdAllocation).filter_by(
network_id=ha_network.network_id,
vr_id=vr_id).delete()
def _set_vr_id(self, context, router, ha_network):
with context.session.begin(subtransactions=True):
router.extra_attributes.ha_vr_id = self._allocate_vr_id(
context, ha_network.network_id, router.id)
def _create_ha_subnet(self, context, network_id, tenant_id):
args = {'subnet':
{'network_id': network_id,
'tenant_id': '',
'name': constants.HA_SUBNET_NAME % tenant_id,
'ip_version': 4,
'cidr': cfg.CONF.l3_ha_net_cidr,
'enable_dhcp': False,
'host_routes': attributes.ATTR_NOT_SPECIFIED,
'dns_nameservers': attributes.ATTR_NOT_SPECIFIED,
'allocation_pools': attributes.ATTR_NOT_SPECIFIED,
'gateway_ip': None}}
return self._core_plugin.create_subnet(context, args)
def _create_ha_network_tenant_binding(self, context, tenant_id,
network_id):
with context.session.begin(subtransactions=True):
ha_network = L3HARouterNetwork(tenant_id=tenant_id,
network_id=network_id)
context.session.add(ha_network)
return ha_network
def _create_ha_network(self, context, tenant_id):
admin_ctx = context.elevated()
args = {'network':
{'name': constants.HA_NETWORK_NAME % tenant_id,
'tenant_id': '',
'shared': False,
'admin_state_up': True,
'status': constants.NET_STATUS_ACTIVE}}
network = self._core_plugin.create_network(context, args)
try:
ha_network = self._create_ha_network_tenant_binding(admin_ctx,
tenant_id,
network['id'])
except Exception:
with excutils.save_and_reraise_exception():
self._core_plugin.delete_network(admin_ctx, network['id'])
try:
self._create_ha_subnet(admin_ctx, network['id'], tenant_id)
except Exception:
with excutils.save_and_reraise_exception():
self._core_plugin.delete_network(admin_ctx, network['id'])
return ha_network
def get_number_of_agents_for_scheduling(self, context):
"""Return the number of agents on which the router will be scheduled.
Raises an exception if there are not enough agents available to honor
the min_agents config parameter. If the max_agents parameter is set to
0 all the agents will be used.
"""
min_agents = cfg.CONF.min_l3_agents_per_router
num_agents = len(self.get_l3_agents(context))
max_agents = cfg.CONF.max_l3_agents_per_router
if max_agents:
if max_agents > num_agents:
LOG.info(_LI("Number of available agents lower than "
"max_l3_agents_per_router. L3 agents "
"available: %s"), num_agents)
else:
num_agents = max_agents
if num_agents < min_agents:
raise l3_ha.HANotEnoughAvailableAgents(min_agents=min_agents,
num_agents=num_agents)
return num_agents
def _create_ha_port_binding(self, context, port_id, router_id):
with context.session.begin(subtransactions=True):
portbinding = L3HARouterAgentPortBinding(port_id=port_id,
router_id=router_id)
context.session.add(portbinding)
return portbinding
def add_ha_port(self, context, router_id, network_id, tenant_id):
port = self._core_plugin.create_port(context, {
'port':
{'tenant_id': '',
'network_id': network_id,
'fixed_ips': attributes.ATTR_NOT_SPECIFIED,
'mac_address': attributes.ATTR_NOT_SPECIFIED,
'admin_state_up': True,
'device_id': router_id,
'device_owner': constants.DEVICE_OWNER_ROUTER_HA_INTF,
'name': constants.HA_PORT_NAME % tenant_id}})
try:
return self._create_ha_port_binding(context, port['id'], router_id)
except Exception:
with excutils.save_and_reraise_exception():
self._core_plugin.delete_port(context, port['id'],
l3_port_check=False)
def _create_ha_interfaces(self, context, router, ha_network):
admin_ctx = context.elevated()
num_agents = self.get_number_of_agents_for_scheduling(context)
port_ids = []
try:
for index in range(num_agents):
binding = self.add_ha_port(admin_ctx, router.id,
ha_network.network['id'],
router.tenant_id)
port_ids.append(binding.port_id)
except Exception:
with excutils.save_and_reraise_exception():
for port_id in port_ids:
self._core_plugin.delete_port(admin_ctx, port_id,
l3_port_check=False)
def _delete_ha_interfaces(self, context, router_id):
admin_ctx = context.elevated()
device_filter = {'device_id': [router_id],
'device_owner':
[constants.DEVICE_OWNER_ROUTER_HA_INTF]}
ports = self._core_plugin.get_ports(admin_ctx, filters=device_filter)
for port in ports:
self._core_plugin.delete_port(admin_ctx, port['id'],
l3_port_check=False)
def _notify_ha_interfaces_updated(self, context, router_id):
self.l3_rpc_notifier.routers_updated(context, [router_id])
@classmethod
def _is_ha(cls, router):
ha = router.get('ha')
if not attributes.is_attr_set(ha):
ha = cfg.CONF.l3_ha
return ha
def _create_router_db(self, context, router, tenant_id):
router['ha'] = self._is_ha(router)
if router['ha'] and l3_dvr_db.is_distributed_router(router):
raise l3_ha.DistributedHARouterNotSupported()
with context.session.begin(subtransactions=True):
router_db = super(L3_HA_NAT_db_mixin, self)._create_router_db(
context, router, tenant_id)
if router['ha']:
try:
ha_network = self.get_ha_network(context,
router_db.tenant_id)
if not ha_network:
ha_network = self._create_ha_network(context,
router_db.tenant_id)
self._set_vr_id(context, router_db, ha_network)
self._create_ha_interfaces(context, router_db, ha_network)
self._notify_ha_interfaces_updated(context, router_db.id)
except Exception:
with excutils.save_and_reraise_exception():
self.delete_router(context, router_db.id)
return router_db
def _update_router_db(self, context, router_id, data, gw_info):
ha = data.pop('ha', None)
if ha and data.get('distributed'):
raise l3_ha.DistributedHARouterNotSupported()
with context.session.begin(subtransactions=True):
router_db = super(L3_HA_NAT_db_mixin, self)._update_router_db(
context, router_id, data, gw_info)
ha_not_changed = ha is None or ha == router_db.extra_attributes.ha
if ha_not_changed:
return router_db
ha_network = self.get_ha_network(context,
router_db.tenant_id)
router_db.extra_attributes.ha = ha
if not ha:
self._delete_vr_id_allocation(
context, ha_network, router_db.extra_attributes.ha_vr_id)
router_db.extra_attributes.ha_vr_id = None
if ha:
if not ha_network:
ha_network = self._create_ha_network(context,
router_db.tenant_id)
self._set_vr_id(context, router_db, ha_network)
self._create_ha_interfaces(context, router_db, ha_network)
self._notify_ha_interfaces_updated(context, router_db.id)
else:
self._delete_ha_interfaces(context, router_db.id)
self._notify_ha_interfaces_updated(context, router_db.id)
return router_db
def update_router_state(self, context, router_id, state, host):
with context.session.begin(subtransactions=True):
bindings = self.get_ha_router_port_bindings(context, [router_id],
host)
if bindings:
if len(bindings) > 1:
LOG.warn(_LW("The router %(router_id)s is bound multiple "
"times on the agent %(host)s"),
{'router_id': router_id, 'host': host})
bindings[0].update({'state': state})
def delete_router(self, context, id):
router_db = self._get_router(context, id)
if router_db.extra_attributes.ha:
ha_network = self.get_ha_network(context,
router_db.tenant_id)
if ha_network:
self._delete_vr_id_allocation(
context, ha_network, router_db.extra_attributes.ha_vr_id)
self._delete_ha_interfaces(context, router_db.id)
return super(L3_HA_NAT_db_mixin, self).delete_router(context, id)
def get_ha_router_port_bindings(self, context, router_ids, host=None):
query = context.session.query(L3HARouterAgentPortBinding)
if host:
query = query.join(agents_db.Agent).filter(
agents_db.Agent.host == host)
query = query.filter(
L3HARouterAgentPortBinding.router_id.in_(router_ids))
return query.all()
def _process_sync_ha_data(self, context, routers, host):
routers_dict = dict((router['id'], router) for router in routers)
bindings = self.get_ha_router_port_bindings(context,
routers_dict.keys(),
host)
for binding in bindings:
port_dict = self._core_plugin._make_port_dict(binding.port)
router = routers_dict.get(binding.router_id)
router[constants.HA_INTERFACE_KEY] = port_dict
router[constants.HA_ROUTER_STATE_KEY] = binding.state
for router in routers_dict.values():
interface = router.get(constants.HA_INTERFACE_KEY)
if interface:
self._populate_subnet_for_ports(context, [interface])
return routers_dict.values()
def get_ha_sync_data_for_host(self, context, host=None, router_ids=None,
active=None):
sync_data = super(L3_HA_NAT_db_mixin, self).get_sync_data(context,
router_ids,
active)
return self._process_sync_ha_data(context, sync_data, host)

86
neutron/db/migration/alembic_migrations/versions/16a27a58e093_ext_l3_ha_mode.py Normal file
View File

@ -0,0 +1,86 @@
# Copyright 2014 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
"""ext_l3_ha_mode
Revision ID: 16a27a58e093
Revises: 86d6d9776e2b
Create Date: 2014-02-01 10:24:12.412733
"""
# revision identifiers, used by Alembic.
revision = '16a27a58e093'
down_revision = '86d6d9776e2b'
from alembic import op
import sqlalchemy as sa
l3_ha_states = sa.Enum('active', 'standby', name='l3_ha_states')
def upgrade(active_plugins=None, options=None):
op.add_column('router_extra_attributes',
sa.Column('ha', sa.Boolean(),
nullable=False,
server_default=sa.sql.false()))
op.add_column('router_extra_attributes',
sa.Column('ha_vr_id', sa.Integer()))
op.create_table('ha_router_agent_port_bindings',
sa.Column('port_id', sa.String(length=36),
nullable=False),
sa.Column('router_id', sa.String(length=36),
nullable=False),
sa.Column('l3_agent_id', sa.String(length=36),
nullable=True),
sa.Column('state', l3_ha_states,
server_default='standby'),
sa.PrimaryKeyConstraint('port_id'),
sa.ForeignKeyConstraint(['port_id'], ['ports.id'],
ondelete='CASCADE'),
sa.ForeignKeyConstraint(['router_id'], ['routers.id'],
ondelete='CASCADE'),
sa.ForeignKeyConstraint(['l3_agent_id'], ['agents.id'],
ondelete='CASCADE'))
op.create_table('ha_router_networks',
sa.Column('tenant_id', sa.String(length=255),
nullable=False, primary_key=True),
sa.Column('network_id', sa.String(length=36),
nullable=False,
primary_key=True),
sa.ForeignKeyConstraint(['network_id'], ['networks.id'],
ondelete='CASCADE'))
op.create_table('ha_router_vrid_allocations',
sa.Column('network_id', sa.String(length=36),
nullable=False,
primary_key=True),
sa.Column('vr_id', sa.Integer(),
nullable=False,
primary_key=True),
sa.ForeignKeyConstraint(['network_id'], ['networks.id'],
ondelete='CASCADE'))
def downgrade(active_plugins=None, options=None):
op.drop_table('ha_router_vrid_allocations')
op.drop_table('ha_router_networks')
op.drop_table('ha_router_agent_port_bindings')
l3_ha_states.drop(op.get_bind(), checkfirst=False)
op.drop_column('router_extra_attributes', 'ha_vr_id')
op.drop_column('router_extra_attributes', 'ha')

2
neutron/db/migration/alembic_migrations/versions/HEAD
View File

@ -1 +1 @@
86d6d9776e2b
16a27a58e093

1
neutron/db/migration/models/head.py
View File

@ -34,6 +34,7 @@ from neutron.db import l3_attrs_db # noqa
from neutron.db import l3_db # noqa
from neutron.db import l3_dvrscheduler_db # noqa
from neutron.db import l3_gwmode_db # noqa
from neutron.db import l3_hamode_db # noqa
from neutron.db.loadbalancer import loadbalancer_db # noqa
from neutron.db.metering import metering_db # noqa
from neutron.db import model_base

91
neutron/extensions/l3_ext_ha_mode.py Normal file
View File

@ -0,0 +1,91 @@
# Copyright (C) 2014 eNovance SAS <licensing@enovance.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
from neutron.api import extensions
from neutron.api.v2 import attributes
from neutron.common import constants
from neutron.common import exceptions
HA_INFO = 'ha'
EXTENDED_ATTRIBUTES_2_0 = {
'routers': {
HA_INFO: {'allow_post': True, 'allow_put': True,
'default': attributes.ATTR_NOT_SPECIFIED, 'is_visible': True,
'enforce_policy': True,
'convert_to': attributes.convert_to_boolean_if_not_none}
}
}
class DistributedHARouterNotSupported(NotImplementedError):
message = _("Currenly distributed HA routers are "
"not supported.")
class MaxVRIDAllocationTriesReached(exceptions.NeutronException):
message = _("Failed to allocate a VRID in the network %(network_id)s "
"for the router %(router_id)s after %(max_tries)s tries.")
class NoVRIDAvailable(exceptions.Conflict):
message = _("No more Virtual Router Identifier (VRID) available when "
"creating router %(router_id)s. The limit of number "
"of HA Routers per tenant is 254.")
class HANetworkCIDRNotValid(exceptions.NeutronException):
message = _("The HA Network CIDR specified in the configuration file "
"isn't valid; %(cidr)s.")
class HANotEnoughAvailableAgents(exceptions.NeutronException):
message = _("Not enough l3 agents available to ensure HA. Minimum "
"required %(min_agents)s, available %(num_agents)s.")
class HAMinimumAgentsNumberNotValid(exceptions.NeutronException):
message = (_("min_l3_agents_per_router config parameter is not valid. "
"It has to be equal to or more than %s for HA.") %
constants.MINIMUM_AGENTS_FOR_HA)
class L3_ext_ha_mode(extensions.ExtensionDescriptor):
"""Extension class supporting virtual router in HA mode."""
@classmethod
def get_name(cls):
return "HA Router extension"
@classmethod
def get_alias(cls):
return constants.L3_HA_MODE_EXT_ALIAS
@classmethod
def get_description(cls):
return "Add HA capability to routers."
@classmethod
def get_namespace(cls):
return ""
@classmethod
def get_updated(cls):
return "2014-04-26T00:00:00-00:00"
def get_extended_resources(self, version):
if version == "2.0":
return EXTENDED_ATTRIBUTES_2_0
else:
return {}

14
neutron/services/l3_router/l3_router_plugin.py
View File

@ -24,18 +24,18 @@ from neutron.common import rpc as n_rpc
from neutron.common import topics
from neutron.db import common_db_mixin
from neutron.db import extraroute_db
from neutron.db import l3_dvr_db
from neutron.db import l3_dvrscheduler_db
from neutron.db import l3_gwmode_db
from neutron.db import l3_hamode_db
from neutron.openstack.common import importutils
from neutron.plugins.common import constants
class L3RouterPlugin(common_db_mixin.CommonDbMixin,
extraroute_db.ExtraRoute_db_mixin,
l3_dvr_db.L3_NAT_with_dvr_db_mixin,
l3_gwmode_db.L3_NAT_db_mixin,
l3_dvrscheduler_db.L3_DVRsch_db_mixin):
l3_dvrscheduler_db.L3_DVRsch_db_mixin,
l3_hamode_db.L3_HA_NAT_db_mixin):
"""Implementation of the Neutron L3 Router Service Plugin.
@ -43,17 +43,19 @@ class L3RouterPlugin(common_db_mixin.CommonDbMixin,
router and floatingip resources and manages associated
request/response.
All DB related work is implemented in classes
l3_db.L3_NAT_db_mixin, l3_dvr_db.L3_NAT_with_dvr_db_mixin, and
extraroute_db.ExtraRoute_db_mixin.
l3_db.L3_NAT_db_mixin, l3_hamode_db.L3_HA_NAT_db_mixin,
l3_dvr_db.L3_NAT_with_dvr_db_mixin, and extraroute_db.ExtraRoute_db_mixin.
"""
supported_extension_aliases = ["dvr", "router", "ext-gw-mode",
"extraroute", "l3_agent_scheduler"]
"extraroute", "l3_agent_scheduler",
"l3-ha"]
def __init__(self):
self.setup_rpc()
self.router_scheduler = importutils.import_object(
cfg.CONF.router_scheduler_driver)
self.start_periodic_agent_status_check()
super(L3RouterPlugin, self).__init__()
def setup_rpc(self):
# RPC support

2
neutron/tests/unit/db/test_l3_dvr_db.py
View File

@ -118,7 +118,7 @@ class L3DvrTestCase(testlib_api.SqlTestCase):
pass_router_id=False)
def _test__is_distributed_router(self, router, expected):
result = l3_dvr_db._is_distributed_router(router)
result = l3_dvr_db.is_distributed_router(router)
self.assertEqual(expected, result)
def test__is_distributed_router_by_db_object(self):

390
neutron/tests/unit/db/test_l3_ha_db.py Normal file
View File

@ -0,0 +1,390 @@
# Copyright (C) 2014 eNovance SAS <licensing@enovance.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
from oslo.config import cfg
from neutron.common import constants
from neutron import context
from neutron.db import agents_db
from neutron.db import common_db_mixin
from neutron.db import l3_hamode_db
from neutron.extensions import l3_ext_ha_mode
from neutron import manager
from neutron.openstack.common import uuidutils
from neutron.tests.unit import testlib_api
from neutron.tests.unit import testlib_plugin
_uuid = uuidutils.generate_uuid
class FakeL3Plugin(common_db_mixin.CommonDbMixin,
l3_hamode_db.L3_HA_NAT_db_mixin):
pass
class FakeL3PluginWithAgents(FakeL3Plugin,
agents_db.AgentDbMixin):
pass
class L3HATestFramework(testlib_api.SqlTestCase,
testlib_plugin.PluginSetupHelper):
def setUp(self):
super(L3HATestFramework, self).setUp()
self.admin_ctx = context.get_admin_context()
self.setup_coreplugin('neutron.plugins.ml2.plugin.Ml2Plugin')
self.core_plugin = manager.NeutronManager.get_plugin()
mock.patch.object(l3_hamode_db.L3_HA_NAT_db_mixin, 'get_l3_agents',
create=True, return_value=[1, 2]).start()
notif_p = mock.patch.object(l3_hamode_db.L3_HA_NAT_db_mixin,
'_notify_ha_interfaces_updated')
self.notif_m = notif_p.start()
cfg.CONF.set_override('allow_overlapping_ips', True)
def _create_router(self, ha=True, tenant_id='tenant1', distributed=None):
router = {'name': 'router1', 'admin_state_up': True}
if ha is not None:
router['ha'] = ha
if distributed is not None:
router['distributed'] = distributed
return self.plugin._create_router_db(self.admin_ctx, router, tenant_id)
def _update_router(self, router_id, ha=True, distributed=None):
data = {'ha': ha} if ha is not None else {}
if distributed is not None:
data['distributed'] = distributed
return self.plugin._update_router_db(self.admin_ctx, router_id,
data, None)
class L3HAGetSyncDataTestCase(L3HATestFramework):
def setUp(self):
super(L3HAGetSyncDataTestCase, self).setUp()
self.plugin = FakeL3PluginWithAgents()
self._register_agents()
def _register_agents(self):
agent_status = {
'agent_type': constants.AGENT_TYPE_L3,
'binary': 'neutron-l3-agent',
'host': 'l3host',
'topic': 'N/A'
}
self.plugin.create_or_update_agent(self.admin_ctx, agent_status)
agent_status['host'] = 'l3host_2'
self.plugin.create_or_update_agent(self.admin_ctx, agent_status)
self.agent1, self.agent2 = self.plugin.get_agents(self.admin_ctx)
def _bind_router(self, router_id):
with self.admin_ctx.session.begin(subtransactions=True):
bindings = self.plugin.get_ha_router_port_bindings(self.admin_ctx,
[router_id])
for agent_id, binding in zip(
[self.agent1['id'], self.agent2['id']], bindings):
binding.l3_agent_id = agent_id
def test_l3_agent_routers_query_interface(self):
router = self._create_router()
self._bind_router(router.id)
routers = self.plugin.get_ha_sync_data_for_host(self.admin_ctx,
self.agent1['host'])
self.assertEqual(1, len(routers))
router = routers[0]
self.assertIsNotNone(router.get('ha'))
interface = router.get(constants.HA_INTERFACE_KEY)
self.assertIsNotNone(interface)
self.assertEqual(constants.DEVICE_OWNER_ROUTER_HA_INTF,
interface['device_owner'])
self.assertEqual(cfg.CONF.l3_ha_net_cidr, interface['subnet']['cidr'])
def test_update_state(self):
router = self._create_router()
self._bind_router(router.id)
routers = self.plugin.get_ha_sync_data_for_host(self.admin_ctx,
self.agent1['host'])
state = routers[0].get(constants.HA_ROUTER_STATE_KEY)
self.assertEqual('standby', state)
self.plugin.update_router_state(self.admin_ctx, router.id, 'active',
self.agent1['host'])
routers = self.plugin.get_ha_sync_data_for_host(self.admin_ctx,
self.agent1['host'])
state = routers[0].get(constants.HA_ROUTER_STATE_KEY)
self.assertEqual('active', state)
class L3HATestCase(L3HATestFramework):
def setUp(self):
super(L3HATestCase, self).setUp()
self.plugin = FakeL3Plugin()
def test_verify_configuration_succeed(self):
# Default configuration should pass
self.plugin._verify_configuration()
def test_verify_configuration_l3_ha_net_cidr_is_not_a_cidr(self):
cfg.CONF.set_override('l3_ha_net_cidr', 'not a cidr')
self.assertRaises(
l3_ext_ha_mode.HANetworkCIDRNotValid,
self.plugin._verify_configuration)
def test_verify_configuration_l3_ha_net_cidr_is_not_a_subnet(self):
cfg.CONF.set_override('l3_ha_net_cidr', '10.0.0.1/8')
self.assertRaises(
l3_ext_ha_mode.HANetworkCIDRNotValid,
self.plugin._verify_configuration)
def test_verify_conifguration_min_l3_agents_per_router_below_minimum(self):
cfg.CONF.set_override('min_l3_agents_per_router', 0)
self.assertRaises(
l3_ext_ha_mode.HAMinimumAgentsNumberNotValid,
self.plugin._verify_configuration)
def test_ha_router_create(self):
router = self._create_router()
self.assertTrue(router.extra_attributes['ha'])
def test_ha_router_create_with_distributed(self):
self.assertRaises(l3_ext_ha_mode.DistributedHARouterNotSupported,
self._create_router,
distributed=True)
def test_no_ha_router_create(self):
router = self._create_router(ha=False)
self.assertFalse(router.extra_attributes['ha'])
def test_router_create_with_ha_conf_enabled(self):
cfg.CONF.set_override('l3_ha', True)
router = self._create_router(ha=None)
self.assertTrue(router.extra_attributes['ha'])
def test_migration_from_ha(self):
router = self._create_router()
self.assertTrue(router.extra_attributes['ha'])
router = self._update_router(router.id, ha=False)
self.assertFalse(router.extra_attributes['ha'])
self.assertIsNone(router.extra_attributes['ha_vr_id'])
def test_migration_to_ha(self):
router = self._create_router(ha=False)
self.assertFalse(router.extra_attributes['ha'])
router = self._update_router(router.id, ha=True)
self.assertTrue(router.extra_attributes['ha'])
self.assertIsNotNone(router.extra_attributes['ha_vr_id'])
def test_migrate_ha_router_to_distributed(self):
router = self._create_router()
self.assertTrue(router.extra_attributes['ha'])
self.assertRaises(l3_ext_ha_mode.DistributedHARouterNotSupported,
self._update_router,
router.id,
distributed=True)
def test_unique_ha_network_per_tenant(self):
tenant1 = _uuid()
tenant2 = _uuid()
self._create_router(tenant_id=tenant1)
self._create_router(tenant_id=tenant2)
ha_network1 = self.plugin.get_ha_network(self.admin_ctx, tenant1)
ha_network2 = self.plugin.get_ha_network(self.admin_ctx, tenant2)
self.assertNotEqual(
ha_network1['network_id'], ha_network2['network_id'])
def _deployed_router_change_ha_flag(self, to_ha):
self._create_router(ha=not to_ha)
routers = self.plugin.get_ha_sync_data_for_host(self.admin_ctx)
router = routers[0]
interface = router.get(constants.HA_INTERFACE_KEY)
if to_ha:
self.assertIsNone(interface)
else:
self.assertIsNotNone(interface)
self._update_router(router['id'], to_ha)
routers = self.plugin.get_ha_sync_data_for_host(self.admin_ctx)
router = routers[0]
interface = router.get(constants.HA_INTERFACE_KEY)
if to_ha:
self.assertIsNotNone(interface)
else:
self.assertIsNone(interface)
def test_deployed_router_can_have_ha_enabled(self):
self._deployed_router_change_ha_flag(to_ha=True)
def test_deployed_router_can_have_ha_disabled(self):
self._deployed_router_change_ha_flag(to_ha=False)
def test_create_ha_router_notifies_agent(self):
self._create_router()
self.assertTrue(self.notif_m.called)
def test_update_router_to_ha_notifies_agent(self):
router = self._create_router(ha=False)
self.notif_m.reset_mock()
self._update_router(router.id, ha=True)
self.assertTrue(self.notif_m.called)
def test_unique_vr_id_between_routers(self):
self._create_router()
self._create_router()
routers = self.plugin.get_ha_sync_data_for_host(self.admin_ctx)
self.assertEqual(2, len(routers))
self.assertNotEqual(routers[0]['ha_vr_id'], routers[1]['ha_vr_id'])
@mock.patch('neutron.db.l3_hamode_db.VR_ID_RANGE', new=set(range(1, 1)))
def test_vr_id_depleted(self):
self.assertRaises(l3_ext_ha_mode.NoVRIDAvailable, self._create_router)
@mock.patch('neutron.db.l3_hamode_db.VR_ID_RANGE', new=set(range(1, 2)))
def test_vr_id_unique_range_per_tenant(self):
self._create_router()
self._create_router(tenant_id=_uuid())
routers = self.plugin.get_ha_sync_data_for_host(self.admin_ctx)
self.assertEqual(2, len(routers))
self.assertEqual(routers[0]['ha_vr_id'], routers[1]['ha_vr_id'])
@mock.patch('neutron.db.l3_hamode_db.MAX_ALLOCATION_TRIES', new=2)
def test_vr_id_allocation_contraint_conflict(self):
router = self._create_router()
network = self.plugin.get_ha_network(self.admin_ctx, router.tenant_id)
with mock.patch.object(self.plugin, '_get_allocated_vr_id',
return_value=set()) as alloc:
self.assertRaises(l3_ext_ha_mode.MaxVRIDAllocationTriesReached,
self.plugin._allocate_vr_id, self.admin_ctx,
network.network_id, router.id)
self.assertEqual(2, len(alloc.mock_calls))
def test_vr_id_allocation_delete_router(self):
router = self._create_router()
network = self.plugin.get_ha_network(self.admin_ctx, router.tenant_id)
allocs_before = self.plugin._get_allocated_vr_id(self.admin_ctx,
network.network_id)
router = self._create_router()
allocs_current = self.plugin._get_allocated_vr_id(self.admin_ctx,
network.network_id)
self.assertNotEqual(allocs_before, allocs_current)
self.plugin.delete_router(self.admin_ctx, router.id)
allocs_after = self.plugin._get_allocated_vr_id(self.admin_ctx,
network.network_id)
self.assertEqual(allocs_before, allocs_after)
def test_vr_id_allocation_router_migration(self):
router = self._create_router()
network = self.plugin.get_ha_network(self.admin_ctx, router.tenant_id)
allocs_before = self.plugin._get_allocated_vr_id(self.admin_ctx,
network.network_id)
router = self._create_router()
self._update_router(router.id, ha=False)
allocs_after = self.plugin._get_allocated_vr_id(self.admin_ctx,
network.network_id)
self.assertEqual(allocs_before, allocs_after)
def test_one_ha_router_one_not(self):
self._create_router(ha=False)
self._create_router()
routers = self.plugin.get_ha_sync_data_for_host(self.admin_ctx)
ha0 = routers[0]['ha']
ha1 = routers[1]['ha']
self.assertNotEqual(ha0, ha1)
def test_add_ha_port_binding_failure_rolls_back_port(self):
router = self._create_router()
device_filter = {'device_id': [router.id]}
ports_before = self.core_plugin.get_ports(
self.admin_ctx, filters=device_filter)
network = self.plugin.get_ha_network(self.admin_ctx, router.tenant_id)
with mock.patch.object(self.plugin, '_create_ha_port_binding',
side_effect=ValueError):
self.assertRaises(ValueError, self.plugin.add_ha_port,
self.admin_ctx, router.id, network.network_id,
router.tenant_id)
ports_after = self.core_plugin.get_ports(
self.admin_ctx, filters=device_filter)
self.assertEqual(ports_before, ports_after)
def test_create_ha_network_binding_failure_rolls_back_network(self):
networks_before = self.core_plugin.get_networks(self.admin_ctx)
with mock.patch.object(self.plugin,
'_create_ha_network_tenant_binding',
side_effect=ValueError):
self.assertRaises(ValueError, self.plugin._create_ha_network,
self.admin_ctx, _uuid())
networks_after = self.core_plugin.get_networks(self.admin_ctx)
self.assertEqual(networks_before, networks_after)
def test_create_ha_network_subnet_failure_rolls_back_network(self):
networks_before = self.core_plugin.get_networks(self.admin_ctx)
with mock.patch.object(self.plugin, '_create_ha_subnet',
side_effect=ValueError):
self.assertRaises(ValueError, self.plugin._create_ha_network,
self.admin_ctx, _uuid())
networks_after = self.core_plugin.get_networks(self.admin_ctx)
self.assertEqual(networks_before, networks_after)
def test_create_ha_interfaces_binding_failure_rolls_back_ports(self):
router = self._create_router()
network = self.plugin.get_ha_network(self.admin_ctx, router.tenant_id)
device_filter = {'device_id': [router.id]}
ports_before = self.core_plugin.get_ports(
self.admin_ctx, filters=device_filter)
with mock.patch.object(self.plugin, '_create_ha_port_binding',
side_effect=ValueError):
self.assertRaises(ValueError, self.plugin._create_ha_interfaces,
self.admin_ctx, router, network)
ports_after = self.core_plugin.get_ports(
self.admin_ctx, filters=device_filter)
self.assertEqual(ports_before, ports_after)
def test_create_router_db_ha_attribute_failure_rolls_back_router(self):
routers_before = self.plugin.get_routers(self.admin_ctx)
for method in ('_set_vr_id',
'_create_ha_interfaces',
'_notify_ha_interfaces_updated'):
with mock.patch.object(self.plugin, method,
side_effect=ValueError):
self.assertRaises(ValueError, self._create_router)
routers_after = self.plugin.get_routers(self.admin_ctx)
self.assertEqual(routers_before, routers_after)