openstack
/
neutron
Code Issues Proposed changes

Add HA support to the l3 agent 

* Add HA mixins used by RouterInfo and LNAT3Agent
* For HA routers: Internal, external and floating IP addresses are no
  longer configured by the agent. Instead the interfaces and addresses
  are passed to a keepalived configuration, which configures the
  addresses when the router transitions to the master state.
* Only the master instance of the router opens the metadata proxy.
  This happens due to keepalived notification scripts that are
  called upon state transitions.
* Extra routes are handled via keepalived virtual routes and are
  no longer configured by the agent.
* HA routers create a 'HA device' on a VRRP-traffic only HA-network.
* Functional testing: Add two new tests to the L3 agent:
    1) Translation of a router configuration to a keepalived
       configuration.
    2) HA specific events when creating a HA router - Assert that
       keepalived is up, etc.

Change-Id: I83f2a5d2af42164c42773b385ba7b00872eed54e
Implements: blueprint l3-high-availability
Co-Authored-By: Assaf Muller <amuller@redhat.com>
This commit is contained in:
Sylvain Afchain 2014-08-06 15:32:51 +03:00 committed by Assaf Muller
parent f095f99330
commit e7ce526ac1
12 changed files with 632 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
etc/l3_agent.ini
View File

@ -88,3 +88,15 @@
# DVR. This mode must be used for an L3 agent running on a centralized
# node (or in single-host deployments, e.g. devstack).
# agent_mode = legacy
# Location to store keepalived and all HA configurations
# ha_confs_path = $state_path/ha_confs
# VRRP authentication type AH/PASS
# ha_vrrp_auth_type = PASS
# VRRP authentication password
# ha_vrrp_auth_password =
# The advertisement interval in seconds
# ha_vrrp_advert_int = 2

162
neutron/agent/l3_agent.py
View File

@ -25,6 +25,7 @@ from oslo.config import cfg
import Queue
from neutron.agent.common import config
from neutron.agent import l3_ha_agent
from neutron.agent.linux import external_process
from neutron.agent.linux import interface
from neutron.agent.linux import ip_lib
@ -237,7 +238,7 @@ class LinkLocalAllocator(object):
return f.readlines()
class RouterInfo(object):
class RouterInfo(l3_ha_agent.RouterMixin):
def __init__(self, router_id, root_helper, use_namespaces, router,
use_ipv6=False):
@ -265,6 +266,8 @@ class RouterInfo(object):
self.rtr_fip_subnet = None
self.dist_fip_count = 0
super(RouterInfo, self).__init__()
@property
def router(self):
return self._router
@ -430,7 +433,9 @@ class RouterProcessingQueue(object):
yield (rp, update)
class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback,
l3_ha_agent.AgentMixin,
manager.Manager):
"""Manager for L3NatAgent
API version history:
@ -734,8 +739,15 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
ri.iptables_manager.ipv4['nat'].add_rule(c, r)
ri.iptables_manager.apply()
self.process_router_add(ri)
if ri.is_ha:
self.process_ha_router_added(ri)
if self.conf.enable_metadata_proxy:
self._spawn_metadata_proxy(ri.router_id, ri.ns_name)
if ri.is_ha:
self._add_keepalived_notifiers(ri)
else:
self._spawn_metadata_proxy(ri.router_id, ri.ns_name)
def _router_removed(self, router_id):
ri = self.router_info.get(router_id)
@ -743,6 +755,10 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
LOG.warn(_("Info for router %s were not found. "
"Skipping router removal"), router_id)
return
if ri.is_ha:
self.process_ha_router_removed(ri)
ri.router['gw_port'] = None
ri.router[l3_constants.INTERFACE_KEY] = []
ri.router[l3_constants.FLOATINGIP_KEY] = []
@ -757,7 +773,8 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
del self.router_info[router_id]
self._destroy_router_namespace(ri.ns_name)
def _spawn_metadata_proxy(self, router_id, ns_name):
def _get_metadata_proxy_callback(self, router_id):
def callback(pid_file):
metadata_proxy_socket = cfg.CONF.metadata_proxy_socket
proxy_cmd = ['neutron-ns-metadata-proxy',
@ -771,19 +788,22 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
router_id))
return proxy_cmd
pm = external_process.ProcessManager(
return callback
def _get_metadata_proxy_process_manager(self, router_id, ns_name):
return external_process.ProcessManager(
self.conf,
router_id,
self.root_helper,
ns_name)
def _spawn_metadata_proxy(self, router_id, ns_name):
callback = self._get_metadata_proxy_callback(router_id)
pm = self._get_metadata_proxy_process_manager(router_id, ns_name)
pm.enable(callback)
def _destroy_metadata_proxy(self, router_id, ns_name):
pm = external_process.ProcessManager(
self.conf,
router_id,
self.root_helper,
ns_name)
pm = self._get_metadata_proxy_process_manager(router_id, ns_name)
pm.disable()
def _set_subnet_arp_info(self, ri, port):
@ -885,10 +905,20 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
if ex_gw_port_id:
interface_name = self.get_external_device_name(ex_gw_port_id)
if ex_gw_port:
def _gateway_ports_equal(port1, port2):
def _get_filtered_dict(d, ignore):
return dict((k, v) for k, v in d.iteritems()
if k not in ignore)
keys_to_ignore = set(['binding:host_id'])
port1_filtered = _get_filtered_dict(port1, keys_to_ignore)
port2_filtered = _get_filtered_dict(port2, keys_to_ignore)
return port1_filtered == port2_filtered
self._set_subnet_info(ex_gw_port)
if not ri.ex_gw_port:
self.external_gateway_added(ri, ex_gw_port, interface_name)
elif ex_gw_port != ri.ex_gw_port:
elif not _gateway_ports_equal(ex_gw_port, ri.ex_gw_port):
self.external_gateway_updated(ri, ex_gw_port, interface_name)
elif not ex_gw_port and ri.ex_gw_port:
self.external_gateway_removed(ri, ri.ex_gw_port, interface_name)
@ -946,6 +976,12 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
ri.snat_ports = snat_ports
ri.enable_snat = ri.router.get('enable_snat')
if ri.is_ha:
if ri.ha_port:
ri.spawn_keepalived()
else:
ri.disable_keepalived()
def _handle_router_snat_rules(self, ri, ex_gw_port, internal_cidrs,
interface_name, action):
# Remove all the rules
@ -1048,32 +1084,39 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
def _add_floating_ip(self, ri, fip, interface_name, device):
fip_ip = fip['floating_ip_address']
ip_cidr = str(fip_ip) + FLOATING_IP_CIDR_SUFFIX
net = netaddr.IPNetwork(ip_cidr)
try:
device.addr.add(net.version, ip_cidr, str(net.broadcast))
except (processutils.UnknownArgumentError,
processutils.ProcessExecutionError):
# any exception occurred here should cause the floating IP
# to be set in error state
LOG.warn(_("Unable to configure IP address for "
"floating IP: %s"), fip['id'])
return l3_constants.FLOATINGIP_STATUS_ERROR
if ri.router['distributed']:
# Special Handling for DVR - update FIP namespace
# and ri.namespace to handle DVR based FIP
self.floating_ip_added_dist(ri, fip)
if ri.is_ha:
self._add_vip(ri, ip_cidr, interface_name)
else:
# As GARP is processed in a distinct thread the call below
# won't raise an exception to be handled.
self._send_gratuitous_arp_packet(
ri.ns_name, interface_name, fip_ip)
return l3_constants.FLOATINGIP_STATUS_ACTIVE
net = netaddr.IPNetwork(ip_cidr)
try:
device.addr.add(net.version, ip_cidr, str(net.broadcast))
except (processutils.UnknownArgumentError,
processutils.ProcessExecutionError):
# any exception occurred here should cause the floating IP
# to be set in error state
LOG.warn(_("Unable to configure IP address for "
"floating IP: %s"), fip['id'])
return l3_constants.FLOATINGIP_STATUS_ERROR
if ri.router['distributed']:
# Special Handling for DVR - update FIP namespace
# and ri.namespace to handle DVR based FIP
self.floating_ip_added_dist(ri, fip)
else:
# As GARP is processed in a distinct thread the call below
# won't raise an exception to be handled.
self._send_gratuitous_arp_packet(
ri.ns_name, interface_name, fip_ip)
return l3_constants.FLOATINGIP_STATUS_ACTIVE
def _remove_floating_ip(self, ri, device, ip_cidr):
net = netaddr.IPNetwork(ip_cidr)
device.addr.delete(net.version, ip_cidr)
if ri.router['distributed']:
self.floating_ip_removed_dist(ri, ip_cidr)
if ri.is_ha:
self._remove_vip(ri, ip_cidr)
else:
net = netaddr.IPNetwork(ip_cidr)
device.addr.delete(net.version, ip_cidr)
if ri.router['distributed']:
self.floating_ip_removed_dist(ri, ip_cidr)
def process_router_floating_ip_addresses(self, ri, ex_gw_port):
"""Configure IP addresses on router's external gateway interface.
@ -1253,6 +1296,9 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
self._external_gateway_added(ri, ex_gw_port, interface_name,
ri.ns_name, preserve_ips)
if ri.is_ha:
self._ha_external_gateway_added(ri, ex_gw_port, interface_name)
def external_gateway_updated(self, ri, ex_gw_port, interface_name):
preserve_ips = []
if ri.router['distributed']:
@ -1272,6 +1318,9 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
self._external_gateway_added(ri, ex_gw_port, interface_name,
ns_name, preserve_ips)
if ri.is_ha:
self._ha_external_gateway_updated(ri, ex_gw_port, interface_name)
def _external_gateway_added(self, ri, ex_gw_port, interface_name,
ns_name, preserve_ips):
if not ip_lib.device_exists(interface_name,
@ -1284,14 +1333,15 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
namespace=ns_name,
prefix=EXTERNAL_DEV_PREFIX)
self.driver.init_l3(interface_name, [ex_gw_port['ip_cidr']],
namespace=ns_name,
gateway=ex_gw_port['subnet'].get('gateway_ip'),
extra_subnets=ex_gw_port.get('extra_subnets', []),
preserve_ips=preserve_ips)
ip_address = ex_gw_port['ip_cidr'].split('/')[0]
self._send_gratuitous_arp_packet(ns_name,
interface_name, ip_address)
if not ri.is_ha:
self.driver.init_l3(
interface_name, [ex_gw_port['ip_cidr']], namespace=ns_name,
gateway=ex_gw_port['subnet'].get('gateway_ip'),
extra_subnets=ex_gw_port.get('extra_subnets', []),
preserve_ips=preserve_ips)
ip_address = ex_gw_port['ip_cidr'].split('/')[0]
self._send_gratuitous_arp_packet(ns_name,
interface_name, ip_address)
def agent_gateway_added(self, ns_name, ex_gw_port,
interface_name):
@ -1343,6 +1393,9 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
else:
ns_name = ri.ns_name
if ri.is_ha:
self._ha_external_gateway_removed(ri, interface_name)
self.driver.unplug(interface_name,
bridge=self.conf.external_network_bridge,
namespace=ns_name,
@ -1404,7 +1457,7 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
def _internal_network_added(self, ns_name, network_id, port_id,
internal_cidr, mac_address,
interface_name, prefix):
interface_name, prefix, is_ha=False):
if not ip_lib.device_exists(interface_name,
root_helper=self.root_helper,
namespace=ns_name):
@ -1412,10 +1465,12 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
namespace=ns_name,
prefix=prefix)
self.driver.init_l3(interface_name, [internal_cidr],
namespace=ns_name)
ip_address = internal_cidr.split('/')[0]
self._send_gratuitous_arp_packet(ns_name, interface_name, ip_address)
if not is_ha:
self.driver.init_l3(interface_name, [internal_cidr],
namespace=ns_name)
ip_address = internal_cidr.split('/')[0]
self._send_gratuitous_arp_packet(ns_name, interface_name,
ip_address)
def internal_network_added(self, ri, port):
network_id = port['network_id']
@ -1427,7 +1482,11 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
self._internal_network_added(ri.ns_name, network_id, port_id,
internal_cidr, mac_address,
interface_name, INTERNAL_DEV_PREFIX)
interface_name, INTERNAL_DEV_PREFIX,
ri.is_ha)
if ri.is_ha:
self._add_vip(ri, internal_cidr, interface_name)
ex_gw_port = self._get_ex_gw_port(ri)
if ri.router['distributed'] and ex_gw_port:
@ -1475,6 +1534,8 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
if ip_lib.device_exists(interface_name,
root_helper=self.root_helper,
namespace=ri.ns_name):
if ri.is_ha:
self._clear_vips(ri, interface_name)
self.driver.unplug(interface_name, namespace=ri.ns_name,
prefix=INTERNAL_DEV_PREFIX)
@ -1843,6 +1904,10 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager):
def routes_updated(self, ri):
new_routes = ri.router['routes']
if ri.is_ha:
self._process_virtual_routes(ri, new_routes)
return
old_routes = ri.routes
adds, removes = common_utils.diff_list_of_dict(old_routes,
new_routes)
@ -1931,6 +1996,7 @@ class L3NATAgentWithStateReport(L3NATAgent):
def _register_opts(conf):
conf.register_opts(L3NATAgent.OPTS)
conf.register_opts(l3_ha_agent.OPTS)
config.register_interface_driver_opts_helper(conf)
config.register_use_namespaces_opts_helper(conf)
config.register_agent_state_opts_helper(conf)

232
neutron/agent/l3_ha_agent.py Normal file
View File

@ -0,0 +1,232 @@
# Copyright (c) 2014 OpenStack Foundation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import shutil
import signal
from oslo.config import cfg
from neutron.agent.linux import keepalived
from neutron.common import constants as l3_constants
from neutron.openstack.common.gettextutils import _LE
from neutron.openstack.common import log as logging
from neutron.openstack.common import periodic_task
LOG = logging.getLogger(__name__)
HA_DEV_PREFIX = 'ha-'
OPTS = [
cfg.StrOpt('ha_confs_path',
default='$state_path/ha_confs',
help=_('Location to store keepalived/conntrackd '
'config files')),
cfg.StrOpt('ha_vrrp_auth_type',
default='PASS',
help=_('VRRP authentication type AH/PASS')),
cfg.StrOpt('ha_vrrp_auth_password',
help=_('VRRP authentication password'),
secret=True),
cfg.IntOpt('ha_vrrp_advert_int',
default=2,
help=_('The advertisement interval in seconds')),
]
class RouterMixin(object):
def __init__(self):
self.ha_port = None
self.keepalived_manager = None
def _verify_ha(self):
if not self.is_ha:
raise ValueError(_('Router %s is not a HA router') %
self.router_id)
@property
def is_ha(self):
return self.router is not None and self.router.get('ha')
@property
def ha_priority(self):
self._verify_ha()
return self.router is not None and self.router.get(
'priority', keepalived.HA_DEFAULT_PRIORITY)
@property
def ha_vr_id(self):
self._verify_ha()
return self.router is not None and self.router.get('ha_vr_id')
@property
def ha_state(self):
self._verify_ha()
ha_state_path = self.keepalived_manager._get_full_config_file_path(
'state')
try:
with open(ha_state_path, 'r') as f:
return f.read()
except (OSError, IOError):
LOG.debug('Error while reading HA state for %s', self.router_id)
return None
def spawn_keepalived(self):
self.keepalived_manager.spawn_or_restart()
def disable_keepalived(self):
self.keepalived_manager.disable()
conf_dir = self.keepalived_manager.get_conf_dir()
shutil.rmtree(conf_dir)
class AgentMixin(object):
def __init__(self, host):
self._init_ha_conf_path()
super(AgentMixin, self).__init__(host)
def _init_ha_conf_path(self):
ha_full_path = os.path.dirname("/%s/" % self.conf.ha_confs_path)
if not os.path.isdir(ha_full_path):
os.makedirs(ha_full_path, 0o755)
def _init_keepalived_manager(self, ri):
ri.keepalived_manager = keepalived.KeepalivedManager(
ri.router['id'],
keepalived.KeepalivedConf(),
conf_path=self.conf.ha_confs_path,
namespace=ri.ns_name,
root_helper=self.root_helper)
config = ri.keepalived_manager.config
interface_name = self.get_ha_device_name(ri.ha_port['id'])
instance = keepalived.KeepalivedInstance(
'BACKUP', interface_name, ri.ha_vr_id, nopreempt=True,
advert_int=self.conf.ha_vrrp_advert_int, priority=ri.ha_priority)
instance.track_interfaces.append(interface_name)
if self.conf.ha_vrrp_auth_password:
# TODO(safchain): use oslo.config types when it will be available
# in order to check the validity of ha_vrrp_auth_type
instance.set_authentication(self.conf.ha_vrrp_auth_type,
self.conf.ha_vrrp_auth_password)
group = keepalived.KeepalivedGroup(ri.ha_vr_id)
group.add_instance(instance)
config.add_group(group)
config.add_instance(instance)
def process_ha_router_added(self, ri):
ha_port = ri.router.get(l3_constants.HA_INTERFACE_KEY)
if not ha_port:
LOG.error(_LE('Unable to process HA router %s without ha port'),
ri.router_id)
return
self._set_subnet_info(ha_port)
self.ha_network_added(ri, ha_port['network_id'], ha_port['id'],
ha_port['ip_cidr'], ha_port['mac_address'])
ri.ha_port = ha_port
self._init_keepalived_manager(ri)
def process_ha_router_removed(self, ri):
self.ha_network_removed(ri)
def get_ha_device_name(self, port_id):
return (HA_DEV_PREFIX + port_id)[:self.driver.DEV_NAME_LEN]
def ha_network_added(self, ri, network_id, port_id, internal_cidr,
mac_address):
interface_name = self.get_ha_device_name(port_id)
self.driver.plug(network_id, port_id, interface_name, mac_address,
namespace=ri.ns_name,
prefix=HA_DEV_PREFIX)
self.driver.init_l3(interface_name, [internal_cidr],
namespace=ri.ns_name)
def ha_network_removed(self, ri):
interface_name = self.get_ha_device_name(ri.ha_port['id'])
self.driver.unplug(interface_name, namespace=ri.ns_name,
prefix=HA_DEV_PREFIX)
ri.ha_port = None
def _add_vip(self, ri, ip_cidr, interface):
instance = ri.keepalived_manager.config.get_instance(ri.ha_vr_id)
instance.add_vip(ip_cidr, interface)
def _remove_vip(self, ri, ip_cidr):
instance = ri.keepalived_manager.config.get_instance(ri.ha_vr_id)
instance.remove_vip_by_ip_address(ip_cidr)
def _clear_vips(self, ri, interface):
instance = ri.keepalived_manager.config.get_instance(ri.ha_vr_id)
instance.remove_vips_vroutes_by_interface(interface)
def _add_keepalived_notifiers(self, ri):
callback = self._get_metadata_proxy_callback(ri.router_id)
pm = self._get_metadata_proxy_process_manager(ri.router_id, ri.ns_name)
pid = pm.get_pid_file_name(ensure_pids_dir=True)
ri.keepalived_manager.add_notifier(
callback(pid), 'master', ri.ha_vr_id)
for state in ('backup', 'fault'):
ri.keepalived_manager.add_notifier(
['kill', '-%s' % signal.SIGKILL,
'$(cat ' + pid + ')'], state, ri.ha_vr_id)
def _ha_external_gateway_updated(self, ri, ex_gw_port, interface_name):
old_gateway_cidr = ri.ex_gw_port['ip_cidr']
self._remove_vip(ri, old_gateway_cidr)
self._ha_external_gateway_added(ri, ex_gw_port, interface_name)
def _add_default_gw_virtual_route(self, ri, ex_gw_port, interface_name):
gw_ip = ex_gw_port['subnet']['gateway_ip']
if gw_ip:
instance = ri.keepalived_manager.config.get_instance(ri.ha_vr_id)
instance.virtual_routes = (
[route for route in instance.virtual_routes
if route.destination != '0.0.0.0/0'])
instance.virtual_routes.append(
keepalived.KeepalivedVirtualRoute(
'0.0.0.0/0', gw_ip, interface_name))
def _ha_external_gateway_added(self, ri, ex_gw_port, interface_name):
self._add_vip(ri, ex_gw_port['ip_cidr'], interface_name)
self._add_default_gw_virtual_route(ri, ex_gw_port, interface_name)
def _ha_external_gateway_removed(self, ri, interface_name):
self._clear_vips(ri, interface_name)
def _process_virtual_routes(self, ri, new_routes):
instance = ri.keepalived_manager.config.get_instance(ri.ha_vr_id)
# Filter out all of the old routes while keeping only the default route
instance.virtual_routes = [route for route in instance.virtual_routes
if route.destination == '0.0.0.0/0']
for route in new_routes:
instance.virtual_routes.append(keepalived.KeepalivedVirtualRoute(
route['destination'],
route['nexthop']))
def get_ha_routers(self):
return (router for router in self.router_info.values() if router.is_ha)
@periodic_task.periodic_task
def _ensure_keepalived_alive(self, context):
# TODO(amuller): Use external_process.ProcessMonitor
for router in self.get_ha_routers():
router.keepalived_manager.revive()

9
neutron/agent/linux/keepalived.py
View File

@ -83,7 +83,7 @@ class KeepalivedGroup(object):
self.ha_vr_id = ha_vr_id
self.name = 'VG_%s' % ha_vr_id
self.instance_names = set()
self.notifiers = {}
self.notifiers = []
def add_instance(self, instance):
self.instance_names.add(instance.name)
@ -91,7 +91,7 @@ class KeepalivedGroup(object):
def set_notify(self, state, path):
if state not in VALID_NOTIFY_STATES:
raise InvalidNotifyStateException(state=state)
self.notifiers[state] = path
self.notifiers.append((state, path))
def build_config(self):
return itertools.chain(['vrrp_sync_group %s {' % self.name,
@ -99,7 +99,7 @@ class KeepalivedGroup(object):
(' %s' % i for i in self.instance_names),
[' }'],
(' notify_%s "%s"' % (state, path)
for state, path in self.notifiers.items()),
for state, path in self.notifiers),
['}'])
@ -132,6 +132,9 @@ class KeepalivedInstance(object):
self.authentication = (auth_type, password)
def add_vip(self, ip_cidr, interface_name):
self.vips.append(KeepalivedVipAddress(ip_cidr, interface_name))
def remove_vips_vroutes_by_interface(self, interface_name):
self.vips = [vip for vip in self.vips
if vip.interface_name != interface_name]

2
neutron/services/firewall/agents/varmour/varmour_router.py
View File

@ -26,6 +26,7 @@ from oslo.config import cfg
from neutron.agent.common import config
from neutron.agent import l3_agent
from neutron.agent import l3_ha_agent
from neutron.agent.linux import external_process
from neutron.agent.linux import interface
from neutron.agent.linux import ip_lib
@ -332,6 +333,7 @@ class vArmourL3NATAgentWithStateReport(vArmourL3NATAgent,
def main():
conf = cfg.CONF
conf.register_opts(vArmourL3NATAgent.OPTS)
conf.register_opts(l3_ha_agent.OPTS)
config.register_interface_driver_opts_helper(conf)
config.register_use_namespaces_opts_helper(conf)
config.register_agent_state_opts_helper(conf)

150
neutron/tests/functional/agent/test_l3_agent.py
View File

@ -13,6 +13,8 @@
# License for the specific language governing permissions and limitations
# under the License.
import copy
import mock
from oslo.config import cfg
@ -60,9 +62,10 @@ class L3AgentTestFramework(base.BaseOVSLinuxTestCase):
mock.patch.object(self.agent, '_send_gratuitous_arp_packet').start()
def manage_router(self):
def manage_router(self, enable_ha):
router = test_l3_agent.prepare_router_data(enable_snat=True,
enable_floating_ip=True)
enable_floating_ip=True,
enable_ha=enable_ha)
self.addCleanup(self._delete_router, router['id'])
ri = self._create_router(router)
return ri
@ -77,6 +80,13 @@ class L3AgentTestFramework(base.BaseOVSLinuxTestCase):
def _delete_router(self, router_id):
self.agent._router_removed(router_id)
def _add_fip(self, router, fip_address, fixed_address='10.0.0.2'):
fip = {'id': _uuid(),
'port_id': _uuid(),
'floating_ip_address': fip_address,
'fixed_ip_address': fixed_address}
router.router[l3_constants.FLOATINGIP_KEY].append(fip)
def _namespace_exists(self, router):
ip = ip_lib.IPWrapper(self.root_helper, router.ns_name)
return ip.netns.exists(router.ns_name)
@ -97,21 +107,137 @@ class L3AgentTestFramework(base.BaseOVSLinuxTestCase):
expected_device['mac_address'],
namespace, self.root_helper)
def get_expected_keepalive_configuration(self, router):
ha_confs_path = cfg.CONF.ha_confs_path
router_id = router.router_id
ha_device_name = self.agent.get_ha_device_name(router.ha_port['id'])
ha_device_cidr = router.ha_port['ip_cidr']
external_port = self.agent._get_ex_gw_port(router)
external_device_name = self.agent.get_external_device_name(
external_port['id'])
external_device_cidr = external_port['ip_cidr']
internal_port = router.router[l3_constants.INTERFACE_KEY][0]
internal_device_name = self.agent.get_internal_device_name(
internal_port['id'])
internal_device_cidr = internal_port['ip_cidr']
floating_ip_cidr = (
self.agent.get_floating_ips(router)[0]
['floating_ip_address'] + l3_agent.FLOATING_IP_CIDR_SUFFIX)
default_gateway_ip = external_port['subnet'].get('gateway_ip')
return """vrrp_sync_group VG_1 {
group {
VR_1
}
notify_master "%(ha_confs_path)s/%(router_id)s/notify_master.sh"
notify_backup "%(ha_confs_path)s/%(router_id)s/notify_backup.sh"
notify_fault "%(ha_confs_path)s/%(router_id)s/notify_fault.sh"
}
vrrp_instance VR_1 {
state BACKUP
interface %(ha_device_name)s
virtual_router_id 1
priority 50
nopreempt
advert_int 2
track_interface {
%(ha_device_name)s
}
virtual_ipaddress {
%(floating_ip_cidr)s dev %(external_device_name)s
}
virtual_ipaddress_excluded {
%(external_device_cidr)s dev %(external_device_name)s
%(internal_device_cidr)s dev %(internal_device_name)s
}
virtual_routes {
0.0.0.0/0 via %(default_gateway_ip)s dev %(external_device_name)s
}
}""" % {
'ha_confs_path': ha_confs_path,
'router_id': router_id,
'ha_device_name': ha_device_name,
'ha_device_cidr': ha_device_cidr,
'external_device_name': external_device_name,
'external_device_cidr': external_device_cidr,
'internal_device_name': internal_device_name,
'internal_device_cidr': internal_device_cidr,
'floating_ip_cidr': floating_ip_cidr,
'default_gateway_ip': default_gateway_ip
}
class L3AgentTestCase(L3AgentTestFramework):
def test_router_lifecycle(self):
router = self.manage_router()
def test_legacy_router_lifecycle(self):
self._router_lifecycle(enable_ha=False)
def test_ha_router_lifecycle(self):
self._router_lifecycle(enable_ha=True)
def test_keepalived_configuration(self):
router = self.manage_router(enable_ha=True)
expected = self.get_expected_keepalive_configuration(router)
self.assertEqual(expected,
router.keepalived_manager.config.get_config_str())
# Add a new FIP and change the GW IP address
router.router = copy.deepcopy(router.router)
existing_fip = '19.4.4.2'
new_fip = '19.4.4.3'
self._add_fip(router, new_fip)
router.router['gw_port']['subnet']['gateway_ip'] = '19.4.4.5'
router.router['gw_port']['fixed_ips'][0]['ip_address'] = '19.4.4.10'
self.agent.process_router(router)
# Get the updated configuration and assert that both FIPs are in,
# and that the GW IP address was updated.
new_config = router.keepalived_manager.config.get_config_str()
old_gw = '0.0.0.0/0 via 19.4.4.1'
new_gw = '0.0.0.0/0 via 19.4.4.5'
old_external_device_ip = '19.4.4.4'
new_external_device_ip = '19.4.4.10'
self.assertIn(existing_fip, new_config)
self.assertIn(new_fip, new_config)
self.assertNotIn(old_gw, new_config)
self.assertIn(new_gw, new_config)
self.assertNotIn(old_external_device_ip, new_config)
self.assertIn(new_external_device_ip, new_config)
def _router_lifecycle(self, enable_ha):
router = self.manage_router(enable_ha)
if enable_ha:
self.wait_until(lambda: router.ha_state == 'master')
# Keepalived notifies of a state transition when it starts,
# not when it ends. Thus, we have to wait until keepalived finishes
# configuring everything. We verify this by waiting until the last
# device has an IP address.
device = router.router[l3_constants.INTERFACE_KEY][-1]
self.wait_until(self.device_exists_with_ip_mac, device,
self.agent.get_internal_device_name,
router.ns_name)
self.assertTrue(self._namespace_exists(router))
self.assertTrue(self._metadata_proxy_exists(router))
self._assert_internal_devices(router)
self._assert_external_device(router)
self._assert_gateway(router)
self._assert_floating_ips(router)
self._assert_snat_chains(router)
self._assert_floating_ip_chains(router)
if enable_ha:
self._assert_ha_device(router)
self.assertTrue(router.keepalived_manager.process.active)
self._delete_router(router.router_id)
self._assert_router_does_not_exist(router)
if enable_ha:
self.assertFalse(router.keepalived_manager.process.active)
def _assert_internal_devices(self, router):
internal_devices = router.router[l3_constants.INTERFACE_KEY]
@ -138,6 +264,17 @@ class L3AgentTestCase(L3AgentTestFramework):
expected_gateway = external_port['subnet']['gateway_ip']
self.assertEqual(expected_gateway, existing_gateway)
def _assert_floating_ips(self, router):
floating_ips = router.router[l3_constants.FLOATINGIP_KEY]
self.assertTrue(len(floating_ips))
external_port = self.agent._get_ex_gw_port(router)
for fip in floating_ips:
self.assertTrue(ip_lib.device_exists_with_ip_mac(
self.agent.get_external_device_name(external_port['id']),
'%s/32' % fip['floating_ip_address'],
external_port['mac_address'],
router.ns_name, self.root_helper))
def _assert_snat_chains(self, router):
self.assertFalse(router.iptables_manager.is_chain_empty(
'nat', 'snat'))
@ -154,3 +291,8 @@ class L3AgentTestCase(L3AgentTestFramework):
# so there's no need to check that explicitly.
self.assertFalse(self._namespace_exists(router))
self.assertFalse(self._metadata_proxy_exists(router))
def _assert_ha_device(self, router):
self.assertTrue(self.device_exists_with_ip_mac(
router.router[l3_constants.HA_INTERFACE_KEY],
self.agent.get_ha_device_name, router.ns_name))

8
neutron/tests/functional/base.py
View File

@ -14,11 +14,14 @@
# under the License.
import os
import time
from neutron.tests import base
SUDO_CMD = 'sudo -n'
TIMEOUT = 60
SLEEP_INTERVAL = 1
class BaseSudoTestCase(base.BaseTestCase):
@ -55,3 +58,8 @@ class BaseSudoTestCase(base.BaseTestCase):
def check_sudo_enabled(self):
if not self.sudo_enabled:
self.skipTest('testing with sudo is not enabled')
def wait_until(self, predicate, *args, **kwargs):
with self.assert_max_execution_time(TIMEOUT):
while not predicate(*args, **kwargs):
time.sleep(SLEEP_INTERVAL)

2
neutron/tests/unit/services/firewall/agents/l3reference/test_firewall_l3_agent.py
View File

@ -24,6 +24,7 @@ from oslo.config import cfg
from neutron.agent.common import config as agent_config
from neutron.agent import l3_agent
from neutron.agent import l3_ha_agent
from neutron.agent.linux import ip_lib
from neutron.common import config as base_config
from neutron import context
@ -58,6 +59,7 @@ class TestFwaasL3AgentRpcCallback(base.BaseTestCase):
self.conf = cfg.ConfigOpts()
self.conf.register_opts(base_config.core_opts)
self.conf.register_opts(l3_agent.L3NATAgent.OPTS)
self.conf.register_opts(l3_ha_agent.OPTS)
agent_config.register_use_namespaces_opts_helper(self.conf)
agent_config.register_root_helper(self.conf)
self.conf.root_helper = 'sudo'

8
neutron/tests/unit/services/firewall/agents/varmour/test_varmour_router.py
View File

@ -18,10 +18,10 @@
import mock
from oslo.config import cfg
from neutron.agent.common import config as agent_config
from neutron.agent import l3_agent
from neutron.agent import l3_ha_agent
from neutron.agent.linux import interface
from neutron.common import config as base_config
from neutron.common import constants as l3_constants
@ -39,9 +39,10 @@ class TestVarmourRouter(base.BaseTestCase):
def setUp(self):
super(TestVarmourRouter, self).setUp()
self.conf = cfg.ConfigOpts()
self.conf = agent_config.setup_conf()
self.conf.register_opts(base_config.core_opts)
self.conf.register_opts(varmour_router.vArmourL3NATAgent.OPTS)
self.conf.register_opts(l3_ha_agent.OPTS)
agent_config.register_interface_driver_opts_helper(self.conf)
agent_config.register_use_namespaces_opts_helper(self.conf)
agent_config.register_root_helper(self.conf)
@ -63,6 +64,9 @@ class TestVarmourRouter(base.BaseTestCase):
'neutron.agent.linux.external_process.ProcessManager')
self.external_process = self.external_process_p.start()
self.makedirs_p = mock.patch('os.makedirs')
self.makedirs = self.makedirs_p.start()
self.dvr_cls_p = mock.patch('neutron.agent.linux.interface.NullDriver')
driver_cls = self.dvr_cls_p.start()
self.mock_driver = mock.MagicMock()

8
neutron/tests/unit/services/firewall/drivers/varmour/test_varmour_fwaas.py
View File

@ -18,10 +18,10 @@
import mock
from oslo.config import cfg
from neutron.agent.common import config as agent_config
from neutron.agent import l3_agent
from neutron.agent import l3_ha_agent
from neutron.agent.linux import interface
from neutron.common import config as base_config
from neutron.common import constants as l3_constants
@ -40,9 +40,10 @@ class TestBasicRouterOperations(base.BaseTestCase):
def setUp(self):
super(TestBasicRouterOperations, self).setUp()
self.conf = cfg.ConfigOpts()
self.conf = agent_config.setup_conf()
self.conf.register_opts(base_config.core_opts)
self.conf.register_opts(varmour_router.vArmourL3NATAgent.OPTS)
self.conf.register_opts(l3_ha_agent.OPTS)
agent_config.register_interface_driver_opts_helper(self.conf)
agent_config.register_use_namespaces_opts_helper(self.conf)
agent_config.register_root_helper(self.conf)
@ -64,6 +65,9 @@ class TestBasicRouterOperations(base.BaseTestCase):
'neutron.agent.linux.external_process.ProcessManager')
self.external_process = self.external_process_p.start()
self.makedirs_p = mock.patch('os.makedirs')
self.makedirs = self.makedirs_p.start()
self.dvr_cls_p = mock.patch('neutron.agent.linux.interface.NullDriver')
driver_cls = self.dvr_cls_p.start()
self.mock_driver = mock.MagicMock()

2
neutron/tests/unit/services/vpn/test_vpn_agent.py
View File

@ -18,6 +18,7 @@ from oslo.config import cfg
from neutron.agent.common import config as agent_config
from neutron.agent import l3_agent
from neutron.agent import l3_ha_agent
from neutron.agent.linux import interface
from neutron.common import config as base_config
from neutron.openstack.common import uuidutils
@ -48,6 +49,7 @@ class TestVPNAgent(base.BaseTestCase):
self.conf = cfg.CONF
self.conf.register_opts(base_config.core_opts)
self.conf.register_opts(l3_agent.L3NATAgent.OPTS)
self.conf.register_opts(l3_ha_agent.OPTS)
self.conf.register_opts(interface.OPTS)
agent_config.register_interface_driver_opts_helper(self.conf)
agent_config.register_use_namespaces_opts_helper(self.conf)

102
neutron/tests/unit/test_l3_agent.py
View File

@ -24,6 +24,7 @@ from testtools import matchers
from neutron.agent.common import config as agent_config
from neutron.agent import l3_agent
from neutron.agent import l3_ha_agent
from neutron.agent.linux import interface
from neutron.common import config as base_config
from neutron.common import constants as l3_constants
@ -230,7 +231,7 @@ def router_append_interface(router, count=1, ip_version=4, ra_mode=None,
def prepare_router_data(ip_version=4, enable_snat=None, num_internal_ports=1,
enable_floating_ip=False):
enable_floating_ip=False, enable_ha=False):
if ip_version == 4:
ip_addr = '19.4.4.4'
cidr = '19.4.4.0/24'
@ -267,6 +268,10 @@ def prepare_router_data(ip_version=4, enable_snat=None, num_internal_ports=1,
router_append_interface(router, count=num_internal_ports,
ip_version=ip_version)
if enable_ha:
router['ha'] = True
router['ha_vr_id'] = 1
router[l3_constants.HA_INTERFACE_KEY] = get_ha_interface()
if enable_snat is not None:
router['enable_snat'] = enable_snat
@ -277,6 +282,26 @@ def _get_subnet_id(port):
return port['fixed_ips'][0]['subnet_id']
def get_ha_interface():
return {'admin_state_up': True,
'device_id': _uuid(),
'device_owner': 'network:router_ha_interface',
'fixed_ips': [{'ip_address': '169.254.0.2',
'subnet_id': _uuid()}],
'id': _uuid(),
'mac_address': '12:34:56:78:2b:5d',
'name': u'L3 HA Admin port 0',
'network_id': _uuid(),
'status': u'ACTIVE',
'subnet': {'cidr': '169.254.0.0/24',
'gateway_ip': '169.254.0.1',
'id': _uuid()},
'tenant_id': '',
'agent_id': _uuid(),
'agent_host': 'aaa',
'priority': 1}
class TestBasicRouterOperations(base.BaseTestCase):
def setUp(self):
@ -284,6 +309,7 @@ class TestBasicRouterOperations(base.BaseTestCase):
self.conf = agent_config.setup_conf()
self.conf.register_opts(base_config.core_opts)
self.conf.register_opts(l3_agent.L3NATAgent.OPTS)
self.conf.register_opts(l3_ha_agent.OPTS)
agent_config.register_interface_driver_opts_helper(self.conf)
agent_config.register_use_namespaces_opts_helper(self.conf)
agent_config.register_root_helper(self.conf)
@ -291,12 +317,19 @@ class TestBasicRouterOperations(base.BaseTestCase):
self.conf.set_override('router_id', 'fake_id')
self.conf.set_override('interface_driver',
'neutron.agent.linux.interface.NullDriver')
self.conf.set_override('send_arp_for_ha', 1)
self.conf.set_override('state_path', '')
self.conf.root_helper = 'sudo'
self.device_exists_p = mock.patch(
'neutron.agent.linux.ip_lib.device_exists')
self.device_exists = self.device_exists_p.start()
mock.patch('neutron.agent.l3_ha_agent.AgentMixin'
'._init_ha_conf_path').start()
mock.patch('neutron.agent.linux.keepalived.KeepalivedNotifierMixin'
'._get_full_config_file_path').start()
self.utils_exec_p = mock.patch(
'neutron.agent.linux.utils.execute')
self.utils_exec = self.utils_exec_p.start()
@ -959,6 +992,60 @@ class TestBasicRouterOperations(base.BaseTestCase):
self.assertFalse(agent.process_router_floating_ip_addresses.called)
self.assertFalse(agent.process_router_floating_ip_nat_rules.called)
def test_ha_router_keepalived_config(self):
agent = l3_agent.L3NATAgent(HOSTNAME, self.conf)
router = prepare_router_data(enable_ha=True)
router['routes'] = [
{'destination': '8.8.8.8/32', 'nexthop': '35.4.0.10'},
{'destination': '8.8.4.4/32', 'nexthop': '35.4.0.11'}]
ri = l3_agent.RouterInfo(router['id'], self.conf.root_helper,
self.conf.use_namespaces, router=router)
ri.router = router
with contextlib.nested(mock.patch.object(agent,
'_spawn_metadata_proxy'),
mock.patch('neutron.agent.linux.'
'utils.replace_file'),
mock.patch('neutron.agent.linux.'
'utils.execute'),
mock.patch('os.makedirs')):
agent.process_ha_router_added(ri)
agent.process_router(ri)
config = ri.keepalived_manager.config
ha_iface = agent.get_ha_device_name(ri.ha_port['id'])
ex_iface = agent.get_external_device_name(ri.ex_gw_port['id'])
int_iface = agent.get_internal_device_name(
ri.internal_ports[0]['id'])
expected = """vrrp_sync_group VG_1 {
group {
VR_1
}
}
vrrp_instance VR_1 {
state BACKUP
interface %(ha_iface)s
virtual_router_id 1
priority 50
nopreempt
advert_int 2
track_interface {
%(ha_iface)s
}
virtual_ipaddress {
19.4.4.4/24 dev %(ex_iface)s
}
virtual_ipaddress_excluded {
35.4.0.4/24 dev %(int_iface)s
}
virtual_routes {
0.0.0.0/0 via 19.4.4.1 dev %(ex_iface)s
8.8.8.8/32 via 35.4.0.10
8.8.4.4/32 via 35.4.0.11
}
}""" % {'ha_iface': ha_iface, 'ex_iface': ex_iface, 'int_iface': int_iface}
self.assertEqual(expected, config.get_config_str())
@mock.patch('neutron.agent.linux.ip_lib.IPDevice')
def _test_process_router_floating_ip_addresses_add(self, ri,
agent, IPDevice):
@ -1047,6 +1134,7 @@ class TestBasicRouterOperations(base.BaseTestCase):
ri = mock.MagicMock()
ri.router.get.return_value = []
type(ri).is_ha = mock.PropertyMock(return_value=False)
ri.router['distributed'].__nonzero__ = lambda self: False
agent = l3_agent.L3NATAgent(HOSTNAME, self.conf)
@ -1081,7 +1169,7 @@ class TestBasicRouterOperations(base.BaseTestCase):
device.addr.list.return_value = [{'cidr': '15.1.2.3/32'}]
ri = mock.MagicMock()
ri.router['distributed'].__nonzero__ = lambda self: False
type(ri).is_ha = mock.PropertyMock(return_value=False)
ri.router.get.return_value = [fip]
agent = l3_agent.L3NATAgent(HOSTNAME, self.conf)
@ -1126,6 +1214,7 @@ class TestBasicRouterOperations(base.BaseTestCase):
'fixed_ip_address': '192.168.0.2'
}
ri = mock.MagicMock()
type(ri).is_ha = mock.PropertyMock(return_value=False)
ri.router.get.return_value = [fip]
ri.router['distributed'].__nonzero__ = lambda self: False
@ -1710,7 +1799,7 @@ class TestBasicRouterOperations(base.BaseTestCase):
agent, '_spawn_metadata_proxy') as spawn_proxy:
agent._router_added(router_id, router)
if enableflag:
spawn_proxy.assert_called_with(mock.ANY, mock.ANY)
spawn_proxy.assert_called_with(router_id, mock.ANY)
else:
self.assertFalse(spawn_proxy.call_count)
agent._router_removed(router_id)
@ -2146,6 +2235,7 @@ class TestL3AgentEventHandler(base.BaseTestCase):
def setUp(self):
super(TestL3AgentEventHandler, self).setUp()
cfg.CONF.register_opts(l3_agent.L3NATAgent.OPTS)
cfg.CONF.register_opts(l3_ha_agent.OPTS)
agent_config.register_interface_driver_opts_helper(cfg.CONF)
agent_config.register_use_namespaces_opts_helper(cfg.CONF)
cfg.CONF.set_override(
@ -2194,12 +2284,12 @@ class TestL3AgentEventHandler(base.BaseTestCase):
cfg.CONF.set_override('debug', True)
self.external_process_p.stop()
ns = 'qrouter-' + router_id
ri = l3_agent.RouterInfo(router_id, None, True, None)
try:
with mock.patch(ip_class_path) as ip_mock:
self.agent._spawn_metadata_proxy(router_id, ns)
self.agent._spawn_metadata_proxy(ri.router_id, ri.ns_name)
ip_mock.assert_has_calls([
mock.call('sudo', ns),
mock.call('sudo', ri.ns_name),
mock.call().netns.execute([
'neutron-ns-metadata-proxy',
mock.ANY,