diff --git a/etc/l3_agent.ini b/etc/l3_agent.ini index c8eafc841b8..94c97147543 100644 --- a/etc/l3_agent.ini +++ b/etc/l3_agent.ini @@ -88,3 +88,15 @@ # DVR. This mode must be used for an L3 agent running on a centralized # node (or in single-host deployments, e.g. devstack). # agent_mode = legacy + +# Location to store keepalived and all HA configurations +# ha_confs_path = $state_path/ha_confs + +# VRRP authentication type AH/PASS +# ha_vrrp_auth_type = PASS + +# VRRP authentication password +# ha_vrrp_auth_password = + +# The advertisement interval in seconds +# ha_vrrp_advert_int = 2 diff --git a/neutron/agent/l3_agent.py b/neutron/agent/l3_agent.py index 88f547e7adb..c12693558e4 100644 --- a/neutron/agent/l3_agent.py +++ b/neutron/agent/l3_agent.py @@ -25,6 +25,7 @@ from oslo.config import cfg import Queue from neutron.agent.common import config +from neutron.agent import l3_ha_agent from neutron.agent.linux import external_process from neutron.agent.linux import interface from neutron.agent.linux import ip_lib @@ -237,7 +238,7 @@ class LinkLocalAllocator(object): return f.readlines() -class RouterInfo(object): +class RouterInfo(l3_ha_agent.RouterMixin): def __init__(self, router_id, root_helper, use_namespaces, router, use_ipv6=False): @@ -265,6 +266,8 @@ class RouterInfo(object): self.rtr_fip_subnet = None self.dist_fip_count = 0 + super(RouterInfo, self).__init__() + @property def router(self): return self._router @@ -430,7 +433,9 @@ class RouterProcessingQueue(object): yield (rp, update) -class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager): +class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, + l3_ha_agent.AgentMixin, + manager.Manager): """Manager for L3NatAgent API version history: @@ -734,8 +739,15 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager): ri.iptables_manager.ipv4['nat'].add_rule(c, r) ri.iptables_manager.apply() self.process_router_add(ri) + + if ri.is_ha: + self.process_ha_router_added(ri) + if self.conf.enable_metadata_proxy: - self._spawn_metadata_proxy(ri.router_id, ri.ns_name) + if ri.is_ha: + self._add_keepalived_notifiers(ri) + else: + self._spawn_metadata_proxy(ri.router_id, ri.ns_name) def _router_removed(self, router_id): ri = self.router_info.get(router_id) @@ -743,6 +755,10 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager): LOG.warn(_("Info for router %s were not found. " "Skipping router removal"), router_id) return + + if ri.is_ha: + self.process_ha_router_removed(ri) + ri.router['gw_port'] = None ri.router[l3_constants.INTERFACE_KEY] = [] ri.router[l3_constants.FLOATINGIP_KEY] = [] @@ -757,7 +773,8 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager): del self.router_info[router_id] self._destroy_router_namespace(ri.ns_name) - def _spawn_metadata_proxy(self, router_id, ns_name): + def _get_metadata_proxy_callback(self, router_id): + def callback(pid_file): metadata_proxy_socket = cfg.CONF.metadata_proxy_socket proxy_cmd = ['neutron-ns-metadata-proxy', @@ -771,19 +788,22 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager): router_id)) return proxy_cmd - pm = external_process.ProcessManager( + return callback + + def _get_metadata_proxy_process_manager(self, router_id, ns_name): + return external_process.ProcessManager( self.conf, router_id, self.root_helper, ns_name) + + def _spawn_metadata_proxy(self, router_id, ns_name): + callback = self._get_metadata_proxy_callback(router_id) + pm = self._get_metadata_proxy_process_manager(router_id, ns_name) pm.enable(callback) def _destroy_metadata_proxy(self, router_id, ns_name): - pm = external_process.ProcessManager( - self.conf, - router_id, - self.root_helper, - ns_name) + pm = self._get_metadata_proxy_process_manager(router_id, ns_name) pm.disable() def _set_subnet_arp_info(self, ri, port): @@ -885,10 +905,20 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager): if ex_gw_port_id: interface_name = self.get_external_device_name(ex_gw_port_id) if ex_gw_port: + def _gateway_ports_equal(port1, port2): + def _get_filtered_dict(d, ignore): + return dict((k, v) for k, v in d.iteritems() + if k not in ignore) + + keys_to_ignore = set(['binding:host_id']) + port1_filtered = _get_filtered_dict(port1, keys_to_ignore) + port2_filtered = _get_filtered_dict(port2, keys_to_ignore) + return port1_filtered == port2_filtered + self._set_subnet_info(ex_gw_port) if not ri.ex_gw_port: self.external_gateway_added(ri, ex_gw_port, interface_name) - elif ex_gw_port != ri.ex_gw_port: + elif not _gateway_ports_equal(ex_gw_port, ri.ex_gw_port): self.external_gateway_updated(ri, ex_gw_port, interface_name) elif not ex_gw_port and ri.ex_gw_port: self.external_gateway_removed(ri, ri.ex_gw_port, interface_name) @@ -946,6 +976,12 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager): ri.snat_ports = snat_ports ri.enable_snat = ri.router.get('enable_snat') + if ri.is_ha: + if ri.ha_port: + ri.spawn_keepalived() + else: + ri.disable_keepalived() + def _handle_router_snat_rules(self, ri, ex_gw_port, internal_cidrs, interface_name, action): # Remove all the rules @@ -1048,32 +1084,39 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager): def _add_floating_ip(self, ri, fip, interface_name, device): fip_ip = fip['floating_ip_address'] ip_cidr = str(fip_ip) + FLOATING_IP_CIDR_SUFFIX - net = netaddr.IPNetwork(ip_cidr) - try: - device.addr.add(net.version, ip_cidr, str(net.broadcast)) - except (processutils.UnknownArgumentError, - processutils.ProcessExecutionError): - # any exception occurred here should cause the floating IP - # to be set in error state - LOG.warn(_("Unable to configure IP address for " - "floating IP: %s"), fip['id']) - return l3_constants.FLOATINGIP_STATUS_ERROR - if ri.router['distributed']: - # Special Handling for DVR - update FIP namespace - # and ri.namespace to handle DVR based FIP - self.floating_ip_added_dist(ri, fip) + + if ri.is_ha: + self._add_vip(ri, ip_cidr, interface_name) else: - # As GARP is processed in a distinct thread the call below - # won't raise an exception to be handled. - self._send_gratuitous_arp_packet( - ri.ns_name, interface_name, fip_ip) - return l3_constants.FLOATINGIP_STATUS_ACTIVE + net = netaddr.IPNetwork(ip_cidr) + try: + device.addr.add(net.version, ip_cidr, str(net.broadcast)) + except (processutils.UnknownArgumentError, + processutils.ProcessExecutionError): + # any exception occurred here should cause the floating IP + # to be set in error state + LOG.warn(_("Unable to configure IP address for " + "floating IP: %s"), fip['id']) + return l3_constants.FLOATINGIP_STATUS_ERROR + if ri.router['distributed']: + # Special Handling for DVR - update FIP namespace + # and ri.namespace to handle DVR based FIP + self.floating_ip_added_dist(ri, fip) + else: + # As GARP is processed in a distinct thread the call below + # won't raise an exception to be handled. + self._send_gratuitous_arp_packet( + ri.ns_name, interface_name, fip_ip) + return l3_constants.FLOATINGIP_STATUS_ACTIVE def _remove_floating_ip(self, ri, device, ip_cidr): - net = netaddr.IPNetwork(ip_cidr) - device.addr.delete(net.version, ip_cidr) - if ri.router['distributed']: - self.floating_ip_removed_dist(ri, ip_cidr) + if ri.is_ha: + self._remove_vip(ri, ip_cidr) + else: + net = netaddr.IPNetwork(ip_cidr) + device.addr.delete(net.version, ip_cidr) + if ri.router['distributed']: + self.floating_ip_removed_dist(ri, ip_cidr) def process_router_floating_ip_addresses(self, ri, ex_gw_port): """Configure IP addresses on router's external gateway interface. @@ -1253,6 +1296,9 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager): self._external_gateway_added(ri, ex_gw_port, interface_name, ri.ns_name, preserve_ips) + if ri.is_ha: + self._ha_external_gateway_added(ri, ex_gw_port, interface_name) + def external_gateway_updated(self, ri, ex_gw_port, interface_name): preserve_ips = [] if ri.router['distributed']: @@ -1272,6 +1318,9 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager): self._external_gateway_added(ri, ex_gw_port, interface_name, ns_name, preserve_ips) + if ri.is_ha: + self._ha_external_gateway_updated(ri, ex_gw_port, interface_name) + def _external_gateway_added(self, ri, ex_gw_port, interface_name, ns_name, preserve_ips): if not ip_lib.device_exists(interface_name, @@ -1284,14 +1333,15 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager): namespace=ns_name, prefix=EXTERNAL_DEV_PREFIX) - self.driver.init_l3(interface_name, [ex_gw_port['ip_cidr']], - namespace=ns_name, - gateway=ex_gw_port['subnet'].get('gateway_ip'), - extra_subnets=ex_gw_port.get('extra_subnets', []), - preserve_ips=preserve_ips) - ip_address = ex_gw_port['ip_cidr'].split('/')[0] - self._send_gratuitous_arp_packet(ns_name, - interface_name, ip_address) + if not ri.is_ha: + self.driver.init_l3( + interface_name, [ex_gw_port['ip_cidr']], namespace=ns_name, + gateway=ex_gw_port['subnet'].get('gateway_ip'), + extra_subnets=ex_gw_port.get('extra_subnets', []), + preserve_ips=preserve_ips) + ip_address = ex_gw_port['ip_cidr'].split('/')[0] + self._send_gratuitous_arp_packet(ns_name, + interface_name, ip_address) def agent_gateway_added(self, ns_name, ex_gw_port, interface_name): @@ -1343,6 +1393,9 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager): else: ns_name = ri.ns_name + if ri.is_ha: + self._ha_external_gateway_removed(ri, interface_name) + self.driver.unplug(interface_name, bridge=self.conf.external_network_bridge, namespace=ns_name, @@ -1404,7 +1457,7 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager): def _internal_network_added(self, ns_name, network_id, port_id, internal_cidr, mac_address, - interface_name, prefix): + interface_name, prefix, is_ha=False): if not ip_lib.device_exists(interface_name, root_helper=self.root_helper, namespace=ns_name): @@ -1412,10 +1465,12 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager): namespace=ns_name, prefix=prefix) - self.driver.init_l3(interface_name, [internal_cidr], - namespace=ns_name) - ip_address = internal_cidr.split('/')[0] - self._send_gratuitous_arp_packet(ns_name, interface_name, ip_address) + if not is_ha: + self.driver.init_l3(interface_name, [internal_cidr], + namespace=ns_name) + ip_address = internal_cidr.split('/')[0] + self._send_gratuitous_arp_packet(ns_name, interface_name, + ip_address) def internal_network_added(self, ri, port): network_id = port['network_id'] @@ -1427,7 +1482,11 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager): self._internal_network_added(ri.ns_name, network_id, port_id, internal_cidr, mac_address, - interface_name, INTERNAL_DEV_PREFIX) + interface_name, INTERNAL_DEV_PREFIX, + ri.is_ha) + + if ri.is_ha: + self._add_vip(ri, internal_cidr, interface_name) ex_gw_port = self._get_ex_gw_port(ri) if ri.router['distributed'] and ex_gw_port: @@ -1475,6 +1534,8 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager): if ip_lib.device_exists(interface_name, root_helper=self.root_helper, namespace=ri.ns_name): + if ri.is_ha: + self._clear_vips(ri, interface_name) self.driver.unplug(interface_name, namespace=ri.ns_name, prefix=INTERNAL_DEV_PREFIX) @@ -1843,6 +1904,10 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, manager.Manager): def routes_updated(self, ri): new_routes = ri.router['routes'] + if ri.is_ha: + self._process_virtual_routes(ri, new_routes) + return + old_routes = ri.routes adds, removes = common_utils.diff_list_of_dict(old_routes, new_routes) @@ -1931,6 +1996,7 @@ class L3NATAgentWithStateReport(L3NATAgent): def _register_opts(conf): conf.register_opts(L3NATAgent.OPTS) + conf.register_opts(l3_ha_agent.OPTS) config.register_interface_driver_opts_helper(conf) config.register_use_namespaces_opts_helper(conf) config.register_agent_state_opts_helper(conf) diff --git a/neutron/agent/l3_ha_agent.py b/neutron/agent/l3_ha_agent.py new file mode 100644 index 00000000000..49cba74153d --- /dev/null +++ b/neutron/agent/l3_ha_agent.py @@ -0,0 +1,232 @@ +# Copyright (c) 2014 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import shutil +import signal + +from oslo.config import cfg + +from neutron.agent.linux import keepalived +from neutron.common import constants as l3_constants +from neutron.openstack.common.gettextutils import _LE +from neutron.openstack.common import log as logging +from neutron.openstack.common import periodic_task + +LOG = logging.getLogger(__name__) + +HA_DEV_PREFIX = 'ha-' + +OPTS = [ + cfg.StrOpt('ha_confs_path', + default='$state_path/ha_confs', + help=_('Location to store keepalived/conntrackd ' + 'config files')), + cfg.StrOpt('ha_vrrp_auth_type', + default='PASS', + help=_('VRRP authentication type AH/PASS')), + cfg.StrOpt('ha_vrrp_auth_password', + help=_('VRRP authentication password'), + secret=True), + cfg.IntOpt('ha_vrrp_advert_int', + default=2, + help=_('The advertisement interval in seconds')), +] + + +class RouterMixin(object): + def __init__(self): + self.ha_port = None + self.keepalived_manager = None + + def _verify_ha(self): + if not self.is_ha: + raise ValueError(_('Router %s is not a HA router') % + self.router_id) + + @property + def is_ha(self): + return self.router is not None and self.router.get('ha') + + @property + def ha_priority(self): + self._verify_ha() + return self.router is not None and self.router.get( + 'priority', keepalived.HA_DEFAULT_PRIORITY) + + @property + def ha_vr_id(self): + self._verify_ha() + return self.router is not None and self.router.get('ha_vr_id') + + @property + def ha_state(self): + self._verify_ha() + ha_state_path = self.keepalived_manager._get_full_config_file_path( + 'state') + try: + with open(ha_state_path, 'r') as f: + return f.read() + except (OSError, IOError): + LOG.debug('Error while reading HA state for %s', self.router_id) + return None + + def spawn_keepalived(self): + self.keepalived_manager.spawn_or_restart() + + def disable_keepalived(self): + self.keepalived_manager.disable() + conf_dir = self.keepalived_manager.get_conf_dir() + shutil.rmtree(conf_dir) + + +class AgentMixin(object): + def __init__(self, host): + self._init_ha_conf_path() + super(AgentMixin, self).__init__(host) + + def _init_ha_conf_path(self): + ha_full_path = os.path.dirname("/%s/" % self.conf.ha_confs_path) + if not os.path.isdir(ha_full_path): + os.makedirs(ha_full_path, 0o755) + + def _init_keepalived_manager(self, ri): + ri.keepalived_manager = keepalived.KeepalivedManager( + ri.router['id'], + keepalived.KeepalivedConf(), + conf_path=self.conf.ha_confs_path, + namespace=ri.ns_name, + root_helper=self.root_helper) + + config = ri.keepalived_manager.config + + interface_name = self.get_ha_device_name(ri.ha_port['id']) + instance = keepalived.KeepalivedInstance( + 'BACKUP', interface_name, ri.ha_vr_id, nopreempt=True, + advert_int=self.conf.ha_vrrp_advert_int, priority=ri.ha_priority) + instance.track_interfaces.append(interface_name) + + if self.conf.ha_vrrp_auth_password: + # TODO(safchain): use oslo.config types when it will be available + # in order to check the validity of ha_vrrp_auth_type + instance.set_authentication(self.conf.ha_vrrp_auth_type, + self.conf.ha_vrrp_auth_password) + + group = keepalived.KeepalivedGroup(ri.ha_vr_id) + group.add_instance(instance) + + config.add_group(group) + config.add_instance(instance) + + def process_ha_router_added(self, ri): + ha_port = ri.router.get(l3_constants.HA_INTERFACE_KEY) + if not ha_port: + LOG.error(_LE('Unable to process HA router %s without ha port'), + ri.router_id) + return + + self._set_subnet_info(ha_port) + self.ha_network_added(ri, ha_port['network_id'], ha_port['id'], + ha_port['ip_cidr'], ha_port['mac_address']) + ri.ha_port = ha_port + + self._init_keepalived_manager(ri) + + def process_ha_router_removed(self, ri): + self.ha_network_removed(ri) + + def get_ha_device_name(self, port_id): + return (HA_DEV_PREFIX + port_id)[:self.driver.DEV_NAME_LEN] + + def ha_network_added(self, ri, network_id, port_id, internal_cidr, + mac_address): + interface_name = self.get_ha_device_name(port_id) + self.driver.plug(network_id, port_id, interface_name, mac_address, + namespace=ri.ns_name, + prefix=HA_DEV_PREFIX) + self.driver.init_l3(interface_name, [internal_cidr], + namespace=ri.ns_name) + + def ha_network_removed(self, ri): + interface_name = self.get_ha_device_name(ri.ha_port['id']) + self.driver.unplug(interface_name, namespace=ri.ns_name, + prefix=HA_DEV_PREFIX) + ri.ha_port = None + + def _add_vip(self, ri, ip_cidr, interface): + instance = ri.keepalived_manager.config.get_instance(ri.ha_vr_id) + instance.add_vip(ip_cidr, interface) + + def _remove_vip(self, ri, ip_cidr): + instance = ri.keepalived_manager.config.get_instance(ri.ha_vr_id) + instance.remove_vip_by_ip_address(ip_cidr) + + def _clear_vips(self, ri, interface): + instance = ri.keepalived_manager.config.get_instance(ri.ha_vr_id) + instance.remove_vips_vroutes_by_interface(interface) + + def _add_keepalived_notifiers(self, ri): + callback = self._get_metadata_proxy_callback(ri.router_id) + pm = self._get_metadata_proxy_process_manager(ri.router_id, ri.ns_name) + pid = pm.get_pid_file_name(ensure_pids_dir=True) + ri.keepalived_manager.add_notifier( + callback(pid), 'master', ri.ha_vr_id) + for state in ('backup', 'fault'): + ri.keepalived_manager.add_notifier( + ['kill', '-%s' % signal.SIGKILL, + '$(cat ' + pid + ')'], state, ri.ha_vr_id) + + def _ha_external_gateway_updated(self, ri, ex_gw_port, interface_name): + old_gateway_cidr = ri.ex_gw_port['ip_cidr'] + self._remove_vip(ri, old_gateway_cidr) + self._ha_external_gateway_added(ri, ex_gw_port, interface_name) + + def _add_default_gw_virtual_route(self, ri, ex_gw_port, interface_name): + gw_ip = ex_gw_port['subnet']['gateway_ip'] + if gw_ip: + instance = ri.keepalived_manager.config.get_instance(ri.ha_vr_id) + instance.virtual_routes = ( + [route for route in instance.virtual_routes + if route.destination != '0.0.0.0/0']) + instance.virtual_routes.append( + keepalived.KeepalivedVirtualRoute( + '0.0.0.0/0', gw_ip, interface_name)) + + def _ha_external_gateway_added(self, ri, ex_gw_port, interface_name): + self._add_vip(ri, ex_gw_port['ip_cidr'], interface_name) + self._add_default_gw_virtual_route(ri, ex_gw_port, interface_name) + + def _ha_external_gateway_removed(self, ri, interface_name): + self._clear_vips(ri, interface_name) + + def _process_virtual_routes(self, ri, new_routes): + instance = ri.keepalived_manager.config.get_instance(ri.ha_vr_id) + + # Filter out all of the old routes while keeping only the default route + instance.virtual_routes = [route for route in instance.virtual_routes + if route.destination == '0.0.0.0/0'] + for route in new_routes: + instance.virtual_routes.append(keepalived.KeepalivedVirtualRoute( + route['destination'], + route['nexthop'])) + + def get_ha_routers(self): + return (router for router in self.router_info.values() if router.is_ha) + + @periodic_task.periodic_task + def _ensure_keepalived_alive(self, context): + # TODO(amuller): Use external_process.ProcessMonitor + for router in self.get_ha_routers(): + router.keepalived_manager.revive() diff --git a/neutron/agent/linux/keepalived.py b/neutron/agent/linux/keepalived.py index cb4aeebcfc8..2c3fc62a3de 100644 --- a/neutron/agent/linux/keepalived.py +++ b/neutron/agent/linux/keepalived.py @@ -83,7 +83,7 @@ class KeepalivedGroup(object): self.ha_vr_id = ha_vr_id self.name = 'VG_%s' % ha_vr_id self.instance_names = set() - self.notifiers = {} + self.notifiers = [] def add_instance(self, instance): self.instance_names.add(instance.name) @@ -91,7 +91,7 @@ class KeepalivedGroup(object): def set_notify(self, state, path): if state not in VALID_NOTIFY_STATES: raise InvalidNotifyStateException(state=state) - self.notifiers[state] = path + self.notifiers.append((state, path)) def build_config(self): return itertools.chain(['vrrp_sync_group %s {' % self.name, @@ -99,7 +99,7 @@ class KeepalivedGroup(object): (' %s' % i for i in self.instance_names), [' }'], (' notify_%s "%s"' % (state, path) - for state, path in self.notifiers.items()), + for state, path in self.notifiers), ['}']) @@ -132,6 +132,9 @@ class KeepalivedInstance(object): self.authentication = (auth_type, password) + def add_vip(self, ip_cidr, interface_name): + self.vips.append(KeepalivedVipAddress(ip_cidr, interface_name)) + def remove_vips_vroutes_by_interface(self, interface_name): self.vips = [vip for vip in self.vips if vip.interface_name != interface_name] diff --git a/neutron/services/firewall/agents/varmour/varmour_router.py b/neutron/services/firewall/agents/varmour/varmour_router.py index 3c36f7e907f..44e3ce17619 100755 --- a/neutron/services/firewall/agents/varmour/varmour_router.py +++ b/neutron/services/firewall/agents/varmour/varmour_router.py @@ -26,6 +26,7 @@ from oslo.config import cfg from neutron.agent.common import config from neutron.agent import l3_agent +from neutron.agent import l3_ha_agent from neutron.agent.linux import external_process from neutron.agent.linux import interface from neutron.agent.linux import ip_lib @@ -332,6 +333,7 @@ class vArmourL3NATAgentWithStateReport(vArmourL3NATAgent, def main(): conf = cfg.CONF conf.register_opts(vArmourL3NATAgent.OPTS) + conf.register_opts(l3_ha_agent.OPTS) config.register_interface_driver_opts_helper(conf) config.register_use_namespaces_opts_helper(conf) config.register_agent_state_opts_helper(conf) diff --git a/neutron/tests/functional/agent/test_l3_agent.py b/neutron/tests/functional/agent/test_l3_agent.py index eeda8eb2761..068e01c5b52 100644 --- a/neutron/tests/functional/agent/test_l3_agent.py +++ b/neutron/tests/functional/agent/test_l3_agent.py @@ -13,6 +13,8 @@ # License for the specific language governing permissions and limitations # under the License. +import copy + import mock from oslo.config import cfg @@ -60,9 +62,10 @@ class L3AgentTestFramework(base.BaseOVSLinuxTestCase): mock.patch.object(self.agent, '_send_gratuitous_arp_packet').start() - def manage_router(self): + def manage_router(self, enable_ha): router = test_l3_agent.prepare_router_data(enable_snat=True, - enable_floating_ip=True) + enable_floating_ip=True, + enable_ha=enable_ha) self.addCleanup(self._delete_router, router['id']) ri = self._create_router(router) return ri @@ -77,6 +80,13 @@ class L3AgentTestFramework(base.BaseOVSLinuxTestCase): def _delete_router(self, router_id): self.agent._router_removed(router_id) + def _add_fip(self, router, fip_address, fixed_address='10.0.0.2'): + fip = {'id': _uuid(), + 'port_id': _uuid(), + 'floating_ip_address': fip_address, + 'fixed_ip_address': fixed_address} + router.router[l3_constants.FLOATINGIP_KEY].append(fip) + def _namespace_exists(self, router): ip = ip_lib.IPWrapper(self.root_helper, router.ns_name) return ip.netns.exists(router.ns_name) @@ -97,21 +107,137 @@ class L3AgentTestFramework(base.BaseOVSLinuxTestCase): expected_device['mac_address'], namespace, self.root_helper) + def get_expected_keepalive_configuration(self, router): + ha_confs_path = cfg.CONF.ha_confs_path + router_id = router.router_id + ha_device_name = self.agent.get_ha_device_name(router.ha_port['id']) + ha_device_cidr = router.ha_port['ip_cidr'] + external_port = self.agent._get_ex_gw_port(router) + external_device_name = self.agent.get_external_device_name( + external_port['id']) + external_device_cidr = external_port['ip_cidr'] + internal_port = router.router[l3_constants.INTERFACE_KEY][0] + internal_device_name = self.agent.get_internal_device_name( + internal_port['id']) + internal_device_cidr = internal_port['ip_cidr'] + floating_ip_cidr = ( + self.agent.get_floating_ips(router)[0] + ['floating_ip_address'] + l3_agent.FLOATING_IP_CIDR_SUFFIX) + default_gateway_ip = external_port['subnet'].get('gateway_ip') + + return """vrrp_sync_group VG_1 { + group { + VR_1 + } + notify_master "%(ha_confs_path)s/%(router_id)s/notify_master.sh" + notify_backup "%(ha_confs_path)s/%(router_id)s/notify_backup.sh" + notify_fault "%(ha_confs_path)s/%(router_id)s/notify_fault.sh" +} +vrrp_instance VR_1 { + state BACKUP + interface %(ha_device_name)s + virtual_router_id 1 + priority 50 + nopreempt + advert_int 2 + track_interface { + %(ha_device_name)s + } + virtual_ipaddress { + %(floating_ip_cidr)s dev %(external_device_name)s + } + virtual_ipaddress_excluded { + %(external_device_cidr)s dev %(external_device_name)s + %(internal_device_cidr)s dev %(internal_device_name)s + } + virtual_routes { + 0.0.0.0/0 via %(default_gateway_ip)s dev %(external_device_name)s + } +}""" % { + 'ha_confs_path': ha_confs_path, + 'router_id': router_id, + 'ha_device_name': ha_device_name, + 'ha_device_cidr': ha_device_cidr, + 'external_device_name': external_device_name, + 'external_device_cidr': external_device_cidr, + 'internal_device_name': internal_device_name, + 'internal_device_cidr': internal_device_cidr, + 'floating_ip_cidr': floating_ip_cidr, + 'default_gateway_ip': default_gateway_ip + } + class L3AgentTestCase(L3AgentTestFramework): - def test_router_lifecycle(self): - router = self.manage_router() + def test_legacy_router_lifecycle(self): + self._router_lifecycle(enable_ha=False) + + def test_ha_router_lifecycle(self): + self._router_lifecycle(enable_ha=True) + + def test_keepalived_configuration(self): + router = self.manage_router(enable_ha=True) + expected = self.get_expected_keepalive_configuration(router) + + self.assertEqual(expected, + router.keepalived_manager.config.get_config_str()) + + # Add a new FIP and change the GW IP address + router.router = copy.deepcopy(router.router) + existing_fip = '19.4.4.2' + new_fip = '19.4.4.3' + self._add_fip(router, new_fip) + router.router['gw_port']['subnet']['gateway_ip'] = '19.4.4.5' + router.router['gw_port']['fixed_ips'][0]['ip_address'] = '19.4.4.10' + + self.agent.process_router(router) + + # Get the updated configuration and assert that both FIPs are in, + # and that the GW IP address was updated. + new_config = router.keepalived_manager.config.get_config_str() + old_gw = '0.0.0.0/0 via 19.4.4.1' + new_gw = '0.0.0.0/0 via 19.4.4.5' + old_external_device_ip = '19.4.4.4' + new_external_device_ip = '19.4.4.10' + self.assertIn(existing_fip, new_config) + self.assertIn(new_fip, new_config) + self.assertNotIn(old_gw, new_config) + self.assertIn(new_gw, new_config) + self.assertNotIn(old_external_device_ip, new_config) + self.assertIn(new_external_device_ip, new_config) + + def _router_lifecycle(self, enable_ha): + router = self.manage_router(enable_ha) + + if enable_ha: + self.wait_until(lambda: router.ha_state == 'master') + + # Keepalived notifies of a state transition when it starts, + # not when it ends. Thus, we have to wait until keepalived finishes + # configuring everything. We verify this by waiting until the last + # device has an IP address. + device = router.router[l3_constants.INTERFACE_KEY][-1] + self.wait_until(self.device_exists_with_ip_mac, device, + self.agent.get_internal_device_name, + router.ns_name) self.assertTrue(self._namespace_exists(router)) self.assertTrue(self._metadata_proxy_exists(router)) self._assert_internal_devices(router) self._assert_external_device(router) self._assert_gateway(router) + self._assert_floating_ips(router) self._assert_snat_chains(router) self._assert_floating_ip_chains(router) + if enable_ha: + self._assert_ha_device(router) + self.assertTrue(router.keepalived_manager.process.active) + self._delete_router(router.router_id) + self._assert_router_does_not_exist(router) + if enable_ha: + self.assertFalse(router.keepalived_manager.process.active) def _assert_internal_devices(self, router): internal_devices = router.router[l3_constants.INTERFACE_KEY] @@ -138,6 +264,17 @@ class L3AgentTestCase(L3AgentTestFramework): expected_gateway = external_port['subnet']['gateway_ip'] self.assertEqual(expected_gateway, existing_gateway) + def _assert_floating_ips(self, router): + floating_ips = router.router[l3_constants.FLOATINGIP_KEY] + self.assertTrue(len(floating_ips)) + external_port = self.agent._get_ex_gw_port(router) + for fip in floating_ips: + self.assertTrue(ip_lib.device_exists_with_ip_mac( + self.agent.get_external_device_name(external_port['id']), + '%s/32' % fip['floating_ip_address'], + external_port['mac_address'], + router.ns_name, self.root_helper)) + def _assert_snat_chains(self, router): self.assertFalse(router.iptables_manager.is_chain_empty( 'nat', 'snat')) @@ -154,3 +291,8 @@ class L3AgentTestCase(L3AgentTestFramework): # so there's no need to check that explicitly. self.assertFalse(self._namespace_exists(router)) self.assertFalse(self._metadata_proxy_exists(router)) + + def _assert_ha_device(self, router): + self.assertTrue(self.device_exists_with_ip_mac( + router.router[l3_constants.HA_INTERFACE_KEY], + self.agent.get_ha_device_name, router.ns_name)) diff --git a/neutron/tests/functional/base.py b/neutron/tests/functional/base.py index fe25535fa92..7ca32372735 100644 --- a/neutron/tests/functional/base.py +++ b/neutron/tests/functional/base.py @@ -14,11 +14,14 @@ # under the License. import os +import time from neutron.tests import base SUDO_CMD = 'sudo -n' +TIMEOUT = 60 +SLEEP_INTERVAL = 1 class BaseSudoTestCase(base.BaseTestCase): @@ -55,3 +58,8 @@ class BaseSudoTestCase(base.BaseTestCase): def check_sudo_enabled(self): if not self.sudo_enabled: self.skipTest('testing with sudo is not enabled') + + def wait_until(self, predicate, *args, **kwargs): + with self.assert_max_execution_time(TIMEOUT): + while not predicate(*args, **kwargs): + time.sleep(SLEEP_INTERVAL) diff --git a/neutron/tests/unit/services/firewall/agents/l3reference/test_firewall_l3_agent.py b/neutron/tests/unit/services/firewall/agents/l3reference/test_firewall_l3_agent.py index 113377b462a..81fa211e063 100644 --- a/neutron/tests/unit/services/firewall/agents/l3reference/test_firewall_l3_agent.py +++ b/neutron/tests/unit/services/firewall/agents/l3reference/test_firewall_l3_agent.py @@ -24,6 +24,7 @@ from oslo.config import cfg from neutron.agent.common import config as agent_config from neutron.agent import l3_agent +from neutron.agent import l3_ha_agent from neutron.agent.linux import ip_lib from neutron.common import config as base_config from neutron import context @@ -58,6 +59,7 @@ class TestFwaasL3AgentRpcCallback(base.BaseTestCase): self.conf = cfg.ConfigOpts() self.conf.register_opts(base_config.core_opts) self.conf.register_opts(l3_agent.L3NATAgent.OPTS) + self.conf.register_opts(l3_ha_agent.OPTS) agent_config.register_use_namespaces_opts_helper(self.conf) agent_config.register_root_helper(self.conf) self.conf.root_helper = 'sudo' diff --git a/neutron/tests/unit/services/firewall/agents/varmour/test_varmour_router.py b/neutron/tests/unit/services/firewall/agents/varmour/test_varmour_router.py index 4b458fb3b9f..113e4c0964c 100644 --- a/neutron/tests/unit/services/firewall/agents/varmour/test_varmour_router.py +++ b/neutron/tests/unit/services/firewall/agents/varmour/test_varmour_router.py @@ -18,10 +18,10 @@ import mock -from oslo.config import cfg from neutron.agent.common import config as agent_config from neutron.agent import l3_agent +from neutron.agent import l3_ha_agent from neutron.agent.linux import interface from neutron.common import config as base_config from neutron.common import constants as l3_constants @@ -39,9 +39,10 @@ class TestVarmourRouter(base.BaseTestCase): def setUp(self): super(TestVarmourRouter, self).setUp() - self.conf = cfg.ConfigOpts() + self.conf = agent_config.setup_conf() self.conf.register_opts(base_config.core_opts) self.conf.register_opts(varmour_router.vArmourL3NATAgent.OPTS) + self.conf.register_opts(l3_ha_agent.OPTS) agent_config.register_interface_driver_opts_helper(self.conf) agent_config.register_use_namespaces_opts_helper(self.conf) agent_config.register_root_helper(self.conf) @@ -63,6 +64,9 @@ class TestVarmourRouter(base.BaseTestCase): 'neutron.agent.linux.external_process.ProcessManager') self.external_process = self.external_process_p.start() + self.makedirs_p = mock.patch('os.makedirs') + self.makedirs = self.makedirs_p.start() + self.dvr_cls_p = mock.patch('neutron.agent.linux.interface.NullDriver') driver_cls = self.dvr_cls_p.start() self.mock_driver = mock.MagicMock() diff --git a/neutron/tests/unit/services/firewall/drivers/varmour/test_varmour_fwaas.py b/neutron/tests/unit/services/firewall/drivers/varmour/test_varmour_fwaas.py index db38e4a4d1b..0b5121afe90 100644 --- a/neutron/tests/unit/services/firewall/drivers/varmour/test_varmour_fwaas.py +++ b/neutron/tests/unit/services/firewall/drivers/varmour/test_varmour_fwaas.py @@ -18,10 +18,10 @@ import mock -from oslo.config import cfg from neutron.agent.common import config as agent_config from neutron.agent import l3_agent +from neutron.agent import l3_ha_agent from neutron.agent.linux import interface from neutron.common import config as base_config from neutron.common import constants as l3_constants @@ -40,9 +40,10 @@ class TestBasicRouterOperations(base.BaseTestCase): def setUp(self): super(TestBasicRouterOperations, self).setUp() - self.conf = cfg.ConfigOpts() + self.conf = agent_config.setup_conf() self.conf.register_opts(base_config.core_opts) self.conf.register_opts(varmour_router.vArmourL3NATAgent.OPTS) + self.conf.register_opts(l3_ha_agent.OPTS) agent_config.register_interface_driver_opts_helper(self.conf) agent_config.register_use_namespaces_opts_helper(self.conf) agent_config.register_root_helper(self.conf) @@ -64,6 +65,9 @@ class TestBasicRouterOperations(base.BaseTestCase): 'neutron.agent.linux.external_process.ProcessManager') self.external_process = self.external_process_p.start() + self.makedirs_p = mock.patch('os.makedirs') + self.makedirs = self.makedirs_p.start() + self.dvr_cls_p = mock.patch('neutron.agent.linux.interface.NullDriver') driver_cls = self.dvr_cls_p.start() self.mock_driver = mock.MagicMock() diff --git a/neutron/tests/unit/services/vpn/test_vpn_agent.py b/neutron/tests/unit/services/vpn/test_vpn_agent.py index 0371cb26d4f..f360dfaa1fb 100644 --- a/neutron/tests/unit/services/vpn/test_vpn_agent.py +++ b/neutron/tests/unit/services/vpn/test_vpn_agent.py @@ -18,6 +18,7 @@ from oslo.config import cfg from neutron.agent.common import config as agent_config from neutron.agent import l3_agent +from neutron.agent import l3_ha_agent from neutron.agent.linux import interface from neutron.common import config as base_config from neutron.openstack.common import uuidutils @@ -48,6 +49,7 @@ class TestVPNAgent(base.BaseTestCase): self.conf = cfg.CONF self.conf.register_opts(base_config.core_opts) self.conf.register_opts(l3_agent.L3NATAgent.OPTS) + self.conf.register_opts(l3_ha_agent.OPTS) self.conf.register_opts(interface.OPTS) agent_config.register_interface_driver_opts_helper(self.conf) agent_config.register_use_namespaces_opts_helper(self.conf) diff --git a/neutron/tests/unit/test_l3_agent.py b/neutron/tests/unit/test_l3_agent.py index 42d0d09ef3f..c9fb026f301 100644 --- a/neutron/tests/unit/test_l3_agent.py +++ b/neutron/tests/unit/test_l3_agent.py @@ -24,6 +24,7 @@ from testtools import matchers from neutron.agent.common import config as agent_config from neutron.agent import l3_agent +from neutron.agent import l3_ha_agent from neutron.agent.linux import interface from neutron.common import config as base_config from neutron.common import constants as l3_constants @@ -230,7 +231,7 @@ def router_append_interface(router, count=1, ip_version=4, ra_mode=None, def prepare_router_data(ip_version=4, enable_snat=None, num_internal_ports=1, - enable_floating_ip=False): + enable_floating_ip=False, enable_ha=False): if ip_version == 4: ip_addr = '19.4.4.4' cidr = '19.4.4.0/24' @@ -267,6 +268,10 @@ def prepare_router_data(ip_version=4, enable_snat=None, num_internal_ports=1, router_append_interface(router, count=num_internal_ports, ip_version=ip_version) + if enable_ha: + router['ha'] = True + router['ha_vr_id'] = 1 + router[l3_constants.HA_INTERFACE_KEY] = get_ha_interface() if enable_snat is not None: router['enable_snat'] = enable_snat @@ -277,6 +282,26 @@ def _get_subnet_id(port): return port['fixed_ips'][0]['subnet_id'] +def get_ha_interface(): + return {'admin_state_up': True, + 'device_id': _uuid(), + 'device_owner': 'network:router_ha_interface', + 'fixed_ips': [{'ip_address': '169.254.0.2', + 'subnet_id': _uuid()}], + 'id': _uuid(), + 'mac_address': '12:34:56:78:2b:5d', + 'name': u'L3 HA Admin port 0', + 'network_id': _uuid(), + 'status': u'ACTIVE', + 'subnet': {'cidr': '169.254.0.0/24', + 'gateway_ip': '169.254.0.1', + 'id': _uuid()}, + 'tenant_id': '', + 'agent_id': _uuid(), + 'agent_host': 'aaa', + 'priority': 1} + + class TestBasicRouterOperations(base.BaseTestCase): def setUp(self): @@ -284,6 +309,7 @@ class TestBasicRouterOperations(base.BaseTestCase): self.conf = agent_config.setup_conf() self.conf.register_opts(base_config.core_opts) self.conf.register_opts(l3_agent.L3NATAgent.OPTS) + self.conf.register_opts(l3_ha_agent.OPTS) agent_config.register_interface_driver_opts_helper(self.conf) agent_config.register_use_namespaces_opts_helper(self.conf) agent_config.register_root_helper(self.conf) @@ -291,12 +317,19 @@ class TestBasicRouterOperations(base.BaseTestCase): self.conf.set_override('router_id', 'fake_id') self.conf.set_override('interface_driver', 'neutron.agent.linux.interface.NullDriver') + self.conf.set_override('send_arp_for_ha', 1) + self.conf.set_override('state_path', '') self.conf.root_helper = 'sudo' self.device_exists_p = mock.patch( 'neutron.agent.linux.ip_lib.device_exists') self.device_exists = self.device_exists_p.start() + mock.patch('neutron.agent.l3_ha_agent.AgentMixin' + '._init_ha_conf_path').start() + mock.patch('neutron.agent.linux.keepalived.KeepalivedNotifierMixin' + '._get_full_config_file_path').start() + self.utils_exec_p = mock.patch( 'neutron.agent.linux.utils.execute') self.utils_exec = self.utils_exec_p.start() @@ -959,6 +992,60 @@ class TestBasicRouterOperations(base.BaseTestCase): self.assertFalse(agent.process_router_floating_ip_addresses.called) self.assertFalse(agent.process_router_floating_ip_nat_rules.called) + def test_ha_router_keepalived_config(self): + agent = l3_agent.L3NATAgent(HOSTNAME, self.conf) + router = prepare_router_data(enable_ha=True) + router['routes'] = [ + {'destination': '8.8.8.8/32', 'nexthop': '35.4.0.10'}, + {'destination': '8.8.4.4/32', 'nexthop': '35.4.0.11'}] + ri = l3_agent.RouterInfo(router['id'], self.conf.root_helper, + self.conf.use_namespaces, router=router) + ri.router = router + with contextlib.nested(mock.patch.object(agent, + '_spawn_metadata_proxy'), + mock.patch('neutron.agent.linux.' + 'utils.replace_file'), + mock.patch('neutron.agent.linux.' + 'utils.execute'), + mock.patch('os.makedirs')): + agent.process_ha_router_added(ri) + agent.process_router(ri) + config = ri.keepalived_manager.config + ha_iface = agent.get_ha_device_name(ri.ha_port['id']) + ex_iface = agent.get_external_device_name(ri.ex_gw_port['id']) + int_iface = agent.get_internal_device_name( + ri.internal_ports[0]['id']) + + expected = """vrrp_sync_group VG_1 { + group { + VR_1 + } +} +vrrp_instance VR_1 { + state BACKUP + interface %(ha_iface)s + virtual_router_id 1 + priority 50 + nopreempt + advert_int 2 + track_interface { + %(ha_iface)s + } + virtual_ipaddress { + 19.4.4.4/24 dev %(ex_iface)s + } + virtual_ipaddress_excluded { + 35.4.0.4/24 dev %(int_iface)s + } + virtual_routes { + 0.0.0.0/0 via 19.4.4.1 dev %(ex_iface)s + 8.8.8.8/32 via 35.4.0.10 + 8.8.4.4/32 via 35.4.0.11 + } +}""" % {'ha_iface': ha_iface, 'ex_iface': ex_iface, 'int_iface': int_iface} + + self.assertEqual(expected, config.get_config_str()) + @mock.patch('neutron.agent.linux.ip_lib.IPDevice') def _test_process_router_floating_ip_addresses_add(self, ri, agent, IPDevice): @@ -1047,6 +1134,7 @@ class TestBasicRouterOperations(base.BaseTestCase): ri = mock.MagicMock() ri.router.get.return_value = [] + type(ri).is_ha = mock.PropertyMock(return_value=False) ri.router['distributed'].__nonzero__ = lambda self: False agent = l3_agent.L3NATAgent(HOSTNAME, self.conf) @@ -1081,7 +1169,7 @@ class TestBasicRouterOperations(base.BaseTestCase): device.addr.list.return_value = [{'cidr': '15.1.2.3/32'}] ri = mock.MagicMock() ri.router['distributed'].__nonzero__ = lambda self: False - + type(ri).is_ha = mock.PropertyMock(return_value=False) ri.router.get.return_value = [fip] agent = l3_agent.L3NATAgent(HOSTNAME, self.conf) @@ -1126,6 +1214,7 @@ class TestBasicRouterOperations(base.BaseTestCase): 'fixed_ip_address': '192.168.0.2' } ri = mock.MagicMock() + type(ri).is_ha = mock.PropertyMock(return_value=False) ri.router.get.return_value = [fip] ri.router['distributed'].__nonzero__ = lambda self: False @@ -1710,7 +1799,7 @@ class TestBasicRouterOperations(base.BaseTestCase): agent, '_spawn_metadata_proxy') as spawn_proxy: agent._router_added(router_id, router) if enableflag: - spawn_proxy.assert_called_with(mock.ANY, mock.ANY) + spawn_proxy.assert_called_with(router_id, mock.ANY) else: self.assertFalse(spawn_proxy.call_count) agent._router_removed(router_id) @@ -2146,6 +2235,7 @@ class TestL3AgentEventHandler(base.BaseTestCase): def setUp(self): super(TestL3AgentEventHandler, self).setUp() cfg.CONF.register_opts(l3_agent.L3NATAgent.OPTS) + cfg.CONF.register_opts(l3_ha_agent.OPTS) agent_config.register_interface_driver_opts_helper(cfg.CONF) agent_config.register_use_namespaces_opts_helper(cfg.CONF) cfg.CONF.set_override( @@ -2194,12 +2284,12 @@ class TestL3AgentEventHandler(base.BaseTestCase): cfg.CONF.set_override('debug', True) self.external_process_p.stop() - ns = 'qrouter-' + router_id + ri = l3_agent.RouterInfo(router_id, None, True, None) try: with mock.patch(ip_class_path) as ip_mock: - self.agent._spawn_metadata_proxy(router_id, ns) + self.agent._spawn_metadata_proxy(ri.router_id, ri.ns_name) ip_mock.assert_has_calls([ - mock.call('sudo', ns), + mock.call('sudo', ri.ns_name), mock.call().netns.execute([ 'neutron-ns-metadata-proxy', mock.ANY,