From 9bae3b1832798a274bef4a77e5acda1abaa51f2a Mon Sep 17 00:00:00 2001 From: Assaf Muller Date: Thu, 12 Mar 2015 19:50:43 -0400 Subject: [PATCH] Replace keepalived notifier bash script with Python ip monitor Previously L3 HA generated a bash script and copied it to a per-router configuration directory that was visible to that router's keepalived instance. This patch changes the in-line generated Bash script to a Python script that can be maintained in the repository. The bash script was used as a keepalived notifier script, that was invoked by keepalived whenever a state transition occured. These notifier scripts may be invoked by keepalived out of order in case it transitions quickly twice. For example, if the master failed and two slaves fight for the new master role. One will transition to master, and the other will often transition to master and then immidiately back to standby. In this case, the transition scripts were often fired out of order, resulting in the wrong state being reported. The proposed approach is to get rid of the keepalived notifier scripts entirely. Instead, monitor IP changes on the HA device. If the omnipresent IP address was configured on the HA device, it means that we're looking at a master instance. If it was deleted, the router transition to standby or fault. In order to keep the L3 agent CPU usage down, it will spawn a process per HA router. That process will start the ip address monitor. Whenever it gets an IP address change event, it will notify the L3 agent via a unix domain socket. Partially-Implements: blueprint report-ha-router-master Change-Id: I2022bced330d5f108fbedd40548a901225d7ea1c Closes-Bug: #1402010 Closes-Bug: #1367705 --- etc/neutron/rootwrap.d/l3.filters | 3 + neutron/agent/l3/ha.py | 82 +++++++++- neutron/agent/l3/ha_router.py | 89 ++++++++--- neutron/agent/l3/keepalived_state_change.py | 144 ++++++++++++++++++ neutron/agent/linux/ip_monitor.py | 5 +- neutron/agent/linux/keepalived.py | 84 ++-------- neutron/cmd/keepalived_state_change.py | 19 +++ neutron/tests/functional/agent/l3/__init__.py | 0 .../agent/l3/test_keepalived_state_change.py | 73 +++++++++ .../tests/functional/agent/test_l3_agent.py | 32 +++- .../tests/unit/agent/linux/test_keepalived.py | 15 +- .../tests/unit/agent/metadata/test_driver.py | 1 + neutron/tests/unit/test_l3_agent.py | 3 +- setup.cfg | 1 + 14 files changed, 428 insertions(+), 123 deletions(-) create mode 100644 neutron/agent/l3/keepalived_state_change.py create mode 100644 neutron/cmd/keepalived_state_change.py create mode 100644 neutron/tests/functional/agent/l3/__init__.py create mode 100644 neutron/tests/functional/agent/l3/test_keepalived_state_change.py diff --git a/etc/neutron/rootwrap.d/l3.filters b/etc/neutron/rootwrap.d/l3.filters index a3818a2dcf7..425ac57725a 100644 --- a/etc/neutron/rootwrap.d/l3.filters +++ b/etc/neutron/rootwrap.d/l3.filters @@ -49,3 +49,6 @@ kill_keepalived: KillFilter, root, /usr/sbin/keepalived, -HUP, -15, -9 # l3 agent to delete floatingip's conntrack state conntrack: CommandFilter, conntrack, root + +# keepalived state change monitor +keepalived_state_change: CommandFilter, neutron-keepalived-state-change, root diff --git a/neutron/agent/l3/ha.py b/neutron/agent/l3/ha.py index 3518e8eef86..7f75864d767 100644 --- a/neutron/agent/l3/ha.py +++ b/neutron/agent/l3/ha.py @@ -15,17 +15,20 @@ import os +import eventlet from oslo_config import cfg from oslo_log import log as logging +import webob from neutron.agent.linux import keepalived -from neutron.agent.linux import utils +from neutron.agent.linux import utils as agent_utils from neutron.common import constants as l3_constants -from neutron.i18n import _LE +from neutron.i18n import _LE, _LI LOG = logging.getLogger(__name__) HA_DEV_PREFIX = 'ha-' +KEEPALIVED_STATE_CHANGE_SERVER_BACKLOG = 4096 OPTS = [ cfg.StrOpt('ha_confs_path', @@ -45,14 +48,83 @@ OPTS = [ ] +class KeepalivedStateChangeHandler(object): + def __init__(self, agent): + self.agent = agent + + @webob.dec.wsgify(RequestClass=webob.Request) + def __call__(self, req): + router_id = req.headers['X-Neutron-Router-Id'] + state = req.headers['X-Neutron-State'] + self.enqueue(router_id, state) + + def enqueue(self, router_id, state): + LOG.debug('Handling notification for router ' + '%(router_id)s, state %(state)s', {'router_id': router_id, + 'state': state}) + self.agent.enqueue_state_change(router_id, state) + + +class L3AgentKeepalivedStateChangeServer(object): + def __init__(self, agent, conf): + self.agent = agent + self.conf = conf + + agent_utils.ensure_directory_exists_without_file( + self.get_keepalived_state_change_socket_path(self.conf)) + + @classmethod + def get_keepalived_state_change_socket_path(cls, conf): + return os.path.join(conf.state_path, 'keepalived-state-change') + + def run(self): + server = agent_utils.UnixDomainWSGIServer( + 'neutron-keepalived-state-change') + server.start(KeepalivedStateChangeHandler(self.agent), + self.get_keepalived_state_change_socket_path(self.conf), + workers=0, + backlog=KEEPALIVED_STATE_CHANGE_SERVER_BACKLOG) + server.wait() + + class AgentMixin(object): def __init__(self, host): self._init_ha_conf_path() super(AgentMixin, self).__init__(host) + eventlet.spawn(self._start_keepalived_notifications_server) + + def _start_keepalived_notifications_server(self): + state_change_server = ( + L3AgentKeepalivedStateChangeServer(self, self.conf)) + state_change_server.run() + + def enqueue_state_change(self, router_id, state): + LOG.info(_LI('Router %(router_id)s transitioned to %(state)s'), + {'router_id': router_id, + 'state': state}) + self._update_metadata_proxy(router_id, state) + + def _update_metadata_proxy(self, router_id, state): + try: + ri = self.router_info[router_id] + except AttributeError: + LOG.info(_LI('Router %s is not managed by this agent. It was ' + 'possibly deleted concurrently.'), router_id) + return + + if state == 'master': + LOG.debug('Spawning metadata proxy for router %s', router_id) + self.metadata_driver.spawn_monitored_metadata_proxy( + self.process_monitor, ri.ns_name, self.conf.metadata_port, + self.conf, router_id=ri.router_id) + else: + LOG.debug('Closing metadata proxy for router %s', router_id) + self.metadata_driver.destroy_monitored_metadata_proxy( + self.process_monitor, ri.router_id, ri.ns_name, self.conf) def _init_ha_conf_path(self): ha_full_path = os.path.dirname("/%s/" % self.conf.ha_confs_path) - utils.ensure_dir(ha_full_path) + agent_utils.ensure_dir(ha_full_path) def process_ha_router_added(self, ri): ha_port = ri.router.get(l3_constants.HA_INTERFACE_KEY) @@ -69,7 +141,9 @@ class AgentMixin(object): ha_port['ip_cidr'], ha_port['mac_address']) - ri._add_keepalived_notifiers() + ri.update_initial_state(self.enqueue_state_change) + ri.spawn_state_change_monitor(self.process_monitor) def process_ha_router_removed(self, ri): + ri.destroy_state_change_monitor(self.process_monitor) ri.ha_network_removed() diff --git a/neutron/agent/l3/ha_router.py b/neutron/agent/l3/ha_router.py index f3a0571b6db..d28c6c6d655 100644 --- a/neutron/agent/l3/ha_router.py +++ b/neutron/agent/l3/ha_router.py @@ -12,21 +12,23 @@ # License for the specific language governing permissions and limitations # under the License. +import os import shutil -import signal import netaddr from oslo_log import log as logging from neutron.agent.l3 import router_info as router +from neutron.agent.linux import external_process from neutron.agent.linux import ip_lib from neutron.agent.linux import keepalived -from neutron.agent.metadata import driver as metadata_driver from neutron.common import constants as n_consts from neutron.common import utils as common_utils +from neutron.i18n import _LE LOG = logging.getLogger(__name__) HA_DEV_PREFIX = 'ha-' +IP_MONITOR_PROCESS_SERVICE = 'ip_monitor' class HaRouter(router.RouterInfo): @@ -69,6 +71,18 @@ class HaRouter(router.RouterInfo): LOG.debug('Error while reading HA state for %s', self.router_id) return None + @ha_state.setter + def ha_state(self, new_state): + self._verify_ha() + ha_state_path = self.keepalived_manager._get_full_config_file_path( + 'state') + try: + with open(ha_state_path, 'w') as f: + f.write(new_state) + except (OSError, IOError): + LOG.error(_LE('Error while writing HA state for %s'), + self.router_id) + def _init_keepalived_manager(self, process_monitor): self.keepalived_manager = keepalived.KeepalivedManager( self.router['id'], @@ -107,27 +121,6 @@ class HaRouter(router.RouterInfo): conf_dir = self.keepalived_manager.get_conf_dir() shutil.rmtree(conf_dir) - def _add_keepalived_notifiers(self): - callback = ( - metadata_driver.MetadataDriver._get_metadata_proxy_callback( - self.agent_conf.metadata_port, self.agent_conf, - router_id=self.router_id)) - # TODO(mangelajo): use the process monitor in keepalived when - # keepalived stops killing/starting metadata - # proxy on its own - pm = ( - metadata_driver.MetadataDriver. - _get_metadata_proxy_process_manager(self.router_id, - self.ns_name, - self.agent_conf)) - pid = pm.get_pid_file_name() - self.keepalived_manager.add_notifier( - callback(pid), 'master', self.ha_vr_id) - for state in ('backup', 'fault'): - self.keepalived_manager.add_notifier( - ['kill', '-%s' % signal.SIGKILL, - '$(cat ' + pid + ')'], state, self.ha_vr_id) - def _get_keepalived_instance(self): return self.keepalived_manager.config.get_instance(self.ha_vr_id) @@ -276,3 +269,53 @@ class HaRouter(router.RouterInfo): interface_name = self.get_internal_device_name(port['id']) self._clear_vips(interface_name) + + def _get_state_change_monitor_process_manager(self): + return external_process.ProcessManager( + self.agent_conf, + '%s.monitor' % self.router_id, + self.ns_name, + default_cmd_callback=self._get_state_change_monitor_callback()) + + def _get_state_change_monitor_callback(self): + ha_device = self.get_ha_device_name(self.ha_port['id']) + ha_cidr = self._get_primary_vip() + + def callback(pid_file): + cmd = [ + 'neutron-keepalived-state-change', + '--router_id=%s' % self.router_id, + '--namespace=%s' % self.ns_name, + '--conf_dir=%s' % self.keepalived_manager.get_conf_dir(), + '--monitor_interface=%s' % ha_device, + '--monitor_cidr=%s' % ha_cidr, + '--pid_file=%s' % pid_file, + '--state_path=%s' % self.agent_conf.state_path, + '--user=%s' % os.geteuid(), + '--group=%s' % os.getegid()] + return cmd + + return callback + + def spawn_state_change_monitor(self, process_monitor): + pm = self._get_state_change_monitor_process_manager() + pm.enable() + process_monitor.register( + self.router_id, IP_MONITOR_PROCESS_SERVICE, pm) + + def destroy_state_change_monitor(self, process_monitor): + pm = self._get_state_change_monitor_process_manager() + process_monitor.unregister( + self.router_id, IP_MONITOR_PROCESS_SERVICE) + pm.disable() + + def update_initial_state(self, callback): + ha_device = ip_lib.IPDevice( + self.get_ha_device_name(self.ha_port['id']), + self.ns_name) + addresses = ha_device.addr.list() + cidrs = (address['cidr'] for address in addresses) + ha_cidr = self._get_primary_vip() + state = 'master' if ha_cidr in cidrs else 'backup' + self.ha_state = state + callback(self.router_id, state) diff --git a/neutron/agent/l3/keepalived_state_change.py b/neutron/agent/l3/keepalived_state_change.py new file mode 100644 index 00000000000..6d33be2a67a --- /dev/null +++ b/neutron/agent/l3/keepalived_state_change.py @@ -0,0 +1,144 @@ +# Copyright (c) 2015 Red Hat Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import sys + +import httplib2 +from oslo_config import cfg +from oslo_log import log as logging +import requests + +from neutron.agent.l3 import ha +from neutron.agent.linux import daemon +from neutron.agent.linux import ip_monitor +from neutron.agent.linux import utils as agent_utils +from neutron.common import config +from neutron.i18n import _LE + + +LOG = logging.getLogger(__name__) + + +class KeepalivedUnixDomainConnection(agent_utils.UnixDomainHTTPConnection): + def __init__(self, *args, **kwargs): + # Old style super initialization is required! + agent_utils.UnixDomainHTTPConnection.__init__( + self, *args, **kwargs) + self.socket_path = ( + ha.L3AgentKeepalivedStateChangeServer. + get_keepalived_state_change_socket_path(cfg.CONF)) + + +class MonitorDaemon(daemon.Daemon): + def __init__(self, pidfile, router_id, user, group, namespace, conf_dir, + interface, cidr): + self.router_id = router_id + self.namespace = namespace + self.conf_dir = conf_dir + self.interface = interface + self.cidr = cidr + super(MonitorDaemon, self).__init__(pidfile, uuid=router_id, + user=user, group=group) + + def run(self, run_as_root=False): + monitor = ip_monitor.IPMonitor(namespace=self.namespace, + run_as_root=run_as_root) + monitor.start() + # Only drop privileges if the process is currently running as root + # (The run_as_root variable name here is unfortunate - It means to + # use a root helper when the running process is NOT already running + # as root + if not run_as_root: + super(MonitorDaemon, self).run() + for iterable in monitor: + self.parse_and_handle_event(iterable) + + def parse_and_handle_event(self, iterable): + try: + event = ip_monitor.IPMonitorEvent.from_text(iterable) + if event.interface == self.interface and event.cidr == self.cidr: + new_state = 'master' if event.added else 'backup' + self.write_state_change(new_state) + self.notify_agent(new_state) + except Exception: + LOG.exception(_LE( + 'Failed to process or handle event for line %s'), iterable) + + def write_state_change(self, state): + with open(os.path.join( + self.conf_dir, 'state'), 'w') as state_file: + state_file.write(state) + LOG.debug('Wrote router %s state %s', self.router_id, state) + + def notify_agent(self, state): + resp, content = httplib2.Http().request( + # Note that the message is sent via a Unix domain socket so that + # the URL doesn't matter. + 'http://127.0.0.1/', + headers={'X-Neutron-Router-Id': self.router_id, + 'X-Neutron-State': state}, + connection_type=KeepalivedUnixDomainConnection) + + if resp.status != requests.codes.ok: + raise Exception(_('Unexpected response: %s') % resp) + + LOG.debug('Notified agent router %s, state %s', self.router_id, state) + + +def register_opts(conf): + conf.register_cli_opt( + cfg.StrOpt('router_id', help=_('ID of the router'))) + conf.register_cli_opt( + cfg.StrOpt('namespace', help=_('Namespace of the router'))) + conf.register_cli_opt( + cfg.StrOpt('conf_dir', help=_('Path to the router directory'))) + conf.register_cli_opt( + cfg.StrOpt('monitor_interface', help=_('Interface to monitor'))) + conf.register_cli_opt( + cfg.StrOpt('monitor_cidr', help=_('CIDR to monitor'))) + conf.register_cli_opt( + cfg.StrOpt('pid_file', help=_('Path to PID file for this process'))) + conf.register_cli_opt( + cfg.StrOpt('user', help=_('User (uid or name) running this process ' + 'after its initialization'))) + conf.register_cli_opt( + cfg.StrOpt('group', help=_('Group (gid or name) running this process ' + 'after its initialization'))) + conf.register_opt( + cfg.StrOpt('metadata_proxy_socket', + default='$state_path/metadata_proxy', + help=_('Location of Metadata Proxy UNIX domain ' + 'socket'))) + + +def configure(conf): + config.init(sys.argv[1:]) + conf.set_override('log_dir', cfg.CONF.conf_dir) + conf.set_override('debug', True) + conf.set_override('verbose', True) + config.setup_logging() + + +def main(): + register_opts(cfg.CONF) + configure(cfg.CONF) + MonitorDaemon(cfg.CONF.pid_file, + cfg.CONF.router_id, + cfg.CONF.user, + cfg.CONF.group, + cfg.CONF.namespace, + cfg.CONF.conf_dir, + cfg.CONF.monitor_interface, + cfg.CONF.monitor_cidr).start() diff --git a/neutron/agent/linux/ip_monitor.py b/neutron/agent/linux/ip_monitor.py index 02bb242cbbf..19323eaaef7 100644 --- a/neutron/agent/linux/ip_monitor.py +++ b/neutron/agent/linux/ip_monitor.py @@ -60,7 +60,7 @@ class IPMonitor(async_process.AsyncProcess): """Wrapper over `ip monitor address`. To monitor and react indefinitely: - m = IPMonitor(namespace='tmp') + m = IPMonitor(namespace='tmp', root_as_root=True) m.start() for iterable in m: event = IPMonitorEvent.from_text(iterable) @@ -69,9 +69,10 @@ class IPMonitor(async_process.AsyncProcess): def __init__(self, namespace=None, + run_as_root=True, respawn_interval=None): super(IPMonitor, self).__init__(['ip', '-o', 'monitor', 'address'], - run_as_root=True, + run_as_root=run_as_root, respawn_interval=respawn_interval, namespace=namespace) diff --git a/neutron/agent/linux/keepalived.py b/neutron/agent/linux/keepalived.py index c51c2ca98a2..52a9377fc81 100644 --- a/neutron/agent/linux/keepalived.py +++ b/neutron/agent/linux/keepalived.py @@ -15,7 +15,6 @@ import errno import itertools import os -import stat import netaddr from oslo_config import cfg @@ -26,7 +25,6 @@ from neutron.agent.linux import utils from neutron.common import exceptions VALID_STATES = ['MASTER', 'BACKUP'] -VALID_NOTIFY_STATES = ['master', 'backup', 'fault'] VALID_AUTH_TYPES = ['AH', 'PASS'] HA_DEFAULT_PRIORITY = 50 PRIMARY_VIP_RANGE_SIZE = 24 @@ -69,16 +67,6 @@ class InvalidInstanceStateException(exceptions.NeutronException): super(InvalidInstanceStateException, self).__init__(**kwargs) -class InvalidNotifyStateException(exceptions.NeutronException): - message = _('Invalid notify state: %(state)s, valid states are: ' - '%(valid_notify_states)s') - - def __init__(self, **kwargs): - if 'valid_notify_states' not in kwargs: - kwargs['valid_notify_states'] = ', '.join(VALID_NOTIFY_STATES) - super(InvalidNotifyStateException, self).__init__(**kwargs) - - class InvalidAuthenticationTypeException(exceptions.NeutronException): message = _('Invalid authentication type: %(auth_type)s, ' 'valid types are: %(valid_auth_types)s') @@ -141,7 +129,6 @@ class KeepalivedInstance(object): self.vips = [] self.virtual_routes = [] self.authentication = None - self.notifiers = [] metadata_cidr = '169.254.169.254/32' self.primary_vip_range = get_free_range( parent_range='169.254.0.0/16', @@ -174,11 +161,6 @@ class KeepalivedInstance(object): return [vip.ip_address for vip in self.vips if vip.interface_name == interface_name] - def set_notify(self, state, path): - if state not in VALID_NOTIFY_STATES: - raise InvalidNotifyStateException(state=state) - self.notifiers.append((state, path)) - def _build_track_interface_config(self): return itertools.chain( [' track_interface {'], @@ -234,10 +216,6 @@ class KeepalivedInstance(object): for route in self.virtual_routes), [' }']) - def _build_notify_scripts(self): - return itertools.chain((' notify_%s "%s"' % (state, path) - for state, path in self.notifiers)) - def build_config(self): config = ['vrrp_instance %s {' % self.name, ' state %s' % self.state, @@ -270,9 +248,6 @@ class KeepalivedInstance(object): if self.virtual_routes: config.extend(self._build_virtual_routes_config()) - if self.notifiers: - config.extend(self._build_notify_scripts()) - config.append('}') return config @@ -309,53 +284,7 @@ class KeepalivedConf(object): return '\n'.join(self.build_config()) -class KeepalivedNotifierMixin(object): - def _get_notifier_path(self, state): - return self._get_full_config_file_path('notify_%s.sh' % state) - - def _write_notify_script(self, state, script): - name = self._get_notifier_path(state) - utils.replace_file(name, script) - st = os.stat(name) - os.chmod(name, st.st_mode | stat.S_IEXEC) - - return name - - def _prepend_shebang(self, script): - return '#!/bin/sh\n%s' % script - - def _append_state(self, script, state): - state_path = self._get_full_config_file_path('state') - return '%s\necho -n %s > %s' % (script, state, state_path) - - def add_notifier(self, script, state, vrouter_id): - """Add a master, backup or fault notifier. - - These notifiers are executed when keepalived invokes a state - transition. Write a notifier to disk and add it to the - configuration. - """ - - script_with_prefix = self._prepend_shebang(' '.join(script)) - full_script = self._append_state(script_with_prefix, state) - self._write_notify_script(state, full_script) - - vr_instance = self.config.get_instance(vrouter_id) - vr_instance.set_notify(state, self._get_notifier_path(state)) - - def get_conf_dir(self): - confs_dir = os.path.abspath(os.path.normpath(self.conf_path)) - conf_dir = os.path.join(confs_dir, self.resource_id) - return conf_dir - - def _get_full_config_file_path(self, filename, ensure_conf_dir=True): - conf_dir = self.get_conf_dir() - if ensure_conf_dir: - utils.ensure_dir(conf_dir) - return os.path.join(conf_dir, filename) - - -class KeepalivedManager(KeepalivedNotifierMixin): +class KeepalivedManager(object): """Wrapper for keepalived. This wrapper permits to write keepalived config files, to start/restart @@ -372,6 +301,17 @@ class KeepalivedManager(KeepalivedNotifierMixin): self.conf_path = conf_path self.process = None + def get_conf_dir(self): + confs_dir = os.path.abspath(os.path.normpath(self.conf_path)) + conf_dir = os.path.join(confs_dir, self.resource_id) + return conf_dir + + def _get_full_config_file_path(self, filename, ensure_conf_dir=True): + conf_dir = self.get_conf_dir() + if ensure_conf_dir: + utils.ensure_dir(conf_dir) + return os.path.join(conf_dir, filename) + def _output_config_file(self): config_str = self.config.get_config_str() config_path = self._get_full_config_file_path('keepalived.conf') diff --git a/neutron/cmd/keepalived_state_change.py b/neutron/cmd/keepalived_state_change.py new file mode 100644 index 00000000000..eca618a2d25 --- /dev/null +++ b/neutron/cmd/keepalived_state_change.py @@ -0,0 +1,19 @@ +# Copyright (c) 2015 Red Hat Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from neutron.agent.l3 import keepalived_state_change + + +def main(): + keepalived_state_change.main() diff --git a/neutron/tests/functional/agent/l3/__init__.py b/neutron/tests/functional/agent/l3/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/neutron/tests/functional/agent/l3/test_keepalived_state_change.py b/neutron/tests/functional/agent/l3/test_keepalived_state_change.py new file mode 100644 index 00000000000..b87b504e86c --- /dev/null +++ b/neutron/tests/functional/agent/l3/test_keepalived_state_change.py @@ -0,0 +1,73 @@ +# Copyright (c) 2015 Red Hat Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +import mock +from oslo_config import cfg + +from neutron.agent.l3 import keepalived_state_change +from neutron.openstack.common import uuidutils +from neutron.tests.functional import base + + +class TestKeepalivedStateChange(base.BaseSudoTestCase): + def setUp(self): + super(TestKeepalivedStateChange, self).setUp() + cfg.CONF.register_opt( + cfg.StrOpt('metadata_proxy_socket', + default='$state_path/metadata_proxy', + help=_('Location of Metadata Proxy UNIX domain ' + 'socket'))) + + self.router_id = uuidutils.generate_uuid() + self.conf_dir = self.get_default_temp_dir().path + self.cidr = '169.254.128.1/24' + self.interface_name = 'interface' + self.monitor = keepalived_state_change.MonitorDaemon( + self.get_temp_file_path('monitor.pid'), + self.router_id, + 1, + 2, + 'namespace', + self.conf_dir, + self.interface_name, + self.cidr) + mock.patch.object(self.monitor, 'notify_agent').start() + self.line = '1: %s inet %s' % (self.interface_name, self.cidr) + + def test_parse_and_handle_event_wrong_device_completes_without_error(self): + self.monitor.parse_and_handle_event( + '1: wrong_device inet wrong_cidr') + + def _get_state(self): + with open(os.path.join(self.monitor.conf_dir, 'state')) as state_file: + return state_file.read() + + def test_parse_and_handle_event_writes_to_file(self): + self.monitor.parse_and_handle_event('Deleted %s' % self.line) + self.assertEqual('backup', self._get_state()) + + self.monitor.parse_and_handle_event(self.line) + self.assertEqual('master', self._get_state()) + + def test_parse_and_handle_event_fails_writing_state(self): + with mock.patch.object( + self.monitor, 'write_state_change', side_effect=OSError): + self.monitor.parse_and_handle_event(self.line) + + def test_parse_and_handle_event_fails_notifying_agent(self): + with mock.patch.object( + self.monitor, 'notify_agent', side_effect=Exception): + self.monitor.parse_and_handle_event(self.line) diff --git a/neutron/tests/functional/agent/test_l3_agent.py b/neutron/tests/functional/agent/test_l3_agent.py index 7777f4fd35b..a822eb2640e 100755 --- a/neutron/tests/functional/agent/test_l3_agent.py +++ b/neutron/tests/functional/agent/test_l3_agent.py @@ -147,7 +147,6 @@ class L3AgentTestFramework(base.BaseOVSLinuxTestCase): expected_device['mac_address'], namespace) def get_expected_keepalive_configuration(self, router): - ha_confs_path = self.agent.conf.ha_confs_path router_id = router.router_id ha_device_name = router.get_ha_device_name(router.ha_port['id']) ha_device_cidr = router.ha_port['ip_cidr'] @@ -191,11 +190,7 @@ class L3AgentTestFramework(base.BaseOVSLinuxTestCase): 0.0.0.0/0 via %(default_gateway_ip)s dev %(external_device_name)s 8.8.8.0/24 via 19.4.4.4 } - notify_master "%(ha_confs_path)s/%(router_id)s/notify_master.sh" - notify_backup "%(ha_confs_path)s/%(router_id)s/notify_backup.sh" - notify_fault "%(ha_confs_path)s/%(router_id)s/notify_fault.sh" }""" % { - 'ha_confs_path': ha_confs_path, 'router_id': router_id, 'ha_device_name': ha_device_name, 'ha_device_cidr': ha_device_cidr, @@ -219,7 +214,8 @@ class L3AgentTestFramework(base.BaseOVSLinuxTestCase): # then the devices and iptable rules have also been deleted, # so there's no need to check that explicitly. self.assertFalse(self._namespace_exists(router.ns_name)) - self.assertFalse(self._metadata_proxy_exists(self.agent.conf, router)) + utils.wait_until_true( + lambda: not self._metadata_proxy_exists(self.agent.conf, router)) def _assert_snat_chains(self, router): self.assertFalse(router.iptables_manager.is_chain_empty( @@ -283,6 +279,25 @@ class L3AgentTestCase(L3AgentTestFramework): def test_observer_notifications_ha_router(self): self._test_observer_notifications(enable_ha=True) + def test_keepalived_state_change_notification(self): + enqueue_mock = mock.patch.object( + self.agent, 'enqueue_state_change').start() + router_info = self.generate_router_info(enable_ha=True) + router = self.manage_router(self.agent, router_info) + utils.wait_until_true(lambda: router.ha_state == 'master') + + device_name = router.get_ha_device_name( + router.router[l3_constants.HA_INTERFACE_KEY]['id']) + ha_device = ip_lib.IPDevice(device_name, router.ns_name) + ha_device.link.set_down() + utils.wait_until_true(lambda: router.ha_state == 'backup') + + utils.wait_until_true(lambda: enqueue_mock.call_count == 3) + calls = [args[0] for args in enqueue_mock.call_args_list] + self.assertEqual((router.router_id, 'backup'), calls[0]) + self.assertEqual((router.router_id, 'master'), calls[1]) + self.assertEqual((router.router_id, 'backup'), calls[2]) + def _test_observer_notifications(self, enable_ha): """Test create, update, delete of router and notifications.""" with mock.patch.object( @@ -419,7 +434,8 @@ class L3AgentTestCase(L3AgentTestFramework): utils.wait_until_true(device_exists) self.assertTrue(self._namespace_exists(router.ns_name)) - self.assertTrue(self._metadata_proxy_exists(self.agent.conf, router)) + utils.wait_until_true( + lambda: self._metadata_proxy_exists(self.agent.conf, router)) self._assert_internal_devices(router) self._assert_external_device(router) if ip_version == 4: @@ -538,7 +554,7 @@ class L3HATestFramework(L3AgentTestFramework): ha_device.link.set_down() utils.wait_until_true(lambda: router2.ha_state == 'master') - utils.wait_until_true(lambda: router1.ha_state == 'fault') + utils.wait_until_true(lambda: router1.ha_state == 'backup') class MetadataFakeProxyHandler(object): diff --git a/neutron/tests/unit/agent/linux/test_keepalived.py b/neutron/tests/unit/agent/linux/test_keepalived.py index b76c29ae1af..ca2c7dbfe86 100644 --- a/neutron/tests/unit/agent/linux/test_keepalived.py +++ b/neutron/tests/unit/agent/linux/test_keepalived.py @@ -66,7 +66,6 @@ class KeepalivedConfBaseMixin(object): advert_int=5) instance1.set_authentication('AH', 'pass123') instance1.track_interfaces.append("eth0") - instance1.set_notify('master', '/tmp/script.sh') vip_address1 = keepalived.KeepalivedVipAddress('192.168.1.0/24', 'eth1') @@ -136,7 +135,6 @@ class KeepalivedConfTestCase(base.BaseTestCase, virtual_routes { 0.0.0.0/0 via 192.168.1.1 dev eth1 } - notify_master "/tmp/script.sh" } vrrp_instance VR_2 { state MASTER @@ -177,20 +175,12 @@ vrrp_instance VR_2 { class KeepalivedStateExceptionTestCase(base.BaseTestCase): def test_state_exception(self): - instance = keepalived.KeepalivedInstance('MASTER', 'eth0', 1, - '169.254.192.0/18') - - invalid_notify_state = 'a seal walks' - self.assertRaises(keepalived.InvalidNotifyStateException, - instance.set_notify, - invalid_notify_state, '/tmp/script.sh') - - invalid_vrrp_state = 'into a club' + invalid_vrrp_state = 'a seal walks' self.assertRaises(keepalived.InvalidInstanceStateException, keepalived.KeepalivedInstance, invalid_vrrp_state, 'eth0', 33, '169.254.192.0/18') - invalid_auth_type = '[hip, hip]' + invalid_auth_type = 'into a club' instance = keepalived.KeepalivedInstance('MASTER', 'eth0', 1, '169.254.192.0/18') self.assertRaises(keepalived.InvalidAuthenticationTypeException, @@ -233,7 +223,6 @@ class KeepalivedInstanceTestCase(base.BaseTestCase, virtual_routes { 0.0.0.0/0 via 192.168.1.1 dev eth1 } - notify_master "/tmp/script.sh" } vrrp_instance VR_2 { state MASTER diff --git a/neutron/tests/unit/agent/metadata/test_driver.py b/neutron/tests/unit/agent/metadata/test_driver.py index afd97d9c2e8..f6545bc48a4 100644 --- a/neutron/tests/unit/agent/metadata/test_driver.py +++ b/neutron/tests/unit/agent/metadata/test_driver.py @@ -63,6 +63,7 @@ class TestMetadataDriverProcess(base.BaseTestCase): def setUp(self): super(TestMetadataDriverProcess, self).setUp() + mock.patch('eventlet.spawn').start() agent_config.register_interface_driver_opts_helper(cfg.CONF) cfg.CONF.set_override('interface_driver', 'neutron.agent.linux.interface.NullDriver') diff --git a/neutron/tests/unit/test_l3_agent.py b/neutron/tests/unit/test_l3_agent.py index fe41f77360f..9736f5b1e40 100644 --- a/neutron/tests/unit/test_l3_agent.py +++ b/neutron/tests/unit/test_l3_agent.py @@ -177,6 +177,7 @@ class BasicRouterOperationsFramework(base.BaseTestCase): def setUp(self): super(BasicRouterOperationsFramework, self).setUp() + mock.patch('eventlet.spawn').start() self.conf = agent_config.setup_conf() self.conf.register_opts(base_config.core_opts) log.register_options(self.conf) @@ -200,7 +201,7 @@ class BasicRouterOperationsFramework(base.BaseTestCase): self.ensure_dir = mock.patch('neutron.agent.linux.utils' '.ensure_dir').start() - mock.patch('neutron.agent.linux.keepalived.KeepalivedNotifierMixin' + mock.patch('neutron.agent.linux.keepalived.KeepalivedManager' '._get_full_config_file_path').start() self.utils_exec_p = mock.patch( diff --git a/setup.cfg b/setup.cfg index a446c015e00..64650725c49 100644 --- a/setup.cfg +++ b/setup.cfg @@ -92,6 +92,7 @@ console_scripts = neutron-debug = neutron.debug.shell:main neutron-dhcp-agent = neutron.cmd.eventlet.agents.dhcp:main neutron-hyperv-agent = neutron.plugins.hyperv.agent.hyperv_neutron_agent:main + neutron-keepalived-state-change = neutron.cmd.keepalived_state_change:main neutron-ibm-agent = neutron.plugins.ibm.agent.sdnve_neutron_agent:main neutron-l3-agent = neutron.cmd.eventlet.agents.l3:main neutron-linuxbridge-agent = neutron.plugins.linuxbridge.agent.linuxbridge_neutron_agent:main