Create utility to clean-up netns.

Fixes bug 1035366

Adds namespace clean up utility called quantum-netns-cleanup which can be used
to remove old namespaces.

The --force option can be used to remove all Quantum namespaces and any
remaining devices.  The force option is should not be run on a live Quantum
systems.  It is intended for cleaning up devstack a after running
unstack.sh (ideally this will be added to unstack.sh in the future).

Example cmd line when cleaning up a devstack install:
quantum-netns-cleanup --config-file /etc/quantum/quantum.conf \
--config-file /etc/quantum/dhcp_agent.ini --force

Change-Id: I6cf153df21e83bff2cde816db12b22102d1ba698
This commit is contained in:
Mark McClain 2012-08-20 10:18:01 -04:00
parent d8160e02bb
commit 8e34320bbc
9 changed files with 541 additions and 4 deletions

20
bin/quantum-netns-cleanup Executable file
View File

@ -0,0 +1,20 @@
#!/usr/bin/env python
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright (c) 2012 Openstack, LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from quantum.agent.netns_cleanup_util import main
main()

View File

@ -154,6 +154,10 @@ class OVSInterfaceDriver(LinuxInterfaceDriver):
bridge = ovs_lib.OVSBridge(bridge, self.conf.root_helper)
bridge.delete_port(device_name)
if namespace:
ip = ip_lib.IPWrapper(self.conf.root_helper, namespace)
ip.garbage_collect_namespace()
class BridgeInterfaceDriver(LinuxInterfaceDriver):
"""Driver for creating bridge interfaces."""
@ -196,6 +200,10 @@ class BridgeInterfaceDriver(LinuxInterfaceDriver):
LOG.error(_("Failed unplugging interface '%s'") %
device_name)
if namespace:
ip = ip_lib.IPWrapper(self.conf.root_helper, namespace)
ip.garbage_collect_namespace()
class RyuInterfaceDriver(OVSInterfaceDriver):
"""Driver for creating a Ryu OVS interface."""

View File

@ -18,6 +18,9 @@ from quantum.agent.linux import utils
from quantum.common import exceptions
LOOPBACK_DEVNAME = 'lo'
class SubProcessBase(object):
def __init__(self, root_helper=None, namespace=None):
self.root_helper = root_helper
@ -62,7 +65,7 @@ class IPWrapper(SubProcessBase):
def device(self, name):
return IPDevice(name, self.root_helper, self.namespace)
def get_devices(self):
def get_devices(self, exclude_loopback=False):
retval = []
output = self._execute('o', 'link', ('list',),
self.root_helper, self.namespace)
@ -71,7 +74,12 @@ class IPWrapper(SubProcessBase):
continue
tokens = line.split(':', 2)
if len(tokens) >= 3:
retval.append(IPDevice(tokens[1].strip(),
name = tokens[1].strip()
if exclude_loopback and name == LOOPBACK_DEVNAME:
continue
retval.append(IPDevice(name,
self.root_helper,
self.namespace))
return retval
@ -90,12 +98,23 @@ class IPWrapper(SubProcessBase):
def ensure_namespace(self, name):
if not self.netns.exists(name):
ip = self.netns.add(name)
lo = ip.device('lo')
lo = ip.device(LOOPBACK_DEVNAME)
lo.link.set_up()
else:
ip = IPWrapper(self.root_helper, name)
return ip
def namespace_is_empty(self):
return not self.get_devices(exclude_loopback=True)
def garbage_collect_namespace(self):
"""Conditionally destroy the namespace if it is empty."""
if self.namespace and self.netns.exists(self.namespace):
if self.namespace_is_empty():
self.netns.delete(self.namespace)
return True
return False
def add_device_to_namespace(self, device):
if self.namespace:
device.link.set_netns(self.namespace)

View File

@ -269,3 +269,12 @@ class OVSBridge:
except Exception, e:
LOG.info("Unable to parse regex results. Exception: %s", e)
return
def get_bridge_for_iface(root_helper, iface):
args = ["ovs-vsctl", "--timeout=2", "iface-to-br", iface]
try:
return utils.execute(args, root_helper=root_helper).strip()
except Exception, e:
LOG.error(_("iface %s not found. Exception: %s"), iface, e)
return None

View File

@ -0,0 +1,162 @@
import logging
import os
import re
import sys
import traceback
import eventlet
from quantum.agent import dhcp_agent
from quantum.agent import l3_agent
from quantum.agent.linux import dhcp
from quantum.agent.linux import ip_lib
from quantum.agent.linux import ovs_lib
from quantum.api.v2 import attributes
from quantum.common import config
from quantum.openstack.common import cfg
from quantum.openstack.common import importutils
LOG = logging.getLogger(__name__)
NS_MANGLING_PATTERN = ('(%s|%s)' % (dhcp_agent.NS_PREFIX, l3_agent.NS_PREFIX) +
attributes.UUID_PATTERN)
class NullDelegate(object):
def __getattribute__(self, name):
def noop(*args, **kwargs):
pass
return noop
class FakeNetwork(object):
def __init__(self, id):
self.id = id
def setup_conf():
"""Setup the cfg for the clean up utility.
Use separate setup_conf for the utility because there are many options
from the main config that do not apply during clean-up.
"""
opts = [
cfg.StrOpt('root_helper', default='sudo'),
cfg.StrOpt('dhcp_driver',
default='quantum.agent.linux.dhcp.Dnsmasq',
help="The driver used to manage the DHCP server."),
cfg.StrOpt('state_path',
default='.',
help='Top-level directory for maintaining dhcp state'),
cfg.BoolOpt('force',
default=False,
help='Delete the namespace by removing all devices.'),
]
conf = cfg.CommonConfigOpts()
conf.register_opts(opts)
conf.register_opts(dhcp.OPTS)
config.setup_logging(conf)
return conf
def kill_dhcp(conf, namespace):
"""Disable DHCP for a network if DHCP is still active."""
network_id = namespace.replace(dhcp_agent.NS_PREFIX, '')
null_delegate = NullDelegate()
dhcp_driver = importutils.import_object(
conf.dhcp_driver,
conf,
FakeNetwork(network_id),
conf.root_helper,
null_delegate)
if dhcp_driver.active:
dhcp_driver.disable()
def eligible_for_deletion(conf, namespace, force=False):
"""Determine whether a namespace is eligible for deletion.
Eligibility is determined by having only the lo device or if force
is passed as a parameter.
"""
# filter out namespaces without UUID as the name
if not re.match(NS_MANGLING_PATTERN, namespace):
return False
ip = ip_lib.IPWrapper(conf.root_helper, namespace)
return force or ip.namespace_is_empty()
def unplug_device(conf, device):
try:
device.link.delete()
except RuntimeError:
# Maybe the device is OVS port, so try to delete
bridge_name = ovs_lib.get_bridge_for_iface(conf.root_helper,
device.name)
if bridge_name:
bridge = ovs_lib.OVSBridge(bridge_name,
conf.root_helper)
bridge.delete_port(device.name)
else:
LOG.debug(_('Unable to find bridge for device: %s') % device.name)
def destroy_namespace(conf, namespace, force=False):
"""Destroy a given namespace.
If force is True, then dhcp (if it exists) will be disabled and all
devices will be forcibly removed.
"""
try:
ip = ip_lib.IPWrapper(conf.root_helper, namespace)
if force:
kill_dhcp(conf, namespace)
# NOTE: The dhcp driver will remove the namespace if is it empty,
# so a second check is required here.
if ip.netns.exists(namespace):
for device in ip.get_devices(exclude_loopback=True):
unplug_device(conf, device)
ip.garbage_collect_namespace()
except Exception, e:
LOG.exception(_('Error unable to destroy namespace: %s') % namespace)
def main():
"""Main method for cleaning up network namespaces.
This method will make two passes checking for namespaces to delete. The
process will identify candidates, sleep, and call garbage collect. The
garbage collection will re-verify that the namespace meets the criteria for
deletion (ie it is empty). The period of sleep and the 2nd pass allow
time for the namespace state to settle, so that the check prior deletion
will re-confirm the namespace is empty.
The utility is designed to clean-up after the forced or unexpected
termination of Quantum agents.
The --force flag should only be used as part of the cleanup of a devstack
installation as it will blindly purge namespaces and their devices. This
option also kills any lingering DHCP instances.
"""
eventlet.monkey_patch()
conf = setup_conf()
conf(sys.argv)
# Identify namespaces that are candidates for deletion.
candidates = [ns for ns in
ip_lib.IPWrapper.get_namespaces(conf.root_helper)
if eligible_for_deletion(conf, ns, conf.force)]
if candidates:
eventlet.sleep(2)
for namespace in candidates:
destroy_namespace(conf, namespace, conf.force)

View File

@ -15,10 +15,10 @@
# under the License.
# @author: Dan Wendlandt, Nicira, Inc.
import unittest
import uuid
import mox
import unittest2 as unittest
from quantum.agent.linux import ovs_lib, utils
@ -292,3 +292,25 @@ class OVS_Lib_Test(unittest.TestCase):
self.assertEqual(vif_id, '5c1321a7-c73f-4a77-95e6-9f86402e5c8f')
self.assertEqual(port_name, 'dhc5c1321a7-c7')
self.assertEqual(ofport, 2)
def test_iface_to_br(self):
iface = 'tap0'
br = 'br-int'
root_helper = 'sudo'
utils.execute(["ovs-vsctl", self.TO, "iface-to-br", iface],
root_helper=root_helper).AndReturn('br-int')
self.mox.ReplayAll()
self.assertEqual(ovs_lib.get_bridge_for_iface(root_helper, iface), br)
self.mox.VerifyAll()
def test_iface_to_br(self):
iface = 'tap0'
br = 'br-int'
root_helper = 'sudo'
utils.execute(["ovs-vsctl", self.TO, "iface-to-br", iface],
root_helper=root_helper).AndRaise(Exception)
self.mox.ReplayAll()
self.assertIsNone(ovs_lib.get_bridge_for_iface(root_helper, iface))
self.mox.VerifyAll()

View File

@ -0,0 +1,230 @@
import mock
import unittest2 as unittest
from quantum.agent import netns_cleanup_util as util
class TestNetnsCleanup(unittest.TestCase):
def test_setup_conf(self):
conf = util.setup_conf()
self.assertFalse(conf.force)
def test_kill_dhcp(self, dhcp_active=True):
conf = mock.Mock()
conf.root_helper = 'sudo',
conf.dhcp_driver = 'driver'
method_to_patch = 'quantum.openstack.common.importutils.import_object'
with mock.patch(method_to_patch) as import_object:
driver = mock.Mock()
driver.active = dhcp_active
import_object.return_value = driver
util.kill_dhcp(conf, 'ns')
import_object.called_once_with('driver', conf, mock.ANY, 'sudo',
mock.ANY)
if dhcp_active:
driver.assert_has_calls([mock.call.disable()])
else:
self.assertFalse(driver.called)
def test_kill_dhcp_no_active(self):
self.test_kill_dhcp(False)
def test_eligible_for_deletion_ns_not_uuid(self):
ns = 'not_a_uuid'
self.assertFalse(util.eligible_for_deletion(mock.Mock(), ns))
def _test_eligible_for_deletion_helper(self, prefix, force, is_empty,
expected):
ns = prefix + '6e322ac7-ab50-4f53-9cdc-d1d3c1164b6d'
conf = mock.Mock()
conf.root_helper = 'sudo'
with mock.patch('quantum.agent.linux.ip_lib.IPWrapper') as ip_wrap:
ip_wrap.return_value.namespace_is_empty.return_value = is_empty
self.assertEqual(util.eligible_for_deletion(conf, ns, force),
expected)
expected_calls = [mock.call('sudo', ns)]
if not force:
expected_calls.append(mock.call().namespace_is_empty())
ip_wrap.assert_has_calls(expected_calls)
def test_eligible_for_deletion_empty(self):
self._test_eligible_for_deletion_helper('qrouter-', False, True, True)
def test_eligible_for_deletion_not_empty(self):
self._test_eligible_for_deletion_helper('qdhcp-', False, False, False)
def test_eligible_for_deletion_not_empty_forced(self):
self._test_eligible_for_deletion_helper('qdhcp-', True, False, True)
def test_unplug_device_regular_device(self):
conf = mock.Mock()
device = mock.Mock()
util.unplug_device(conf, device)
device.assert_has_calls([mock.call.link.delete()])
def test_unplug_device_ovs_port(self):
conf = mock.Mock()
conf.ovs_integration_bridge = 'br-int'
conf.root_helper = 'sudo'
device = mock.Mock()
device.name = 'tap1'
device.link.delete.side_effect = RuntimeError
with mock.patch('quantum.agent.linux.ovs_lib.OVSBridge') as ovs_br_cls:
br_patch = mock.patch(
'quantum.agent.linux.ovs_lib.get_bridge_for_iface')
with br_patch as mock_get_bridge_for_iface:
mock_get_bridge_for_iface.return_value = 'br-int'
ovs_bridge = mock.Mock()
ovs_br_cls.return_value = ovs_bridge
util.unplug_device(conf, device)
mock_get_bridge_for_iface.assert_called_once_with(
conf.root_helper, 'tap1')
ovs_br_cls.called_once_with('br-int', 'sudo')
ovs_bridge.assert_has_calls(
[mock.call.delete_port(device.name)])
def test_unplug_device_cannot_determine_bridge_port(self):
conf = mock.Mock()
conf.ovs_integration_bridge = 'br-int'
conf.root_helper = 'sudo'
device = mock.Mock()
device.name = 'tap1'
device.link.delete.side_effect = RuntimeError
with mock.patch('quantum.agent.linux.ovs_lib.OVSBridge') as ovs_br_cls:
br_patch = mock.patch(
'quantum.agent.linux.ovs_lib.get_bridge_for_iface')
with br_patch as mock_get_bridge_for_iface:
with mock.patch.object(util.LOG, 'debug') as debug:
mock_get_bridge_for_iface.return_value = None
ovs_bridge = mock.Mock()
ovs_br_cls.return_value = ovs_bridge
util.unplug_device(conf, device)
mock_get_bridge_for_iface.assert_called_once_with(
conf.root_helper, 'tap1')
self.assertEquals(ovs_br_cls.mock_calls, [])
self.assertTrue(debug.called)
def _test_destroy_namespace_helper(self, force, num_devices):
ns = 'qrouter-6e322ac7-ab50-4f53-9cdc-d1d3c1164b6d'
conf = mock.Mock()
conf.root_helper = 'sudo'
lo_device = mock.Mock()
lo_device.name = 'lo'
devices = [lo_device]
while num_devices:
dev = mock.Mock()
dev.name = 'tap%d' % num_devices
devices.append(dev)
num_devices -= 1
with mock.patch('quantum.agent.linux.ip_lib.IPWrapper') as ip_wrap:
ip_wrap.return_value.get_devices.return_value = devices
ip_wrap.return_value.netns.exists.return_value = True
with mock.patch.object(util, 'unplug_device') as unplug:
with mock.patch.object(util, 'kill_dhcp') as kill_dhcp:
util.destroy_namespace(conf, ns, force)
expected = [mock.call('sudo', ns)]
if force:
expected.extend([
mock.call().netns.exists(ns),
mock.call().get_devices(exclude_loopback=True)])
self.assertTrue(kill_dhcp.called)
unplug.assert_has_calls(
[mock.call(conf, d) for d in
devices[1:]])
expected.append(mock.call().garbage_collect_namespace())
ip_wrap.assert_has_calls(expected)
def test_destory_namespace_empty(self):
self._test_destroy_namespace_helper(False, 0)
def test_destory_namespace_not_empty(self):
self._test_destroy_namespace_helper(False, 1)
def test_destory_namespace_not_empty_forced(self):
self._test_destroy_namespace_helper(True, 2)
def test_main(self):
namespaces = ['ns1', 'ns2']
with mock.patch('quantum.agent.linux.ip_lib.IPWrapper') as ip_wrap:
ip_wrap.get_namespaces.return_value = namespaces
with mock.patch('eventlet.sleep') as eventlet_sleep:
conf = mock.Mock()
conf.root_helper = 'sudo'
conf.force = False
methods_to_mock = dict(
eligible_for_deletion=mock.DEFAULT,
destroy_namespace=mock.DEFAULT,
setup_conf=mock.DEFAULT)
with mock.patch.multiple(util, **methods_to_mock) as mocks:
mocks['eligible_for_deletion'].return_value = True
mocks['setup_conf'].return_value = conf
util.main()
mocks['eligible_for_deletion'].assert_has_calls(
[mock.call(conf, 'ns1', False),
mock.call(conf, 'ns2', False)])
mocks['destroy_namespace'].assert_has_calls(
[mock.call(conf, 'ns1', False),
mock.call(conf, 'ns2', False)])
ip_wrap.assert_has_calls(
[mock.call.get_namespaces('sudo')])
eventlet_sleep.assert_called_once_with(2)
def test_main_no_candidates(self):
namespaces = ['ns1', 'ns2']
with mock.patch('quantum.agent.linux.ip_lib.IPWrapper') as ip_wrap:
ip_wrap.get_namespaces.return_value = namespaces
with mock.patch('eventlet.sleep') as eventlet_sleep:
conf = mock.Mock()
conf.root_helper = 'sudo'
conf.force = False
methods_to_mock = dict(
eligible_for_deletion=mock.DEFAULT,
destroy_namespace=mock.DEFAULT,
setup_conf=mock.DEFAULT)
with mock.patch.multiple(util, **methods_to_mock) as mocks:
mocks['eligible_for_deletion'].return_value = False
mocks['setup_conf'].return_value = conf
util.main()
ip_wrap.assert_has_calls(
[mock.call.get_namespaces('sudo')])
mocks['eligible_for_deletion'].assert_has_calls(
[mock.call(conf, 'ns1', False),
mock.call(conf, 'ns2', False)])
self.assertFalse(mocks['destroy_namespace'].called)
self.assertFalse(eventlet_sleep.called)

View File

@ -206,6 +206,72 @@ class TestIpWrapper(unittest.TestCase):
self.assertFalse(self.execute.called)
self.assertEqual(ns.namespace, 'ns')
def test_namespace_is_empty_no_devices(self):
ip = ip_lib.IPWrapper('sudo', 'ns')
with mock.patch.object(ip, 'get_devices') as get_devices:
get_devices.return_value = []
self.assertTrue(ip.namespace_is_empty())
get_devices.assert_called_once_with(exclude_loopback=True)
def test_namespace_is_empty(self):
ip = ip_lib.IPWrapper('sudo', 'ns')
with mock.patch.object(ip, 'get_devices') as get_devices:
get_devices.return_value = [mock.Mock()]
self.assertFalse(ip.namespace_is_empty())
get_devices.assert_called_once_with(exclude_loopback=True)
def test_garbage_collect_namespace_does_not_exist(self):
with mock.patch.object(ip_lib, 'IpNetnsCommand') as ip_ns_cmd_cls:
ip_ns_cmd_cls.return_value.exists.return_value = False
ip = ip_lib.IPWrapper('sudo', 'ns')
with mock.patch.object(ip, 'namespace_is_empty') as mock_is_empty:
self.assertFalse(ip.garbage_collect_namespace())
ip_ns_cmd_cls.assert_has_calls([mock.call().exists('ns')])
self.assertNotIn(mock.call().delete('ns'),
ip_ns_cmd_cls.return_value.mock_calls)
self.assertEqual(mock_is_empty.mock_calls, [])
def test_garbage_collect_namespace_existing_empty_ns(self):
with mock.patch.object(ip_lib, 'IpNetnsCommand') as ip_ns_cmd_cls:
ip_ns_cmd_cls.return_value.exists.return_value = True
ip = ip_lib.IPWrapper('sudo', 'ns')
with mock.patch.object(ip, 'namespace_is_empty') as mock_is_empty:
mock_is_empty.return_value = True
self.assertTrue(ip.garbage_collect_namespace())
mock_is_empty.assert_called_once_with()
expected = [mock.call().exists('ns'),
mock.call().delete('ns')]
ip_ns_cmd_cls.assert_has_calls(expected)
def test_garbage_collect_namespace_existing_not_empty(self):
lo_device = mock.Mock()
lo_device.name = 'lo'
tap_device = mock.Mock()
tap_device.name = 'tap1'
with mock.patch.object(ip_lib, 'IpNetnsCommand') as ip_ns_cmd_cls:
ip_ns_cmd_cls.return_value.exists.return_value = True
ip = ip_lib.IPWrapper('sudo', 'ns')
with mock.patch.object(ip, 'namespace_is_empty') as mock_is_empty:
mock_is_empty.return_value = False
self.assertFalse(ip.garbage_collect_namespace())
mock_is_empty.assert_called_once_with()
expected = [mock.call(ip),
mock.call().exists('ns')]
self.assertEqual(ip_ns_cmd_cls.mock_calls, expected)
self.assertNotIn(mock.call().delete('ns'),
ip_ns_cmd_cls.mock_calls)
def test_add_device_to_namespace(self):
dev = mock.Mock()
ip_lib.IPWrapper('sudo', 'ns').add_device_to_namespace(dev)

View File

@ -102,6 +102,7 @@ setuptools.setup(
'quantum-dhcp-agent = quantum.agent.dhcp_agent:main',
'quantum-dhcp-agent-dnsmasq-lease-update ='
'quantum.agent.linux.dhcp:Dnsmasq.lease_update',
'quantum-netns-cleanup = quantum.agent.netns_cleanup_util:main',
'quantum-l3-agent = quantum.agent.l3_nat_agent:main',
'quantum-linuxbridge-agent ='
'quantum.plugins.linuxbridge.agent.linuxbridge_quantum_agent:main',