From bebc0a4b2ea915fa214518ea667cd25812cda058 Mon Sep 17 00:00:00 2001 From: Huan Xie Date: Mon, 30 Nov 2015 09:24:54 +0000 Subject: [PATCH] XenAPI: Support neutron security group This implementation is to give support on neutron security group with XenServer as compute driver. When using neutron+openvswitch, the ovs agent on compute node cannot run correctly due to lack of qbr linux bridge on compute node. This change will add qbr linux bridge when xenserver as hypervisor Xenserver driver now doesn't have linux bridge, the connection is: compute node: vm-vif -> br-int -> br-eth network node: br-eth -> br-int -> br-ex With this implemented, linux bridge(qbr) will be added in compute node. Thus the security group rules can be applied on qbr bridge. The connection will look like: compute node: vm-vif -> qbr(linux bridge) -> br-int -> br-eth network node: br-eth -> br-int -> br-ex Closes-Bug: #1526138 Implements: blueprint support-neutron-security-group DocImpact: /etc/modprobe.d/blacklist-bridge file in dom0 should be deleted since it prevent loading linux bridge module in dom0 Depends-On: I377f8ad51e1d2725c3e0153e64322055fcce7b54 Change-Id: Id9b39aa86558a9f7099caedabd2d517bf8ad3d68 --- nova/tests/unit/virt/xenapi/test_vif.py | 24 ++- nova/virt/xenapi/client/session.py | 2 +- nova/virt/xenapi/fake.py | 2 +- nova/virt/xenapi/vif.py | 171 ++++++++++++++++-- .../etc/xapi.d/plugins/nova_plugin_version | 3 +- .../xenapi/etc/xapi.d/plugins/xenhost | 95 +++++++++- 6 files changed, 266 insertions(+), 31 deletions(-) diff --git a/nova/tests/unit/virt/xenapi/test_vif.py b/nova/tests/unit/virt/xenapi/test_vif.py index ebd3a9a85cc1..320a92a115c4 100644 --- a/nova/tests/unit/virt/xenapi/test_vif.py +++ b/nova/tests/unit/virt/xenapi/test_vif.py @@ -22,6 +22,7 @@ from nova.tests.unit.virt.xenapi import stubs from nova.virt.xenapi import network_utils from nova.virt.xenapi import vif + fake_vif = { 'created_at': None, 'updated_at': None, @@ -190,6 +191,10 @@ class XenAPIOpenVswitchDriverTestCase(XenVIFDriverTestBase): self.assertTrue(mock_create_vif.called) self.assertEqual('fake_vif_ref', ret_vif_ref) + @mock.patch.object(vif.XenAPIOpenVswitchDriver, '_delete_linux_bridge') + @mock.patch.object(vif.XenAPIOpenVswitchDriver, '_delete_linux_port') + @mock.patch.object(vif.XenAPIOpenVswitchDriver, '_device_exists', + return_value=True) @mock.patch.object(vif.XenAPIOpenVswitchDriver, '_ovs_del_br') @mock.patch.object(vif.XenAPIOpenVswitchDriver, '_ovs_del_port') @mock.patch.object(network_utils, 'find_network_with_name_label', @@ -198,7 +203,10 @@ class XenAPIOpenVswitchDriverTestCase(XenVIFDriverTestBase): def test_unplug(self, mock_super_unplug, mock_find_network_with_name_label, mock_ovs_del_port, - mock_ovs_del_br): + mock_ovs_del_br, + mock_device_exists, + mock_delete_linux_port, + mock_delete_linux_bridge): instance = {'name': "fake_instance"} vm_ref = "fake_vm_ref" @@ -241,10 +249,12 @@ class XenAPIOpenVswitchDriverTestCase(XenVIFDriverTestBase): self.ovs_driver.unplug, instance, fake_vif, vm_ref) - @mock.patch.object(vif.XenAPIOpenVswitchDriver, '_ovs_add_patch_port') - @mock.patch.object(vif.XenAPIOpenVswitchDriver, '_ovs_map_external_ids') - def test_post_start_actions(self, mock_ovs_map_external_ids, - mock_ovs_add_patch_port): + @mock.patch.object(vif.XenAPIOpenVswitchDriver, '_brctl_add_if') + @mock.patch.object(vif.XenAPIOpenVswitchDriver, '_create_linux_bridge') + @mock.patch.object(vif.XenAPIOpenVswitchDriver, '_ovs_add_port') + def test_post_start_actions(self, mock_ovs_add_port, + mock_create_linux_bridge, + mock_brctl_add_if): vif_ref = "fake_vif_ref" instance = {'name': 'fake_instance_name'} fake_vif_rec = {'uuid': fake_vif['uuid'], @@ -267,8 +277,8 @@ class XenAPIOpenVswitchDriverTestCase(XenVIFDriverTestBase): self.assertTrue(mock_VIF_get_record.called) self.assertTrue(mock_network_get_bridge.called) self.assertTrue(mock_network_get_uuid.called) - self.assertEqual(mock_ovs_add_patch_port.call_count, 2) - self.assertTrue(mock_ovs_map_external_ids.called) + self.assertEqual(mock_ovs_add_port.call_count, 1) + self.assertTrue(mock_brctl_add_if.called) @mock.patch.object(network_utils, 'find_network_with_name_label', return_value="exist_network_ref") diff --git a/nova/virt/xenapi/client/session.py b/nova/virt/xenapi/client/session.py index ada1b7b927cb..483c63a6f0ef 100644 --- a/nova/virt/xenapi/client/session.py +++ b/nova/virt/xenapi/client/session.py @@ -70,7 +70,7 @@ class XenAPISession(object): # changed in development environments. # MAJOR VERSION: Incompatible changes with the plugins # MINOR VERSION: Compatible changes, new plguins, etc - PLUGIN_REQUIRED_VERSION = '1.5' + PLUGIN_REQUIRED_VERSION = '1.6' def __init__(self, url, user, pw): version_string = version.version_string_with_package() diff --git a/nova/virt/xenapi/fake.py b/nova/virt/xenapi/fake.py index b368dac2fad4..12c038921c8e 100644 --- a/nova/virt/xenapi/fake.py +++ b/nova/virt/xenapi/fake.py @@ -760,7 +760,7 @@ class SessionBase(object): return base64.b64encode(zlib.compress("dom_id: %s" % dom_id)) def _plugin_nova_plugin_version_get_version(self, method, args): - return pickle.dumps("1.5") + return pickle.dumps("1.6") def _plugin_xenhost_query_gc(self, method, args): return pickle.dumps("False") diff --git a/nova/virt/xenapi/vif.py b/nova/virt/xenapi/vif.py index 79d27fcb56a7..48293091496d 100644 --- a/nova/virt/xenapi/vif.py +++ b/nova/virt/xenapi/vif.py @@ -225,11 +225,11 @@ class XenAPIOpenVswitchDriver(XenVIFDriver): def unplug(self, instance, vif, vm_ref): """unplug vif: - 1. unplug and destroy vif. - 2. delete the patch port pair between the integration bridge and - the interim network. - 3. destroy the interim network - 4. delete the OVS bridge service for the interim network + 1. delete the patch port pair between the integration bridge and + the qbr linux bridge(if exist) and the interim network. + 2. destroy the interim network + 3. delete the OVS bridge service for the interim network + 4. delete linux bridge qbr and related ports if exist """ super(XenAPIOpenVswitchDriver, self).unplug(instance, vif, vm_ref) @@ -248,12 +248,10 @@ class XenAPIOpenVswitchDriver(XenVIFDriver): LOG.debug('destroying patch port pair for vif: vif_id=%(vif_id)s', {'vif_id': vif['id']}) bridge_name = self._session.network.get_bridge(network) - patch_port1, patch_port2 = self._get_patch_port_pair_names(vif['id']) + patch_port1, tap_name = self._get_patch_port_pair_names(vif['id']) try: # delete the patch port pair self._ovs_del_port(bridge_name, patch_port1) - self._ovs_del_port(CONF.xenserver.ovs_integration_bridge, - patch_port2) except Exception as e: LOG.warn(_LW("Failed to delete patch port pair for vif %(if)s," " exception:%(exception)s"), @@ -272,6 +270,18 @@ class XenAPIOpenVswitchDriver(XenVIFDriver): # won't be destroyed automatically by XAPI. So let's destroy it # at here. self._ovs_del_br(bridge_name) + + qbr_name = self._get_qbr_name(vif['id']) + qvb_name, qvo_name = self._get_veth_pair_names(vif['id']) + if self._device_exists(qbr_name): + # delete tap port, qvb port and qbr + LOG.debug( + "destroy linux bridge %(qbr)s when unplug vif %(vif)s", + {'qbr': qbr_name, 'vif': vif['id']}) + self._delete_linux_port(qbr_name, tap_name) + self._delete_linux_port(qbr_name, qvb_name) + self._delete_linux_bridge(qbr_name) + self._ovs_del_port(CONF.xenserver.ovs_integration_bridge, qvo_name) except Exception as e: LOG.warn(_LW("Failed to delete bridge for vif %(if)s, " "exception:%(exception)s"), @@ -279,6 +289,88 @@ class XenAPIOpenVswitchDriver(XenVIFDriver): raise exception.VirtualInterfaceUnplugException( reason=_("Failed to delete bridge")) + def _get_qbr_name(self, iface_id): + return ("qbr" + iface_id)[:network_model.NIC_NAME_LEN] + + def _get_veth_pair_names(self, iface_id): + return (("qvb%s" % iface_id)[:network_model.NIC_NAME_LEN], + ("qvo%s" % iface_id)[:network_model.NIC_NAME_LEN]) + + def _device_exists(self, device): + """Check if ethernet device exists.""" + try: + cmd = 'ip_link_get_dev' + args = {'device_name': device} + self._exec_dom0_cmd(cmd, args) + return True + except Exception: + # Swallow exception from plugin, since this indicates the device + # doesn't exist + return False + + def _delete_net_dev(self, dev): + """Delete a network device only if it exists.""" + if self._device_exists(dev): + LOG.debug("delete network device '%s'", dev) + args = {'device_name': dev} + self._exec_dom0_cmd('ip_link_del_dev', args) + + def _create_veth_pair(self, dev1_name, dev2_name): + """Create a pair of veth devices with the specified names, + deleting any previous devices with those names. + """ + LOG.debug("Create veth pair, port1:%(qvb)s, port2:%(qvo)s", + {'qvb': dev1_name, 'qvo': dev2_name}) + for dev in [dev1_name, dev2_name]: + self._delete_net_dev(dev) + args = {'dev1_name': dev1_name, 'dev2_name': dev2_name} + self._exec_dom0_cmd('ip_link_add_veth_pair', args) + for dev in [dev1_name, dev2_name]: + args = {'device_name': dev, 'option': 'up'} + self._exec_dom0_cmd('ip_link_set_dev', args) + args = {'device_name': dev, 'option': 'on'} + self._exec_dom0_cmd('ip_link_set_promisc', args) + + def _create_linux_bridge(self, vif_rec): + """create a qbr linux bridge for neutron security group + """ + iface_id = vif_rec['other_config']['nicira-iface-id'] + linux_br_name = self._get_qbr_name(iface_id) + if not self._device_exists(linux_br_name): + LOG.debug("Create linux bridge %s", linux_br_name) + self._brctl_add_br(linux_br_name) + self._brctl_set_fd(linux_br_name, '0') + self._brctl_set_stp(linux_br_name, 'off') + args = {'device_name': linux_br_name, 'option': 'up'} + self._exec_dom0_cmd('ip_link_set_dev', args) + + qvb_name, qvo_name = self._get_veth_pair_names(iface_id) + if not self._device_exists(qvo_name): + self._create_veth_pair(qvb_name, qvo_name) + self._brctl_add_if(linux_br_name, qvb_name) + self._ovs_add_port(CONF.xenserver.ovs_integration_bridge, qvo_name) + self._ovs_map_external_ids(qvo_name, vif_rec) + return linux_br_name + + def _delete_linux_port(self, qbr_name, port_name): + try: + # delete port in linux bridge + self._brctl_del_if(qbr_name, port_name) + self._delete_net_dev(port_name) + except Exception: + LOG.debug("Fail to delete linux port %(port_name)s on bridge" + "%(qbr_name)s", + {'port_name': port_name, 'qbr_name': qbr_name}) + + def _delete_linux_bridge(self, qbr_name): + try: + # delete linux bridge qbrxxx + args = {'device_name': qbr_name, 'option': 'down'} + self._exec_dom0_cmd('ip_link_set_dev', args) + self._brctl_del_br(qbr_name) + except Exception: + LOG.debug("Fail to delete linux bridge %s", qbr_name) + def post_start_actions(self, instance, vif_ref): """Do needed actions post vif start: plug the interim ovs bridge to the integration bridge; @@ -290,19 +382,25 @@ class XenAPIOpenVswitchDriver(XenVIFDriver): bridge_name = self._session.network.get_bridge(network_ref) network_uuid = self._session.network.get_uuid(network_ref) iface_id = vif_rec['other_config']['nicira-iface-id'] - patch_port1, patch_port2 = self._get_patch_port_pair_names(iface_id) + patch_port1, tap_name = self._get_patch_port_pair_names(iface_id) LOG.debug('plug_ovs_bridge: port1=%(port1)s, port2=%(port2)s,' 'network_uuid=%(uuid)s, bridge_name=%(bridge_name)s', - {'port1': patch_port1, 'port2': patch_port2, + {'port1': patch_port1, 'port2': tap_name, 'uuid': network_uuid, 'bridge_name': bridge_name}) if bridge_name is None: raise exception.VirtualInterfacePlugException( _("Failed to find bridge for vif")) - self._ovs_add_patch_port(bridge_name, patch_port1, patch_port2) - self._ovs_add_patch_port(CONF.xenserver.ovs_integration_bridge, - patch_port2, patch_port1) - self._ovs_map_external_ids(patch_port2, vif_rec) + # Create Linux bridge qbrXXX + linux_br_name = self._create_linux_bridge(vif_rec) + LOG.debug("create veth pair for interim bridge %(interim_bridge)s and " + "linux bridge %(linux_bridge)s", + {'interim_bridge': bridge_name, + 'linux_bridge': linux_br_name}) + self._create_veth_pair(tap_name, patch_port1) + self._brctl_add_if(linux_br_name, tap_name) + # Add port to interim bridge + self._ovs_add_port(bridge_name, patch_port1) def get_vif_interim_net_name(self, vif): return ("net-" + vif['id'])[:network_model.NIC_NAME_LEN] @@ -330,14 +428,13 @@ class XenAPIOpenVswitchDriver(XenVIFDriver): return network_ref def _get_patch_port_pair_names(self, iface_id): - return (("pp1-%s" % iface_id)[:network_model.NIC_NAME_LEN], - ("pp2-%s" % iface_id)[:network_model.NIC_NAME_LEN]) + return (("vif%s" % iface_id)[:network_model.NIC_NAME_LEN], + ("tap%s" % iface_id)[:network_model.NIC_NAME_LEN]) - def _ovs_add_patch_port(self, bridge_name, port_name, peer_port_name): - cmd = 'ovs_add_patch_port' + def _ovs_add_port(self, bridge_name, port_name): + cmd = 'ovs_add_port' args = {'bridge_name': bridge_name, - 'port_name': port_name, - 'peer_port_name': peer_port_name + 'port_name': port_name } self._exec_dom0_cmd(cmd, args) @@ -373,6 +470,40 @@ class XenAPIOpenVswitchDriver(XenVIFDriver): self._ovs_set_if_external_id(interface, 'xs-vif-uuid', vif_uuid) self._ovs_set_if_external_id(interface, 'iface-status', status) + def _brctl_add_if(self, bridge_name, interface_name): + cmd = 'brctl_add_if' + args = {'bridge_name': bridge_name, + 'interface_name': interface_name} + self._exec_dom0_cmd(cmd, args) + + def _brctl_del_if(self, bridge_name, interface_name): + cmd = 'brctl_del_if' + args = {'bridge_name': bridge_name, + 'interface_name': interface_name} + self._exec_dom0_cmd(cmd, args) + + def _brctl_del_br(self, bridge_name): + cmd = 'brctl_del_br' + args = {'bridge_name': bridge_name} + self._exec_dom0_cmd(cmd, args) + + def _brctl_add_br(self, bridge_name): + cmd = 'brctl_add_br' + args = {'bridge_name': bridge_name} + self._exec_dom0_cmd(cmd, args) + + def _brctl_set_fd(self, bridge_name, fd): + cmd = 'brctl_set_fd' + args = {'bridge_name': bridge_name, + 'fd': fd} + self._exec_dom0_cmd(cmd, args) + + def _brctl_set_stp(self, bridge_name, stp_opt): + cmd = 'brctl_set_stp' + args = {'bridge_name': bridge_name, + 'option': stp_opt} + self._exec_dom0_cmd(cmd, args) + def _exec_dom0_cmd(self, cmd, cmd_args): args = {'cmd': cmd, 'args': cmd_args diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/nova_plugin_version b/plugins/xenserver/xenapi/etc/xapi.d/plugins/nova_plugin_version index e82a41396a7a..f700c6f6dbb7 100755 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/nova_plugin_version +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/nova_plugin_version @@ -31,7 +31,8 @@ import utils # 1.3 - Add vhd2 functions for doing glance operations by url # 1.4 - Add support of Glance v2 api # 1.5 - Added function for network configuration on ovs bridge -PLUGIN_VERSION = "1.5" +# 1.6 - Add function for network configuration on Linux bridge +PLUGIN_VERSION = "1.6" def get_version(session): diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/xenhost b/plugins/xenserver/xenapi/etc/xapi.d/plugins/xenhost index ed0ff0fd13de..045f3623e6aa 100755 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/xenhost +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/xenhost @@ -252,12 +252,105 @@ def _ovs_set_if_external_id(args): return _run_command(cmd_args) +def _ovs_add_port(args): + bridge_name = pluginlib.exists(args, 'bridge_name') + port_name = pluginlib.exists(args, 'port_name') + cmd_args = ['ovs-vsctl', '--', '--if-exists', 'del-port', port_name, + '--', 'add-port', bridge_name, port_name] + return _run_command(cmd_args) + + +def _ip_link_get_dev(args): + device_name = pluginlib.exists(args, 'device_name') + cmd_args = ['ip', 'link', 'show', device_name] + return _run_command(cmd_args) + + +def _ip_link_del_dev(args): + device_name = pluginlib.exists(args, 'device_name') + cmd_args = ['ip', 'link', 'delete', device_name] + return _run_command(cmd_args) + +def _ip_link_add_veth_pair(args): + dev1_name = pluginlib.exists(args, 'dev1_name') + dev2_name = pluginlib.exists(args, 'dev2_name') + cmd_args = ['ip', 'link', 'add', dev1_name, 'type', 'veth', 'peer', + 'name', dev2_name] + return _run_command(cmd_args) + + +def _ip_link_set_dev(args): + device_name = pluginlib.exists(args, 'device_name') + option = pluginlib.exists(args, 'option') + cmd_args = ['ip', 'link', 'set', device_name, option] + return _run_command(cmd_args) + + +def _ip_link_set_promisc(args): + device_name = pluginlib.exists(args, 'device_name') + option = pluginlib.exists(args, 'option') + cmd_args = ['ip', 'link', 'set', device_name, 'promisc', option] + return _run_command(cmd_args) + + +def _brctl_add_br(args): + bridge_name = pluginlib.exists(args, 'bridge_name') + cmd_args = ['brctl', 'addbr', bridge_name] + return _run_command(cmd_args) + + +def _brctl_del_br(args): + bridge_name = pluginlib.exists(args, 'bridge_name') + cmd_args = ['brctl', 'delbr', bridge_name] + return _run_command(cmd_args) + + +def _brctl_set_fd(args): + bridge_name = pluginlib.exists(args, 'bridge_name') + fd = pluginlib.exists(args, 'fd') + cmd_args = ['brctl', 'setfd', bridge_name, fd] + return _run_command(cmd_args) + + +def _brctl_set_stp(args): + bridge_name = pluginlib.exists(args, 'bridge_name') + option = pluginlib.exists(args, 'option') + cmd_args = ['brctl', 'stp', bridge_name, option] + return _run_command(cmd_args) + + +def _brctl_add_if(args): + bridge_name = pluginlib.exists(args, 'bridge_name') + if_name = pluginlib.exists(args, 'interface_name') + cmd_args = ['brctl', 'addif', bridge_name, if_name] + return _run_command(cmd_args) + + +def _brctl_del_if(args): + bridge_name = pluginlib.exists(args, 'bridge_name') + if_name = pluginlib.exists(args, 'interface_name') + cmd_args = ['brctl', 'delif', bridge_name, if_name] + return _run_command(cmd_args) + + ALLOWED_NETWORK_CMDS = { # allowed cmds to config OVS bridge 'ovs_add_patch_port': _ovs_add_patch_port, + 'ovs_add_port': _ovs_add_port, 'ovs_del_port': _ovs_del_port, 'ovs_del_br': _ovs_del_br, - 'ovs_set_if_external_id': _ovs_set_if_external_id + 'ovs_set_if_external_id': _ovs_set_if_external_id, + 'ip_link_add_veth_pair': _ip_link_add_veth_pair, + 'ip_link_del_dev': _ip_link_del_dev, + 'ip_link_get_dev': _ip_link_get_dev, + 'ip_link_set_dev': _ip_link_set_dev, + 'ip_link_set_promisc': _ip_link_set_promisc, + 'brctl_add_br': _brctl_add_br, + 'brctl_add_if': _brctl_add_if, + 'brctl_del_br': _brctl_del_br, + 'brctl_del_if': _brctl_del_if, + 'brctl_set_fd': _brctl_set_fd, + 'brctl_set_stp': _brctl_set_stp }