Browse Source

Enable VxLAN with XenServer

1. Remove the restriction of vxlan on yaml file
2. Configure iptables in Dom0 to enable vxlan port 4789
3. Move br-mesh from compute node to Dom0
4. Add mos-vxlan.service to take use of systemctl and make
   sure when XenServer host reboot, the br-mesh and related
   configurations is still there
5. Persist iptables rules
6. Add cleanup of br-mesh if it exist in in Dom0 when the
   neutron network type is not vxlan

Change-Id: I1ab8ca56714167a2e20513b0f9b0ed4a82d9648e
Huan Xie 2 years ago
parent
commit
6689a9f9c9

+ 0
- 2
plugin_source/components.yaml View File

@@ -6,8 +6,6 @@
6 6
       description: ''
7 7
     - name: 'hypervisor:qemu'
8 8
       description: ''
9
-    - name: 'network:neutron:ml2:tun'
10
-      description: ''
11 9
     - name: 'additional_service:sahara'
12 10
       description: ''
13 11
     - name: 'additional_service:murano'

+ 182
- 11
plugin_source/deployment_scripts/compute_post_deployment.py View File

@@ -2,6 +2,7 @@
2 2
 
3 3
 import ConfigParser
4 4
 from distutils.version import LooseVersion
5
+import ipaddress
5 6
 import netifaces
6 7
 import os
7 8
 import re
@@ -14,6 +15,10 @@ from utils import HIMN_IP
14 15
 
15 16
 
16 17
 INT_BRIDGE = 'br-int'
18
+MESH_BRIDGE = 'br-mesh'
19
+AUTO_START_SERVICE = 'mos-vxlan.service'
20
+AUTO_START_SERVICE_TEMPLATE = 'mos-vxlan-template.service'
21
+AUTO_SCRIPT = 'fuel-xs-vxlan.sh'
17 22
 XS_PLUGIN_ISO = 'xenapi-plugins-mitaka.iso'
18 23
 CONNTRACK_CONF_SAMPLE =\
19 24
     '/usr/share/doc/conntrack-tools-1.4.2/doc/stats/conntrackd.conf'
@@ -264,7 +269,7 @@ def modify_neutron_rootwrap_conf(himn, username, password):
264 269
     LOG.info('Modify file %s successfully', filename)
265 270
 
266 271
 
267
-def modify_neutron_ovs_agent_conf(int_br, br_mappings):
272
+def modify_neutron_ovs_agent_conf(int_br, br_mappings=None, local_ip=None):
268 273
     filename = '/etc/neutron/plugins/ml2/openvswitch_agent.ini'
269 274
     cf = ConfigParser.ConfigParser()
270 275
     try:
@@ -274,7 +279,10 @@ def modify_neutron_ovs_agent_conf(int_br, br_mappings):
274 279
         cf.set('agent', 'root_helper_daemon', '')
275 280
         cf.set('agent', 'minimize_polling', False)
276 281
         cf.set('ovs', 'integration_bridge', int_br)
277
-        cf.set('ovs', 'bridge_mappings', br_mappings)
282
+        if br_mappings:
283
+            cf.set('ovs', 'bridge_mappings', br_mappings)
284
+        if local_ip:
285
+            cf.set('ovs', 'local_ip', local_ip)
278 286
         with open(filename, 'w') as configfile:
279 287
             cf.write(configfile)
280 288
     except Exception:
@@ -282,26 +290,28 @@ def modify_neutron_ovs_agent_conf(int_br, br_mappings):
282 290
     LOG.info('Modify %s successfully', filename)
283 291
 
284 292
 
285
-def get_private_network_ethX():
293
+def get_network_ethX(bridge_name):
286 294
     # find out ethX in DomU which connect to private network
287 295
     # br-aux is the auxiliary bridge and in normal case there will be a patch
288 296
     # between br-prv and br-aux
289 297
     values = astute['network_scheme']['transformations']
290 298
     for item in values:
291
-        if item['action'] == 'add-port' and item['bridge'] == 'br-aux':
299
+        if item['action'] == 'add-port' and item['bridge'] == bridge_name:
292 300
             return item['name']
293
-    # If cannot find br-aux, the network topo should be public and private
294
-    # connect to the same network and "Assign public network to all nodes"
295
-    # is checked, we need to use br-ex to find ethX in domU
301
+    # If cannot find given bridge, the network topo should be public and
302
+    # private connecting to the same network and the checkbox
303
+    # "Assign public network to all nodes" is checked, we need to use br-ex
304
+    # to find ethX in domU
296 305
     for item in values:
297 306
         if item['action'] == 'add-port' and item['bridge'] == 'br-ex':
298 307
             return item['name']
299 308
 
300 309
 
301
-def find_bridge_mappings(astute, himn, username):
302
-    ethX = get_private_network_ethX()
310
+def find_dom0_bridge(himn, username, bridge_name):
311
+    ethX = get_network_ethX(bridge_name)
303 312
     if not ethX:
304 313
         utils.reportError("Cannot find eth used for private network")
314
+    ethX = ethX.split('.')[0]
305 315
 
306 316
     # find the ethX mac in /sys/class/net/ethX/address
307 317
     with open('/sys/class/net/%s/address' % ethX, 'r') as fo:
@@ -312,6 +322,12 @@ def find_bridge_mappings(astute, himn, username):
312 322
     bridge = utils.ssh(himn, username,
313 323
                        ('xe network-param-get param-name=bridge '
314 324
                         'uuid=%s') % network_uuid)
325
+    return bridge
326
+
327
+
328
+def find_physical_network_mappings(astute, himn, username):
329
+    # find corresponding bridge in Dom0
330
+    bridge = find_dom0_bridge(himn, username, 'br-aux')
315 331
 
316 332
     # find physical network name
317 333
     phynet_setting = astute['quantum_settings']['L2']['phys_nets']
@@ -458,6 +474,142 @@ def enable_conntrack_service(himn, username):
458 474
     utils.ssh(himn, username, 'service', 'conntrackd', 'restart')
459 475
 
460 476
 
477
+def configure_dom0_iptables(himn, username):
478
+    xs_chain = 'XenServer-Neutron-INPUT'
479
+
480
+    # Check XenServer specific chain, create if not exist
481
+    commands = ('iptables -t filter -L %s' % xs_chain,
482
+                'iptables -t filter --new %s' % xs_chain,
483
+                'iptables -t filter -I INPUT -j %s' % xs_chain)
484
+    execute_iptables_commands(himn, username, commands)
485
+
486
+    # Check XenServer rule for ovs native mode, create if not exist
487
+    commands = ('iptables -t filter -C %s -p tcp -m tcp --dport 6640 -j ACCEPT'
488
+                % xs_chain,
489
+                'iptables -t filter -I %s -p tcp --dport 6640 -j ACCEPT'
490
+                % xs_chain)
491
+    execute_iptables_commands(himn, username, commands)
492
+
493
+    # Check XenServer rule for vxlan, create if not exist
494
+    commands = ('iptables -t filter -C %s -p udp -m multiport --dports 4789 '
495
+                '-j ACCEPT' % xs_chain,
496
+                'iptables -t filter -I %s -p udp -m multiport --dport 4789 -j '
497
+                'ACCEPT' % xs_chain)
498
+    execute_iptables_commands(himn, username, commands)
499
+
500
+    # Persist iptables rules
501
+    utils.ssh(himn, username, 'service', 'iptables', 'save')
502
+
503
+
504
+def execute_iptables_commands(himn, username, command_list):
505
+    # Execute first command and continue based on first command result
506
+    exitcode, _, _ = utils.ssh_detailed(
507
+        himn, username, command_list[0], allowed_return_codes=[0, 1])
508
+    if exitcode == 1:
509
+        for command in command_list[1:]:
510
+            LOG.info('Execute iptables command %s', command)
511
+            utils.ssh(himn, username, command)
512
+
513
+
514
+def create_dom0_mesh_bridge(himn, username, dom0_bridge, mesh_info):
515
+    # Create br-mesh and veth pair if not exist in Dom0
516
+    exitcode, out, _ = utils.ssh_detailed(himn, username,
517
+                                          'ip', 'addr', 'show', MESH_BRIDGE,
518
+                                          allowed_return_codes=[0, 1])
519
+    if exitcode == 1:
520
+        # create mesh bridge if it not exist in Dom0
521
+        create_mesh_bridge = True
522
+    else:
523
+        # if mesh bridge exist in Dom0, check its ip, re-configure ip if
524
+        # it's not the same as what we want to set
525
+        bridge_info = out.split()
526
+        try:
527
+            index_inet = bridge_info.index('inet')
528
+            # get inet info like '192.168.2.2/24'
529
+            ipaddr = bridge_info[index_inet + 1].split('/')[0]
530
+            current_ip = ipaddress.ip_address(unicode(ipaddr))
531
+            configured_ip = ipaddress.ip_address(unicode(mesh_info['ipaddr']))
532
+            if current_ip == configured_ip:
533
+                LOG.info('Bridge %s already exist in Dom0' % MESH_BRIDGE)
534
+                return
535
+            else:
536
+                create_mesh_bridge = True
537
+                remove_old_mesh_bridge(himn, username, MESH_BRIDGE)
538
+        except ValueError:
539
+            create_mesh_bridge = True
540
+            remove_old_mesh_bridge(himn, username, MESH_BRIDGE)
541
+
542
+    if create_mesh_bridge:
543
+        LOG.debug("Create mesh bridge in Dom0")
544
+        utils.scp(himn, username, '/etc/sysconfig/network-scripts/',
545
+                  AUTO_SCRIPT)
546
+        utils.ssh(himn, username, 'chmod', '+x',
547
+                  '/etc/sysconfig/network-scripts/%s' % AUTO_SCRIPT)
548
+
549
+        start_param = '%(bridge)s %(ip)s %(netmask)s %(broadcast)s %(tag)s' \
550
+                      % {'bridge': dom0_bridge,
551
+                         'ip': mesh_info['ipaddr'],
552
+                         'netmask': mesh_info['netmask'],
553
+                         'broadcast': mesh_info['broadcast'],
554
+                         'tag': mesh_info['tag']}
555
+        with open(AUTO_START_SERVICE_TEMPLATE) as f:
556
+            contents = f.read()
557
+        contents = contents.replace('@MESH_INFO@', start_param)
558
+        with open(AUTO_START_SERVICE, 'w') as f:
559
+            f.write(contents)
560
+        utils.scp(himn, username, '/etc/systemd/system', AUTO_START_SERVICE)
561
+        utils.ssh(himn, username, 'systemctl', 'daemon-reload')
562
+        utils.ssh(himn, username, 'systemctl', 'enable', AUTO_START_SERVICE)
563
+        utils.ssh(himn, username, 'systemctl', 'start', AUTO_START_SERVICE)
564
+
565
+
566
+def disable_local_mesh_bridge(bridge):
567
+    iface_list = netifaces.interfaces()
568
+    if bridge in iface_list:
569
+        utils.execute('ifconfig', bridge, '0.0.0.0')
570
+        utils.execute('ip', 'link', 'set', bridge, 'down')
571
+
572
+    filename = '/etc/network/interfaces.d/ifcfg-%s' % bridge
573
+    if os.path.isfile(filename):
574
+        utils.execute('rm', '-f', filename)
575
+
576
+
577
+def get_mesh_info(astute, bridge):
578
+    mesh_nets = astute['network_scheme']['endpoints'][bridge]['IP'][0]
579
+    mesh_ip = mesh_nets.split('/')[0]
580
+    ipv4_net = ipaddress.ip_network(unicode(mesh_nets), strict=False)
581
+    mesh_broadcast = str(ipv4_net.broadcast_address)
582
+    network_netmask = str(ipv4_net.with_netmask).split('/')
583
+    mesh_netmask = network_netmask[1]
584
+    mesh_network = network_netmask[0]
585
+    mesh_eth = get_network_ethX(bridge)
586
+    mesh_tag = "''"
587
+    index = mesh_eth.index('.')
588
+    if index > 0:
589
+        mesh_tag = mesh_eth[index+1:]
590
+    mesh_info = {'ipaddr': mesh_ip, 'network': mesh_network,
591
+                 'netmask': mesh_netmask, 'broadcast': mesh_broadcast,
592
+                 'tag': mesh_tag}
593
+    return mesh_info
594
+
595
+
596
+def remove_old_mesh_bridge(himn, username, bridge):
597
+    exitcode, _, _ = utils.ssh_detailed(himn, username, 'ip', 'link', 'show',
598
+                                        bridge, allowed_return_codes=[0, 1])
599
+    if exitcode == 0:
600
+        # Allow return code 5 to make sure it won't fail when
601
+        # mos-vxlan.service isn't exist
602
+        utils.ssh_detailed(himn, username, 'systemctl', 'stop',
603
+                           AUTO_START_SERVICE, allowed_return_codes=[0, 5])
604
+        utils.ssh_detailed(himn, username, 'systemctl', 'disable',
605
+                           AUTO_START_SERVICE, allowed_return_codes=[0, 1])
606
+        utils.ssh(himn, username, 'rm', '-f',
607
+                  '/etc/systemd/system/%s' % AUTO_START_SERVICE)
608
+        utils.ssh(himn, username, 'rm', '-f',
609
+                  '/etc/sysconfig/network-scripts/%s' % AUTO_SCRIPT)
610
+        utils.ssh(himn, username, 'systemctl', 'daemon-reload')
611
+
612
+
461 613
 if __name__ == '__main__':
462 614
     install_xenapi_sdk()
463 615
     astute = utils.get_astute()
@@ -493,10 +645,29 @@ if __name__ == '__main__':
493 645
             # enable conntrackd service in Dom0
494 646
             enable_conntrack_service(HIMN_IP, username)
495 647
 
648
+            # configure iptables in Dom0 to support ovs native mode and VxLAN
649
+            configure_dom0_iptables(HIMN_IP, username)
650
+
496 651
             # neutron-l2-agent in compute node
497 652
             modify_neutron_rootwrap_conf(HIMN_IP, username, password)
498
-            br_mappings = find_bridge_mappings(astute, HIMN_IP, username)
499
-            modify_neutron_ovs_agent_conf(INT_BRIDGE, br_mappings)
653
+            l2_net_type = astute['quantum_settings']['predefined_networks'][
654
+                'admin_internal_net']['L2']['network_type']
655
+            br_mappings = None
656
+            if l2_net_type == 'vlan':
657
+                br_mappings = find_physical_network_mappings(astute, HIMN_IP,
658
+                                                             username)
659
+                remove_old_mesh_bridge(HIMN_IP, username, MESH_BRIDGE)
660
+            ip = None
661
+            if l2_net_type == 'tun':
662
+                dom0_priv_bridge = find_dom0_bridge(HIMN_IP, username,
663
+                                                    MESH_BRIDGE)
664
+                mesh_info = get_mesh_info(astute, MESH_BRIDGE)
665
+                ip = mesh_info['ipaddr']
666
+                disable_local_mesh_bridge(MESH_BRIDGE)
667
+                create_dom0_mesh_bridge(HIMN_IP, username, dom0_priv_bridge,
668
+                                        mesh_info)
669
+            modify_neutron_ovs_agent_conf(INT_BRIDGE, br_mappings=br_mappings,
670
+                                          local_ip=ip)
500 671
             patch_neutron_ovs_agent()
501 672
             restart_services('neutron-openvswitch-agent')
502 673
 

+ 63
- 0
plugin_source/deployment_scripts/fuel-xs-vxlan.sh View File

@@ -0,0 +1,63 @@
1
+#!/bin/bash
2
+
3
+OP=$1
4
+COUNT=$#
5
+
6
+function create_mesh_bridge {
7
+    local dom0_bridge=$1
8
+    local mesh_ip=$2
9
+    local mesh_netmask=$3
10
+    local mesh_broadcast=$4
11
+    local tag=$5
12
+
13
+    ip link show br-mesh
14
+    exitcode=$?
15
+    if [ "$exitcode" == "1" ]; then
16
+        brctl addbr br-mesh
17
+        brctl setfd br-mesh 0
18
+        brctl stp br-mesh off
19
+        ip link set br-mesh up
20
+        ip link delete mesh_ovs
21
+        ip link add mesh_ovs type veth peer name mesh_linux
22
+        ip link set mesh_ovs up
23
+        ip link set mesh_ovs promisc on
24
+        ip link set mesh_linux up
25
+        ip link set mesh_linux promisc on
26
+        brctl addif br-mesh mesh_linux
27
+        ovs-vsctl -- --if-exists del-port mesh_ovs -- add-port $dom0_bridge mesh_ovs
28
+        ip addr add $mesh_ip/$mesh_netmask broadcast $mesh_broadcast dev br-mesh
29
+        if [ -n "$tag" ]; then
30
+            ovs-vsctl -- set Port mesh_ovs tag=$tag
31
+        fi
32
+    fi
33
+}
34
+
35
+function delete_mesh_bridge {
36
+    ip link show br-mesh
37
+    exitcode=$?
38
+    if [ "$exitcode" == "0" ]; then
39
+        ip link set br-mesh down
40
+        ip link set mesh_ovs down
41
+        ip link delete mesh_ovs
42
+        ovs-vsctl -- --if-exist del-port mesh_ovs
43
+        brctl delbr br-mesh
44
+    fi
45
+}
46
+
47
+
48
+if [ "$OP" == "start" ]; then
49
+    if [ $COUNT -lt 6 ]; then
50
+        echo "usage: fuel-xs-vlan.sh start BRIDGE IP NETMASK BROADCAST [VLAN_TAG]"
51
+        echo "Exit due to lack of parameters!"
52
+        exit 0
53
+    fi
54
+
55
+    dom0_bridge=$2
56
+    mesh_ip=$3
57
+    mesh_netmask=$4
58
+    mesh_broadcast=$5
59
+    tag=$6
60
+    create_mesh_bridge $dom0_bridge $mesh_ip $mesh_netmask $mesh_broadcast $tag
61
+elif [ "$OP" == "stop" ]; then
62
+    delete_mesh_bridge
63
+fi

+ 14
- 0
plugin_source/deployment_scripts/mos-vxlan-template.service View File

@@ -0,0 +1,14 @@
1
+[Unit]
2
+Description=Configure Mirantis OpenStack mesh bridge
3
+Requires=xcp-networkd.service openvswitch-xapi-sync.service
4
+After=xcp-networkd.service openvswitch-xapi-sync.service
5
+AssertPathExists=/etc/sysconfig/network-scripts/
6
+
7
+[Service]
8
+Type=oneshot
9
+RemainAfterExit=yes
10
+ExecStart=/bin/bash /etc/sysconfig/network-scripts/fuel-xs-vxlan.sh start @MESH_INFO@
11
+ExecStop=/bin/bash /etc/sysconfig/network-scripts/fuel-xs-vxlan.sh stop
12
+
13
+[Install]
14
+WantedBy=multi-user.target

+ 1
- 0
plugin_source/deployment_scripts/utils.py View File

@@ -67,6 +67,7 @@ def detailed_execute(*cmd, **kwargs):
67 67
             LOG.info('Swallowed acceptable return code of %d',
68 68
                      proc.returncode)
69 69
         else:
70
+            LOG.warn('proc.returncode: %s', proc.returncode)
70 71
             raise ExecutionError(err)
71 72
 
72 73
     return proc.returncode, out, err

Loading…
Cancel
Save