Enable VxLAN with XenServer
1. Remove the restriction of vxlan on yaml file 2. Configure iptables in Dom0 to enable vxlan port 4789 3. Move br-mesh from compute node to Dom0 4. Add mos-vxlan.service to take use of systemctl and make sure when XenServer host reboot, the br-mesh and related configurations is still there 5. Persist iptables rules 6. Add cleanup of br-mesh if it exist in in Dom0 when the neutron network type is not vxlan Change-Id: I1ab8ca56714167a2e20513b0f9b0ed4a82d9648e
This commit is contained in:
parent
9a2beb2950
commit
6689a9f9c9
|
@ -6,8 +6,6 @@
|
||||||
description: ''
|
description: ''
|
||||||
- name: 'hypervisor:qemu'
|
- name: 'hypervisor:qemu'
|
||||||
description: ''
|
description: ''
|
||||||
- name: 'network:neutron:ml2:tun'
|
|
||||||
description: ''
|
|
||||||
- name: 'additional_service:sahara'
|
- name: 'additional_service:sahara'
|
||||||
description: ''
|
description: ''
|
||||||
- name: 'additional_service:murano'
|
- name: 'additional_service:murano'
|
||||||
|
|
|
@ -2,6 +2,7 @@
|
||||||
|
|
||||||
import ConfigParser
|
import ConfigParser
|
||||||
from distutils.version import LooseVersion
|
from distutils.version import LooseVersion
|
||||||
|
import ipaddress
|
||||||
import netifaces
|
import netifaces
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
@ -14,6 +15,10 @@ from utils import HIMN_IP
|
||||||
|
|
||||||
|
|
||||||
INT_BRIDGE = 'br-int'
|
INT_BRIDGE = 'br-int'
|
||||||
|
MESH_BRIDGE = 'br-mesh'
|
||||||
|
AUTO_START_SERVICE = 'mos-vxlan.service'
|
||||||
|
AUTO_START_SERVICE_TEMPLATE = 'mos-vxlan-template.service'
|
||||||
|
AUTO_SCRIPT = 'fuel-xs-vxlan.sh'
|
||||||
XS_PLUGIN_ISO = 'xenapi-plugins-mitaka.iso'
|
XS_PLUGIN_ISO = 'xenapi-plugins-mitaka.iso'
|
||||||
CONNTRACK_CONF_SAMPLE =\
|
CONNTRACK_CONF_SAMPLE =\
|
||||||
'/usr/share/doc/conntrack-tools-1.4.2/doc/stats/conntrackd.conf'
|
'/usr/share/doc/conntrack-tools-1.4.2/doc/stats/conntrackd.conf'
|
||||||
|
@ -264,7 +269,7 @@ def modify_neutron_rootwrap_conf(himn, username, password):
|
||||||
LOG.info('Modify file %s successfully', filename)
|
LOG.info('Modify file %s successfully', filename)
|
||||||
|
|
||||||
|
|
||||||
def modify_neutron_ovs_agent_conf(int_br, br_mappings):
|
def modify_neutron_ovs_agent_conf(int_br, br_mappings=None, local_ip=None):
|
||||||
filename = '/etc/neutron/plugins/ml2/openvswitch_agent.ini'
|
filename = '/etc/neutron/plugins/ml2/openvswitch_agent.ini'
|
||||||
cf = ConfigParser.ConfigParser()
|
cf = ConfigParser.ConfigParser()
|
||||||
try:
|
try:
|
||||||
|
@ -274,7 +279,10 @@ def modify_neutron_ovs_agent_conf(int_br, br_mappings):
|
||||||
cf.set('agent', 'root_helper_daemon', '')
|
cf.set('agent', 'root_helper_daemon', '')
|
||||||
cf.set('agent', 'minimize_polling', False)
|
cf.set('agent', 'minimize_polling', False)
|
||||||
cf.set('ovs', 'integration_bridge', int_br)
|
cf.set('ovs', 'integration_bridge', int_br)
|
||||||
cf.set('ovs', 'bridge_mappings', br_mappings)
|
if br_mappings:
|
||||||
|
cf.set('ovs', 'bridge_mappings', br_mappings)
|
||||||
|
if local_ip:
|
||||||
|
cf.set('ovs', 'local_ip', local_ip)
|
||||||
with open(filename, 'w') as configfile:
|
with open(filename, 'w') as configfile:
|
||||||
cf.write(configfile)
|
cf.write(configfile)
|
||||||
except Exception:
|
except Exception:
|
||||||
|
@ -282,26 +290,28 @@ def modify_neutron_ovs_agent_conf(int_br, br_mappings):
|
||||||
LOG.info('Modify %s successfully', filename)
|
LOG.info('Modify %s successfully', filename)
|
||||||
|
|
||||||
|
|
||||||
def get_private_network_ethX():
|
def get_network_ethX(bridge_name):
|
||||||
# find out ethX in DomU which connect to private network
|
# find out ethX in DomU which connect to private network
|
||||||
# br-aux is the auxiliary bridge and in normal case there will be a patch
|
# br-aux is the auxiliary bridge and in normal case there will be a patch
|
||||||
# between br-prv and br-aux
|
# between br-prv and br-aux
|
||||||
values = astute['network_scheme']['transformations']
|
values = astute['network_scheme']['transformations']
|
||||||
for item in values:
|
for item in values:
|
||||||
if item['action'] == 'add-port' and item['bridge'] == 'br-aux':
|
if item['action'] == 'add-port' and item['bridge'] == bridge_name:
|
||||||
return item['name']
|
return item['name']
|
||||||
# If cannot find br-aux, the network topo should be public and private
|
# If cannot find given bridge, the network topo should be public and
|
||||||
# connect to the same network and "Assign public network to all nodes"
|
# private connecting to the same network and the checkbox
|
||||||
# is checked, we need to use br-ex to find ethX in domU
|
# "Assign public network to all nodes" is checked, we need to use br-ex
|
||||||
|
# to find ethX in domU
|
||||||
for item in values:
|
for item in values:
|
||||||
if item['action'] == 'add-port' and item['bridge'] == 'br-ex':
|
if item['action'] == 'add-port' and item['bridge'] == 'br-ex':
|
||||||
return item['name']
|
return item['name']
|
||||||
|
|
||||||
|
|
||||||
def find_bridge_mappings(astute, himn, username):
|
def find_dom0_bridge(himn, username, bridge_name):
|
||||||
ethX = get_private_network_ethX()
|
ethX = get_network_ethX(bridge_name)
|
||||||
if not ethX:
|
if not ethX:
|
||||||
utils.reportError("Cannot find eth used for private network")
|
utils.reportError("Cannot find eth used for private network")
|
||||||
|
ethX = ethX.split('.')[0]
|
||||||
|
|
||||||
# find the ethX mac in /sys/class/net/ethX/address
|
# find the ethX mac in /sys/class/net/ethX/address
|
||||||
with open('/sys/class/net/%s/address' % ethX, 'r') as fo:
|
with open('/sys/class/net/%s/address' % ethX, 'r') as fo:
|
||||||
|
@ -312,6 +322,12 @@ def find_bridge_mappings(astute, himn, username):
|
||||||
bridge = utils.ssh(himn, username,
|
bridge = utils.ssh(himn, username,
|
||||||
('xe network-param-get param-name=bridge '
|
('xe network-param-get param-name=bridge '
|
||||||
'uuid=%s') % network_uuid)
|
'uuid=%s') % network_uuid)
|
||||||
|
return bridge
|
||||||
|
|
||||||
|
|
||||||
|
def find_physical_network_mappings(astute, himn, username):
|
||||||
|
# find corresponding bridge in Dom0
|
||||||
|
bridge = find_dom0_bridge(himn, username, 'br-aux')
|
||||||
|
|
||||||
# find physical network name
|
# find physical network name
|
||||||
phynet_setting = astute['quantum_settings']['L2']['phys_nets']
|
phynet_setting = astute['quantum_settings']['L2']['phys_nets']
|
||||||
|
@ -458,6 +474,142 @@ def enable_conntrack_service(himn, username):
|
||||||
utils.ssh(himn, username, 'service', 'conntrackd', 'restart')
|
utils.ssh(himn, username, 'service', 'conntrackd', 'restart')
|
||||||
|
|
||||||
|
|
||||||
|
def configure_dom0_iptables(himn, username):
|
||||||
|
xs_chain = 'XenServer-Neutron-INPUT'
|
||||||
|
|
||||||
|
# Check XenServer specific chain, create if not exist
|
||||||
|
commands = ('iptables -t filter -L %s' % xs_chain,
|
||||||
|
'iptables -t filter --new %s' % xs_chain,
|
||||||
|
'iptables -t filter -I INPUT -j %s' % xs_chain)
|
||||||
|
execute_iptables_commands(himn, username, commands)
|
||||||
|
|
||||||
|
# Check XenServer rule for ovs native mode, create if not exist
|
||||||
|
commands = ('iptables -t filter -C %s -p tcp -m tcp --dport 6640 -j ACCEPT'
|
||||||
|
% xs_chain,
|
||||||
|
'iptables -t filter -I %s -p tcp --dport 6640 -j ACCEPT'
|
||||||
|
% xs_chain)
|
||||||
|
execute_iptables_commands(himn, username, commands)
|
||||||
|
|
||||||
|
# Check XenServer rule for vxlan, create if not exist
|
||||||
|
commands = ('iptables -t filter -C %s -p udp -m multiport --dports 4789 '
|
||||||
|
'-j ACCEPT' % xs_chain,
|
||||||
|
'iptables -t filter -I %s -p udp -m multiport --dport 4789 -j '
|
||||||
|
'ACCEPT' % xs_chain)
|
||||||
|
execute_iptables_commands(himn, username, commands)
|
||||||
|
|
||||||
|
# Persist iptables rules
|
||||||
|
utils.ssh(himn, username, 'service', 'iptables', 'save')
|
||||||
|
|
||||||
|
|
||||||
|
def execute_iptables_commands(himn, username, command_list):
|
||||||
|
# Execute first command and continue based on first command result
|
||||||
|
exitcode, _, _ = utils.ssh_detailed(
|
||||||
|
himn, username, command_list[0], allowed_return_codes=[0, 1])
|
||||||
|
if exitcode == 1:
|
||||||
|
for command in command_list[1:]:
|
||||||
|
LOG.info('Execute iptables command %s', command)
|
||||||
|
utils.ssh(himn, username, command)
|
||||||
|
|
||||||
|
|
||||||
|
def create_dom0_mesh_bridge(himn, username, dom0_bridge, mesh_info):
|
||||||
|
# Create br-mesh and veth pair if not exist in Dom0
|
||||||
|
exitcode, out, _ = utils.ssh_detailed(himn, username,
|
||||||
|
'ip', 'addr', 'show', MESH_BRIDGE,
|
||||||
|
allowed_return_codes=[0, 1])
|
||||||
|
if exitcode == 1:
|
||||||
|
# create mesh bridge if it not exist in Dom0
|
||||||
|
create_mesh_bridge = True
|
||||||
|
else:
|
||||||
|
# if mesh bridge exist in Dom0, check its ip, re-configure ip if
|
||||||
|
# it's not the same as what we want to set
|
||||||
|
bridge_info = out.split()
|
||||||
|
try:
|
||||||
|
index_inet = bridge_info.index('inet')
|
||||||
|
# get inet info like '192.168.2.2/24'
|
||||||
|
ipaddr = bridge_info[index_inet + 1].split('/')[0]
|
||||||
|
current_ip = ipaddress.ip_address(unicode(ipaddr))
|
||||||
|
configured_ip = ipaddress.ip_address(unicode(mesh_info['ipaddr']))
|
||||||
|
if current_ip == configured_ip:
|
||||||
|
LOG.info('Bridge %s already exist in Dom0' % MESH_BRIDGE)
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
create_mesh_bridge = True
|
||||||
|
remove_old_mesh_bridge(himn, username, MESH_BRIDGE)
|
||||||
|
except ValueError:
|
||||||
|
create_mesh_bridge = True
|
||||||
|
remove_old_mesh_bridge(himn, username, MESH_BRIDGE)
|
||||||
|
|
||||||
|
if create_mesh_bridge:
|
||||||
|
LOG.debug("Create mesh bridge in Dom0")
|
||||||
|
utils.scp(himn, username, '/etc/sysconfig/network-scripts/',
|
||||||
|
AUTO_SCRIPT)
|
||||||
|
utils.ssh(himn, username, 'chmod', '+x',
|
||||||
|
'/etc/sysconfig/network-scripts/%s' % AUTO_SCRIPT)
|
||||||
|
|
||||||
|
start_param = '%(bridge)s %(ip)s %(netmask)s %(broadcast)s %(tag)s' \
|
||||||
|
% {'bridge': dom0_bridge,
|
||||||
|
'ip': mesh_info['ipaddr'],
|
||||||
|
'netmask': mesh_info['netmask'],
|
||||||
|
'broadcast': mesh_info['broadcast'],
|
||||||
|
'tag': mesh_info['tag']}
|
||||||
|
with open(AUTO_START_SERVICE_TEMPLATE) as f:
|
||||||
|
contents = f.read()
|
||||||
|
contents = contents.replace('@MESH_INFO@', start_param)
|
||||||
|
with open(AUTO_START_SERVICE, 'w') as f:
|
||||||
|
f.write(contents)
|
||||||
|
utils.scp(himn, username, '/etc/systemd/system', AUTO_START_SERVICE)
|
||||||
|
utils.ssh(himn, username, 'systemctl', 'daemon-reload')
|
||||||
|
utils.ssh(himn, username, 'systemctl', 'enable', AUTO_START_SERVICE)
|
||||||
|
utils.ssh(himn, username, 'systemctl', 'start', AUTO_START_SERVICE)
|
||||||
|
|
||||||
|
|
||||||
|
def disable_local_mesh_bridge(bridge):
|
||||||
|
iface_list = netifaces.interfaces()
|
||||||
|
if bridge in iface_list:
|
||||||
|
utils.execute('ifconfig', bridge, '0.0.0.0')
|
||||||
|
utils.execute('ip', 'link', 'set', bridge, 'down')
|
||||||
|
|
||||||
|
filename = '/etc/network/interfaces.d/ifcfg-%s' % bridge
|
||||||
|
if os.path.isfile(filename):
|
||||||
|
utils.execute('rm', '-f', filename)
|
||||||
|
|
||||||
|
|
||||||
|
def get_mesh_info(astute, bridge):
|
||||||
|
mesh_nets = astute['network_scheme']['endpoints'][bridge]['IP'][0]
|
||||||
|
mesh_ip = mesh_nets.split('/')[0]
|
||||||
|
ipv4_net = ipaddress.ip_network(unicode(mesh_nets), strict=False)
|
||||||
|
mesh_broadcast = str(ipv4_net.broadcast_address)
|
||||||
|
network_netmask = str(ipv4_net.with_netmask).split('/')
|
||||||
|
mesh_netmask = network_netmask[1]
|
||||||
|
mesh_network = network_netmask[0]
|
||||||
|
mesh_eth = get_network_ethX(bridge)
|
||||||
|
mesh_tag = "''"
|
||||||
|
index = mesh_eth.index('.')
|
||||||
|
if index > 0:
|
||||||
|
mesh_tag = mesh_eth[index+1:]
|
||||||
|
mesh_info = {'ipaddr': mesh_ip, 'network': mesh_network,
|
||||||
|
'netmask': mesh_netmask, 'broadcast': mesh_broadcast,
|
||||||
|
'tag': mesh_tag}
|
||||||
|
return mesh_info
|
||||||
|
|
||||||
|
|
||||||
|
def remove_old_mesh_bridge(himn, username, bridge):
|
||||||
|
exitcode, _, _ = utils.ssh_detailed(himn, username, 'ip', 'link', 'show',
|
||||||
|
bridge, allowed_return_codes=[0, 1])
|
||||||
|
if exitcode == 0:
|
||||||
|
# Allow return code 5 to make sure it won't fail when
|
||||||
|
# mos-vxlan.service isn't exist
|
||||||
|
utils.ssh_detailed(himn, username, 'systemctl', 'stop',
|
||||||
|
AUTO_START_SERVICE, allowed_return_codes=[0, 5])
|
||||||
|
utils.ssh_detailed(himn, username, 'systemctl', 'disable',
|
||||||
|
AUTO_START_SERVICE, allowed_return_codes=[0, 1])
|
||||||
|
utils.ssh(himn, username, 'rm', '-f',
|
||||||
|
'/etc/systemd/system/%s' % AUTO_START_SERVICE)
|
||||||
|
utils.ssh(himn, username, 'rm', '-f',
|
||||||
|
'/etc/sysconfig/network-scripts/%s' % AUTO_SCRIPT)
|
||||||
|
utils.ssh(himn, username, 'systemctl', 'daemon-reload')
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
install_xenapi_sdk()
|
install_xenapi_sdk()
|
||||||
astute = utils.get_astute()
|
astute = utils.get_astute()
|
||||||
|
@ -493,10 +645,29 @@ if __name__ == '__main__':
|
||||||
# enable conntrackd service in Dom0
|
# enable conntrackd service in Dom0
|
||||||
enable_conntrack_service(HIMN_IP, username)
|
enable_conntrack_service(HIMN_IP, username)
|
||||||
|
|
||||||
|
# configure iptables in Dom0 to support ovs native mode and VxLAN
|
||||||
|
configure_dom0_iptables(HIMN_IP, username)
|
||||||
|
|
||||||
# neutron-l2-agent in compute node
|
# neutron-l2-agent in compute node
|
||||||
modify_neutron_rootwrap_conf(HIMN_IP, username, password)
|
modify_neutron_rootwrap_conf(HIMN_IP, username, password)
|
||||||
br_mappings = find_bridge_mappings(astute, HIMN_IP, username)
|
l2_net_type = astute['quantum_settings']['predefined_networks'][
|
||||||
modify_neutron_ovs_agent_conf(INT_BRIDGE, br_mappings)
|
'admin_internal_net']['L2']['network_type']
|
||||||
|
br_mappings = None
|
||||||
|
if l2_net_type == 'vlan':
|
||||||
|
br_mappings = find_physical_network_mappings(astute, HIMN_IP,
|
||||||
|
username)
|
||||||
|
remove_old_mesh_bridge(HIMN_IP, username, MESH_BRIDGE)
|
||||||
|
ip = None
|
||||||
|
if l2_net_type == 'tun':
|
||||||
|
dom0_priv_bridge = find_dom0_bridge(HIMN_IP, username,
|
||||||
|
MESH_BRIDGE)
|
||||||
|
mesh_info = get_mesh_info(astute, MESH_BRIDGE)
|
||||||
|
ip = mesh_info['ipaddr']
|
||||||
|
disable_local_mesh_bridge(MESH_BRIDGE)
|
||||||
|
create_dom0_mesh_bridge(HIMN_IP, username, dom0_priv_bridge,
|
||||||
|
mesh_info)
|
||||||
|
modify_neutron_ovs_agent_conf(INT_BRIDGE, br_mappings=br_mappings,
|
||||||
|
local_ip=ip)
|
||||||
patch_neutron_ovs_agent()
|
patch_neutron_ovs_agent()
|
||||||
restart_services('neutron-openvswitch-agent')
|
restart_services('neutron-openvswitch-agent')
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,63 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
OP=$1
|
||||||
|
COUNT=$#
|
||||||
|
|
||||||
|
function create_mesh_bridge {
|
||||||
|
local dom0_bridge=$1
|
||||||
|
local mesh_ip=$2
|
||||||
|
local mesh_netmask=$3
|
||||||
|
local mesh_broadcast=$4
|
||||||
|
local tag=$5
|
||||||
|
|
||||||
|
ip link show br-mesh
|
||||||
|
exitcode=$?
|
||||||
|
if [ "$exitcode" == "1" ]; then
|
||||||
|
brctl addbr br-mesh
|
||||||
|
brctl setfd br-mesh 0
|
||||||
|
brctl stp br-mesh off
|
||||||
|
ip link set br-mesh up
|
||||||
|
ip link delete mesh_ovs
|
||||||
|
ip link add mesh_ovs type veth peer name mesh_linux
|
||||||
|
ip link set mesh_ovs up
|
||||||
|
ip link set mesh_ovs promisc on
|
||||||
|
ip link set mesh_linux up
|
||||||
|
ip link set mesh_linux promisc on
|
||||||
|
brctl addif br-mesh mesh_linux
|
||||||
|
ovs-vsctl -- --if-exists del-port mesh_ovs -- add-port $dom0_bridge mesh_ovs
|
||||||
|
ip addr add $mesh_ip/$mesh_netmask broadcast $mesh_broadcast dev br-mesh
|
||||||
|
if [ -n "$tag" ]; then
|
||||||
|
ovs-vsctl -- set Port mesh_ovs tag=$tag
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
function delete_mesh_bridge {
|
||||||
|
ip link show br-mesh
|
||||||
|
exitcode=$?
|
||||||
|
if [ "$exitcode" == "0" ]; then
|
||||||
|
ip link set br-mesh down
|
||||||
|
ip link set mesh_ovs down
|
||||||
|
ip link delete mesh_ovs
|
||||||
|
ovs-vsctl -- --if-exist del-port mesh_ovs
|
||||||
|
brctl delbr br-mesh
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
if [ "$OP" == "start" ]; then
|
||||||
|
if [ $COUNT -lt 6 ]; then
|
||||||
|
echo "usage: fuel-xs-vlan.sh start BRIDGE IP NETMASK BROADCAST [VLAN_TAG]"
|
||||||
|
echo "Exit due to lack of parameters!"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
dom0_bridge=$2
|
||||||
|
mesh_ip=$3
|
||||||
|
mesh_netmask=$4
|
||||||
|
mesh_broadcast=$5
|
||||||
|
tag=$6
|
||||||
|
create_mesh_bridge $dom0_bridge $mesh_ip $mesh_netmask $mesh_broadcast $tag
|
||||||
|
elif [ "$OP" == "stop" ]; then
|
||||||
|
delete_mesh_bridge
|
||||||
|
fi
|
|
@ -0,0 +1,14 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Configure Mirantis OpenStack mesh bridge
|
||||||
|
Requires=xcp-networkd.service openvswitch-xapi-sync.service
|
||||||
|
After=xcp-networkd.service openvswitch-xapi-sync.service
|
||||||
|
AssertPathExists=/etc/sysconfig/network-scripts/
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
RemainAfterExit=yes
|
||||||
|
ExecStart=/bin/bash /etc/sysconfig/network-scripts/fuel-xs-vxlan.sh start @MESH_INFO@
|
||||||
|
ExecStop=/bin/bash /etc/sysconfig/network-scripts/fuel-xs-vxlan.sh stop
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
|
@ -67,6 +67,7 @@ def detailed_execute(*cmd, **kwargs):
|
||||||
LOG.info('Swallowed acceptable return code of %d',
|
LOG.info('Swallowed acceptable return code of %d',
|
||||||
proc.returncode)
|
proc.returncode)
|
||||||
else:
|
else:
|
||||||
|
LOG.warn('proc.returncode: %s', proc.returncode)
|
||||||
raise ExecutionError(err)
|
raise ExecutionError(err)
|
||||||
|
|
||||||
return proc.returncode, out, err
|
return proc.returncode, out, err
|
||||||
|
|
Loading…
Reference in New Issue