From 491e5ade3bcde161127bf90ef89ed934776e1cb0 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Tue, 28 Oct 2014 16:57:00 -0700 Subject: [PATCH] Separate public and flat interfaces for multinode Expand the nova network gre bridging used in aiopcpu testing to create two layer two networks. Without this nova's networking sets up ebtables such that the two layer 3 networks cannot talk properly on the same layer 2 gre setup. Change-Id: I814d234bc30d67af40c2fe55be90a5e6b0024135 --- devstack-vm-gate.sh | 23 ++++++++++++++--- functions.sh | 61 +++++++++++++++++++++++++++++++++------------ 2 files changed, 64 insertions(+), 20 deletions(-) diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 26222dce..a51b7ae1 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -34,7 +34,21 @@ echo $PPID > $WORKSPACE/gate.pid source `dirname "$(readlink -f "$0")"`/functions.sh FIXED_RANGE=${DEVSTACK_GATE_FIXED_RANGE:-10.1.0.0/20} -FLOATING_RANGE=${DEVSTACK_GATE_FLOATING_RANGE:-172.24.4.0/24} +FLOATING_RANGE=${DEVSTACK_GATE_FLOATING_RANGE:-172.24.5.0/24} +PUBLIC_NETWORK_GATEWAY=${DEVSTACK_GATE_PUBLIC_NETWORK_GATEWAY:-172.24.5.1} +# The next two values are used in multinode testing and are related +# to the floating range. For multinode test envs to know how to route +# packets to floating IPs on other hosts we put addresses on the compute +# node interfaces on a network that overlaps the FLOATING_RANGE. This +# automagically sets up routing in a sane way. By default we put floating +# IPs on 172.24.5.0/24 and compute nodes get addresses in the 172.24.4/23 +# space. Note that while the FLOATING_RANGE should overlap the +# FLOATING_HOST_* space you should have enough sequential room starting at +# the beginning of your FLOATING_HOST range to give one IP address to each +# compute host without letting compute host IPs run into the FLOATING_RANGE. +# By default this lets us have 255 compute hosts (172.24.4.1 - 172.24.4.255). +FLOATING_HOST_PREFIX=${DEVSTACK_GATE_FLOATING_HOST_PREFIX:-172.24.4} +FLOATING_HOST_MASK=${DEVSTACK_GATE_FLOATING_HOST_MASK:-23} function setup_localrc { local localrc_oldnew=$1; @@ -124,6 +138,7 @@ LOGFILE=$BASE/$localrc_oldnew/devstacklog.txt VERBOSE=True FIXED_RANGE=$FIXED_RANGE FLOATING_RANGE=$FLOATING_RANGE +PUBLIC_NETWORK_GATEWAY=$PUBLIC_NETWORK_GATEWAY FIXED_NETWORK_SIZE=4096 VIRT_DRIVER=$DEVSTACK_GATE_VIRT_DRIVER SWIFT_REPLICAS=1 @@ -436,13 +451,13 @@ else NODES="$PRIMARY_NODE $SUB_NODES" if [[ "$DEVSTACK_GATE_NEUTRON" -ne '1' ]]; then (source $BASE/new/devstack/functions-common; install_package bridge-utils) - gre_bridge "pub_if" 1 $NODES + gre_bridge "flat_if" "pub_if" 1 $FLOATING_HOST_PREFIX $FLOATING_HOST_MASK $NODES cat <>"$BASE/new/devstack/sub_localrc" -FLAT_INTERFACE=pub_if +FLAT_INTERFACE=flat_if PUBLIC_INTERFACE=pub_if EOF cat <>"$BASE/new/devstack/localrc" -FLAT_INTERFACE=pub_if +FLAT_INTERFACE=flat_if PUBLIC_INTERFACE=pub_if EOF fi diff --git a/functions.sh b/functions.sh index 38f78e66..24d22093 100644 --- a/functions.sh +++ b/functions.sh @@ -770,30 +770,59 @@ function remote_copy_file { scp $ssh_opts "$src" "$dest" } -# if_name: Interface name on each host +# flat_if_name: Interface name on each host for the "flat" network +# pub_if_name: Interface name on each host for the "public" network. +# IPv4 addresses will be assigned to these interfaces using +# the details provided below. # offset: starting key value for the gre tunnels (MUST not be overlapping) +# note that two keys are used for each subnode. one for flat +# interface and the other for the pub interface. +# pub_addr_prefix: The IPv4 address three octet prefix used to give compute +# nodes non conflicting addresses on the pub_if_name'd +# network. Should be provided as X.Y.Z. Offset will be +# applied to this as well as the below mask to get the +# resulting address. +# pub_addr_mask: the CIDR mask less the '/' for the IPv4 addresses used +# above. # host_ip: ip address of the bridge host which is reachable for all peer # every additinal paramater is considered as a peer host function gre_bridge { - local if_name=$1 - local offset=$2 - local host_ip=$3 - shift 3 + local flat_if_name=$1 + local pub_if_name=$2 + local offset=$3 + local pub_addr_prefix=$4 + local pub_addr_mask=$5 + local host_ip=$6 + shift 6 local peer_ips=$@ - sudo brctl addbr ${if_name}_br + sudo brctl addbr gre_${flat_if_name}_br + sudo brctl addbr gre_${pub_if_name}_br sudo iptables -I FORWARD -m physdev --physdev-is-bridged -j ACCEPT local key=$offset for node in $peer_ips; do - sudo ip link add gretap_$key type gretap local $host_ip remote $node key $key - sudo ip link set gretap_$key up - remote_command $node sudo -i ip link add ${if_name} type gretap local $node remote $host_ip key $key - remote_command $node sudo -i ip link set ${if_name} up - sudo brctl addif ${if_name}_br gretap_$key + sudo ip link add gretap_${flat_if_name}${key} type gretap local $host_ip remote $node key $key + sudo ip link set gretap_${flat_if_name}${key} up + remote_command $node sudo -i ip link add ${flat_if_name} type gretap local $node remote $host_ip key $key + remote_command $node sudo -i ip link set ${flat_if_name} up + sudo brctl addif gre_${flat_if_name}_br gretap_${flat_if_name}${key} + (( key++ )) + sudo ip link add gretap_${pub_if_name}${key} type gretap local $host_ip remote $node key $key + sudo ip link set gretap_${pub_if_name}${key} up + remote_command $node sudo -i ip link add ${pub_if_name} type gretap local $node remote $host_ip key $key + remote_command $node sudo -i ip link set ${pub_if_name} up + remote_command $node sudo -i ip address add ${pub_addr_prefix}.${key}/${pub_addr_mask} brd + dev ${pub_if_name} + sudo brctl addif gre_${pub_if_name}_br gretap_${pub_if_name}${key} (( key++ )) done - sudo ip link add ${if_name}_br_if type veth peer name ${if_name} - sudo brctl addif ${if_name}_br ${if_name}_br_if - sudo ip link set ${if_name}_br_if up - sudo ip link set ${if_name} up - sudo ip link set ${if_name}_br up + sudo ip link add ${flat_if_name}_br_if type veth peer name ${flat_if_name} + sudo brctl addif gre_${flat_if_name}_br ${flat_if_name}_br_if + sudo ip link set ${flat_if_name}_br_if up + sudo ip link set ${flat_if_name} up + sudo ip link set gre_${flat_if_name}_br up + sudo ip link add ${pub_if_name}_br_if type veth peer name ${pub_if_name} + sudo brctl addif gre_${pub_if_name}_br ${pub_if_name}_br_if + sudo ip link set ${pub_if_name}_br_if up + sudo ip link set ${pub_if_name} up + sudo ip link set gre_${pub_if_name}_br up + sudo ip address add ${pub_addr_prefix}.${offset}/${pub_addr_mask} brd + dev ${pub_if_name} }