Merge "Fix issues with provider networks and public ips"

This commit is contained in:
Jenkins 2014-08-20 19:21:43 +00:00 committed by Gerrit Code Review
commit 0894278a82
5 changed files with 245 additions and 54 deletions

View File

@ -67,3 +67,6 @@ Note that if you are feeding this Metadata to ControllerResource it
will not be fed into the process until the Heat Metadata is refreshed,
since the initial Metadata copy will have '0.0.0.0' (as we don't know
the address until after we create a server record).
Some configuration is tied into the neutron-openvswitch-agent - see the
README.md there as well.

View File

@ -4,9 +4,17 @@ Currently only installs a single script
ensure-bridge : A bridge configuration script which can be used to create
a ovs bridge and place a network device on it. Transferring ip addresses and
routes to the bridge. The script takes 3 parameters
$ ensure-bridge EXTERNAL_BRIDGE PHYSICAL_INTERFACE [PUBLIC_INTERFACE_ROUTE]
EXTERNAL_BRIDGE : The name of the bridge to create.
PHYSICAL_INTERFACE : The physical interface to place on the bridge.
PUBLIC_INTERFACE_ROUTE : Add a default route via this for all IP's except
routes to the bridge. The script takes 3 parameters:
$ ensure-bridge EXTERNAL_BRIDGE PHYSICAL_INTERFACE [PUBLIC_IP_CIDR [PUBLIC_INTERFACE_ROUTE]]
EXTERNAL_BRIDGE : The name of the bridge to create.
PHYSICAL_INTERFACE : The physical interface to place on the bridge.
PUBLIC_IP_CIDR : Optional static IP address in CIDR notation - 1.2.3.4/5
PUBLIC_INTERFACE_ROUTE : Add a default route via this for all IP's except
169.254.169.254/32
ensure-bridge also accepts:
* --public-tag: A VLAN tag to use for creating a public IP access port on a VLAN.
* --public-tag-ip: An IP address to put on the access port public-tag creates.
* --bridge-route: A route "prefix via" to add to the bridge.

View File

@ -32,14 +32,24 @@ function show_options () {
echo "If IP is not empty, it will be set as the IP address for the bridge."
echo "Otherwise the bridge will be configured for DHCP. If route is supplied"
echo "it will be used as the default route."
echo "Public-tag and public-tag-ip must both be empty, or both set."
echo
echo "Options:"
echo " -h|--help -- this help."
echo " -h|--help -- this help."
echo " --bridge-route -- Add a route to the bridge, e.g. to IPMI network."
echo " Accepts one parameter, the prefix and via with a"
echo " space between them."
echo " --public-tag -- Make int_public an access port with this tag."
echo " --public-tag-ip -- Give int_public this IP address."
echo
exit $1
}
TEMP=$(getopt -o h -l help -n $SCRIPT_NAME -- "$@")
BRIDGE_ROUTE=
PUBLIC_TAG=
PUBLIC_TAG_IP=
TEMP=$(getopt -o h -l bridge-route:,help,public-tag:,public-tag-ip: -n $SCRIPT_NAME -- "$@")
if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi
# Note the quotes around `$TEMP': they are essential!
@ -48,6 +58,9 @@ eval set -- "$TEMP"
while true ; do
case "$1" in
-h|--help) show_options 0;;
--bridge-route) BRIDGE_ROUTE=$2; shift 2;;
--public-tag) PUBLIC_TAG=$2; shift 2;;
--public-tag-ip) PUBLIC_TAG_IP=$2; shift 2;;
--) shift ; break ;;
*) echo "Error: unsupported option $1." ; exit 1 ;;
esac
@ -63,6 +76,25 @@ if [ -z "$PHYSICAL_INTERFACE" -o -n "$EXTRA" ]; then
show_options 1
fi
if [ \( -n "$PUBLIC_TAG" -a -z "$PUBLIC_TAG_IP" \) -o \
\( -z "$PUBLIC_TAG" -a -n "$PUBLIC_TAG_IP" \) ]; then
# For now, we don't support DHCP on vlans - the next gen stuff will be well
# layered and do that.
echo "Only one of --public-tag and --public-tag-ip supplied." >&2
show_options 1
fi
if [ -n "$BRIDGE_ROUTE" ]; then
read -s BRIDGE_ROUTE_PREFIX BRIDGE_ROUTE_VIA <<< $BRIDGE_ROUTE
if [ -z "$BRIDGE_ROUTE_PREFIX" -o -z "$BRIDGE_ROUTE_VIA" ]; then
echo "Invalid route '$BRIDGE_ROUTE'" >&2
show_options 1
fi
else
BRIDGE_ROUTE_PREFIX=
BRIDGE_ROUTE_VIA=
fi
set -x
# network scripts function used on Fedora/RHEL/Centos, etc.
@ -70,13 +102,22 @@ function configure_bridge_interface_dhcp_netscripts() {
local bridge=$1
local interface=$2
local bridge_ip_addr=${3:-''}
local bridge_netmask=${4:-''}
local public_ip_addr=${3:-''}
local public_ip_netmask=${4:-''}
local interface_mac=${5}
local public_tag=${6}
local public_tag_ip=${7}
local public_tag_ip_netmask=${8}
local bridge_route_prefix=${9}
local bridge_route_via=${10}
local tmp_bridge_config=$(mktemp)
local tmp_bridge_route=$(mktemp)
local tmp_interface_config=$(mktemp)
local tmp_int_public_config=$(mktemp)
local bridge_config="/etc/sysconfig/network-scripts/ifcfg-$bridge"
local bridge_route="/etc/sysconfig/network-scripts/route-$bridge"
local interface_config="/etc/sysconfig/network-scripts/ifcfg-$interface"
local int_public_config="/etc/sysconfig/network-scripts/ifcfg-int_public"
#interface config
cat > $tmp_interface_config <<EOF_CAT
@ -90,7 +131,8 @@ HOTPLUG=no
EOF_CAT
#bridge config
if [ -z "$bridge_ip_addr" ]; then
if [ -z "$public_ip_addr" ]; then
# DHCP for the bridge itself.
cat > $tmp_bridge_config <<EOF_CAT
DEVICE=$bridge
ONBOOT=yes
@ -109,24 +151,53 @@ DEVICETYPE=ovs
TYPE=OVSBridge
OVS_EXTRA="set bridge $bridge other-config:hwaddr=$interface_mac"
BOOTPROTO=static
IPADDR=$bridge_ip_addr
NETMASK=$bridge_netmask
IPADDR=$public_ip_addr
NETMASK=$public_ip_netmask
HOTPLUG=no
EOF_CAT
fi
if [ -n "$bridge_route_prefix" ]; then
echo $bridge_route_prefix via $bridge_route_via dev $bridge > $tmp_bridge_route
fi
if [ -n "$public_tag" ]; then
# Setup the access port
cat > $tmp_int_public_config <<EOF_CAT
DEVICE=int_public
ONBOOT=yes
DEVICETYPE=ovs
TYPE=OVSIntPort
BOOTPROTO=static
IPADDR=$public_tag_ip
NETMASK=$public_tag_ip_netmask
OVS_BRIDGE=$bridge
OVS_OPTIONS="tag=$public_tag"
HOTPLUG=no
EOF_CAT
fi
if ! diff $tmp_interface_config $interface_config &>/dev/null || \
! diff $tmp_bridge_config $bridge_config &>/dev/null; then
! diff $tmp_bridge_config $bridge_config &>/dev/null || \
! diff $tmp_int_public_config $int_public_config &>/dev/null || \
! diff $tmp_bridge_route $bridge_route &>/dev/null ; then
ifdown int_public &>/dev/null || true
ifdown $interface &>/dev/null || true
ifdown $bridge &>/dev/null || true
cp $tmp_interface_config $interface_config
cp $tmp_bridge_config $bridge_config
cp $tmp_bridge_route $bridge_route
if [ -n "$public_tag" ]; then
cp $tmp_int_public_config $int_public_config
else
rm -f $int_public_config
fi
ifup $bridge
ifup $interface
if [ -n "$public_tag" ]; then
ifup int_public
fi
fi
rm $tmp_bridge_config
@ -139,26 +210,45 @@ function configure_bridge_interface_dhcp_eni() {
local bridge=$1
local interface=$2
local bridge_ip_addr=${3:-''}
local bridge_netmask=${4:-''}
local public_ip_addr=${3:-''}
local public_ip_netmask=${4:-''}
local interface_mac=${5}
local public_tag=${6}
local public_tag_ip=${7}
local public_tag_ip_netmask=${8}
local bridge_route_prefix=${9}
local bridge_route_via=${10}
local tmp_config=$(mktemp)
local config="/etc/network/interfaces"
cp $config $tmp_config
sed -e "/auto $interface\$/,/^$/d" -i $tmp_config
sed -e "/auto $bridge\$/,/^$/d" -i $tmp_config
sed -e "/auto int_public\$/,/^$/d" -i $tmp_config
if [ -n "$bridge_route_prefix" ]; then
local route_line="post-up ip route replace $bridge_route_prefix via $bridge_route_via"
else
local route_line=
fi
ovs_ports="$interface"
if [ -n "$public_tag" ]; then
ovs_ports="$ovs_ports int_public"
fi
#bridge config
if [ -z "$bridge_ip_addr" ]; then
if [ -z "$public_ip_addr" ]; then
# DHCP for the bridge itself.
cat >> $tmp_config <<EOF_CAT
auto $bridge
allow-ovs $bridge
iface $bridge inet dhcp
iface $bridge inet dhcp
pre-up ip addr flush dev $interface
ovs_type OVSBridge
ovs_ports $interface
ovs_ports $ovs_ports
ovs_extra set bridge $bridge other-config:hwaddr=$interface_mac
$route_line
EOF_CAT
else
@ -167,25 +257,40 @@ auto $bridge
allow-ovs $bridge
iface $bridge inet static
pre-up ip addr flush dev $interface
address $bridge_ip_addr
netmask $bridge_netmask
address $public_ip_addr
netmask $public_ip_netmask
ovs_type OVSBridge
ovs_extra set bridge $bridge other-config:hwaddr=$interface_mac
$route_line
EOF_CAT
fi
if [ -n "$public_tag" ]; then
# Setup the access port
cat >> $tmp_config <<EOF_CAT
auto int_public
allow-$bridge int_public
iface int_public inet static
address $public_tag_ip
netmask $public_tag_ip_netmask
ovs_bridge $bridge
ovs_type OVSIntPort
ovs_options tag=$public_tag
EOF_CAT
fi
#interface config
cat >> $tmp_config <<-EOF_CAT
cat >> $tmp_config <<EOF_CAT
auto $interface
allow-$bridge $interface
iface $interface inet manual
iface $interface inet manual
ovs_bridge $bridge
ovs_type OVSPort
EOF_CAT
if ! diff $tmp_config $config &>/dev/null; then
ifdown int_public &>/dev/null || true
ifdown $interface &>/dev/null || true
ifdown $bridge &>/dev/null || true
@ -193,7 +298,9 @@ EOF_CAT
ifup $bridge
ifup $interface
if [ -n "$public_tag" ]; then
ifup int_public
fi
fi
rm $tmp_config
@ -208,12 +315,19 @@ else
NETMASK=''
fi
if [ -n "$PUBLIC_TAG_IP" ]; then
PUBLIC_TAG_IP_NETMASK=$(python -c "import netaddr; print netaddr.IPNetwork('$PUBLIC_TAG_IP').netmask")
PUBLIC_TAG_IP=$(python -c "import netaddr; print netaddr.IPNetwork('$PUBLIC_TAG_IP').ip")
else
PUBLIC_TAG_IP_NETMASK=
fi
interface_mac=$(ip link show dev "$PHYSICAL_INTERFACE" | awk '/ether/ {print $2}')
if [ -d "/etc/sysconfig/network-scripts/" ]; then
configure_bridge_interface_dhcp_netscripts $EXTERNAL_BRIDGE $PHYSICAL_INTERFACE "$IP" "$NETMASK" $interface_mac
configure_bridge_interface_dhcp_netscripts $EXTERNAL_BRIDGE $PHYSICAL_INTERFACE "$IP" "$NETMASK" $interface_mac "$PUBLIC_TAG" "$PUBLIC_TAG_IP" "$PUBLIC_TAG_IP_NETMASK" "$BRIDGE_ROUTE_PREFIX" "$BRIDGE_ROUTE_VIA"
elif [ -d "/etc/network" ]; then
configure_bridge_interface_dhcp_eni $EXTERNAL_BRIDGE $PHYSICAL_INTERFACE "$IP" "$NETMASK" $interface_mac
configure_bridge_interface_dhcp_eni $EXTERNAL_BRIDGE $PHYSICAL_INTERFACE "$IP" "$NETMASK" $interface_mac "$PUBLIC_TAG" "$PUBLIC_TAG_IP" "$PUBLIC_TAG_IP_NETMASK" "$BRIDGE_ROUTE_PREFIX" "$BRIDGE_ROUTE_VIA"
else
echo "Unsupported network configuration type!"
exit 1

View File

@ -15,18 +15,83 @@ configured via Heat Metadata. For example:
physical_network: ctlplane
network_vlan_ranges: ctlplane
bridge_mappings: ctlplane:br-ctlplane
bootstack:
public_interface_ip: 12.34.56.79/24
If public\_interface and physical\_bridge are not set, no bridges will be
connected directly. This is normal for neutron hosting virtual machines
when using an overlay network (e.g. GRE tunnelling). Some of the
other fields will be ignored in this case. Most of them map 1:1 with their
counterparts in the OVS section of ovs\_neutron\_plugin.ini If
public\_interface\_raw\_device is set, public\_interface must be a vlan device,
and the vlan device will be created using the raw device during
os-collect-config configuration.
when using an overlay network (e.g. GRE tunnelling) with no provider networks.
Some of the other fields will be ignored in this case. Most of them map 1:1
with their counterparts in the OVS section of ovs\_neutron\_plugin.ini.
Once the public interface is configured, public\_interface\_route (if set)
will replace the default route's next hop. The hop this replaces will be
added as the next hop for 169.254.169.254/32 (unless one already exists).
This permits routing default traffic out through a hardware router without
breaking the ability to contact a bare metal metadata server.
Public\_interface\_ip is used to add an *additional* ip address to the machine.
This is set on the bridge device. Our current scripts write a static
configuration with either DHCP or one IP address per interface, so its not
very flexible.
There are two ways to setup VLANs. The old deprecated way using
public\_interface\_raw\_device creates a VLAN device under the bridge, which
prevents the use of provider networks other than that for the same VLAN. It
also sometimes leads to issues with access to the metadata server. In this
configuration the raw device is still configured using DHCP and the public IP
is put on the bridge device itself using public\_interface\_ip.
The new way is to use public\_interface\_tag and public\_interface\_tag\_ip to
create a VLAN access port on top of the bridge. This allows the use of any
provider network desired, as the traffic tagging and filtering occurs in the
bridge rather than below it. In this configuration the access port is given the
public IP address, the bridge is (usually) configured for DHCP, and the
underlying device is no longer given an IP address at all. This can be used
together with public\_interface\_ip to assign a static ip address to the bridge
(which we use for the seed VM as part of bootstrapping an environment).
Routing on the control plane network can be complex, and we have a new feature
coming in to do arbitrary routes, but for now, we offer the ability to add a
single static route via the physical\_bridge\_route key.
For instance:
neutron:
ovs:
public_interface: eth2
public_interface_route: 45.67.89.1
public_interface_tag: 25
public_interface_tag_ip: 45.67.89.10/24
physical_bridge: br-ctlplane
physical_bridge_route:
prefix: 12.34.0.0/16
via: 12.34.56.1
bootstack:
public_interface_ip: 12.34.56.79/24
will result in br-ctlplane being created on eth2, a tagged port (`int\_public`)
added to br-ctlplane with tag 25, ip address 45.67.89.10/24, default route
45.67.89.1 and the bridge device itself being assigned 12.34.56.78/24.
public\_interface\_tag must be an int, or null, which like not present, means
untagged. When public\_interface\_tag is not set, public\_interface\_tag\_ip
must also not be set. The recommended approach is to set the tag, tag\_ip and
\_route options together, or not at all. public\_interface\_ip should only be
used in the seed, as using it elsewhere will usually result in the metadata
service being inaccessible.
For the deprecated behaviour where public\_interface\_raw\_device is set,
public\_interface must be a vlan device, and the vlan device will be created
using the raw device during os-collect-config configuration. We suggest not
using this and migrating to public\_interface\_ip\_tag as soon as possible as
that will fix tag provider networks.
When public\_interface\_raw\_device is not set, setting an IP address without
setting a tag for it will result in an invalid configuration where metadata
access is not possible, as the source IP address will be wrong. This may be
useful where metadata access is not an issue (such as the seed VM).
The bridge is always configured to use the MAC address of the public\_interface
device as its MAC address.
Once the bridge and access port (if configured) are set up, the
public\_interface\_route (if set) will replace the default route's next hop.
The hop this replaces will be added as the next hop for 169.254.169.254/32
(unless one already exists). This permits routing default traffic out through
a hardware router without breaking the ability to contact a local subnet bare
metal metadata server.

View File

@ -18,10 +18,8 @@
#
# If a physical bridge is defined then ensure-bridge is called to set it up.
#
# Note that no persistent config file is written to the OS : ovs-vsctl modifies
# a persistent database so the bridge device will persist across reboots, but
# [on Ubuntu at least] early boot does not bring up ovs-vswitch early enough,
# and metadata access will fail.
# If there is configuration (tag + IP) for an access port, one is created on
# top of the bridge, again by ensure-bridge.
set -eux
@ -32,8 +30,20 @@ PHYSICAL_INTERFACE=$(os-apply-config --key neutron.ovs.public_interface --type r
PHYSICAL_INTERFACE_IP=$(os-apply-config --key bootstack.public_interface_ip --type netaddress --key-default '')
PHYSICAL_INTERFACE_RAW_DEVICE=$(os-apply-config --key neutron.ovs.public_interface_raw_device --type raw --key-default '')
PUBLIC_INTERFACE_ROUTE=$(os-apply-config --key neutron.ovs.public_interface_route --type netaddress --key-default '')
# TAG is type raw because we can't do an absent key as a default in Heat, and '' is not an int.
PUBLIC_INTERFACE_TAG=$(os-apply-config --key neutron.ovs.public_interface_tag --type raw --key-default '')
PUBLIC_INTERFACE_TAG_IP=$(os-apply-config --key neutron.ovs.public_interface_tag_ip --type netaddress --key-default '')
PHYSICAL_ROUTE_PREFIX=$(os-apply-config --key neutron.ovs.physical_bridge_route.prefix --type netaddress --key-default '')
PHYSICAL_ROUTE_VIA=$(os-apply-config --key neutron.ovs.physical_bridge_route.via --type netaddress --key-default '')
if [ -n "$PHYSICAL_INTERFACE_RAW_DEVICE" -a -n "$PUBLIC_INTERFACE_TAG" ]; then
echo "ERROR: cannot specify a raw device and a tagged access port at the same time." >&2
exit 1
fi
if [ -n "$PHYSICAL_INTERFACE_RAW_DEVICE" ]; then
echo "DEPRECATED: please use neutron.ovs.public_interface_ip_tag instead." >&2
if ! (ip link show dev $PHYSICAL_INTERFACE) ; then
VLAN_ID=$(echo $PHYSICAL_INTERFACE | sed s/vlan//)
vconfig set_name_type VLAN_PLUS_VID_NO_PAD
@ -42,17 +52,6 @@ if [ -n "$PHYSICAL_INTERFACE_RAW_DEVICE" ]; then
ip link set $PHYSICAL_INTERFACE up
fi
# NOTE: ensure-bridge actually takes care of this now but it
# as been requested that we leave this in so as to not break
# anything. AFAIK there is no use case in TripleO right now
# where we don't actually call ensure-bridge when using
# init-neutron-ovs so this code should be mute.
if [ -n "$PHYSICAL_INTERFACE_IP" -a -n "$PHYSICAL_INTERFACE" ] ; then
if ! (ip addr show | grep -q $PHYSICAL_INTERFACE_IP) ; then
ip addr add $PHYSICAL_INTERFACE_IP dev $PHYSICAL_INTERFACE
fi
fi
# Hacky: ensure the switch is running : we should instead arrange for this
# script to not be run before it's started.
service openvswitch-switch restart || service openvswitch restart
@ -64,5 +63,7 @@ if [ -z "$EXTERNAL_BRIDGE" ] ; then
exit 0
fi
ensure-bridge "$EXTERNAL_BRIDGE" "$PHYSICAL_INTERFACE" "$PHYSICAL_INTERFACE_IP" "$PUBLIC_INTERFACE_ROUTE"
ensure-bridge "$EXTERNAL_BRIDGE" "$PHYSICAL_INTERFACE" "$PHYSICAL_INTERFACE_IP" \
"$PUBLIC_INTERFACE_ROUTE" --public-tag "$PUBLIC_INTERFACE_TAG" \
--public-tag-ip "$PUBLIC_INTERFACE_TAG_IP" \
${PHYSICAL_ROUTE_PREFIX:+--bridge-route "${PHYSICAL_ROUTE_PREFIX} ${PHYSICAL_ROUTE_VIA}"}