From 48c1290c339e7497ac5f8e73cac70b674e12f5cf Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Mon, 30 Jun 2014 15:25:25 +1200 Subject: [PATCH] Fix issues with provider networks and public ips The current public IP address implementation layers a VLAN access port underneath the exterior bridge - br-ex/br-ctlplane. This prevents the use of Neutron provider network tagging because the access port overrides everything. Instead, put public IP addresses in an access port on top of br-ex, allowing provider networks to be tagged by Neutron openflow rules. As part of this we allow a new non-default route, so that IPMI connectivity is not lost when the default route is put onto a VLAN. This is only needed for the seed - the undercloud can receive this via DHCP. The change is backwards compatible - though the old method really is broken so I've marked it as deprecated in the docs and the code. We will need to change our heat templates and then incubator glue code to use the new option before the bug can be closed. Partial-Bug #1325114 Change-Id: I3f77f72ac623792e844dbb4d501b6ab269141f8e --- elements/boot-stack/README.md | 3 + elements/network-utils/README.md | 18 +- elements/network-utils/bin/ensure-bridge | 156 +++++++++++++++--- elements/neutron-openvswitch-agent/README.md | 87 ++++++++-- .../bin/init-neutron-ovs | 35 ++-- 5 files changed, 245 insertions(+), 54 deletions(-) diff --git a/elements/boot-stack/README.md b/elements/boot-stack/README.md index 09df5cdc4..2e7af30cb 100644 --- a/elements/boot-stack/README.md +++ b/elements/boot-stack/README.md @@ -67,3 +67,6 @@ Note that if you are feeding this Metadata to ControllerResource it will not be fed into the process until the Heat Metadata is refreshed, since the initial Metadata copy will have '0.0.0.0' (as we don't know the address until after we create a server record). + +Some configuration is tied into the neutron-openvswitch-agent - see the +README.md there as well. diff --git a/elements/network-utils/README.md b/elements/network-utils/README.md index 259382818..1ab2ce23a 100644 --- a/elements/network-utils/README.md +++ b/elements/network-utils/README.md @@ -4,9 +4,17 @@ Currently only installs a single script ensure-bridge : A bridge configuration script which can be used to create a ovs bridge and place a network device on it. Transferring ip addresses and - routes to the bridge. The script takes 3 parameters - $ ensure-bridge EXTERNAL_BRIDGE PHYSICAL_INTERFACE [PUBLIC_INTERFACE_ROUTE] - EXTERNAL_BRIDGE : The name of the bridge to create. - PHYSICAL_INTERFACE : The physical interface to place on the bridge. - PUBLIC_INTERFACE_ROUTE : Add a default route via this for all IP's except + routes to the bridge. The script takes 3 parameters: + + $ ensure-bridge EXTERNAL_BRIDGE PHYSICAL_INTERFACE [PUBLIC_IP_CIDR [PUBLIC_INTERFACE_ROUTE]] + EXTERNAL_BRIDGE : The name of the bridge to create. + PHYSICAL_INTERFACE : The physical interface to place on the bridge. + PUBLIC_IP_CIDR : Optional static IP address in CIDR notation - 1.2.3.4/5 + PUBLIC_INTERFACE_ROUTE : Add a default route via this for all IP's except 169.254.169.254/32 + + ensure-bridge also accepts: + + * --public-tag: A VLAN tag to use for creating a public IP access port on a VLAN. + * --public-tag-ip: An IP address to put on the access port public-tag creates. + * --bridge-route: A route "prefix via" to add to the bridge. diff --git a/elements/network-utils/bin/ensure-bridge b/elements/network-utils/bin/ensure-bridge index 79ed44367..ad5442fa0 100755 --- a/elements/network-utils/bin/ensure-bridge +++ b/elements/network-utils/bin/ensure-bridge @@ -32,14 +32,24 @@ function show_options () { echo "If IP is not empty, it will be set as the IP address for the bridge." echo "Otherwise the bridge will be configured for DHCP. If route is supplied" echo "it will be used as the default route." + echo "Public-tag and public-tag-ip must both be empty, or both set." echo echo "Options:" - echo " -h|--help -- this help." + echo " -h|--help -- this help." + echo " --bridge-route -- Add a route to the bridge, e.g. to IPMI network." + echo " Accepts one parameter, the prefix and via with a" + echo " space between them." + echo " --public-tag -- Make int_public an access port with this tag." + echo " --public-tag-ip -- Give int_public this IP address." echo exit $1 } -TEMP=$(getopt -o h -l help -n $SCRIPT_NAME -- "$@") +BRIDGE_ROUTE= +PUBLIC_TAG= +PUBLIC_TAG_IP= + +TEMP=$(getopt -o h -l bridge-route:,help,public-tag:,public-tag-ip: -n $SCRIPT_NAME -- "$@") if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi # Note the quotes around `$TEMP': they are essential! @@ -48,6 +58,9 @@ eval set -- "$TEMP" while true ; do case "$1" in -h|--help) show_options 0;; + --bridge-route) BRIDGE_ROUTE=$2; shift 2;; + --public-tag) PUBLIC_TAG=$2; shift 2;; + --public-tag-ip) PUBLIC_TAG_IP=$2; shift 2;; --) shift ; break ;; *) echo "Error: unsupported option $1." ; exit 1 ;; esac @@ -63,6 +76,25 @@ if [ -z "$PHYSICAL_INTERFACE" -o -n "$EXTRA" ]; then show_options 1 fi +if [ \( -n "$PUBLIC_TAG" -a -z "$PUBLIC_TAG_IP" \) -o \ + \( -z "$PUBLIC_TAG" -a -n "$PUBLIC_TAG_IP" \) ]; then + # For now, we don't support DHCP on vlans - the next gen stuff will be well + # layered and do that. + echo "Only one of --public-tag and --public-tag-ip supplied." >&2 + show_options 1 +fi + +if [ -n "$BRIDGE_ROUTE" ]; then + read -s BRIDGE_ROUTE_PREFIX BRIDGE_ROUTE_VIA <<< $BRIDGE_ROUTE + if [ -z "$BRIDGE_ROUTE_PREFIX" -o -z "$BRIDGE_ROUTE_VIA" ]; then + echo "Invalid route '$BRIDGE_ROUTE'" >&2 + show_options 1 + fi +else + BRIDGE_ROUTE_PREFIX= + BRIDGE_ROUTE_VIA= +fi + set -x # network scripts function used on Fedora/RHEL/Centos, etc. @@ -70,13 +102,22 @@ function configure_bridge_interface_dhcp_netscripts() { local bridge=$1 local interface=$2 - local bridge_ip_addr=${3:-''} - local bridge_netmask=${4:-''} + local public_ip_addr=${3:-''} + local public_ip_netmask=${4:-''} local interface_mac=${5} + local public_tag=${6} + local public_tag_ip=${7} + local public_tag_ip_netmask=${8} + local bridge_route_prefix=${9} + local bridge_route_via=${10} local tmp_bridge_config=$(mktemp) + local tmp_bridge_route=$(mktemp) local tmp_interface_config=$(mktemp) + local tmp_int_public_config=$(mktemp) local bridge_config="/etc/sysconfig/network-scripts/ifcfg-$bridge" + local bridge_route="/etc/sysconfig/network-scripts/route-$bridge" local interface_config="/etc/sysconfig/network-scripts/ifcfg-$interface" + local int_public_config="/etc/sysconfig/network-scripts/ifcfg-int_public" #interface config cat > $tmp_interface_config < $tmp_bridge_config < $tmp_bridge_route + fi + if [ -n "$public_tag" ]; then + # Setup the access port + cat > $tmp_int_public_config </dev/null || \ - ! diff $tmp_bridge_config $bridge_config &>/dev/null; then + ! diff $tmp_bridge_config $bridge_config &>/dev/null || \ + ! diff $tmp_int_public_config $int_public_config &>/dev/null || \ + ! diff $tmp_bridge_route $bridge_route &>/dev/null ; then + ifdown int_public &>/dev/null || true ifdown $interface &>/dev/null || true ifdown $bridge &>/dev/null || true cp $tmp_interface_config $interface_config cp $tmp_bridge_config $bridge_config + cp $tmp_bridge_route $bridge_route + if [ -n "$public_tag" ]; then + cp $tmp_int_public_config $int_public_config + else + rm -f $int_public_config + fi ifup $bridge ifup $interface - + if [ -n "$public_tag" ]; then + ifup int_public + fi fi rm $tmp_bridge_config @@ -139,26 +210,45 @@ function configure_bridge_interface_dhcp_eni() { local bridge=$1 local interface=$2 - local bridge_ip_addr=${3:-''} - local bridge_netmask=${4:-''} + local public_ip_addr=${3:-''} + local public_ip_netmask=${4:-''} local interface_mac=${5} + local public_tag=${6} + local public_tag_ip=${7} + local public_tag_ip_netmask=${8} + local bridge_route_prefix=${9} + local bridge_route_via=${10} local tmp_config=$(mktemp) local config="/etc/network/interfaces" cp $config $tmp_config sed -e "/auto $interface\$/,/^$/d" -i $tmp_config sed -e "/auto $bridge\$/,/^$/d" -i $tmp_config + sed -e "/auto int_public\$/,/^$/d" -i $tmp_config + + if [ -n "$bridge_route_prefix" ]; then + local route_line="post-up ip route replace $bridge_route_prefix via $bridge_route_via" + else + local route_line= + fi + + ovs_ports="$interface" + if [ -n "$public_tag" ]; then + ovs_ports="$ovs_ports int_public" + fi #bridge config - if [ -z "$bridge_ip_addr" ]; then + if [ -z "$public_ip_addr" ]; then + # DHCP for the bridge itself. cat >> $tmp_config <> $tmp_config <> $tmp_config <<-EOF_CAT + cat >> $tmp_config </dev/null; then + ifdown int_public &>/dev/null || true ifdown $interface &>/dev/null || true ifdown $bridge &>/dev/null || true @@ -193,7 +298,9 @@ EOF_CAT ifup $bridge ifup $interface - + if [ -n "$public_tag" ]; then + ifup int_public + fi fi rm $tmp_config @@ -208,12 +315,19 @@ else NETMASK='' fi +if [ -n "$PUBLIC_TAG_IP" ]; then + PUBLIC_TAG_IP_NETMASK=$(python -c "import netaddr; print netaddr.IPNetwork('$PUBLIC_TAG_IP').netmask") + PUBLIC_TAG_IP=$(python -c "import netaddr; print netaddr.IPNetwork('$PUBLIC_TAG_IP').ip") +else + PUBLIC_TAG_IP_NETMASK= +fi + interface_mac=$(ip link show dev "$PHYSICAL_INTERFACE" | awk '/ether/ {print $2}') if [ -d "/etc/sysconfig/network-scripts/" ]; then - configure_bridge_interface_dhcp_netscripts $EXTERNAL_BRIDGE $PHYSICAL_INTERFACE "$IP" "$NETMASK" $interface_mac + configure_bridge_interface_dhcp_netscripts $EXTERNAL_BRIDGE $PHYSICAL_INTERFACE "$IP" "$NETMASK" $interface_mac "$PUBLIC_TAG" "$PUBLIC_TAG_IP" "$PUBLIC_TAG_IP_NETMASK" "$BRIDGE_ROUTE_PREFIX" "$BRIDGE_ROUTE_VIA" elif [ -d "/etc/network" ]; then - configure_bridge_interface_dhcp_eni $EXTERNAL_BRIDGE $PHYSICAL_INTERFACE "$IP" "$NETMASK" $interface_mac + configure_bridge_interface_dhcp_eni $EXTERNAL_BRIDGE $PHYSICAL_INTERFACE "$IP" "$NETMASK" $interface_mac "$PUBLIC_TAG" "$PUBLIC_TAG_IP" "$PUBLIC_TAG_IP_NETMASK" "$BRIDGE_ROUTE_PREFIX" "$BRIDGE_ROUTE_VIA" else echo "Unsupported network configuration type!" exit 1 diff --git a/elements/neutron-openvswitch-agent/README.md b/elements/neutron-openvswitch-agent/README.md index 8c2122d20..9199f2bf9 100644 --- a/elements/neutron-openvswitch-agent/README.md +++ b/elements/neutron-openvswitch-agent/README.md @@ -15,18 +15,83 @@ configured via Heat Metadata. For example: physical_network: ctlplane network_vlan_ranges: ctlplane bridge_mappings: ctlplane:br-ctlplane + bootstack: + public_interface_ip: 12.34.56.79/24 If public\_interface and physical\_bridge are not set, no bridges will be connected directly. This is normal for neutron hosting virtual machines -when using an overlay network (e.g. GRE tunnelling). Some of the -other fields will be ignored in this case. Most of them map 1:1 with their -counterparts in the OVS section of ovs\_neutron\_plugin.ini If -public\_interface\_raw\_device is set, public\_interface must be a vlan device, -and the vlan device will be created using the raw device during -os-collect-config configuration. +when using an overlay network (e.g. GRE tunnelling) with no provider networks. +Some of the other fields will be ignored in this case. Most of them map 1:1 +with their counterparts in the OVS section of ovs\_neutron\_plugin.ini. -Once the public interface is configured, public\_interface\_route (if set) -will replace the default route's next hop. The hop this replaces will be -added as the next hop for 169.254.169.254/32 (unless one already exists). -This permits routing default traffic out through a hardware router without -breaking the ability to contact a bare metal metadata server. +Public\_interface\_ip is used to add an *additional* ip address to the machine. +This is set on the bridge device. Our current scripts write a static +configuration with either DHCP or one IP address per interface, so its not +very flexible. + +There are two ways to setup VLANs. The old deprecated way using +public\_interface\_raw\_device creates a VLAN device under the bridge, which +prevents the use of provider networks other than that for the same VLAN. It +also sometimes leads to issues with access to the metadata server. In this +configuration the raw device is still configured using DHCP and the public IP +is put on the bridge device itself using public\_interface\_ip. + +The new way is to use public\_interface\_tag and public\_interface\_tag\_ip to +create a VLAN access port on top of the bridge. This allows the use of any +provider network desired, as the traffic tagging and filtering occurs in the +bridge rather than below it. In this configuration the access port is given the +public IP address, the bridge is (usually) configured for DHCP, and the +underlying device is no longer given an IP address at all. This can be used +together with public\_interface\_ip to assign a static ip address to the bridge +(which we use for the seed VM as part of bootstrapping an environment). + +Routing on the control plane network can be complex, and we have a new feature +coming in to do arbitrary routes, but for now, we offer the ability to add a +single static route via the physical\_bridge\_route key. + +For instance: + + neutron: + ovs: + public_interface: eth2 + public_interface_route: 45.67.89.1 + public_interface_tag: 25 + public_interface_tag_ip: 45.67.89.10/24 + physical_bridge: br-ctlplane + physical_bridge_route: + prefix: 12.34.0.0/16 + via: 12.34.56.1 + bootstack: + public_interface_ip: 12.34.56.79/24 + +will result in br-ctlplane being created on eth2, a tagged port (`int\_public`) +added to br-ctlplane with tag 25, ip address 45.67.89.10/24, default route +45.67.89.1 and the bridge device itself being assigned 12.34.56.78/24. + +public\_interface\_tag must be an int, or null, which like not present, means +untagged. When public\_interface\_tag is not set, public\_interface\_tag\_ip +must also not be set. The recommended approach is to set the tag, tag\_ip and +\_route options together, or not at all. public\_interface\_ip should only be +used in the seed, as using it elsewhere will usually result in the metadata +service being inaccessible. + +For the deprecated behaviour where public\_interface\_raw\_device is set, +public\_interface must be a vlan device, and the vlan device will be created +using the raw device during os-collect-config configuration. We suggest not +using this and migrating to public\_interface\_ip\_tag as soon as possible as +that will fix tag provider networks. + +When public\_interface\_raw\_device is not set, setting an IP address without +setting a tag for it will result in an invalid configuration where metadata +access is not possible, as the source IP address will be wrong. This may be +useful where metadata access is not an issue (such as the seed VM). + +The bridge is always configured to use the MAC address of the public\_interface +device as its MAC address. + +Once the bridge and access port (if configured) are set up, the +public\_interface\_route (if set) will replace the default route's next hop. +The hop this replaces will be added as the next hop for 169.254.169.254/32 +(unless one already exists). This permits routing default traffic out through +a hardware router without breaking the ability to contact a local subnet bare +metal metadata server. diff --git a/elements/neutron-openvswitch-agent/bin/init-neutron-ovs b/elements/neutron-openvswitch-agent/bin/init-neutron-ovs index 4cb4ee17f..3db37e14b 100755 --- a/elements/neutron-openvswitch-agent/bin/init-neutron-ovs +++ b/elements/neutron-openvswitch-agent/bin/init-neutron-ovs @@ -18,10 +18,8 @@ # # If a physical bridge is defined then ensure-bridge is called to set it up. # -# Note that no persistent config file is written to the OS : ovs-vsctl modifies -# a persistent database so the bridge device will persist across reboots, but -# [on Ubuntu at least] early boot does not bring up ovs-vswitch early enough, -# and metadata access will fail. +# If there is configuration (tag + IP) for an access port, one is created on +# top of the bridge, again by ensure-bridge. set -eux @@ -32,8 +30,20 @@ PHYSICAL_INTERFACE=$(os-apply-config --key neutron.ovs.public_interface --type r PHYSICAL_INTERFACE_IP=$(os-apply-config --key bootstack.public_interface_ip --type netaddress --key-default '') PHYSICAL_INTERFACE_RAW_DEVICE=$(os-apply-config --key neutron.ovs.public_interface_raw_device --type raw --key-default '') PUBLIC_INTERFACE_ROUTE=$(os-apply-config --key neutron.ovs.public_interface_route --type netaddress --key-default '') +# TAG is type raw because we can't do an absent key as a default in Heat, and '' is not an int. +PUBLIC_INTERFACE_TAG=$(os-apply-config --key neutron.ovs.public_interface_tag --type raw --key-default '') +PUBLIC_INTERFACE_TAG_IP=$(os-apply-config --key neutron.ovs.public_interface_tag_ip --type netaddress --key-default '') +PHYSICAL_ROUTE_PREFIX=$(os-apply-config --key neutron.ovs.physical_bridge_route.prefix --type netaddress --key-default '') +PHYSICAL_ROUTE_VIA=$(os-apply-config --key neutron.ovs.physical_bridge_route.via --type netaddress --key-default '') + + +if [ -n "$PHYSICAL_INTERFACE_RAW_DEVICE" -a -n "$PUBLIC_INTERFACE_TAG" ]; then + echo "ERROR: cannot specify a raw device and a tagged access port at the same time." >&2 + exit 1 +fi if [ -n "$PHYSICAL_INTERFACE_RAW_DEVICE" ]; then + echo "DEPRECATED: please use neutron.ovs.public_interface_ip_tag instead." >&2 if ! (ip link show dev $PHYSICAL_INTERFACE) ; then VLAN_ID=$(echo $PHYSICAL_INTERFACE | sed s/vlan//) vconfig set_name_type VLAN_PLUS_VID_NO_PAD @@ -42,17 +52,6 @@ if [ -n "$PHYSICAL_INTERFACE_RAW_DEVICE" ]; then ip link set $PHYSICAL_INTERFACE up fi -# NOTE: ensure-bridge actually takes care of this now but it -# as been requested that we leave this in so as to not break -# anything. AFAIK there is no use case in TripleO right now -# where we don't actually call ensure-bridge when using -# init-neutron-ovs so this code should be mute. -if [ -n "$PHYSICAL_INTERFACE_IP" -a -n "$PHYSICAL_INTERFACE" ] ; then - if ! (ip addr show | grep -q $PHYSICAL_INTERFACE_IP) ; then - ip addr add $PHYSICAL_INTERFACE_IP dev $PHYSICAL_INTERFACE - fi -fi - # Hacky: ensure the switch is running : we should instead arrange for this # script to not be run before it's started. service openvswitch-switch restart || service openvswitch restart @@ -64,5 +63,7 @@ if [ -z "$EXTERNAL_BRIDGE" ] ; then exit 0 fi -ensure-bridge "$EXTERNAL_BRIDGE" "$PHYSICAL_INTERFACE" "$PHYSICAL_INTERFACE_IP" "$PUBLIC_INTERFACE_ROUTE" - +ensure-bridge "$EXTERNAL_BRIDGE" "$PHYSICAL_INTERFACE" "$PHYSICAL_INTERFACE_IP" \ + "$PUBLIC_INTERFACE_ROUTE" --public-tag "$PUBLIC_INTERFACE_TAG" \ + --public-tag-ip "$PUBLIC_INTERFACE_TAG_IP" \ + ${PHYSICAL_ROUTE_PREFIX:+--bridge-route "${PHYSICAL_ROUTE_PREFIX} ${PHYSICAL_ROUTE_VIA}"}