From 5a56cd6283f918e6022dcc47906fb86a739496dc Mon Sep 17 00:00:00 2001 From: Mate Lakat Date: Mon, 17 Jun 2013 13:54:43 +0100 Subject: [PATCH] xenapi: Cleanup networking OpenStack VM had 4 interfaces, from which eth0 was connected to HIMN (Host Internal Management Network), and not used at all. The reason for not using this interface according to johngarbutt is: - VNC console would not work - migrations would not work - many other things This change removes the interface, thus devstack is providing the same architecture as described in the official documentation, YAGNI applied on the HIMN: http://goo.gl/dmrNF The change also modifies some defaults, so it's easier to get started: - The Ubuntu VM is network-installed through the management network - OS VM is getting its address on the management network through DHCP - Default Public IP addresses aligned to stack.sh defaults - OS Services are listening on Management Interface Related to blueprint xenapi-devstack-cleanup Change-Id: I73c9751ade6f4786b8b682a1994cb87f0a624379 --- lib/nova | 2 +- stack.sh | 3 ++ stackrc | 1 - tools/xen/README.md | 25 +++-------- tools/xen/build_xva.sh | 72 +++++++++++++++---------------- tools/xen/functions | 16 +++++++ tools/xen/install_os_domU.sh | 19 +++++--- tools/xen/templates/interfaces.in | 23 ---------- tools/xen/xenrc | 32 ++++++++------ 9 files changed, 96 insertions(+), 97 deletions(-) delete mode 100644 tools/xen/templates/interfaces.in diff --git a/lib/nova b/lib/nova index cac6330cc2..afc540e7c6 100644 --- a/lib/nova +++ b/lib/nova @@ -77,7 +77,7 @@ SPICE_DIR=$DEST/spice-html5 # Set defaults according to the virt driver if [ "$VIRT_DRIVER" = 'xenserver' ]; then - PUBLIC_INTERFACE_DEFAULT=eth3 + PUBLIC_INTERFACE_DEFAULT=eth2 GUEST_INTERFACE_DEFAULT=eth1 # Allow ``build_domU.sh`` to specify the flat network bridge via kernel args FLAT_NETWORK_BRIDGE_DEFAULT=$(sed -e 's/.* flat_network_bridge=\([[:alnum:]]*\).*$/\1/g' /proc/cmdline) diff --git a/stack.sh b/stack.sh index 4089531710..9321b423d5 100755 --- a/stack.sh +++ b/stack.sh @@ -1001,6 +1001,9 @@ if is_service_enabled nova; then if [ "$VIRT_DRIVER" = 'xenserver' ]; then echo_summary "Using XenServer virtualization driver" + if [ -z "$XENAPI_CONNECTION_URL" ]; then + die $LINENO "XENAPI_CONNECTION_URL is not specified" + fi read_password XENAPI_PASSWORD "ENTER A PASSWORD TO USE FOR XEN." iniset $NOVA_CONF DEFAULT compute_driver "xenapi.XenAPIDriver" iniset $NOVA_CONF DEFAULT xenapi_connection_url "$XENAPI_CONNECTION_URL" diff --git a/stackrc b/stackrc index ef39710035..4263a9d41d 100644 --- a/stackrc +++ b/stackrc @@ -178,7 +178,6 @@ case "$VIRT_DRIVER" in ;; xenserver) # Xen config common to nova and quantum - XENAPI_CONNECTION_URL=${XENAPI_CONNECTION_URL:-"http://169.254.0.1"} XENAPI_USER=${XENAPI_USER:-"root"} ;; *) diff --git a/tools/xen/README.md b/tools/xen/README.md index 8f0c10d0d7..9f3908028d 100644 --- a/tools/xen/README.md +++ b/tools/xen/README.md @@ -34,6 +34,13 @@ The `install_os_domU.sh` script will: `eth0`. - eth3 - Public interface, connected to `PUB_BRIDGE_OR_NET_NAME` defaults to `"OpenStack Public Network"`. + - After the Ubuntu install process finished, the network configuration is + modified to: + - eth0 - Management interface, connected to `MGT_BRIDGE_OR_NET_NAME` + - eth1 - VM interface, connected to `VM_BRIDGE_OR_NET_NAME` + - eth2 - Public interface, connected to `PUB_BRIDGE_OR_NET_NAME` + - (eth3) - Optional network interface if quantum is used, to enforce xapi to + create the underlying bridge. - Start devstack inside the created OpenStack VM ## Step 1: Install Xenserver @@ -92,30 +99,12 @@ Of course, use real passwords if this machine is exposed. MULTI_HOST=1 # Give extra time for boot ACTIVE_TIMEOUT=45 - # Host Interface, i.e. the interface on the OpenStack vm you want to expose - # the services on. The default is eth3, which means the public network, but - # as the public network is going to be virtual, we are setting the services - # to listen on the management network, which defaults to 'xenbr0', the - # XenServer's network. - HOST_IP_IFACE=eth2 - - # Use DHCP server to configure the Management IP of OpenStack VM - MGT_IP="dhcp" # Settings for netinstalling Ubuntu UBUNTU_INST_RELEASE=precise - # First time Ubuntu network install params, use the DHCP server on the - # management network - UBUNTU_INST_IFACE="eth2" - UBUNTU_INST_IP="dhcp" - # NOTE: the value of FLAT_NETWORK_BRIDGE will automatically be determined # by install_os_domU.sh script. - - # Public IP address is aligned with the devstack defaults (see FLOATING_RANGE) - PUB_IP=172.24.4.10 - PUB_NETMASK=255.255.255.0 EOF ## Step 4: Run `./install_os_domU.sh` from the `tools/xen` directory diff --git a/tools/xen/build_xva.sh b/tools/xen/build_xva.sh index b0fd003d52..d0cdf17391 100755 --- a/tools/xen/build_xva.sh +++ b/tools/xen/build_xva.sh @@ -32,6 +32,41 @@ source xenrc # GUEST_NAME="$1" +function _print_interface_config() { + local device_nr + local ip_address + local netmask + + device_nr="$1" + ip_address="$2" + netmask="$3" + + local device + + device="eth${device_nr}" + + echo "auto $device" + if [ $ip_address == "dhcp" ]; then + echo "iface $device inet dhcp" + else + echo "iface $device inet static" + echo " address $ip_address" + echo " netmask $netmask" + fi + + # Turn off tx checksumming for better performance + echo " post-up ethtool -K $device tx off" +} + +function print_interfaces_config() { + echo "auto lo" + echo "iface lo inet loopback" + + _print_interface_config $PUB_DEV_NR $PUB_IP $PUB_NETMASK + _print_interface_config $VM_DEV_NR $VM_IP $VM_NETMASK + _print_interface_config $MGT_DEV_NR $MGT_IP $MGT_NETMASK +} + # # Mount the VDI # @@ -81,42 +116,7 @@ $HOSTS_FILE_IP $GUEST_NAME EOF # Configure the network -INTERFACES=$STAGING_DIR/etc/network/interfaces -TEMPLATES_DIR=$TOP_DIR/templates -cp $TEMPLATES_DIR/interfaces.in $INTERFACES -if [ $VM_IP == "dhcp" ]; then - echo 'eth1 on dhcp' - sed -e "s,iface eth1 inet static,iface eth1 inet dhcp,g" -i $INTERFACES - sed -e '/@ETH1_/d' -i $INTERFACES -else - sed -e "s,@ETH1_IP@,$VM_IP,g" -i $INTERFACES - sed -e "s,@ETH1_NETMASK@,$VM_NETMASK,g" -i $INTERFACES -fi - -if [ $MGT_IP == "dhcp" ]; then - echo 'eth2 on dhcp' - sed -e "s,iface eth2 inet static,iface eth2 inet dhcp,g" -i $INTERFACES - sed -e '/@ETH2_/d' -i $INTERFACES -else - sed -e "s,@ETH2_IP@,$MGT_IP,g" -i $INTERFACES - sed -e "s,@ETH2_NETMASK@,$MGT_NETMASK,g" -i $INTERFACES -fi - -if [ $PUB_IP == "dhcp" ]; then - echo 'eth3 on dhcp' - sed -e "s,iface eth3 inet static,iface eth3 inet dhcp,g" -i $INTERFACES - sed -e '/@ETH3_/d' -i $INTERFACES -else - sed -e "s,@ETH3_IP@,$PUB_IP,g" -i $INTERFACES - sed -e "s,@ETH3_NETMASK@,$PUB_NETMASK,g" -i $INTERFACES -fi - -if [ "$ENABLE_GI" == "true" ]; then - cat <>$INTERFACES -auto eth0 -iface eth0 inet dhcp -EOF -fi +print_interfaces_config > $STAGING_DIR/etc/network/interfaces # Gracefully cp only if source file/dir exists function cp_it { diff --git a/tools/xen/functions b/tools/xen/functions index ebfd4835a2..f22a561688 100644 --- a/tools/xen/functions +++ b/tools/xen/functions @@ -224,3 +224,19 @@ function append_kernel_cmdline() pv_args=$(xe vm-param-get param-name=PV-args uuid=$vm) xe vm-param-set PV-args="$pv_args $kernel_args" uuid=$vm } + +function destroy_all_vifs_of() +{ + local vm_name_label + + vm_name_label="$1" + + local vm + + vm=$(_vm_uuid "$vm_name_label") + IFS=, + for vif in $(xe vif-list vm-uuid=$vm --minimal); do + xe vif-destroy uuid="$vif" + done + unset IFS +} diff --git a/tools/xen/install_os_domU.sh b/tools/xen/install_os_domU.sh index a744869288..be4aa16394 100755 --- a/tools/xen/install_os_domU.sh +++ b/tools/xen/install_os_domU.sh @@ -238,6 +238,15 @@ else vm_uuid=$(xe vm-install template="$TNAME" new-name-label="$GUEST_NAME") fi +## Setup network cards +# Wipe out all +destroy_all_vifs_of "$GUEST_NAME" +# Tenant network +add_interface "$GUEST_NAME" "$VM_BRIDGE_OR_NET_NAME" "$VM_DEV_NR" +# Management network +add_interface "$GUEST_NAME" "$MGT_BRIDGE_OR_NET_NAME" "$MGT_DEV_NR" +# Public network +add_interface "$GUEST_NAME" "$PUB_BRIDGE_OR_NET_NAME" "$PUB_DEV_NR" # # Inject DevStack inside VM disk @@ -248,7 +257,7 @@ $THIS_DIR/build_xva.sh "$GUEST_NAME" # is created by XenServer). This is required for Quantum. Also pass that as a # kernel parameter for DomU if is_service_enabled quantum; then - add_interface "$GUEST_NAME" "$XEN_INT_BRIDGE_OR_NET_NAME" "4" + add_interface "$GUEST_NAME" "$XEN_INT_BRIDGE_OR_NET_NAME" $XEN_INT_DEV_NR XEN_INTEGRATION_BRIDGE=$(bridge_for "$XEN_INT_BRIDGE_OR_NET_NAME") append_kernel_cmdline \ @@ -275,19 +284,19 @@ function ssh_no_check() { # Get hold of the Management IP of OpenStack VM OS_VM_MANAGEMENT_ADDRESS=$MGT_IP if [ $OS_VM_MANAGEMENT_ADDRESS == "dhcp" ]; then - OS_VM_MANAGEMENT_ADDRESS=$(find_ip_by_name $GUEST_NAME 2) + OS_VM_MANAGEMENT_ADDRESS=$(find_ip_by_name $GUEST_NAME $MGT_DEV_NR) fi # Get hold of the Service IP of OpenStack VM -if [ $HOST_IP_IFACE == "eth2" ]; then +if [ $HOST_IP_IFACE == "eth${MGT_DEV_NR}" ]; then OS_VM_SERVICES_ADDRESS=$MGT_IP if [ $MGT_IP == "dhcp" ]; then - OS_VM_SERVICES_ADDRESS=$(find_ip_by_name $GUEST_NAME 2) + OS_VM_SERVICES_ADDRESS=$(find_ip_by_name $GUEST_NAME $MGT_DEV_NR) fi else OS_VM_SERVICES_ADDRESS=$PUB_IP if [ $PUB_IP == "dhcp" ]; then - OS_VM_SERVICES_ADDRESS=$(find_ip_by_name $GUEST_NAME 3) + OS_VM_SERVICES_ADDRESS=$(find_ip_by_name $GUEST_NAME $PUB_DEV_NR) fi fi diff --git a/tools/xen/templates/interfaces.in b/tools/xen/templates/interfaces.in deleted file mode 100644 index 74b41ccfcb..0000000000 --- a/tools/xen/templates/interfaces.in +++ /dev/null @@ -1,23 +0,0 @@ -auto lo -iface lo inet loopback - -# If eth3 is static, the order should not matter -# and eth0 will have the default gateway. If not, -# we probably want the default gateway to be -# what is on the public interface. Hence changed -# the order here. -auto eth3 -iface eth3 inet static - address @ETH3_IP@ - netmask @ETH3_NETMASK@ - -auto eth1 -iface eth1 inet static - address @ETH1_IP@ - netmask @ETH1_NETMASK@ -post-up ethtool -K eth1 tx off - -auto eth2 -iface eth2 inet static - address @ETH2_IP@ - netmask @ETH2_NETMASK@ diff --git a/tools/xen/xenrc b/tools/xen/xenrc index 0ed3a6a7e5..2bd7be7e37 100644 --- a/tools/xen/xenrc +++ b/tools/xen/xenrc @@ -30,32 +30,35 @@ XEN_INT_BRIDGE_OR_NET_NAME="OpenStack VM Integration Network" # VM Password GUEST_PASSWORD=${GUEST_PASSWORD:-secrete} +# Extracted variables for OpenStack VM network device numbers. +# Make sure, they form a continous sequence starting from 0 +MGT_DEV_NR=0 +VM_DEV_NR=1 +PUB_DEV_NR=2 +XEN_INT_DEV_NR=3 + # Host Interface, i.e. the interface on the nova vm you want to expose the -# services on. Usually eth2 (management network) or eth3 (public network) and -# not eth0 (private network with XenServer host) or eth1 (VM traffic network) -HOST_IP_IFACE=${HOST_IP_IFACE:-eth3} +# services on. Usually the device connected to the management network or the +# one connected to the public network is used. +HOST_IP_IFACE=${HOST_IP_IFACE:-"eth${MGT_DEV_NR}"} # # Our nova host's network info # # Management network -MGT_IP=${MGT_IP:-172.16.100.55} -MGT_NETMASK=${MGT_NETMASK:-255.255.255.0} +MGT_IP=${MGT_IP:-dhcp} +MGT_NETMASK=${MGT_NETMASK:-ignored} # VM Network VM_IP=${VM_IP:-10.255.255.255} VM_NETMASK=${VM_NETMASK:-255.255.255.0} # Public network -PUB_IP=${PUB_IP:-192.168.1.55} +# Aligned with stack.sh - see FLOATING_RANGE +PUB_IP=${PUB_IP:-172.24.4.10} PUB_NETMASK=${PUB_NETMASK:-255.255.255.0} -# Decide if you should enable eth0, -# the guest installer network -# You need to disable this on xcp-xapi on Ubuntu 12.04 -ENABLE_GI=true - # Ubuntu install settings UBUNTU_INST_RELEASE="oneiric" UBUNTU_INST_TEMPLATE_NAME="Ubuntu 11.10 (64-bit) for DevStack" @@ -69,8 +72,11 @@ UBUNTU_INST_HTTP_DIRECTORY="/ubuntu" UBUNTU_INST_HTTP_PROXY="" UBUNTU_INST_LOCALE="en_US" UBUNTU_INST_KEYBOARD="us" -# network configuration for ubuntu netinstall -UBUNTU_INST_IFACE="eth3" +# network configuration for ubuntu netinstall. +# TODO(matelakat): get rid of legacy network interfaces +# specify "eth2" to use the management network +# specify "eth3" to use the public network +UBUNTU_INST_IFACE="eth2" UBUNTU_INST_IP="dhcp" UBUNTU_INST_NAMESERVERS="" UBUNTU_INST_NETMASK=""