From da97896e8dce4ac85675e1d5605fbbd39a6b83ad Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Date: Fri, 14 Oct 2016 13:06:06 +0200 Subject: [PATCH] Enable support for multinode in the gate hook This deploys two controller nodes with an haproxy in fron of the main API endpoint and workers on both nodes: primary and subnode1. Change-Id: Iecc38aa950e8b5a4c42db6b609820443efe437c2 Related-Bug: 1628481 --- devstack/etc/octavia/haproxy.cfg | 1 + devstack/plugin.sh | 43 +++++++++++++++++++++--------- devstack/settings | 1 + octavia/tests/contrib/gate_hook.sh | 33 +++++++++++++++++++++++ 4 files changed, 66 insertions(+), 12 deletions(-) diff --git a/devstack/etc/octavia/haproxy.cfg b/devstack/etc/octavia/haproxy.cfg index ddef1ed0e9..54fcf651fb 100644 --- a/devstack/etc/octavia/haproxy.cfg +++ b/devstack/etc/octavia/haproxy.cfg @@ -27,3 +27,4 @@ backend octavia-backend-api # server octavia-main : weight 1 # server octavia-second : weight 1 # + diff --git a/devstack/plugin.sh b/devstack/plugin.sh index abaa4d3418..e78c1ee82a 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -15,6 +15,12 @@ function octavia_install { fi } +function set_octavia_worker_image_owner_id { + image_id=$(openstack image list --property name=${OCTAVIA_AMP_IMAGE_NAME} -f value -c ID) + owner_id=$(openstack image show ${image_id} -c owner -f value) + iniset $OCTAVIA_CONF controller_worker amp_image_owner_id ${owner_id} +} + function build_octavia_worker_image { # pull the agent code from the current code zuul has a reference to @@ -33,10 +39,6 @@ function build_octavia_worker_image { fi upload_image file://${OCTAVIA_AMP_IMAGE_FILE} $TOKEN - image_id=$(openstack image list --property name=${OCTAVIA_AMP_IMAGE_NAME} -f value -c ID) - owner_id=$(openstack image show ${image_id} -c owner -f value) - iniset $OCTAVIA_CONF controller_worker amp_image_owner_id ${owner_id} - } function create_octavia_accounts { @@ -155,12 +157,13 @@ function octavia_configure { } function create_mgmt_network_interface { - # Create security group and rules - openstack security group create lb-health-mgr-sec-grp - openstack security group rule create --protocol udp --dst-port $OCTAVIA_HM_LISTEN_PORT lb-health-mgr-sec-grp + if [ $OCTAVIA_MGMT_PORT_IP != 'auto' ]; then + SUBNET_ID=$(neutron subnet-show lb-mgmt-subnet | awk '/ id / {print $4}') + PORT_FIXED_IP="--fixed-ip subnet_id=$SUBNET_ID,ip_address=$OCTAVIA_MGMT_PORT_IP" + fi # TODO(johnsom) Change this to OSC when security group is working - id_and_mac=$(neutron port-create --name octavia-health-manager-$OCTAVIA_NODE-listen-port --security-group lb-health-mgr-sec-grp --device-owner Octavia:health-mgr --binding:host_id=$(hostname) lb-mgmt-net | awk '/ id | mac_address / {print $4}') + id_and_mac=$(neutron port-create --name octavia-health-manager-$OCTAVIA_NODE-listen-port --security-group lb-health-mgr-sec-grp --device-owner Octavia:health-mgr --binding:host_id=$(hostname) lb-mgmt-net $PORT_FIXED_IP | awk '/ id | mac_address / {print $4}') id_and_mac=($id_and_mac) MGMT_PORT_ID=${id_and_mac[0]} @@ -190,6 +193,7 @@ function create_mgmt_network_interface { iniset $OCTAVIA_CONF health_manager bind_ip $MGMT_PORT_IP iniset $OCTAVIA_CONF health_manager bind_port $OCTAVIA_HM_LISTEN_PORT + } function build_mgmt_network { @@ -203,9 +207,14 @@ function build_mgmt_network { openstack security group rule create --protocol tcp --dst-port 22 lb-mgmt-sec-grp openstack security group rule create --protocol tcp --dst-port 9443 lb-mgmt-sec-grp + # Create security group and rules + openstack security group create lb-health-mgr-sec-grp + openstack security group rule create --protocol udp --dst-port $OCTAVIA_HM_LISTEN_PORT lb-health-mgr-sec-grp +} + +function configure_lb_mgmt_sec_grp { OCTAVIA_MGMT_SEC_GRP_ID=$(openstack security group list | awk ' / lb-mgmt-sec-grp / {print $2}') iniset ${OCTAVIA_CONF} controller_worker amp_secgroup_list ${OCTAVIA_MGMT_SEC_GRP_ID} - } function configure_octavia_tempest { @@ -225,12 +234,12 @@ function create_amphora_flavor { function configure_octavia_api_haproxy { + install_package haproxy + cp ${OCTAVIA_DIR}/devstack/etc/octavia/haproxy.cfg ${OCTAVIA_CONF_DIR}/haproxy.cfg sed -i.bak "s/OCTAVIA_PORT/${OCTAVIA_PORT}/" ${OCTAVIA_CONF_DIR}/haproxy.cfg - iniset $OCTAVIA_CONF DEFAULT bind_port ${OCTAVIA_HA_PORT} - NODES=(${OCTAVIA_NODES//,/ }) for NODE in ${NODES[@]}; do @@ -270,6 +279,7 @@ function octavia_start { if ! [ "$DISABLE_AMP_IMAGE_BUILD" == 'True' ]; then build_octavia_worker_image + set_octavia_worker_image_owner_id fi OCTAVIA_AMP_IMAGE_ID=$(openstack image list -f value --property name=${OCTAVIA_AMP_IMAGE_NAME} -c ID) @@ -289,10 +299,11 @@ function octavia_start { configure_octavia_tempest ${OCTAVIA_AMP_NETWORK_ID} fi else - OCTAVIA_AMP_IMAGE_ID=$(openstack image list -f value --property name=${OCTAVIA_AMP_IMAGE_NAME} -c ID) + set_octavia_worker_image_owner_id fi create_mgmt_network_interface + configure_lb_mgmt_sec_grp iniset $OCTAVIA_CONF controller_worker amp_image_tag ${OCTAVIA_AMP_IMAGE_TAG} @@ -303,6 +314,14 @@ function octavia_start { if [ $OCTAVIA_NODE == 'main' ]; then configure_octavia_api_haproxy run_process $OCTAVIA_API_HAPROXY "/usr/sbin/haproxy -db -V -f ${OCTAVIA_CONF_DIR}/haproxy.cfg" + # make sure octavia is reachable from haproxy + iniset $OCTAVIA_CONF DEFAULT bind_port ${OCTAVIA_HA_PORT} + iniset $OCTAVIA_CONF DEFAULT bind_host 0.0.0.0 + fi + if [ $OCTAVIA_NODE != 'main' ] && [ $OCTAVIA_NODE != 'standalone' ] ; then + # make sure octavia is reachable from haproxy from main node + iniset $OCTAVIA_CONF DEFAULT bind_port ${OCTAVIA_HA_PORT} + iniset $OCTAVIA_CONF DEFAULT bind_host 0.0.0.0 fi run_process $OCTAVIA_API "$OCTAVIA_API_BINARY $OCTAVIA_API_ARGS" diff --git a/devstack/settings b/devstack/settings index 7583306a98..b8b45f65b6 100644 --- a/devstack/settings +++ b/devstack/settings @@ -86,3 +86,4 @@ OCTAVIA_USE_PREGENERATED_CERTS=${OCTAVIA_USE_PREGENERATED_CERTS:-"False"} OCTAVIA_PREGENERATED_CERTS_DIR=${OCTAVIA_PREGENERATED_CERTS_DIR:-"${OCTAVIA_DIR}/devstack/pregenerated/certs"} OCTAVIA_NODE=${OCTAVIA_NODE:-"standalone"} OCTAVIA_CONTROLLER_IP_PORT_LIST=${OCTAVIA_CONTROLLER_IP_PORT_LIST:-"auto"} +OCTAVIA_MGMT_PORT_IP=${OCTAVIA_MGMT_PORT_IP:-"auto"} diff --git a/octavia/tests/contrib/gate_hook.sh b/octavia/tests/contrib/gate_hook.sh index e8242699ed..4e637e421b 100755 --- a/octavia/tests/contrib/gate_hook.sh +++ b/octavia/tests/contrib/gate_hook.sh @@ -16,6 +16,36 @@ if egrep --quiet '(vmx|svm)' /proc/cpuinfo; then export DEVSTACK_GATE_LIBVIRT_TYPE=kvm fi +function _setup_octavia_multinode { + + PRIMARY_NODE_IP=$(cat /etc/nodepool/primary_node_private) + SUBNODE_IP=$(head -n1 /etc/nodepool/sub_nodes_private) + + # OCTAVIA_CONTROLLER_IP_PORT_LIST are the ips inside the + # lb-mgmt network that the amphoras reach via heartbeats + COMMON_MULTINODE_CONFIG=" + OCTAVIA_USE_PREGENERATED_CERTS=True + OCTAVIA_USE_PREGENERATED_SSH_KEY=True + OCTAVIA_CONTROLLER_IP_PORT_LIST=192.168.0.3:5555,192.168.0.4:5555 + + # Embedded fix for devstack bug/1629133 , this line can be removed once + # that bug is fixed + SUBNETPOOL_PREFIX_V4=10.0.0.0/16 + " + + export DEVSTACK_LOCAL_CONFIG+="$COMMON_MULTINODE_CONFIG + OCTAVIA_NODE=main + OCTAVIA_NODES=main:$PRIMARY_NODE_IP,second:$SUBNODE_IP + enable_service o-api-ha + OCTAVIA_MGMT_PORT_IP=192.168.0.3 + " + + export DEVSTACK_SUBNODE_CONFIG+="$COMMON_MULTINODE_CONFIG + OCTAVIA_NODE=second + enable_plugin octavia https://git.openstack.org/openstack/octavia + OCTAVIA_MGMT_PORT_IP=192.168.0.4 + " +} function _setup_octavia { export DEVSTACK_LOCAL_CONFIG+=" @@ -29,6 +59,9 @@ function _setup_octavia { fi if [ "$testenv" != "apiv1" ]; then ENABLED_SERVICES+="octavia,o-cw,o-hk,o-hm,o-api," + if [ "$DEVSTACK_GATE_TOPOLOGY" == "multinode" ]; then + _setup_octavia_multinode + fi fi if [ "$testenv" = "apiv1" ]; then cat > "$DEVSTACK_PATH/local.conf" <