Add mulitnode CI job support to tripleo-ci

This would allow using multinode jobs spawned by nodepool, but that is
not the only use case. Any cloud provider could be used by this setup,
including a 3rd party CI setup.

The scripts here are modified to support installing the undercloud
directly on the node running the toci_gate_test.sh script, which is
often the jenkins slave. When deploying an all in one Overcloud this
results in needing 2 nodes total to do the basic TripleO deployment.

There is a new script named toci_instack_multinode.sh to run the
multinode setup.

Change-Id: Icc67a289df207348536c0a2419b3fc81f03fe758

heat-templates/net-config-multinode.yaml

@@ -0,0 +1,64 @@
+heat_template_version: 2015-04-30
+
+description: >
+  Software Config to drive os-net-config for a simple bridge configured
+  with a static IP address for the ctlplane network.
+
+parameters:
+  ControlPlaneIp:
+    default: ''
+    description: IP address/subnet on the ctlplane network
+    type: string
+  ExternalIpSubnet:
+    default: ''
+    description: IP address/subnet on the external network
+    type: string
+  InternalApiIpSubnet:
+    default: ''
+    description: IP address/subnet on the internal API network
+    type: string
+  StorageIpSubnet:
+    default: ''
+    description: IP address/subnet on the storage network
+    type: string
+  StorageMgmtIpSubnet:
+    default: ''
+    description: IP address/subnet on the storage mgmt network
+    type: string
+  TenantIpSubnet:
+    default: ''
+    description: IP address/subnet on the tenant network
+    type: string
+  ManagementIpSubnet:
+    default: ''
+    description: IP address/subnet on the management network
+    type: string
+  ControlPlaneSubnetCidr: # Override this via parameter_defaults
+    default: '24'
+    description: The subnet CIDR of the control plane network.
+    type: string
+
+resources:
+  OsNetConfigImpl:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      config:
+        str_replace:
+          template: |
+            #!/bin/bash
+            ip addr add CONTROLPLANEIP/CONTROLPLANESUBNETCIDR dev $bridge_name
+          params:
+            CONTROLPLANEIP: {get_param: ControlPlaneIp}
+            CONTROLPLANESUBNETCIDR: {get_param: ControlPlaneSubnetCidr}
+      inputs:
+        -
+          name: bridge_name
+          default: br-ex
+          description: bridge-name
+          type: String
+
+outputs:
+  OS::stack_id:
+    description: The OsNetConfigImpl resource.
+    value: {get_resource: OsNetConfigImpl}

scripts/bootstrap-overcloud-full.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+
+set -eux
+
+# Source deploy.env if it exists. It should exist if we are running under
+# tripleo-ci
+export TRIPLEO_ROOT=${TRIPLEO_ROOT:-"/opt/stack/new"}
+if [ -f "$TRIPLEO_ROOT/tripleo-ci/deploy.env" ]; then
+    source $TRIPLEO_ROOT/tripleo-ci/deploy.env
+fi
+
+# Copied from toci_gate_test.sh...need to apply this fix on subnodes as well
+# TODO(pabelanger): Why is python-requests installed from pip?
+sudo rm -rf /usr/lib/python2.7/site-packages/requests
+
+# Clear out any puppet modules on the node placed their by infra configuration
+sudo rm -rf /etc/puppet/modules/*
+
+# This will remove any puppet configuration done my infra setup
+sudo yum -y remove puppet facter hiera
+
+# Update everything
+sudo yum -y update
+# instack-undercloud will pull in all the needed deps
+# git needed since puppet modules installed from source
+# openstack-tripleo-common needed for the tripleo-build-images command
+sudo yum -y install instack-undercloud git openstack-tripleo-common
+
+export ELEMENTS_PATH="/usr/share/diskimage-builder/elements:/usr/share/instack-undercloud:/usr/share/tripleo-image-elements:/usr/share/tripleo-puppet-elements:/usr/share/openstack-heat-templates/software-config/elements"
+
+export DIB_INSTALLTYPE_puppet_modules=source
+
+sudo yum -y install openstack-tripleo-common
+
+ELEMENTS=$(\
+tripleo-build-images \
+  --image-json-output \
+  --image-config-file /usr/share/tripleo-common/image-yaml/overcloud-images-centos7.yaml \
+  --image-config-file /usr/share/tripleo-common/image-yaml/overcloud-images.yaml \
+  | jq '. | map(select(.imagename == "overcloud")) | .[0].elements | map(.+" ") | add' \
+  | sed 's/"//g')
+
+# delorean-repo is excluded b/c we've already run --repo-setup on this node and
+# we don't want to overwrite that.
+sudo -E instack \
+  -e centos7 \
+     enable-packages-install \
+     install-types \
+     $ELEMENTS \
+  -k extra-data \
+     pre-install \
+     install \
+     post-install \
+  -b 05-fstab-rootfs-label \
+     00-fix-requiretty \
+     90-rebuild-ramdisk \
+     00-usr-local-bin-secure-path \
+  -x delorean-repo \
+  -d
+
+PACKAGES=$(\
+tripleo-build-images \
+  --image-json-output \
+  --image-config-file /usr/share/tripleo-common/image-yaml/overcloud-images-centos7.yaml \
+  --image-config-file /usr/share/tripleo-common/image-yaml/overcloud-images.yaml \
+  | jq '. | map(select(.imagename == "overcloud")) | .[0].packages | .[] | tostring' \
+  | sed 's/"//g')
+
+# Install additional packages expected by the image
+sudo yum -y install $PACKAGES
+
+sudo sed -i 's/SELINUX=enforcing/SELINUX=permissive/' /etc/selinux/config
+sudo setenforce 0

scripts/common_functions.sh

@@ -187,6 +187,7 @@ function extract_logs(){
 }
 
 function postci(){
+    set -x
     set +e
     stop_metric "tripleo.ci.total.seconds"
     if [ -e $TRIPLEO_ROOT/delorean/data/repos/ ] ; then
@@ -219,6 +220,23 @@ function postci(){
         if [ -z "${LEAVE_RUNNING:-}" ] && [ -n "${HOST_IP:-}" ] ; then
             destroy_vms &> $WORKSPACE/logs/destroy_vms.log
         fi
+    elif [ "$MULTINODE" = "1" ] ; then
+        local i=2
+        sudo /opt/stack/new/tripleo-ci/scripts/get_host_info.sh
+        $TARCMD > $WORKSPACE/logs/primary_node.tar.xz
+        for ip in $(cat /etc/nodepool/sub_nodes_private); do
+            mkdir $WORKSPACE/logs/subnode-$i/
+            ssh $SSH_OPTIONS -i /etc/nodepool/id_rsa $ip \
+                sudo /opt/stack/new/tripleo-ci/scripts/get_host_info.sh
+            ssh $SSH_OPTIONS -i /etc/nodepool/id_rsa $ip \
+                $TARCMD > $WORKSPACE/logs/subnode-$i/subnode-$i.tar.xz
+            # TODO: For some reason, these files are causing the publish logs ansible
+            # task to fail with an rsync error:
+            # "symlink has no referent"
+            rm -f $WORKSPACE/logs/subnode-$i/etc/sahara/rootwrap.d/sahara.filters
+            rm -f $WORKSPACE/logs/subnode-$i/etc/cinder/rootwrap.d/os-brick.filters
+            let i+=1
+        done
     fi
 
     sudo chown -R $USER $WORKSPACE
@@ -296,7 +314,7 @@ function layer_ci_repo {
 
 
 function echo_vars_to_deploy_env {
-    for VAR in CENTOS_MIRROR EPEL_MIRROR http_proxy INTROSPECT MY_IP no_proxy NODECOUNT OVERCLOUD_DEPLOY_ARGS OVERCLOUD_UPDATE_ARGS PACEMAKER SSH_OPTIONS STABLE_RELEASE TRIPLEO_ROOT TRIPLEO_SH_ARGS NETISO_V4 NETISO_V6 TOCI_JOBTYPE UNDERCLOUD_SSL RUN_TEMPEST_TESTS RUN_PING_TEST JOB_NAME OVB UNDERCLOUD_IDEMPOTENT; do
+    for VAR in CENTOS_MIRROR EPEL_MIRROR http_proxy INTROSPECT MY_IP no_proxy NODECOUNT OVERCLOUD_DEPLOY_ARGS OVERCLOUD_UPDATE_ARGS PACEMAKER SSH_OPTIONS STABLE_RELEASE TRIPLEO_ROOT TRIPLEO_SH_ARGS NETISO_V4 NETISO_V6 TOCI_JOBTYPE UNDERCLOUD_SSL RUN_TEMPEST_TESTS RUN_PING_TEST JOB_NAME OVB UNDERCLOUD_IDEMPOTENT MULTINODE CONTROLLER_HOSTS COMPUTE_HOSTS SUBNODES_SSH_KEY; do
         echo "export $VAR=\"${!VAR}\"" >> $TRIPLEO_ROOT/tripleo-ci/deploy.env
     done
 }

scripts/deploy.sh

@@ -26,6 +26,12 @@ echo "hieradata_override = $HOME/undercloud-hieradata-override.yaml" >> ~/underc
 if [ $UNDERCLOUD_SSL == 1 ] ; then
     echo 'generate_service_certificate = True' >> ~/undercloud.conf
 fi
+# TODO: fix this in instack-undercloud
+sudo mkdir -p /etc/puppet/hieradata
+
+if [ "$MULTINODE" = 1 ]; then
+    echo 'net_config_override = /opt/stack/new/tripleo-ci/undercloud-configs/net-config-multinode.json.template' >> ~/undercloud.conf
+fi
 
 sudo yum install -y moreutils
 echo "INFO: Check /var/log/undercloud_install.txt for undercloud install output"
@@ -95,31 +101,34 @@ EOF_CAT
     sudo os-net-config -c /tmp/eth6.cfg -v
 fi
 
-# Our ci underclouds don't have enough RAM to allow us to use a tmpfs
+if [ "$MULTINODE" = "0" ]; then
-export DIB_NO_TMPFS=1
+    # Our ci underclouds don't have enough RAM to allow us to use a tmpfs
-# Override the default repositories set by tripleo.sh, to add the delorean-ci repository
+    export DIB_NO_TMPFS=1
-export OVERCLOUD_IMAGES_DIB_YUM_REPO_CONF=$(ls /etc/yum.repos.d/delorean*)
+    # Override the default repositories set by tripleo.sh, to add the delorean-ci repository
-# Directing the output of this command to a file as its extreemly verbose
+    export OVERCLOUD_IMAGES_DIB_YUM_REPO_CONF=$(ls /etc/yum.repos.d/delorean*)
-echo "INFO: Check /var/log/image_build.txt for image build output"
+    # Directing the output of this command to a file as its extreemly verbose
-echo "INFO: This file can be found in logs/undercloud.tar.xz in the directory containing console.log"
+    echo "INFO: Check /var/log/image_build.txt for image build output"
-start_metric "tripleo.overcloud.${TOCI_JOBTYPE}.images.seconds"
+    echo "INFO: This file can be found in logs/undercloud.tar.xz in the directory containing console.log"
-$TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --overcloud-images | ts '%Y-%m-%d %H:%M:%S.000 |' | sudo dd of=/var/log/image_build.txt || (tail -n 50 /var/log/image_build.txt && false)
+    start_metric "tripleo.overcloud.${TOCI_JOBTYPE}.images.seconds"
-stop_metric "tripleo.overcloud.${TOCI_JOBTYPE}.images.seconds"
+    $TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --overcloud-images | ts '%Y-%m-%d %H:%M:%S.000 |' | sudo dd of=/var/log/image_build.txt || (tail -n 50 /var/log/image_build.txt && false)
+    stop_metric "tripleo.overcloud.${TOCI_JOBTYPE}.images.seconds"
 
-OVERCLOUD_IMAGE_MB=$(du -ms overcloud-full.qcow2 | cut -f 1 | sed 's|.$||')
+    OVERCLOUD_IMAGE_MB=$(du -ms overcloud-full.qcow2 | cut -f 1 | sed 's|.$||')
-record_metric "tripleo.overcloud.${TOCI_JOBTYPE}.image.size_mb" "$OVERCLOUD_IMAGE_MB"
+    record_metric "tripleo.overcloud.${TOCI_JOBTYPE}.image.size_mb" "$OVERCLOUD_IMAGE_MB"
 
-start_metric "tripleo.register.nodes.seconds"
+    start_metric "tripleo.register.nodes.seconds"
-$TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --register-nodes
+    $TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --register-nodes
-stop_metric "tripleo.register.nodes.seconds"
+    stop_metric "tripleo.register.nodes.seconds"
 
-if [ $INTROSPECT == 1 ] ; then
+    if [ $INTROSPECT == 1 ] ; then
-   start_metric "tripleo.introspect.seconds"
+       start_metric "tripleo.introspect.seconds"
-   $TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --introspect-nodes
+       $TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --introspect-nodes
-   stop_metric "tripleo.introspect.seconds"
+       stop_metric "tripleo.introspect.seconds"
+    fi
+
+    sleep 60
 fi
 
-sleep 60
 
 if [ -n "${OVERCLOUD_UPDATE_ARGS:-}" ] ; then
     # Reinstall openstack-tripleo-heat-templates from delorean-current.
@@ -130,6 +139,23 @@ if [ -n "${OVERCLOUD_UPDATE_ARGS:-}" ] ; then
     sudo yum -y --disablerepo=* --enablerepo=delorean,delorean-current install openstack-tripleo-heat-templates
 fi
 
+if [ "$MULTINODE" = "1" ]; then
+    # Start the script that will configure os-collect-config on the subnodes
+    source ~/stackrc
+    /usr/share/openstack-tripleo-heat-templates/deployed-server/scripts/get-occ-config.sh 2>&1 | sudo dd of=/var/log/deployed-server-os-collect-config.log &
+
+    # Create dummy overcloud-full image since there is no way (yet) to disable
+    # this constraint in the heat templates
+    qemu-img create -f qcow2 overcloud-full.qcow2 1G
+    if ! glance image-show overcloud-full; then
+        glance image-create \
+            --container-format bare \
+            --disk-format qcow2 \
+            --name overcloud-full \
+            --file overcloud-full.qcow2
+    fi
+fi
+
 export OVERCLOUD_DEPLOY_ARGS="$OVERCLOUD_DEPLOY_ARGS -e $TRIPLEO_ROOT/tripleo-ci/test-environments/worker-config.yaml"
 if [[ "${STABLE_RELEASE}" =~ ^(liberty|mitaka)$ ]] ; then
     OVERCLOUD_DEPLOY_ARGS="$OVERCLOUD_DEPLOY_ARGS -e $TRIPLEO_ROOT/tripleo-ci/test-environments/worker-config-mitaka-and-below.yaml"
@@ -159,27 +185,29 @@ if [ -n "${OVERCLOUD_UPDATE_ARGS:-}" ] ; then
     http_proxy= $TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --overcloud-update ${TRIPLEO_SH_ARGS:-}
 fi
 
-# Sanity test we deployed what we said we would
+if [ "$MULTINODE" = "0" ]; then
-source ~/stackrc
+    # Sanity test we deployed what we said we would
-[ "$NODECOUNT" != $(nova list | grep ACTIVE | wc -l | cut -f1 -d " ") ] && echo "Wrong number of nodes deployed" && exit 1
+    source ~/stackrc
+    [ "$NODECOUNT" != $(nova list | grep ACTIVE | wc -l | cut -f1 -d " ") ] && echo "Wrong number of nodes deployed" && exit 1
 
-if [ $PACEMAKER == 1 ] ; then
+    if [ $PACEMAKER == 1 ] ; then
-    # Wait for the pacemaker cluster to settle and all resources to be
+        # Wait for the pacemaker cluster to settle and all resources to be
-    # available. heat-{api,engine} are the best candidates since due to the
+        # available. heat-{api,engine} are the best candidates since due to the
-    # constraint ordering they are typically started last. We'll wait up to
+        # constraint ordering they are typically started last. We'll wait up to
-    # 180s.
+        # 180s.
-    start_metric "tripleo.overcloud.${TOCI_JOBTYPE}.settle.seconds"
+        start_metric "tripleo.overcloud.${TOCI_JOBTYPE}.settle.seconds"
-    timeout -k 10 240 ssh $SSH_OPTIONS heat-admin@$(nova list | grep controller-0 | awk '{print $12}' | cut -d'=' -f2) sudo crm_resource -r openstack-heat-api --wait || {
+        timeout -k 10 240 ssh $SSH_OPTIONS heat-admin@$(nova list | grep controller-0 | awk '{print $12}' | cut -d'=' -f2) sudo crm_resource -r openstack-heat-api --wait || {
-        exitcode=$?
+            exitcode=$?
-        echo "crm_resource for openstack-heat-api has failed!"
+            echo "crm_resource for openstack-heat-api has failed!"
-        exit $exitcode
+            exit $exitcode
-        }
+            }
-    timeout -k 10 240 ssh $SSH_OPTIONS heat-admin@$(nova list | grep controller-0 | awk '{print $12}' | cut -d'=' -f2) sudo crm_resource -r openstack-heat-engine --wait|| {
+        timeout -k 10 240 ssh $SSH_OPTIONS heat-admin@$(nova list | grep controller-0 | awk '{print $12}' | cut -d'=' -f2) sudo crm_resource -r openstack-heat-engine --wait|| {
-        exitcode=$?
+            exitcode=$?
-        echo "crm_resource for openstack-heat-engine has failed!"
+            echo "crm_resource for openstack-heat-engine has failed!"
-        exit $exitcode
+            exit $exitcode
-        }
+            }
-     stop_metric "tripleo.overcloud.${TOCI_JOBTYPE}.settle.seconds"
+         stop_metric "tripleo.overcloud.${TOCI_JOBTYPE}.settle.seconds"
+    fi
 fi
 
 source ~/overcloudrc

scripts/tripleo.sh

@@ -53,6 +53,7 @@ function show_options {
     echo "      --repo-setup            -- Perform repository setup."
     echo "      --delorean-setup        -- Install local delorean build environment."
     echo "      --delorean-build        -- Build a delorean package locally"
+    echo "      --multinode-setup       -- Perform multinode setup."
     echo "      --undercloud            -- Install the undercloud."
     echo "      --overcloud-images      -- Build and load overcloud images."
     echo "      --register-nodes        -- Register and configure nodes."
@@ -78,7 +79,7 @@ if [ ${#@} = 0 ]; then
 fi
 
 TEMP=$(getopt -o ,h \
-        -l,help,repo-setup,delorean-setup,delorean-build,undercloud,overcloud-images,register-nodes,introspect-nodes,overcloud-deploy,overcloud-update,overcloud-delete,use-containers,overcloud-pingtest,skip-pingtest-cleanup,all,enable-check,run-tempest \
+        -l,help,repo-setup,delorean-setup,delorean-build,multinode-setup,undercloud,overcloud-images,register-nodes,introspect-nodes,overcloud-deploy,overcloud-update,overcloud-delete,use-containers,overcloud-pingtest,skip-pingtest-cleanup,all,enable-check,run-tempest \
         -o,x,h,a \
         -n $SCRIPT_NAME -- "$@")
 
@@ -106,7 +107,7 @@ OVERCLOUD_DEPLOY=${OVERCLOUD_DEPLOY:-""}
 OVERCLOUD_DELETE=${OVERCLOUD_DELETE:-""}
 OVERCLOUD_DELETE_TIMEOUT=${OVERCLOUD_DELETE_TIMEOUT:-"300"}
 OVERCLOUD_DEPLOY_ARGS=${OVERCLOUD_DEPLOY_ARGS:-""}
-OVERCLOUD_VALIDATE_ARGS=${OVERCLOUD_VALIDATE_ARGS:-"--validation-errors-fatal --validation-warnings-fatal"}
+OVERCLOUD_VALIDATE_ARGS=${OVERCLOUD_VALIDATE_ARGS-"--validation-errors-fatal --validation-warnings-fatal"}
 OVERCLOUD_UPDATE=${OVERCLOUD_UPDATE:-""}
 OVERCLOUD_UPDATE_RM_FILES=${OVERCLOUD_UPDATE_RM_FILES:-"1"}
 OVERCLOUD_UPDATE_ARGS=${OVERCLOUD_UPDATE_ARGS:-"$OVERCLOUD_DEPLOY_ARGS $OVERCLOUD_VALIDATE_ARGS"}
@@ -125,6 +126,7 @@ OVERCLOUD_IMAGES_DIB_YUM_REPO_CONF=${OVERCLOUD_IMAGES_DIB_YUM_REPO_CONF:-"\
     $REPO_PREFIX/delorean-deps.repo"}
 DELOREAN_SETUP=${DELOREAN_SETUP:-""}
 DELOREAN_BUILD=${DELOREAN_BUILD:-""}
+MULTINODE_SETUP=${MULTINODE_SETUP:-""}
 STDERR=/dev/null
 UNDERCLOUD=${UNDERCLOUD:-""}
 UNDERCLOUD_CONF=${UNDERCLOUD_CONF:-"/usr/share/instack-undercloud/undercloud.conf.sample"}
@@ -135,6 +137,8 @@ TEMPEST_ARGS=${TEMPEST_ARGS:-"--parallel --subunit"}
 TEMPEST_ADD_CONFIG=${TEMPEST_ADD_CONFIG:-}
 TEMPEST_REGEX=${TEMPEST_REGEX:-"^(?=(.*smoke))(?!(tempest.api.orchestration.stacks|tempest.scenario.test_volume_boot_pattern|tempest.api.telemetry))"}
 TEMPEST_PINNED="fb77374ddeeb1642bffa086311d5f281e15142b2"
+SSH_OPTIONS=${SSH_OPTIONS:-'-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=Verbose -o PasswordAuthentication=no -o ConnectionAttempts=32'}
+export SCRIPTS_DIR=$(dirname ${BASH_SOURCE[0]:-$0})
 
 # TODO: remove this when Image create in openstackclient supports the v2 API
 export OS_IMAGE_API_VERSION=1
@@ -159,6 +163,7 @@ while true ; do
         --delorean-setup) DELOREAN_SETUP="1"; shift 1;;
         --delorean-build) DELOREAN_BUILD="1"; shift 1;;
         --undercloud) UNDERCLOUD="1"; shift 1;;
+        --multinode-setup) MULTINODE_SETUP="1"; shift 1;;
         -x) set -x; STDERR=/dev/stderr; shift 1;;
         -h | --help) show_options 0;;
         --) shift ; break ;;
@@ -783,6 +788,103 @@ function tempest_run {
     exit ${exitval}
 }
 
+function clone {
+
+    local repo=$1
+
+    log "$0 requires $repo to be cloned at \$TRIPLEO_ROOT ($TRIPLEO_ROOT)"
+
+    if [ ! -d $TRIPLEO_ROOT/$(basename $repo) ]; then
+        echo "$repo not found at $TRIPLEO_ROOT/$repo, git cloning."
+        pushd $TRIPLEO_ROOT
+        git clone https://git.openstack.org/$repo
+        popd
+    else
+        echo "$repo found at $TRIPLEO_ROOT/$repo, nothing to do."
+    fi
+
+}
+
+function multinode_setup {
+
+    log "Multinode Setup"
+
+    clone openstack-dev/devstack
+    clone openstack-infra/devstack-gate
+
+    # $BASE is expected by devstack/functions-common
+    # which is sourced by devstack-gate/functions.sh
+    # It should be the parent directory of the "new" directory where
+    # zuul-cloner has checked out the repositories
+    log "Sourcing devstack-gate/functions.sh"
+    export BASE=${BASE:-"/opt/stack"}
+    set +u
+    source $TRIPLEO_ROOT/devstack-gate/functions.sh
+    set -u
+
+    PUB_BRIDGE_NAME=${PUB_BRIDGE_NAME:-"br-ex"}
+
+    local primary_node
+    primary_node=$(cat /etc/nodepool/primary_node_private)
+    local sub_nodes
+    sub_nodes=$(cat /etc/nodepool/sub_nodes_private)
+
+    for ip in $sub_nodes; do
+        # Do repo setup
+        ssh $SSH_OPTIONS -t -i /etc/nodepool/id_rsa $ip \
+            $TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --repo-setup
+
+        # Run overcloud full bootstrap script
+        log "Running overcloud-full boostrap script on $ip"
+        ssh $SSH_OPTIONS -t -i /etc/nodepool/id_rsa $ip \
+            $TRIPLEO_ROOT/tripleo-ci/scripts/bootstrap-overcloud-full.sh
+    done
+
+    # Create OVS vxlan bridges
+    # If br-ctlplane already exists on this node, we need to bring it down
+    # first, then bring it back up after calling ovs_vxlan_bridge. This ensures
+    # that the route added to br-ex by ovs_vxlan_bridge will be preferred over
+    # the br-ctlplane route. If it's not preferred, multinode connectivity
+    # across the vxlan bridge will not work.
+    if [ -f /etc/sysconfig/network-scripts/ifcfg-br-ctlplane ]; then
+        sudo ifdown br-ctlplane
+    fi
+    set +u
+    ovs_vxlan_bridge $PUB_BRIDGE_NAME $primary_node "True" 2 192.0.2 24 $sub_nodes
+    set -u
+
+    log "Setting $PUB_BRIDGE_NAME up on $primary_node"
+    sudo ip link set dev $PUB_BRIDGE_NAME up
+
+    if [ -f /etc/sysconfig/network-scripts/ifcfg-br-ctlplane ]; then
+        sudo ifup br-ctlplane
+    fi
+    # Restart neutron-openvswitch-agent if it's enabled, since it may have
+    # terminated when br-ctlplane was down
+    if [ $(sudo systemctl is-enabled neutron-openvswitch-agent) = 'enabled' ]; then
+        sudo systemctl reset-failed neutron-openvswitch-agent
+        sudo systemctl restart neutron-openvswitch-agent
+    fi
+
+    local ping_command="ping -c 3 -W 3 192.0.2.2"
+
+    for ip in $sub_nodes; do
+        log "Setting $PUB_BRIDGE_NAME up on $ip"
+        ssh $SSH_OPTIONS -t -i /etc/nodepool/id_rsa $ip \
+            sudo ip link set dev $PUB_BRIDGE_NAME up
+        if ! remote_command $ip $ping_command; then
+            log "Pinging from $ip failed, restarting openvswitch"
+            remote_command $ip sudo systemctl restart openvswitch
+            if ! remote_command $ip $ping_command; then
+                log "Pinging from $ip still failed after restarting openvswitch"
+                exit 1
+            fi
+        fi
+    done
+
+    log "Multinode Setup - DONE".
+}
+
 if [ "$REPO_SETUP" = 1 ]; then
     repo_setup
 fi
@@ -841,6 +943,10 @@ if [ "$TEMPEST_RUN" = 1 ]; then
     tempest_run
 fi
 
+if [ "$MULTINODE_SETUP" = 1 ]; then
+    multinode_setup
+fi
+
 if [ "$ALL" = 1 ]; then
     repo_setup
     undercloud

test-environments/multinode.yaml

@@ -0,0 +1,60 @@
+resource_registry:
+  OS::TripleO::Controller::Net::SoftwareConfig: ../heat-templates/net-config-multinode.yaml
+  OS::TripleO::Compute::Net::SoftwareConfig: ../heat-templates/net-config-multinode.yaml
+
+parameter_defaults:
+  ControllerServices:
+    - OS::TripleO::Services::CephMon
+    - OS::TripleO::Services::CephExternal
+    - OS::TripleO::Services::CinderApi
+    - OS::TripleO::Services::CinderScheduler
+    - OS::TripleO::Services::CinderVolume
+    - OS::TripleO::Services::Kernel
+    - OS::TripleO::Services::Keystone
+    - OS::TripleO::Services::GlanceApi
+    - OS::TripleO::Services::GlanceRegistry
+    - OS::TripleO::Services::HeatApi
+    - OS::TripleO::Services::HeatApiCfn
+    - OS::TripleO::Services::HeatApiCloudwatch
+    - OS::TripleO::Services::HeatEngine
+    - OS::TripleO::Services::MySQL
+    - OS::TripleO::Services::NeutronDhcpAgent
+    - OS::TripleO::Services::NeutronL3Agent
+    - OS::TripleO::Services::NeutronMetadataAgent
+    - OS::TripleO::Services::NeutronServer
+    - OS::TripleO::Services::NeutronCorePlugin
+    - OS::TripleO::Services::NeutronOvsAgent
+    - OS::TripleO::Services::RabbitMQ
+    - OS::TripleO::Services::HAproxy
+    - OS::TripleO::Services::Keepalived
+    - OS::TripleO::Services::Memcached
+    - OS::TripleO::Services::Pacemaker
+    - OS::TripleO::Services::Redis
+    - OS::TripleO::Services::NovaConductor
+    - OS::TripleO::Services::MongoDb
+    - OS::TripleO::Services::NovaApi
+    - OS::TripleO::Services::NovaScheduler
+    - OS::TripleO::Services::NovaConsoleauth
+    - OS::TripleO::Services::NovaVncproxy
+    - OS::TripleO::Services::Ntp
+    - OS::TripleO::Services::SwiftProxy
+    - OS::TripleO::Services::SwiftStorage
+    - OS::TripleO::Services::Snmp
+    - OS::TripleO::Services::Timezone
+    - OS::TripleO::Services::CeilometerApi
+    - OS::TripleO::Services::CeilometerCollector
+    - OS::TripleO::Services::CeilometerExpirer
+    - OS::TripleO::Services::CeilometerAgentCentral
+    - OS::TripleO::Services::CeilometerAgentNotification
+    - OS::TripleO::Services::Horizon
+    - OS::TripleO::Services::GnocchiApi
+    - OS::TripleO::Services::GnocchiMetricd
+    - OS::TripleO::Services::GnocchiStatsd
+    - OS::TripleO::Services::CephClient
+    - OS::TripleO::Services::CephExternal
+    - OS::TripleO::Services::NovaCompute
+    - OS::TripleO::Services::NovaLibvirt
+    - OS::TripleO::Services::ComputeCeilometerAgent
+  ControllerExtraConfig:
+    nova::compute::libvirt::services::libvirt_virt_type: qemu
+    nova::compute::libvirt::libvirt_virt_type: qemu

toci_gate_test.sh

@@ -71,6 +71,7 @@ export PACEMAKER=0
 # the deploy.  Hopefully that's enough, while still leaving some cushion to come
 # in under the gate timeout so we can collect logs.
 OVERCLOUD_DEPLOY_TIMEOUT=$((DEVSTACK_GATE_TIMEOUT-90))
+export OVERCLOUD_SSH_USER=${OVERCLOUD_SSH_USER:-"jenkins"}
 export OVERCLOUD_DEPLOY_ARGS=${OVERCLOUD_DEPLOY_ARGS:-""}
 export OVERCLOUD_DEPLOY_ARGS="$OVERCLOUD_DEPLOY_ARGS --libvirt-type=qemu -t $OVERCLOUD_DEPLOY_TIMEOUT"
 export OVERCLOUD_UPDATE_ARGS=
@@ -84,6 +85,10 @@ export RUN_TEMPEST_TESTS=0
 export OVB=0
 export UCINSTANCEID=NULL
 export TOCIRUNNER="./toci_instack.sh"
+export MULTINODE=0
+export CONTROLLER_HOSTS=
+export COMPUTE_HOSTS=
+export SUBNODES_SSH_KEY=
 
 # NOTE(pabelanger): Once we bring AFS mirrors online, we no longer need to do this.
 sudo sed -i -e "s|^#baseurl=http://mirror.centos.org/centos/|baseurl=$CENTOS_MIRROR|;/^mirrorlist/d" /etc/yum.repos.d/CentOS-Base.repo
@@ -140,6 +145,18 @@ for JOB_TYPE_PART in $(sed 's/-/ /g' <<< "${TOCI_JOBTYPE:-}") ; do
             # The test env broker needs to know the instanceid of the this node so it can attach it to the provisioning network
             UCINSTANCEID=$(http_proxy= curl http://169.254.169.254/openstack/2015-10-15/meta_data.json | python -c 'import json, sys; print json.load(sys.stdin)["uuid"]')
             ;;
+        multinode)
+            MULTINODE=1
+            TOCIRUNNER="./toci_instack_multinode.sh"
+            NODECOUNT=1
+            PACEMAKER=1
+            CONTROLLER_HOSTS=$(sed -n 1,1p /etc/nodepool/sub_nodes)
+            SUBNODES_SSH_KEY=/etc/nodepool/id_rsa
+            UNDERCLOUD_SSL=0
+            INTROSPECT=0
+            OVERCLOUD_DEPLOY_ARGS="--libvirt-type=qemu -t $OVERCLOUD_DEPLOY_TIMEOUT"
+            OVERCLOUD_DEPLOY_ARGS="$OVERCLOUD_DEPLOY_ARGS -e /usr/share/openstack-tripleo-heat-templates/environments/deployed-server-environment.yaml -e /opt/stack/new/tripleo-ci/test-environments/multinode.yaml --compute-scale 0 --overcloud-ssh-user $OVERCLOUD_SSH_USER"
+            ;;
         periodic)
             export DELOREAN_REPO_URL=http://trunk.rdoproject.org/centos7/consistent
             CACHEUPLOAD=1
@@ -173,7 +190,7 @@ set -m
 
 source /opt/stack/new/tripleo-ci/scripts/metrics.bash
 start_metric "tripleo.testenv.wait.seconds"
-if [ -z ${TE_DATAFILE:-} ] ; then
+if [ -z "${TE_DATAFILE:-}" -a "$MULTINODE" = "0" ] ; then
     # NOTE(pabelanger): We need gear for testenv, but this really should be
     # handled by tox.
     sudo pip install gear

toci_instack_multinode.sh

@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+set -eux
+
+export CURRENT_DIR=$(dirname ${BASH_SOURCE[0]:-$0})
+export TRIPLEO_CI_DIR=$CURRENT_DIR/../
+
+export IP_DEVICE=${IP_DEVICE:-"eth0"}
+
+
+source $TRIPLEO_CI_DIR/tripleo-ci/scripts/common_vars.bash
+source $TRIPLEO_CI_DIR/tripleo-ci/scripts/common_functions.sh
+source $TRIPLEO_CI_DIR/tripleo-ci/scripts/metrics.bash
+start_metric "tripleo.ci.total.seconds"
+
+mkdir -p $WORKSPACE/logs
+
+MY_IP=$(ip addr show dev $IP_DEVICE | awk '/inet / {gsub("/.*", "") ; print $2}')
+MY_IP_eth1=$(ip addr show dev eth1 | awk '/inet / {gsub("/.*", "") ; print $2}')
+
+export http_proxy=""
+export no_proxy=192.0.2.1,$MY_IP,$MY_IP_eth1
+
+# Copy nodepool ssh keys for the jenkins user because apparently id_rsa.pub is
+# missing from /home/jenkins/.ssh
+cp /etc/nodepool/id_rsa  ~/.ssh/
+cp /etc/nodepool/id_rsa.pub  ~/.ssh/
+
+# Clear out any puppet modules on the node placed their by infra configuration
+sudo rm -rf /etc/puppet/modules/*
+
+# Setup delorean
+$TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --delorean-setup
+
+dummy_ci_repo
+
+# Install all of the repositories we need
+$TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --repo-setup
+
+# Install wget and moreutils for timestamping postci.log with ts
+sudo yum -y install wget moreutils python-simplejson dstat yum-plugin-priorities
+# Need to reinstall requests since it's rm'd in toci_gate_test.sh
+sudo rpm -e --nodeps python-requests || :
+sudo yum -y install python-requests
+# Open up port for delorean yum repo server
+sudo iptables -I INPUT -p tcp --dport 8766 -j ACCEPT
+
+trap "[ \$? != 0 ] && echo ERROR DURING PREVIOUS COMMAND ^^^ && echo 'See postci.txt in the logs directory for debugging details'; postci 2>&1 | ts '%Y-%m-%d %H:%M:%S.000 |' > $WORKSPACE/logs/postci.log 2>&1" EXIT
+
+delorean_build_and_serve
+
+# Since we've moved a few commands from this spot before the wget, we need to
+# sleep a few seconds in order for the SimpleHTTPServer to get setup.
+sleep 3
+
+layer_ci_repo
+
+create_dib_vars_for_puppet
+
+echo_vars_to_deploy_env
+# We need to override $OVERCLOUD_VALIDATE_ARGS to be empty so that the
+# validations that check for the correct number of ironic nodes does not fail
+# the deploy.
+echo 'export OVERCLOUD_VALIDATE_ARGS=""' >> $TRIPLEO_ROOT/tripleo-ci/deploy.env
+
+source $TRIPLEO_ROOT/tripleo-ci/deploy.env
+
+deploy_env=$(cat $TRIPLEO_ROOT/tripleo-ci/deploy.env)
+
+for ip in $(cat /etc/nodepool/sub_nodes); do
+    ssh $SSH_OPTIONS -tt -i /etc/nodepool/id_rsa $ip \
+        sudo yum -y install wget
+    ssh $SSH_OPTIONS -tt -i /etc/nodepool/id_rsa $ip \
+        sudo wget http://$MY_IP:8766/current/delorean-ci.repo -O /etc/yum.repos.d/delorean-ci.repo
+    ssh $SSH_OPTIONS -tt -i /etc/nodepool/id_rsa $ip \
+        sudo sed -i -e \"s%baseurl=.*%baseurl=http://$MY_IP:8766/current/%\" /etc/yum.repos.d/delorean-ci.repo
+    ssh $SSH_OPTIONS -tt -i /etc/nodepool/id_rsa $ip \
+        sudo sed -i -e 's%priority=.*%priority=1%' /etc/yum.repos.d/delorean-ci.repo
+    ssh $SSH_OPTIONS -tt -i /etc/nodepool/id_rsa $ip \
+        sudo mkdir -p $TRIPLEO_ROOT/tripleo-ci
+    ssh $SSH_OPTIONS -tt -i /etc/nodepool/id_rsa $ip \
+        /bin/bash -c "echo \"$deploy_env\" > deploy.env"
+    ssh $SSH_OPTIONS -tt -i /etc/nodepool/id_rsa $ip \
+        sudo cp deploy.env $TRIPLEO_ROOT/tripleo-ci/deploy.env
+done
+
+# TODO: remove later, this is for live debugging
+sudo cat /etc/nodepool/*
+
+# This will remove any puppet configuration done by infra setup
+sudo yum -y remove puppet facter hiera
+
+$TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --multinode
+
+# This needs to be done after the --multinode setup otherwise /etc/hosts will
+# get overwritten
+hosts='127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
+::1         localhost localhost.localdomain localhost6 localhost6.localdomain6'
+for ip in $(cat /etc/nodepool/sub_nodes); do
+    ssh $SSH_OPTIONS -tt -i /etc/nodepool/id_rsa $ip \
+        /bin/bash -c "echo \"$hosts\" > hosts"
+    ssh $SSH_OPTIONS -tt -i /etc/nodepool/id_rsa $ip \
+        sudo cp hosts /etc/hosts
+done
+
+# Add a simple system utilisation logger process
+sudo dstat -tcmndrylpg --output /var/log/dstat-csv.log >/dev/null &
+# Install our test cert so SSL tests work
+sudo cp $TRIPLEO_ROOT/tripleo-ci/test-environments/overcloud-cacert.pem /etc/pki/ca-trust/source/anchors/
+sudo update-ca-trust extract
+
+$TRIPLEO_ROOT/tripleo-ci/scripts/deploy.sh
+
+exit 0
+echo 'Run completed.'

undercloud-configs/net-config-multinode.json.template

@@ -0,0 +1,15 @@
+"network_config": [
+ {
+  "type": "ovs_bridge",
+  "name": "br-ctlplane",
+  "ovs_extra": [
+   "br-set-external-id br-ctlplane bridge-id br-ctlplane"
+  ],
+  "addresses": [
+    {
+      "ip_netmask": "{{PUBLIC_INTERFACE_IP}}"
+    }
+  ],
+  "mtu": {{LOCAL_MTU}}
+}
+]