diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index a51201b0..a6d282dc 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -141,10 +141,6 @@ if [ -z "$SKIP_DEVSTACK_GATE_PROJECT" ]; then fi fi -# Make a directory to store logs -rm -rf $WORKSPACE/logs -mkdir -p $WORKSPACE/logs - # The feature matrix to select devstack-gate components export DEVSTACK_GATE_FEATURE_MATRIX=${DEVSTACK_GATE_FEATURE_MATRIX:-features.yaml} @@ -382,6 +378,17 @@ export DEVSTACK_GATE_CEILOMETER_BACKEND=${DEVSTACK_GATE_CEILOMETER_BACKEND:-mysq # Set Zaqar backend to override the default one. It could be mongodb, redis. export DEVSTACK_GATE_ZAQAR_BACKEND=${DEVSTACK_GATE_ZAQAR_BACKEND:-mongodb} +# The topology of the system determinates the service distribution +# among the nodes. +# aio: `all in one` just only one node used +# aiopcpu: `all in one plus compute` one node will be installed as aio +# the extra nodes will gets only limited set of services +# ctrlpcpu: `controller plus compute` One node will gets the controller type +# services without the compute type of services, the others gets, +# the compute style services several services can be common, +# the networking services also presents on the controller [WIP] +export DEVSTACK_GATE_TOPOLOGY=${DEVSTACK_GATE_TOPOLOGY:-aio} + # Set to a space-separated list of projects to prepare in the # workspace, e.g. 'openstack-dev/devstack openstack/neutron'. # Minimizing the number of targeted projects can reduce the setup cost @@ -405,45 +412,76 @@ echo "Pipeline: $ZUUL_PIPELINE" echo "Available disk space on this host:" indent df -h -echo "Setting up the host" +# Enable tracing while we transition to using ansible to run +# setup across multiple nodes. +set -x +# Install ansible +sudo -H pip install virtualenv +virtualenv /tmp/ansible +/tmp/ansible/bin/pip install ansible +export ANSIBLE=/tmp/ansible/bin/ansible + +# Write inventory file with groupings +echo "[primary]" > "$WORKSPACE/inventory" +echo "localhost ansible_connection=local" >> "$WORKSPACE/inventory" +echo "[subnodes]" >> "$WORKSPACE/inventory" +cat /etc/nodepool/sub_nodes_private >> "$WORKSPACE/inventory" + +# NOTE(clarkb): for simplicity we evaluate all bash vars in ansible commands +# on the node running these scripts, we do not pass through unexpanded +# vars to ansible shell commands. This may need to change in the future but +# for now the current setup is simple, consistent and easy to understand. + +# Copy bootstrap to remote hosts +# It is in brackets for avoiding inheriting a huge environment variable +(export PROJECTS; export > "$WORKSPACE/test_env.sh") +$ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" -m copy \ + -a "src='$WORKSPACE/devstack-gate' dest='$WORKSPACE'" +$ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" -m copy \ + -a "src='$WORKSPACE/test_env.sh' dest='$WORKSPACE/test_env.sh'" + +# Make a directory to store logs +$ANSIBLE all -f 5 -i "$WORKSPACE/inventory" -m file \ + -a "path='$WORKSPACE/logs' state=absent" +$ANSIBLE all -f 5 -i "$WORKSPACE/inventory" -m file \ + -a "path='$WORKSPACE/logs' state=directory" + +# Run ansible to do setup_host on all nodes. +echo "Setting up the hosts" echo "... this takes a few seconds (logs at logs/devstack-gate-setup-host.txt.gz)" -tsfilter setup_host &> $WORKSPACE/logs/devstack-gate-setup-host.txt +$ANSIBLE all -f 5 -i "$WORKSPACE/inventory" -m shell \ + -a "source '$WORKSPACE/test_env.sh' && source '$WORKSPACE/devstack-gate/functions.sh' && tsfilter setup_host executable=/bin/bash" \ + &> "$WORKSPACE/logs/devstack-gate-setup-host.txt" if [ -n "$DEVSTACK_GATE_GRENADE" ]; then echo "Setting up the new (migrate to) workspace" echo "... this takes 3 - 5 minutes (logs at logs/devstack-gate-setup-workspace-new.txt.gz)" - tsfilter setup_workspace "$GRENADE_NEW_BRANCH" "$BASE/new" copycache &> \ - $WORKSPACE/logs/devstack-gate-setup-workspace-new.txt + $ANSIBLE all -f 5 -i "$WORKSPACE/inventory" -m shell \ + -a "source '$WORKSPACE/test_env.sh' && source '$WORKSPACE/devstack-gate/functions.sh' && tsfilter setup_workspace '$GRENADE_NEW_BRANCH' '$BASE/new' copycache executable=/bin/bash" \ + &> "$WORKSPACE/logs/devstack-gate-setup-workspace-new.txt" echo "Setting up the old (migrate from) workspace ..." echo "... this takes 3 - 5 minutes (logs at logs/devstack-gate-setup-workspace-old.txt.gz)" - tsfilter setup_workspace "$GRENADE_OLD_BRANCH" "$BASE/old" &> \ - $WORKSPACE/logs/devstack-gate-setup-workspace-old.txt + $ANSIBLE all -f 5 -i "$WORKSPACE/inventory" -m shell \ + -a "source '$WORKSPACE/test_env.sh' && source '$WORKSPACE/devstack-gate/functions.sh' && tsfilter setup_workspace '$GRENADE_OLD_BRANCH' '$BASE/old' executable=/bin/bash" \ + &> "$WORKSPACE/logs/devstack-gate-setup-workspace-old.txt" else echo "Setting up the workspace" echo "... this takes 3 - 5 minutes (logs at logs/devstack-gate-setup-workspace-new.txt.gz)" - tsfilter setup_workspace "$OVERRIDE_ZUUL_BRANCH" "$BASE/new" &> \ - $WORKSPACE/logs/devstack-gate-setup-workspace-new.txt + $ANSIBLE all -f 5 -i "$WORKSPACE/inventory" -m shell \ + -a "source '$WORKSPACE/test_env.sh' && source '$WORKSPACE/devstack-gate/functions.sh' && tsfilter setup_workspace '$OVERRIDE_ZUUL_BRANCH' '$BASE/new' executable=/bin/bash" \ + &> "$WORKSPACE/logs/devstack-gate-setup-workspace-new.txt" fi -# It is in brackets for avoiding inheriting a huge environment variable -(export PROJECTS; export >$WORKSPACE/test_env.sh) # relocate and symlink logs into $BASE to save space on the root filesystem -if [ -d "$WORKSPACE/logs" -a \! -e "$BASE/logs" ]; then - sudo mv $WORKSPACE/logs $BASE/ - ln -s $BASE/logs $WORKSPACE/ -fi - -# The topology of the system determinates the service distribution -# among the nodes. -# aio: `all in one` just only one node used -# aiopcpu: `all in one plus compute` one node will be installed as aio -# the extra nodes will gets only limited set of services -# ctrlpcpu: `controller plus compute` One node will gets the controller type -# services without the compute type of services, the others gets, -# the compute style services several services can be common, -# the networking services also presents on the controller [WIP] -export DEVSTACK_GATE_TOPOLOGY=${DEVSTACK_GATE_TOPOLOGY:-aio} +# TODO: make this more ansibley +$ANSIBLE all -f 5 -i "$WORKSPACE/inventory" -m shell -a " +if [ -d '$WORKSPACE/logs' -a \! -e '$BASE/logs' ]; then + sudo mv '$WORKSPACE/logs' '$BASE/' + ln -s '$BASE/logs' '$WORKSPACE/' +fi executable=/bin/bash" +# Note that hooks should be multihost aware if necessary. +# devstack-vm-gate-wrap.sh will not automagically run the hooks on each node. # Run pre test hook if we have one call_hook_if_defined "pre_test_hook" @@ -473,12 +511,15 @@ fi echo "Cleaning up host" echo "... this takes 3 - 4 minutes (logs at logs/devstack-gate-cleanup-host.txt.gz)" -tsfilter cleanup_host &> $WORKSPACE/devstack-gate-cleanup-host.txt +$ANSIBLE all -f 5 -i "$WORKSPACE/inventory" -m shell \ + -a "source '$WORKSPACE/test_env.sh' && source '$WORKSPACE/devstack-gate/functions.sh' && tsfilter cleanup_host executable=/bin/bash" \ + &> "$WORKSPACE/devstack-gate-cleanup-host.txt" +# TODO(clark) ansiblify this. Fetch can only fetch specifc files no +# recursive dir fetches. if [[ "$DEVSTACK_GATE_TOPOLOGY" != "aio" ]]; then COUNTER=1 for NODE in `cat /etc/nodepool/sub_nodes_private`; do echo "Collecting logs from $NODE" - remote_command $NODE "source $WORKSPACE/test_env.sh; source $BASE/new/devstack-gate/functions.sh; tsfilter cleanup_host &> $WORKSPACE/devstack-gate-cleanup-host.txt" rsync -avz "$NODE:$BASE/logs/" "$BASE/logs/subnode-$COUNTER/" let COUNTER=COUNTER+1 done diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 1ac3797d..4baa1ad4 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -50,6 +50,16 @@ PUBLIC_NETWORK_GATEWAY=${DEVSTACK_GATE_PUBLIC_NETWORK_GATEWAY:-172.24.5.1} FLOATING_HOST_PREFIX=${DEVSTACK_GATE_FLOATING_HOST_PREFIX:-172.24.4} FLOATING_HOST_MASK=${DEVSTACK_GATE_FLOATING_HOST_MASK:-23} +function setup_ssh { + local path=$1 + $ANSIBLE all --sudo -f 5 -i "$WORKSPACE/inventory" -m file \ + -a "path='$path' mode=0700 state=directory" + $ANSIBLE all --sudo -f 5 -i "$WORKSPACE/inventory" -m copy \ + -a "src=/etc/nodepool/id_rsa.pub dest='$path/authorized_keys' mode=0600" + $ANSIBLE all --sudo -f 5 -i "$WORKSPACE/inventory" -m copy \ + -a "src=/etc/nodepool/id_rsa dest='$path/id_rsa' mode=0400" +} + function setup_localrc { local localrc_oldnew=$1; local localrc_branch=$2; @@ -438,7 +448,9 @@ EOF fi # Make the workspace owned by the stack user - sudo chown -R stack:stack $BASE + # It is not clear if the ansible file module can do this for us + $ANSIBLE all --sudo -f 5 -i "$WORKSPACE/inventory" -m shell \ + -a "chown -R stack:stack '$BASE'" cd $BASE/new/grenade echo "Running grenade ..." @@ -454,28 +466,10 @@ else set -x # for now enabling debug and do not turn it off echo -e "[[post-config|\$NOVA_CONF]]\n[libvirt]\ncpu_mode=custom\ncpu_model=gate64" >> local.conf setup_localrc "new" "$OVERRIDE_ZUUL_BRANCH" "sub_localrc" "sub" - sudo mkdir -p $BASE/new/.ssh - sudo cp /etc/nodepool/id_rsa.pub $BASE/new/.ssh/authorized_keys - sudo cp /etc/nodepool/id_rsa $BASE/new/.ssh/ - sudo chmod 600 $BASE/new/.ssh/authorized_keys - sudo chmod 400 $BASE/new/.ssh/id_rsa - sudo mkdir -p ~root/.ssh - sudo cp /etc/nodepool/id_rsa ~root/.ssh/ - sudo chmod 400 ~root/.ssh/id_rsa - for NODE in `cat /etc/nodepool/sub_nodes_private`; do - echo "Copy Files to $NODE" - remote_copy_dir $NODE $BASE/new/devstack-gate $WORKSPACE - remote_copy_file $WORKSPACE/test_env.sh $NODE:$WORKSPACE/test_env.sh - echo "Preparing $NODE" - remote_command $NODE "source $WORKSPACE/test_env.sh; $WORKSPACE/devstack-gate/sub_node_prepare.sh" - remote_copy_file /etc/nodepool/id_rsa "$NODE:$BASE/new/.ssh/" - remote_command $NODE sudo chmod 400 "$BASE/new/.ssh/*" - remote_command $NODE sudo mkdir -p ~root/.ssh - remote_command $NODE sudo cp $BASE/new/.ssh/id_rsa ~root/.ssh/id_rsa - done PRIMARY_NODE=`cat /etc/nodepool/primary_node_private` SUB_NODES=`cat /etc/nodepool/sub_nodes_private` if [[ "$DEVSTACK_GATE_NEUTRON" -ne '1' ]]; then + # TODO (clarkb): figure out how to make bridge setup sane with ansible. ovs_gre_bridge "br_pub" $PRIMARY_NODE "True" 1 \ $FLOATING_HOST_PREFIX $FLOATING_HOST_MASK \ $SUB_NODES @@ -496,14 +490,54 @@ EOF $FLOATING_HOST_PREFIX $FLOATING_HOST_MASK \ $SUB_NODES fi + + echo "Preparing cross node connectivity" + setup_ssh $BASE/new/.ssh + setup_ssh ~root/.ssh + # TODO (clarkb) ansiblify the /etc/hosts and known_hosts changes + # set up ssh_known_hosts by IP and /etc/hosts + for NODE in $SUB_NODES; do + ssh-keyscan $NODE >> /tmp/tmp_ssh_known_hosts + echo $NODE `remote_command $NODE hostname -f | tr -d '\r'` >> /tmp/tmp_hosts + done + ssh-keyscan `cat /etc/nodepool/primary_node_private` >> /tmp/tmp_ssh_known_hosts + echo `cat /etc/nodepool/primary_node_private` `hostname -f` >> /tmp/tmp_hosts + cat /tmp/tmp_hosts | sudo tee --append /etc/hosts + + # set up ssh_known_host files based on hostname + for HOSTNAME in `cat /tmp/tmp_hosts | cut -d' ' -f2`; do + ssh-keyscan $HOSTNAME >> /tmp/tmp_ssh_known_hosts + done + $ANSIBLE all --sudo -f 5 -i "$WORKSPACE/inventory" -m copy \ + -a "src=/tmp/tmp_ssh_known_hosts dest=/etc/ssh/ssh_known_hosts mode=0444" + + for NODE in $SUB_NODES; do + remote_copy_file /tmp/tmp_hosts $NODE:/tmp/tmp_hosts + remote_command $NODE "cat /tmp/tmp_hosts | sudo tee --append /etc/hosts > /dev/null" + cp sub_localrc /tmp/tmp_sub_localrc + echo "HOST_IP=$NODE" >> /tmp/tmp_sub_localrc + remote_copy_file /tmp/tmp_sub_localrc $NODE:$BASE/new/devstack/localrc + remote_copy_file local.conf $NODE:$BASE/new/devstack/local.conf + done + fi # Make the workspace owned by the stack user - sudo chown -R stack:stack $BASE + # It is not clear if the ansible file module can do this for us + $ANSIBLE all --sudo -f 5 -i "$WORKSPACE/inventory" -m shell \ + -a "chown -R stack:stack '$BASE'" echo "Running devstack" echo "... this takes 10 - 15 minutes (logs in logs/devstacklog.txt.gz)" start=$(date +%s) - sudo -H -u stack stdbuf -oL -eL ./stack.sh > /dev/null + $ANSIBLE primary -f 5 -i "$WORKSPACE/inventory" -m shell \ + -a "cd '$BASE/new/devstack' && sudo -H -u stack stdbuf -oL -eL ./stack.sh executable=/bin/bash" \ + > /dev/null + # Run non controller setup after controller is up. This is necessary + # because services like nova apparently expect to have the controller in + # place before anything else. + $ANSIBLE subnodes -f 5 -i "$WORKSPACE/inventory" -m shell \ + -a "cd '$BASE/new/devstack' && sudo -H -u stack stdbuf -oL -eL ./stack.sh executable=/bin/bash" \ + > /dev/null end=$(date +%s) took=$((($end - $start) / 60)) if [[ "$took" -gt 20 ]]; then @@ -531,59 +565,33 @@ EOF fi fi - if [[ "$DEVSTACK_GATE_TOPOLOGY" != "aio" ]]; then - echo "Preparing cross node connectivity" - # set up ssh_known_hosts by IP and /etc/hosts - for NODE in `cat /etc/nodepool/sub_nodes_private`; do - ssh-keyscan $NODE | sudo tee --append tmp_ssh_known_hosts > /dev/null - echo $NODE `remote_command $NODE hostname -f | tr -d '\r'` | sudo tee --append tmp_hosts > /dev/null - done - ssh-keyscan `cat /etc/nodepool/primary_node_private` | sudo tee --append tmp_ssh_known_hosts > /dev/null - echo `cat /etc/nodepool/primary_node_private` `hostname -f` | sudo tee --append tmp_hosts > /dev/null - cat tmp_hosts | sudo tee --append /etc/hosts - - # set up ssh_known_host files based on hostname - for HOSTNAME in `cat tmp_hosts | cut -d' ' -f2`; do - ssh-keyscan $HOSTNAME | sudo tee --append tmp_ssh_known_hosts > /dev/null - done - sudo cp tmp_ssh_known_hosts /etc/ssh/ssh_known_hosts - sudo chmod 444 /etc/ssh/ssh_known_hosts - - for NODE in `cat /etc/nodepool/sub_nodes_private`; do - remote_copy_file tmp_ssh_known_hosts $NODE:$BASE/new/tmp_ssh_known_hosts - remote_copy_file tmp_hosts $NODE:$BASE/new/tmp_hosts - remote_command $NODE "cat $BASE/new/tmp_hosts | sudo tee --append /etc/hosts > /dev/null" - remote_command $NODE "sudo mv $BASE/new/tmp_ssh_known_hosts /etc/ssh/ssh_known_hosts" - remote_command $NODE "sudo chmod 444 /etc/ssh/ssh_known_hosts" - sudo cp sub_localrc tmp_sub_localrc - echo "HOST_IP=$NODE" | sudo tee --append tmp_sub_localrc > /dev/null - remote_copy_file tmp_sub_localrc $NODE:$BASE/new/devstack/localrc - remote_copy_file local.conf $NODE:$BASE/new/devstack/local.conf - remote_command $NODE sudo chown -R stack:stack $BASE - echo "Running devstack on $NODE" - remote_command $NODE "cd $BASE/new/devstack; source $WORKSPACE/test_env.sh; export -n PROJECTS; sudo -H -u stack stdbuf -oL -eL ./stack.sh > /dev/null" - done - - if [[ $DEVSTACK_GATE_NEUTRON -eq "1" ]]; then - # NOTE(afazekas): The cirros lp#1301958 does not support MTU setting via dhcp, - # simplest way the have tunneling working, with dvsm, without increasing the host system MTU - # is to decreasion the MTU on br-ex - # TODO(afazekas): Configure the mtu smarter on the devstack side - sudo ip link set mtu 1450 dev br-ex + if [[ "$DEVSTACK_GATE_TOPOLOGY" != "aio" ]] && [[ $DEVSTACK_GATE_NEUTRON -eq "1" ]]; then + # NOTE(afazekas): The cirros lp#1301958 does not support MTU setting via dhcp, + # simplest way the have tunneling working, with dvsm, without increasing the host system MTU + # is to decreasion the MTU on br-ex + # TODO(afazekas): Configure the mtu smarter on the devstack side + MTU_NODES=primary + if [[ "$DEVSTACK_GATE_NEUTRON_DVR" -eq "1" ]]; then + MTU_NODES=all fi + $ANSIBLE "$MTU_NODES" -f 5 -i "$WORKSPACE/inventory" -m shell \ + -a "sudo ip link set mtu 1450 dev br-ex" fi fi if [[ "$DEVSTACK_GATE_UNSTACK" -eq "1" ]]; then - sudo -H -u stack ./unstack.sh + $ANSIBLE all -f 5 -i "$WORKSPACE/inventory" -m shell \ + -a "cd '$BASE/new/devstack' && sudo -H -u stack ./unstack.sh" fi echo "Removing sudo privileges for devstack user" -sudo rm /etc/sudoers.d/50_stack_sh +$ANSIBLE all --sudo -f 5 -i "$WORKSPACE/inventory" -m file \ + -a "path=/etc/sudoers.d/50_stack_sh state=absent" if [[ "$DEVSTACK_GATE_EXERCISES" -eq "1" ]]; then echo "Running devstack exercises" - sudo -H -u stack ./exercise.sh + $ANSIBLE all -f 5 -i "$WORKSPACE/inventory" -m shell \ + -a "cd '$BASE/new/devstack' && sudo -H -u stack ./exercise.sh" fi function load_subunit_stream { diff --git a/sub_node_prepare.sh b/sub_node_prepare.sh deleted file mode 100755 index a0bd67fd..00000000 --- a/sub_node_prepare.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# -# See the License for the specific language governing permissions and -# limitations under the License. - -set -x -GIT_BASE=${GIT_BASE:-https://git.openstack.org} -GIT_BRANCH=${GIT_BRANCH:-master} - -source $WORKSPACE/devstack-gate/functions.sh - -export BASE=/opt/stack - -# Make a directory to store logs -rm -rf $WORKSPACE/logs -mkdir -p $WORKSPACE/logs - -echo "Available disk space on this host:" -indent df -h - -echo "Setting up the host" -echo "... this takes a few seconds (logs at logs/node_ip/devstack-gate-setup-host.txt.gz)" -tsfilter setup_host &> $WORKSPACE/logs/devstack-gate-setup-host.txt - -if [[ -n "$DEVSTACK_GATE_GRENADE" ]]; then - echo "Setting up the new (migrate to) workspace" - echo "... this takes 3 - 5 minutes (logs at logs/node_ip/devstack-gate-setup-workspace-new.txt.gz)" - tsfilter setup_workspace "$GRENADE_NEW_BRANCH" "$BASE/new" copycache &> \ - $WORKSPACE/logs/devstack-gate-setup-workspace-new.txt - echo "Setting up the old (migrate from) workspace ..." - echo "... this takes 3 - 5 minutes (logs at logs/node_ip/devstack-gate-setup-workspace-old.txt.gz)" - tsfilter setup_workspace "$GRENADE_OLD_BRANCH" "$BASE/old" &> \ - $WORKSPACE/logs/devstack-gate-setup-workspace-old.txt -else - echo "Setting up the workspace" - echo "... this takes 3 - 5 minutes (logs at logs/node_ip/devstack-gate-setup-workspace-new.txt.gz)" - tsfilter setup_workspace "$OVERRIDE_ZUUL_BRANCH" "$BASE/new" &> \ - $WORKSPACE/logs/devstack-gate-setup-workspace-new.txt -fi - -mkdir -p $BASE/new/.ssh -sudo cp /etc/nodepool/id_rsa.pub $BASE/new/.ssh/authorized_keys -sudo chmod 600 $BASE/new/.ssh/authorized_keys - -# relocate and symlink logs into $BASE to save space on the root filesystem -if [ -d "$WORKSPACE/logs" -a \! -e "$BASE/logs" ]; then - sudo mv $WORKSPACE/logs $BASE/ - ln -s $BASE/logs $WORKSPACE/ -fi - -# Run pre test hook if we have one -if function_exists "pre_test_hook"; then - echo "Running pre_test_hook" - xtrace=$(set +o | grep xtrace) - set -o xtrace - tsfilter pre_test_hook | tee $WORKSPACE/devstack-gate-pre-test-hook.txt - sudo mv $WORKSPACE/devstack-gate-pre-test-hook.txt $BASE/logs/ - $xtrace -fi