7874c0cec0
Add ping.py so we can test when a host is up. ping -c 2 -w 90 will almost do the right thing, but will exit on a icmp host unreachable. We need to continue to wait. ping.py just waits for an answer to a ping. Parallelize building the compute nodes (but not the head node). Kill ping process in ping.py. Copy PIP cache to host before running. Add syslog artifacts. Remove known_host weirdness, depend on .ssh/config instead. Change-Id: I65a12074cf80feb3ae69d15c81ba32a81566b22a
44 lines
1.2 KiB
Bash
Executable File
44 lines
1.2 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
set -x
|
|
#HEAD_HOST=${HEAD_HOST:-10.14.247.43}
|
|
#COMPUTE_HOSTS=${COMPUTE_HOSTS:-10.14.247.44,10.14.247.45}
|
|
|
|
WORKSPACE=`pwd`
|
|
mkdir -p logs
|
|
rm -f logs/*
|
|
cd `dirname "$0"`
|
|
|
|
echo "Jenkins: resetting hosts..."
|
|
for host in $HEAD_HOST ${COMPUTE_HOSTS//,/ }; do
|
|
scp -o lvm-kexec-reset.sh root@$host:/var/tmp/
|
|
ssh -o root@$host /var/tmp/lvm-kexec-reset.sh
|
|
sudo rm -f /var/log/orchestra/rsyslog/$host/syslog
|
|
done
|
|
|
|
# Have rsyslog reopen log files we rm'd from under it
|
|
sudo restart rsyslog
|
|
|
|
# wait for the host to come up (2 ping responses or timeout after 5 minutes)
|
|
echo "Jenkins: Waiting for head host to return after reset..."
|
|
if ! timeout 300 ./ping.py $HEAD_HOST; then
|
|
echo "Jenkins: ERROR: Head node did not come back up after reset"
|
|
exit 1
|
|
fi
|
|
|
|
echo "Jenkins: Pre-populating PIP cache"
|
|
for host in $HEAD_HOST ${COMPUTE_HOSTS//,/ }; do
|
|
scp -r ~/cache/pip root@$host:/var/cache/pip
|
|
done
|
|
|
|
echo "Jenkins: Executing build_bm_multi.sh."
|
|
|
|
cd ~/devstack
|
|
source ./functions.sh
|
|
cache_images ~/devstack/files
|
|
bash build_bm_multi.sh
|
|
|
|
for host in $HEAD_HOST ${COMPUTE_HOSTS//,/ }; do
|
|
cp /var/log/orchestra/rsyslog/$host/syslog $WORKSPACE/logs/$host-syslog.txt
|
|
done
|