180 lines
5.9 KiB
Bash
Executable File
180 lines
5.9 KiB
Bash
Executable File
#!/bin/bash -xe
|
|
|
|
ADMIN_USER=${ADMIN_USER:-vagrant}
|
|
|
|
WORKSPACE=${WORKSPACE:-.}
|
|
SSH_OPTIONS="-A -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
|
|
|
|
# Default deployment settings
|
|
COMMON_DEFAULTS_YAML="kargo_default_common.yaml"
|
|
COMMON_DEFAULTS_SRC="${BASH_SOURCE%/*}/../kargo/${COMMON_DEFAULTS_YAML}"
|
|
|
|
NODE_BASE_OS=${NODE_BASE_OS:-ubuntu}
|
|
OS_SPECIFIC_DEFAULTS_YAML="kargo_default_${NODE_BASE_OS}.yaml"
|
|
OS_SPECIFIC_DEFAULTS_SRC="${BASH_SOURCE%/*}/../kargo/${OS_SPECIFIC_DEFAULTS_YAML}"
|
|
|
|
SLAVE_IPS=( $SLAVE_IPS )
|
|
ADMIN_IP=${ADMIN_IP:-${SLAVE_IPS[0]}}
|
|
|
|
required_ansible_version="2.1.0"
|
|
|
|
function exit_gracefully {
|
|
local exit_code=$?
|
|
set +e
|
|
# set exit code if it is a param
|
|
[[ -n "$1" ]] && exit_code=$1
|
|
# Kill current ssh-agent
|
|
if [ -z "$INHERIT_SSH_AGENT" ]; then
|
|
eval $(ssh-agent -k)
|
|
fi
|
|
exit $exit_code
|
|
}
|
|
|
|
function with_retries {
|
|
set +e
|
|
local retries=3
|
|
for try in $(seq 1 $retries); do
|
|
${@}
|
|
[ $? -eq 0 ] && break
|
|
if [[ "$try" == "$retries" ]]; then
|
|
exit 1
|
|
fi
|
|
done
|
|
set -e
|
|
}
|
|
|
|
function admin_node_command {
|
|
if [[ "$ADMIN_IP" == "local" ]];then
|
|
eval "$@"
|
|
else
|
|
ssh $SSH_OPTIONS $ADMIN_USER@$ADMIN_IP "$@"
|
|
fi
|
|
}
|
|
|
|
function wait_for_nodes {
|
|
for IP in $@; do
|
|
elapsed_time=0
|
|
master_wait_time=30
|
|
while true; do
|
|
report=$(ssh ${SSH_OPTIONS} -o PasswordAuthentication=no ${ADMIN_USER}@${IP} echo ok || echo not ready)
|
|
if [ "${report}" = "ok" ]; then
|
|
break
|
|
fi
|
|
|
|
if [ "${elapsed_time}" -gt "${master_wait_time}" ]; then
|
|
exit 2
|
|
fi
|
|
|
|
sleep 1
|
|
let elapsed_time+=1
|
|
done
|
|
done
|
|
}
|
|
|
|
function with_ansible {
|
|
local tries=1
|
|
until admin_node_command /usr/bin/ansible-playbook \
|
|
--ssh-extra-args "-A\ -o\ StrictHostKeyChecking=no" -u ${ADMIN_USER} -b \
|
|
--become-user=root -i $ADMIN_WORKSPACE/inventory/inventory.cfg \
|
|
$@ $KARGO_DEFAULTS_OPT $COMMON_DEFAULTS_OPT \
|
|
$OS_SPECIFIC_DEFAULTS_OPT $custom_opts; do
|
|
if [[ $tries > 1 ]]; then
|
|
(( tries-- ))
|
|
echo "Deployment failed! Trying $tries more times..."
|
|
else
|
|
exit_gracefully 1
|
|
fi
|
|
done
|
|
}
|
|
|
|
mkdir -p tmp logs
|
|
|
|
|
|
# Trap errors during env preparation stage
|
|
trap exit_gracefully ERR INT TERM
|
|
|
|
# FIXME(mattymo): Should be part of underlay
|
|
echo "Checking local SSH environment..."
|
|
if ssh-add -l &>/dev/null; then
|
|
echo "Local SSH agent detected with at least one identity."
|
|
INHERIT_SSH_AGENT="yes"
|
|
else
|
|
echo "No SSH agent available. Using precreated SSH key..."
|
|
if ! [ -f $WORKSPACE/id_rsa ]; then
|
|
echo "ERROR: This script expects an active SSH agent or a key already \
|
|
available at $WORKSPACE/id_rsa. Exiting."
|
|
exit_gracefully 1
|
|
fi
|
|
eval $(ssh-agent)
|
|
ssh-add $WORKSPACE/id_rsa
|
|
fi
|
|
|
|
# Install missing packages on the host running this script
|
|
if ! type sshpass > /dev/null; then
|
|
sudo apt-get update && sudo apt-get install -y sshpass
|
|
fi
|
|
|
|
|
|
echo "Preparing admin node..."
|
|
if [[ "$ADMIN_IP" != "local" ]]; then
|
|
ADMIN_WORKSPACE="workspace"
|
|
else
|
|
ADMIN_WORKSPACE="$WORKSPACE"
|
|
fi
|
|
admin_node_command mkdir -p $ADMIN_WORKSPACE/utils/kargo
|
|
tar cz ${BASH_SOURCE%/*}/../kargo | admin_node_command tar xzf - -C $ADMIN_WORKSPACE/utils/
|
|
|
|
echo "Setting up ansible and required dependencies..."
|
|
installed_ansible_version=$(admin_node_command dpkg-query -W -f='\${Version}\\n' ansible || echo "0.0")
|
|
if ! admin_node_command type ansible > /dev/null || \
|
|
dpkg --compare-versions "$installed_ansible_version" "lt" "$required_ansible_version"; then
|
|
# Wait for apt lock in case it is updating from cron job
|
|
while admin_node_command pgrep -a -f apt; do echo 'Waiting for apt lock...'; sleep 30; done
|
|
case $ADMIN_NODE_BASE_OS in
|
|
ubuntu)
|
|
with_retries admin_node_command -- sudo apt-get update
|
|
with_retries admin_node_command -- sudo apt-get install -y software-properties-common
|
|
with_retries admin_node_command -- sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com --recv-keys 7BB9C367
|
|
with_retries admin_node_command -- "sh -c \"sudo apt-add-repository -y 'deb http://ppa.launchpad.net/ansible/ansible/ubuntu xenial main'\""
|
|
with_retries admin_node_command -- sudo apt-get update
|
|
;;
|
|
debian)
|
|
cat ${BASH_SOURCE%/*}/files/debian_backports_repo.list | admin_node_command "sudo sh -c 'cat - > /etc/apt/sources.list.d/backports.list'"
|
|
cat ${BASH_SOURCE%/*}/files/debian_pinning | admin_node_command "sudo sh -c 'cat - > /etc/apt/preferences.d/backports'"
|
|
with_retries admin_node_command sudo apt-get update
|
|
with_retries admin_node_command sudo apt-get -y install --only-upgrade python-setuptools
|
|
;;
|
|
esac
|
|
admin_node_command sudo apt-get install -y ansible python-netaddr git
|
|
fi
|
|
|
|
# Ensure inventory exists
|
|
if ! admin_node_command test -f $ADMIN_WORKSPACE/inventory/inventory.cfg; then
|
|
echo "ERROR: $ADMIN_WORKSPACE/inventory/inventory.cfg does not exist. \
|
|
Cannot proceed."
|
|
exit_gracefully 1
|
|
fi
|
|
|
|
if [ -n "$SLAVE_IPS" ]; then
|
|
SLAVE_IPS=($(admin_node_command CONFIG_FILE=$ADMIN_WORKSPACE/inventory/inventory.cfg python3 $ADMIN_WORKSPACE/utils/kargo/inventory.py print_ips))
|
|
fi
|
|
|
|
COMMON_DEFAULTS_OPT="-e @$ADMIN_WORKSPACE/inventory/${COMMON_DEFAULTS_YAML}"
|
|
OS_SPECIFIC_DEFAULTS_OPT="-e @$ADMIN_WORKSPACE/inventory/${OS_SPECIFIC_DEFAULTS_YAML}"
|
|
# Kargo opts are not needed for this
|
|
KARGO_DEFAULTS_OPT=""
|
|
if admin_node_command test -f $ADMIN_WORKSPACE/inventory/custom.yaml; then
|
|
custom_opts="-e @$ADMIN_WORKSPACE/inventory/custom.yaml"
|
|
fi
|
|
|
|
echo "Waiting for all nodes to be reachable by SSH..."
|
|
wait_for_nodes ${SLAVE_IPS[@]}
|
|
|
|
# Stop trapping pre-setup tasks
|
|
set +e
|
|
|
|
echo "Running e2e conformance tests via ansible..."
|
|
with_ansible $ADMIN_WORKSPACE/utils/kargo/e2e_conformance.yml
|
|
|
|
exit_gracefully ${deploy_res}
|