Merge "Introduce wait_backoff, fail_if_empty wrappers"
This commit is contained in:
commit
ecbe72ba9b
|
@ -2,7 +2,7 @@
|
||||||
|
|
||||||
set -eux
|
set -eux
|
||||||
|
|
||||||
SLEEP_TIME=5
|
SLEEP_TIME=2
|
||||||
|
|
||||||
CONTROLLER_HOSTS=${CONTROLLER_HOSTS:-""}
|
CONTROLLER_HOSTS=${CONTROLLER_HOSTS:-""}
|
||||||
COMPUTE_HOSTS=${COMPUTE_HOSTS:-""}
|
COMPUTE_HOSTS=${COMPUTE_HOSTS:-""}
|
||||||
|
@ -22,46 +22,92 @@ BlockStorage_hosts=${BlockStorage_hosts:-"$BLOCKSTORAGE_HOSTS"}
|
||||||
ObjectStorage_hosts=${ObjectStorage_hosts:-"$OBJECTSTORAGE_HOSTS"}
|
ObjectStorage_hosts=${ObjectStorage_hosts:-"$OBJECTSTORAGE_HOSTS"}
|
||||||
CephStorage_hosts=${CephStorage_hosts:-"$CEPHSTORAGE_HOSTS"}
|
CephStorage_hosts=${CephStorage_hosts:-"$CEPHSTORAGE_HOSTS"}
|
||||||
|
|
||||||
|
#######################################
|
||||||
|
# Retry with backoff interval
|
||||||
|
#######################################
|
||||||
|
function with_backoff {
|
||||||
|
local max_attempts=${ATTEMPTS:-10}
|
||||||
|
local sleep_timeout=${SLEEP_TIME:-2}
|
||||||
|
local attempt=0
|
||||||
|
local rc=0
|
||||||
|
|
||||||
|
while [ ${attempt} -lt ${max_attempts} ]; do
|
||||||
|
set +e
|
||||||
|
set -o pipefail
|
||||||
|
"$@"
|
||||||
|
rc=$?
|
||||||
|
set +o pipefail
|
||||||
|
set -e
|
||||||
|
|
||||||
|
if [ ${rc} -eq 0 ]; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
echo "Warning! Retrying in ${sleep_timeout} seconds ..." 1>&2
|
||||||
|
sleep ${sleep_timeout}
|
||||||
|
attempt=$(( attempt + 1 ))
|
||||||
|
sleep_timeout=$(( sleep_timeout * 2 ))
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ ${rc} -ne 0 ]; then
|
||||||
|
echo "Warning! Return code is not 0 on the last try for ($@)" 1>&2
|
||||||
|
fi
|
||||||
|
|
||||||
|
return ${rc}
|
||||||
|
}
|
||||||
|
|
||||||
|
#######################################
|
||||||
|
# Return 1 if empty output received
|
||||||
|
#######################################
|
||||||
|
function fail_if_empty {
|
||||||
|
local output="$(${@})"
|
||||||
|
if [ -z "${output}" ]; then
|
||||||
|
echo "Warning! Empty output for ($@)" 1>&2
|
||||||
|
return 1
|
||||||
|
else
|
||||||
|
echo "${output}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
function check_stack {
|
||||||
|
local stack_to_check=${1:-""}
|
||||||
|
local rc=0
|
||||||
|
|
||||||
|
if [ -z "${stack_to_check}" ]; then
|
||||||
|
echo No Stacks received.
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
with_backoff openstack stack resource list $stack_to_check
|
||||||
|
rc=${?}
|
||||||
|
|
||||||
|
if [ ${rc} -ne 0 ]; then
|
||||||
|
echo Stack ${stack_to_check} not yet created
|
||||||
|
fi
|
||||||
|
|
||||||
|
return ${rc}
|
||||||
|
}
|
||||||
|
|
||||||
# Set the _hosts_a vars for each role defined
|
# Set the _hosts_a vars for each role defined
|
||||||
for role in $OVERCLOUD_ROLES; do
|
for role in $OVERCLOUD_ROLES; do
|
||||||
eval "hosts=\${${role}_hosts}"
|
eval "hosts=\${${role}_hosts}"
|
||||||
read -a ${role}_hosts_a <<< $hosts
|
read -a ${role}_hosts_a <<< $hosts
|
||||||
done
|
done
|
||||||
|
|
||||||
function check_stack {
|
|
||||||
local stack_to_check=${1:-""}
|
|
||||||
|
|
||||||
if [ "$stack_to_check" = "" ]; then
|
|
||||||
echo Stack not created
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo Checking if $1 stack is created
|
|
||||||
set +e
|
|
||||||
openstack stack resource list $stack_to_check
|
|
||||||
rc=$?
|
|
||||||
set -e
|
|
||||||
|
|
||||||
if [ ! "$rc" = "0" ]; then
|
|
||||||
echo Stack $1 not yet created
|
|
||||||
fi
|
|
||||||
|
|
||||||
return $rc
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
for role in $OVERCLOUD_ROLES; do
|
for role in $OVERCLOUD_ROLES; do
|
||||||
while ! check_stack $STACK_NAME; do
|
while ! check_stack $STACK_NAME; do
|
||||||
sleep $SLEEP_TIME
|
sleep $SLEEP_TIME
|
||||||
done
|
done
|
||||||
|
|
||||||
rg_stack=$(openstack stack resource show $STACK_NAME $role -c physical_resource_id -f value)
|
rg_stack=$(with_backoff fail_if_empty openstack stack resource show $STACK_NAME $role -c physical_resource_id -f value)
|
||||||
while ! check_stack $rg_stack; do
|
while ! check_stack $rg_stack; do
|
||||||
sleep $SLEEP_TIME
|
rg_stack=$(with_backoff fail_if_empty openstack stack resource show $STACK_NAME $role -c physical_resource_id -f value)
|
||||||
rg_stack=$(openstack stack resource show $STACK_NAME $role -c physical_resource_id -f value)
|
|
||||||
done
|
done
|
||||||
|
|
||||||
stacks=$(openstack stack resource list $rg_stack -c resource_name -c physical_resource_id -f json | jq -r "sort_by(.resource_name) | .[] | .physical_resource_id")
|
stacks=$(with_backoff fail_if_empty openstack stack resource list $rg_stack -c resource_name -c physical_resource_id -f json | jq -r "sort_by(.resource_name) | .[] | .physical_resource_id")
|
||||||
|
rc=${?}
|
||||||
|
while [ ${rc} -ne 0 ]; do
|
||||||
|
stacks=$(with_backoff fail_if_empty openstack stack resource list $rg_stack -c resource_name -c physical_resource_id -f json | jq -r "sort_by(.resource_name) | .[] | .physical_resource_id")
|
||||||
|
done
|
||||||
|
|
||||||
i=0
|
i=0
|
||||||
|
|
||||||
|
@ -71,19 +117,17 @@ for role in $OVERCLOUD_ROLES; do
|
||||||
server_resource_name="NovaCompute"
|
server_resource_name="NovaCompute"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
server_stack=$(openstack stack resource show $stack $server_resource_name -c physical_resource_id -f value)
|
server_stack=$(with_backoff fail_if_empty openstack stack resource show $stack $server_resource_name -c physical_resource_id -f value)
|
||||||
while ! check_stack $server_stack; do
|
while ! check_stack $server_stack; do
|
||||||
sleep $SLEEP_TIME
|
server_stack=$(with_backoff fail_if_empty openstack stack resource show $stack $server_resource_name -c physical_resource_id -f value)
|
||||||
server_stack=$(openstack stack resource show $stack $server_resource_name -c physical_resource_id -f value)
|
|
||||||
done
|
done
|
||||||
|
|
||||||
while true; do
|
while true; do
|
||||||
deployed_server_metadata_url=$(openstack stack resource metadata $server_stack deployed-server | jq -r '.["os-collect-config"].request.metadata_url')
|
deployed_server_metadata_url=$(with_backoff openstack stack resource metadata $server_stack deployed-server | jq -r '.["os-collect-config"].request.metadata_url')
|
||||||
if [ "$deployed_server_metadata_url" = "null" ]; then
|
if [ "$deployed_server_metadata_url" != "null" ]; then
|
||||||
continue
|
|
||||||
else
|
|
||||||
break
|
break
|
||||||
fi
|
fi
|
||||||
|
sleep $SLEEP_TIME
|
||||||
done
|
done
|
||||||
|
|
||||||
echo "======================"
|
echo "======================"
|
||||||
|
|
Loading…
Reference in New Issue