openstack/tripleo-upgrade
Code Issues Proposed changes

Merge "Assert workloads are spawned in different stages"

Browse Source
This commit is contained in:
Zuul 2018-12-01 12:23:22 +00:00 committed by Gerrit Code Review
commit 59e3d8b1b2
4 changed files with 164 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
defaults/main.yml
View File

@ -51,6 +51,8 @@ overcloud_rc: "{{ working_dir }}/{{ overcloud_stack_name }}rc"
timestamper_cmd: >-
| awk '{ print strftime("%Y-%m-%d %H:%M:%S |"), $0; fflush(); }'
launch_sanity_workload: true
# launch workload before update/upgrade
workload_launch: false
external_network_name: "public"

4
templates/overcloud_upgrade_converge.sh.j2
View File

@ -90,3 +90,7 @@ source {{ overcloud_rc }}
kill -9 $( lsof -t {{ working_dir }}/fip_http_check_start.sh )
bash {{ working_dir }}/fip_http_check_stop.sh
{% endif %}
{% if launch_sanity_workload|bool %}
bash {{ working_dir }}/workload_launch.sh 'sanity'
{% endif %}

11
templates/overcloud_upgrade_run.sh.j2
View File

@ -40,6 +40,8 @@ openstack overcloud upgrade run \
echo "[$(date)] Finished major upgrade {{ playbook }} playbook for {{ item }} role"
{%- endfor %}
{# validation scripts should run after all playbooks are finished #}
{% if l3_agent_connectivity_check|bool %}
if [[ -e {{ working_dir }}/l3_agent_stop_ping.sh ]]; then
source {{ overcloud_rc }}
@ -60,8 +62,10 @@ kill -9 $( lsof -t {{ working_dir }}/fip_http_check_start.sh )
bash {{ working_dir }}/fip_http_check_stop.sh
{% endif %}
{%- endfor %}
{% if launch_sanity_workload|bool %}
bash {{ working_dir }}/workload_launch.sh 'sanity'
{% endif %}
{# finished overcloud_upgrade_multibooks section #}
{% else %}
{% if l3_agent_connectivity_check|bool %}
@ -116,4 +120,7 @@ source {{ overcloud_rc }}
kill -9 $( lsof -t {{ working_dir }}/fip_http_check_start.sh )
bash {{ working_dir }}/fip_http_check_stop.sh
{% endif %}
{% if launch_sanity_workload|bool %}
bash {{ working_dir }}/workload_launch.sh 'sanity'
{% endif %}
{% endif %}

167
templates/workload_launch.sh.j2
View File

@ -1,15 +1,89 @@
#!/bin/bash
#
# Script that spawns an instance
# Has 2 operational modes:
# - workload: spawn VM with floating ip (default)
# - sanity: spanw VM with floating ip assert it's ssh'able,
# delete VM and associated resources
#
if [[ -n ${1} ]]; then
MODE="${1}"
else
MODE="workload"
fi
function sanity_teardown {
# remove auxiliary resources
local timeout_seconds=180
local elapsed_seconds=0
echo "Remove ${INSTANCE_FIP} from ${INSTANCE_NAME}"
openstack server remove floating ip ${INSTANCE_NAME} ${INSTANCE_FIP}
echo "Delete floating ip ${INSTANCE_FIP}"
openstack floating ip delete ${INSTANCE_FIP}
echo "Remove VM ${INSTANCE_NAME}"
openstack server delete ${INSTANCE_NAME}
while true; do
openstack server list -f value -c Name | grep -q ${INSTANCE_NAME}
if [[ ${?} -eq 1 ]]; then
echo "Instance ${INSTANCE_NAME} successfully removed"
break
fi
elapsed_seconds=$(( ${elapsed_seconds} + 5 ))
if [ $elapsed_seconds -ge $timeout_seconds ]; then
echo "FAILURE: Failed to remove instance ${INSTANCE_NAME}"
exit 1
fi
done
echo "Remove cinder volume ${CINDER_VOL_ID}"
openstack volume delete ${CINDER_VOL_ID}
echo "Clear default gateway from ${TENANT_NET_NAME}_router"
openstack router unset --external-gateway ${TENANT_NET_NAME}_router
echo "Remove subnet ${TENANT_NET_NAME}_subnet from router ${TENANT_NET_NAME}_router"
openstack router remove subnet ${TENANT_NET_NAME}_router ${TENANT_NET_NAME}_subnet
echo "Remove router ${TENANT_NET_NAME}_router"
openstack router delete ${TENANT_NET_NAME}_router
echo "Remove subnet ${TENANT_NET_NAME}_subnet"
openstack subnet delete ${TENANT_NET_NAME}_subnet
echo "Remove network ${TENANT_NET_NAME}"
openstack network delete ${TENANT_NET_NAME}
echo "Remove security group ${SECGROUP_NAME}"
openstack security group delete ${SECGROUP_NAME}
echo "Remove keypair ${KEYPAIR_NAME}"
openstack keypair delete ${KEYPAIR_NAME}
echo "Remove image ${IMAGE_NAME}"
openstack image delete ${IMAGE_NAME}
echo "Remove file ${IMAGE_FILE}"
rm -Rf $(dirname ${IMAGE_FILE})
}
SUFFIX=$(openssl rand -hex 5)
OVERCLOUD_RC='{{ overcloud_rc }}'
IMAGE_URL='{{ workload_image_url }}'
IMAGE_NAME='upgrade_workload'
IMAGE_FILE=~/upgrade_workload.qcow2
KEYPAIR_NAME=userkey
FLAVOR_NAME='v1-1G-{{ workload_disk}}G'
SECGROUP_NAME='allow-icmp-ssh'
TENANT_NET_NAME='internal_net'
IMAGE_NAME="upgrade_workload_${SUFFIX}"
IMAGE_FILE="$(mktemp -d )/upgrade_workload_image.qcow2"
INSTANCE_NAME="instance_${SUFFIX}"
INSTANCE_USER='{{ workload_user }}'
KEYPAIR_NAME="userkey_${SUFFIX}"
FLAVOR_NAME="v1-1G-{{ workload_disk}}G-${SUFFIX}"
SECGROUP_NAME="allow-icmp-ssh-${SUFFIX}"
TENANT_NET_NAME="internal_net_${SUFFIX}"
EXTERNAL_NET_NAME='{{ external_network_name }}'
source ${OVERCLOUD_RC}
@ -18,7 +92,15 @@ source ${OVERCLOUD_RC}
openstack image list | grep ${IMAGE_NAME}
if [ $? -ne 0 ]; then
curl --silent -L -4 -o ${IMAGE_FILE} ${IMAGE_URL}
echo "Downloading image ${IMAGE_URL}"
curl --silent --retry 3 -L -4 -o ${IMAGE_FILE} ${IMAGE_URL}
if [ $? -ne 0 ]; then
echo "Failed to download ${IMAGE_URL}"
exit 1
fi
echo "Uploading ${IMAGE_NAME} to Glance"
openstack image create \
--file ${IMAGE_FILE} \
--disk-format qcow2 \
@ -29,12 +111,14 @@ fi
## create user key
openstack keypair list | grep ${KEYPAIR_NAME}
if [ $? -ne 0 ]; then
echo "Creating keypair ${KEYPAIR_NAME}"
openstack keypair create --public-key ~/.ssh/id_rsa.pub ${KEYPAIR_NAME}
fi
## create flavor
openstack flavor list | grep ${FLAVOR_NAME}
if [ $? -ne 0 ]; then
echo "Creating flavor ${FLAVOR_NAME}"
openstack flavor create --vcpus {{ workload_vcpu }} \
--ram {{ workload_memory }} \
--disk {{ workload_disk }} \
@ -46,8 +130,13 @@ fi
openstack network list | grep ${TENANT_NET_NAME}
if [ $? -ne 0 ]; then
NAMESERVER=$(grep -Eo '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' /etc/resolv.conf | head -1)
echo "Creating router ${TENANT_NET_NAME}_router"
openstack router create ${TENANT_NET_NAME}_router
echo "Creating network ${TENANT_NET_NAME}"
openstack network create ${TENANT_NET_NAME}
echo "Creating subnet ${TENANT_NET_NAME}_subnet"
openstack subnet create \
--subnet-range 192.168.0.0/24 \
--allocation-pool start=192.168.0.10,end=192.168.0.100 \
@ -55,14 +144,21 @@ if [ $? -ne 0 ]; then
--dns-nameserver ${NAMESERVER} \
--network ${TENANT_NET_NAME} \
${TENANT_NET_NAME}_subnet
echo "Add subnet ${TENANT_NET_NAME}_subnet to router ${TENANT_NET_NAME}_router"
openstack router add subnet ${TENANT_NET_NAME}_router ${TENANT_NET_NAME}_subnet
echo "Set external-gateway for ${TENANT_NET_NAME}_router"
openstack router set --external-gateway ${EXTERNAL_NET_NAME} ${TENANT_NET_NAME}_router
fi
## create security group
openstack security group list | grep ${SECGROUP_NAME}
if [ $? -ne 0 ]; then
echo "Creating security group ${SECGROUP_NAME}"
openstack security group create ${SECGROUP_NAME}
echo "Creating rules for ports 22,80,443 in security group ${SECGROUP_NAME}"
openstack security group rule create --proto icmp ${SECGROUP_NAME}
openstack security group rule create --proto tcp --dst-port 22 ${SECGROUP_NAME}
openstack security group rule create --proto tcp --dst-port 80 ${SECGROUP_NAME}
@ -70,39 +166,74 @@ if [ $? -ne 0 ]; then
fi
## create instance
INSTANCE_NAME="instance_$(openssl rand -hex 5)"
TENANT_NET_ID=$(openstack network list -f json | jq -r -c ".[] | select(.Name | contains(\"$TENANT_NET_NAME\")) | .ID")
TENANT_NET_ID=$( openstack network show -f json ${TENANT_NET_NAME} | jq -r -c '.id' )
echo "Creating overcloud instance ${INSTANCE_NAME}"
openstack server create \
--image ${IMAGE_NAME} \
--flavor ${FLAVOR_NAME} \
--security-group ${SECGROUP_NAME} \
--key-name ${KEYPAIR_NAME} \
--nic net-id=${TENANT_NET_ID} \
$INSTANCE_NAME
${INSTANCE_NAME}
timeout_seconds=120
elapsed_seconds=0
while true; do
INSTANCE_ACTIVE=$(openstack server show $INSTANCE_NAME -f json | jq -r .status)
if [ $INSTANCE_ACTIVE == 'ACTIVE' ]; then
INSTANCE_ACTIVE=$(openstack server show ${INSTANCE_NAME} -f json | jq -r '.status')
if [ "${INSTANCE_ACTIVE}" == "ACTIVE" ]; then
echo "${INSTANCE_NAME} reached 'ACTIVE' status"
break
fi
sleep 3
elapsed_seconds=$(expr $elapsed_seconds + 3)
if [ $elapsed_seconds -ge $timeout_seconds ]; then
echo "FAILURE: Instance failed to boot."
openstack server show ${INSTANCE_NAME} -f json 2>&1
exit 1
fi
done
## assign floating ip
INSTANCE_FIP=$(openstack floating ip create ${EXTERNAL_NET_NAME} -f json | jq -r .id)
INSTANCE_IP=$(openstack server show $INSTANCE_NAME -f json | jq -r .addresses | grep -oP '[0-9.]+')
INSTANCE_PORT=$(openstack port list -f json | jq -r -c ".[] | select(.[\"Fixed IP Addresses\"] | contains(\"${INSTANCE_IP}\")) | .ID")
neutron floatingip-associate ${INSTANCE_FIP} ${INSTANCE_PORT}
INSTANCE_FIP=$(openstack floating ip create ${EXTERNAL_NET_NAME} -f json | jq -r -c '.floating_ip_address' )
echo "Assign FIP[${INSTANCE_FIP}] to server ${INSTANCE_NAME}"
openstack server add floating ip ${INSTANCE_NAME} ${INSTANCE_FIP}
## create and attach a volume
CINDER_VOL_ID=$(openstack volume create --size 1 vol_$(openssl rand -hex 5) -f json | jq -r .id)
CINDER_VOL_ID=$(openstack volume create --size 1 vol_${SUFFIX} -f json | jq -r .id)
echo "Attach volume vol_${SUFFIX} to instance ${INSTANCE_NAME}"
openstack server add volume ${INSTANCE_NAME} ${CINDER_VOL_ID}
echo "floating-ip: $(openstack floating ip show ${INSTANCE_FIP} -f json | jq -r .floating_ip_address)" > ~/${INSTANCE_NAME}
## SSH to VM through it's floating ip
if [[ "${MODE}" == "sanity" ]]; then
timeout_seconds=180
elapsed_seconds=0
while true; do
# assert instance is reachable via FIP
echo " [$(date)] Trying to ssh to ${INSTANCE_FIP}"
ssh -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
${INSTANCE_USER}@${INSTANCE_FIP} 'whoami'
if [[ ${?} -eq 0 ]]; then
echo "Instance ${INSTANCE_NAME} is reachable via ${INSTANCE_FIP}"
break
fi
sleep 3
elapsed_seconds=$(expr $elapsed_seconds + 3)
if [ $elapsed_seconds -ge $timeout_seconds ]; then
echo "FAILURE: Instance failed to boot."
sanity_teardown
exit 1
fi
done
fi
if [[ "${MODE}" == "workload" ]]; then
echo "floating-ip: ${INSTANCE_FIP}" > ~/${INSTANCE_NAME}
fi
if [[ "${MODE}" == "sanity" ]]; then
sanity_teardown
fi