105 lines
4.1 KiB
Django/Jinja
105 lines
4.1 KiB
Django/Jinja
source {{ undercloud_rc }}
|
|
NODE_IP=$(openstack server show {{ node_name | splitext | first | splitext | first }} -f json | jq -r .addresses | grep -oP '[0-9.]+')
|
|
|
|
OVS_RUNNING=$(ssh -q -o StrictHostKeyChecking=no heat-admin@$NODE_IP 'sudo ovs-vsctl show' | grep ovs_version | awk -F \" {'print $2'} | awk -F "." '{print $1"."$2}')
|
|
OVS_INSTALLED=$(ssh -q -o StrictHostKeyChecking=no heat-admin@$NODE_IP 'sudo rpm --queryformat %{VERSION} -q openvswitch' | awk -F "." '{print $1"."$2}')
|
|
|
|
## change
|
|
if [[ $OVS_RUNNING != $OVS_INSTALLED ]]; then
|
|
|
|
echo "Upgraded OVS detected. Rebooting {{ node_name | splitext | first | splitext | first }}"
|
|
ssh -q -o StrictHostKeyChecking=no heat-admin@$NODE_IP 'sudo reboot'
|
|
|
|
## wait for node to go down
|
|
timeout_seconds={{ node_reboot_timeout }}
|
|
elapsed_seconds=0
|
|
while true; do
|
|
echo "Waiting for {{ node_name | splitext | first | splitext | first }} to go down ..."
|
|
NODE_DOWN=$(ping -c1 $NODE_IP)
|
|
if [ $? != 0 ]; then
|
|
break
|
|
fi
|
|
sleep 3
|
|
(( elapsed_seconds += 3 ))
|
|
if [ $elapsed_seconds -ge $timeout_seconds ]; then
|
|
echo "FAILURE: Node {{ node_name | splitext | first | splitext | first }} didn't reboot"
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
{% if 'compute' in node_name %}
|
|
## wait for node to get back online
|
|
timeout_seconds={{ node_reboot_timeout }}
|
|
elapsed_seconds=0
|
|
while true; do
|
|
echo "Waiting for {{ node_name | splitext | first | splitext | first }} to boot ..."
|
|
NOVACOMPUTE_STATUS=$(ssh -q -o StrictHostKeyChecking=no heat-admin@$NODE_IP 'sudo docker inspect --format="{{ "{{" }} .State.Running {{ "}}" }}" nova_compute')
|
|
if [[ $NOVACOMPUTE_STATUS == 'true' ]]; then
|
|
break
|
|
fi
|
|
sleep 3
|
|
(( elapsed_seconds += 3 ))
|
|
if [ $elapsed_seconds -ge $timeout_seconds ]; then
|
|
echo "FAILURE: Nova compute container didn't start on {{ node_name | splitext | first | splitext | first }}"
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
## wait for nove compute service to be reported as up
|
|
source {{ overcloud_rc }}
|
|
timeout_seconds={{ node_reboot_timeout }}
|
|
elapsed_seconds=0
|
|
while true; do
|
|
echo "Waiting for nova-compute service on {{ node_name | splitext | first | splitext | first }} to go up ..."
|
|
NOVACOMPUTE_ENABLED=$(openstack compute service list --host {{ node_name }} -f json | jq -r -c '.[] | select(.Binary | contains("nova-compute")) | .State' | head -1)
|
|
if [[ $NOVACOMPUTE_ENABLED == 'up' ]]; then
|
|
break
|
|
fi
|
|
sleep 3
|
|
(( elapsed_seconds += 3 ))
|
|
if [ $elapsed_seconds -ge $timeout_seconds ]; then
|
|
echo "FAILURE: Nova compute service didn't come up on {{ node_name | splitext | first | splitext | first }}"
|
|
exit 1
|
|
fi
|
|
done
|
|
{% endif %}
|
|
fi
|
|
|
|
{% if compute_evacuate %}
|
|
{% if 'compute' in node_name %}
|
|
source {{ overcloud_rc }}
|
|
## Detect if ceph shared storage was used
|
|
STORAGE_BACKEND=$(openstack volume service list -f json | jq -r -c '.[] | select(.Binary | contains("cinder-volume")) | .Host' | sed s/hostgroup@tripleo_//)
|
|
{% for instance in node_instances.stdout_lines %}
|
|
if [ $STORAGE_BACKEND == 'ceph' ]; then
|
|
nova live-migration {{ instance }} {{ node_name }}
|
|
else
|
|
nova live-migration --block-migrate {{ instance }} {{ node_name }}
|
|
fi
|
|
timeout_seconds=120
|
|
elapsed_seconds=0
|
|
while true; do
|
|
if [ $STORAGE_BACKEND == 'ceph' ]; then
|
|
echo "Shared storage live migrating {{ instance }} back to {{ node_name }} ..."
|
|
## Live migration might not complete on the first run so we run it multiple times
|
|
## until the instance ends back on the originating host
|
|
nova live-migration {{ instance }} {{ node_name }}
|
|
else
|
|
echo "Block migrating {{ instance }} back to {{ node_name }} ..."
|
|
nova live-migration --block-migrate {{ instance }} {{ node_name }}
|
|
fi
|
|
INSTANCE_HOST=$(openstack server show {{ instance }} -f json | jq -r -c '. | .["OS-EXT-SRV-ATTR:host"]')
|
|
if [ $INSTANCE_HOST == '{{ node_name }}' ]; then
|
|
break
|
|
fi
|
|
sleep 3
|
|
(( elapsed_seconds += 3 ))
|
|
if [ $elapsed_seconds -ge $timeout_seconds ]; then
|
|
echo "FAILURE: Could not live migrate instance back to {{ node_name }}"
|
|
exit 1
|
|
fi
|
|
done
|
|
{% endfor %}
|
|
{% endif %}
|
|
{% endif %}
|