9f578c85e0
Adds HAcluster Ansible role. This role contains High Availability clustering solution composed of Corosync, Pacemaker and Pacemaker Remote. HAcluster is added as a helper role for Masakari which requires it for its host monitoring, allowing to provide HA to instances on a failed compute host. Kolla hacluster images merged in [1]. [1] https://review.opendev.org/#/c/668765/ Change-Id: I91e5c1840ace8f567daf462c4eb3ec1f0c503823 Implements: blueprint ansible-pacemaker-support Co-Authored-By: Radosław Piliszek <radoslaw.piliszek@gmail.com> Co-Authored-By: Mark Goddard <mark@stackhpc.com>
96 lines
2.7 KiB
Bash
Executable File
96 lines
2.7 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
set -o xtrace
|
|
set -o errexit
|
|
set -o pipefail
|
|
|
|
# Enable unbuffered output for Ansible in Jenkins.
|
|
export PYTHONUNBUFFERED=1
|
|
|
|
function test_hacluster_logged {
|
|
local cluster_failure
|
|
cluster_failure=0
|
|
|
|
# NOTE(yoctozepto): repeated -V in commands below is used to get 'debug'
|
|
# output; the right amount differs between command sets; the next level is
|
|
# 'trace' which is overly verbose; PCMK_debug=no is used to revert the env
|
|
# var setting from the container which would cause these commands to log up
|
|
# to 'trace' (likely a pacemaker bug)
|
|
|
|
if ! sudo docker exec hacluster_pacemaker cibadmin -VVVVVV --query --local; then
|
|
cluster_failure=1
|
|
fi
|
|
|
|
local mon_output
|
|
|
|
if ! mon_output=$(sudo docker exec -e PCMK_debug=no hacluster_pacemaker crm_mon -VVVVV --one-shot); then
|
|
cluster_failure=1
|
|
fi
|
|
|
|
if ! sudo docker exec -e PCMK_debug=no hacluster_pacemaker crm_verify -VVVVV --live-check; then
|
|
cluster_failure=1
|
|
fi
|
|
|
|
# NOTE(yoctozepto): crm_mon output should include:
|
|
# * Online: [ primary secondary ]
|
|
# * RemoteOnline: [ ternary1 ternary2 ]
|
|
|
|
if ! echo "$mon_output" | grep 'Online: \[ primary secondary \]'; then
|
|
echo 'Full members missing' >&2
|
|
cluster_failure=1
|
|
fi
|
|
|
|
if ! echo "$mon_output" | grep 'RemoteOnline: \[ ternary1 ternary2 \]'; then
|
|
echo 'Remote members missing' >&2
|
|
cluster_failure=1
|
|
fi
|
|
|
|
if [[ $cluster_failure -eq 1 ]]; then
|
|
echo "HAcluster failed"
|
|
return 1
|
|
else
|
|
echo "HAcluster healthy"
|
|
fi
|
|
}
|
|
|
|
function test_masakari_logged {
|
|
# Source OpenStack credentials
|
|
. /etc/kolla/admin-openrc.sh
|
|
|
|
# Activate virtualenv to access Masakari client
|
|
. ~/openstackclient-venv/bin/activate
|
|
|
|
# Create Masakari segment
|
|
if ! openstack segment create test_segment auto COMPUTE; then
|
|
echo "Unable to create Masakari segment"
|
|
return 1
|
|
fi
|
|
|
|
openstack segment host create ternary1 COMPUTE SSH test_segment
|
|
openstack segment host create ternary2 COMPUTE SSH test_segment
|
|
|
|
# Delete Masakari segment
|
|
if ! openstack segment delete test_segment; then
|
|
echo "Unable to delete Masakari segment"
|
|
return 1
|
|
fi
|
|
|
|
# Exit virtualenv
|
|
deactivate
|
|
}
|
|
|
|
function test_masakari {
|
|
echo "Testing Masakari"
|
|
test_hacluster_logged > /tmp/logs/ansible/test-hacluster 2>&1
|
|
test_masakari_logged > /tmp/logs/ansible/test-masakari 2>&1
|
|
result=$?
|
|
if [[ $result != 0 ]]; then
|
|
echo "Testing Masakari failed. See ansible/test-masakari for details"
|
|
else
|
|
echo "Successfully tested Masakari. See ansible/test-masakari for details"
|
|
fi
|
|
return $result
|
|
}
|
|
|
|
test_masakari
|