CI: add prometheus-efk scenario

Tests prometheus, grafana, and centralised logging.

The tests could be improved in future by querying logs in elasticsearch,
and metrics in prometheus.

Change-Id: Iabad035d583d291169f23be3d71931cb260e87ae
This commit is contained in:
Mark Goddard 2020-07-08 19:32:23 +01:00
parent 5142a2bec4
commit f44876c406
7 changed files with 198 additions and 1 deletions

View File

@ -10,7 +10,7 @@
- name: set facts for commonly used variables
vars:
# NOTE(yoctozepto): needed here to use in other facts too
openstack_core_enabled: "{{ scenario not in ['bifrost', 'mariadb'] }}"
openstack_core_enabled: "{{ scenario not in ['bifrost', 'mariadb', 'prometheus-efk'] }}"
set_fact:
kolla_inventory_path: "/etc/kolla/inventory"
logs_dir: "/tmp/logs"
@ -408,6 +408,14 @@
executable: /bin/bash
chdir: "{{ kolla_ansible_src_dir }}"
when: scenario == "mariadb"
- name: Run test-prometheus-efk.sh script
script:
cmd: test-prometheus-efk.sh
executable: /bin/bash
chdir: "{{ kolla_ansible_src_dir }}"
when: scenario == "prometheus-efk"
when: scenario != "bifrost"
# NOTE(yoctozepto): each host checks itself

View File

@ -137,3 +137,11 @@ neutron_plugin_agent: "linuxbridge"
{% if scenario == "ovn" %}
neutron_plugin_agent: "ovn"
{% endif %}
{% if scenario == "prometheus-efk" %}
enable_chrony: "no"
enable_central_logging: "yes"
enable_grafana: "yes"
enable_prometheus: "yes"
enable_prometheus_openstack_exporter: "no"
{% endif %}

150
tests/test-prometheus-efk.sh Executable file
View File

@ -0,0 +1,150 @@
#!/bin/bash
set -o xtrace
set -o errexit
set -o pipefail
# Enable unbuffered output
export PYTHONUNBUFFERED=1
function check_kibana {
# Query kibana, and check that the returned page looks like a kibana page.
KIBANA_URL=${OS_AUTH_URL%:*}:5601
output_path=$1
kibana_password=$(awk '$1 == "kibana_password:" { print $2 }' /etc/kolla/passwords.yml)
args=(
--include
--location
--fail
--user
kibana:$kibana_password
)
if [[ "$TLS_ENABLED" = "True" ]]; then
args+=(--cacert $OS_CACERT)
fi
if ! curl "${args[@]}" $KIBANA_URL > $output_path; then
return 1
fi
if ! grep '<title>Kibana</title>' $output_path >/dev/null; then
return 1
fi
}
function check_grafana {
# Query grafana, and check that the returned page looks like a grafana page.
GRAFANA_URL=${OS_AUTH_URL%:*}:3000
output_path=$1
grafana_password=$(awk '$1 == "grafana_admin_password:" { print $2 }' /etc/kolla/passwords.yml)
args=(
--include
--location
--fail
--user
admin:$grafana_password
)
if [[ "$TLS_ENABLED" = "True" ]]; then
args+=(--cacert $OS_CACERT)
fi
if ! curl "${args[@]}" $GRAFANA_URL > $output_path; then
return 1
fi
if ! grep '<title>Grafana</title>' $output_path >/dev/null; then
return 1
fi
}
function check_prometheus {
# Query prometheus graph, and check that the returned page looks like a
# prometheus page.
PROMETHEUS_URL=${OS_AUTH_URL%:*}:9091/graph
output_path=$1
args=(
--include
--location
--fail
)
if [[ "$TLS_ENABLED" = "True" ]]; then
args+=(--cacert $OS_CACERT)
fi
if ! curl "${args[@]}" $PROMETHEUS_URL > $output_path; then
return 1
fi
if ! grep '<title>Prometheus' $output_path >/dev/null; then
return 1
fi
}
function test_kibana {
# TODO(mgoddard): Query elasticsearch for logs.
echo "TESTING: Kibana"
output_path=$(mktemp)
attempt=1
while ! check_kibana $output_path; do
echo "Kibana not accessible yet"
attempt=$((attempt+1))
if [[ $attempt -eq 12 ]]; then
echo "FAILED: Kibana did not become accessible. Response:"
cat $output_path
return 1
fi
sleep 10
done
echo "SUCCESS: Kibana"
}
function test_grafana {
echo "TESTING: Grafana"
output_path=$(mktemp)
attempt=1
while ! check_grafana $output_path; do
echo "Grafana not accessible yet"
attempt=$((attempt+1))
if [[ $attempt -eq 12 ]]; then
echo "FAILED: Grafana did not become accessible. Response:"
cat $output_path
return 1
fi
sleep 10
done
echo "SUCCESS: Grafana"
}
function test_prometheus {
# TODO(mgoddard): Query metrics.
echo "TESTING: Prometheus"
output_path=$(mktemp)
attempt=1
while ! check_prometheus $output_path; do
echo "Prometheus not accessible yet"
attempt=$((attempt+1))
if [[ $attempt -eq 12 ]]; then
echo "FAILED: Prometheus did not become accessible. Response:"
cat $output_path
return 1
fi
sleep 10
done
echo "SUCCESS: Prometheus"
}
function test_prometheus_efk_logged {
. /etc/kolla/admin-openrc.sh
test_kibana
test_grafana
test_prometheus
}
function test_prometheus_efk {
echo "Testing prometheus and EFK"
test_prometheus_efk_logged > /tmp/logs/ansible/test-prometheus-efk 2>&1
result=$?
if [[ $result != 0 ]]; then
echo "Testing prometheus and EFK failed. See ansible/test-prometheus-efk for details"
else
echo "Successfully tested prometheus and EFK. See ansible/test-prometheus-efk for details"
fi
return $result
}
test_prometheus_efk

View File

@ -75,6 +75,10 @@ function prepare_images {
GATE_IMAGES="^cron,^fluentd,^haproxy,^keepalived,^kolla-toolbox,^mariadb"
fi
if [[ $SCENARIO == "prometheus-efk" ]]; then
GATE_IMAGES="^cron,^elasticsearch,^fluentd,^grafana,^haproxy,^keepalived,^kibana,^kolla-toolbox,^mariadb,^memcached,^prometheus,^rabbitmq"
fi
# NOTE(yoctozepto): we cannot build and push at the same time on debian
# buster see https://github.com/docker/for-linux/issues/711.
PUSH="true"

View File

@ -173,3 +173,12 @@
- ^tests/test-core-openstack.sh
vars:
scenario: ovn
- job:
name: kolla-ansible-prometheus-efk-base
parent: kolla-ansible-base
voting: false
files:
- ^ansible/roles/(common|elasticsearch|grafana|kibana|prometheus)/
vars:
scenario: prometheus-efk

View File

@ -280,3 +280,19 @@
vars:
base_distro: ubuntu
install_type: source
- job:
name: kolla-ansible-centos8-source-prometheus-efk
parent: kolla-ansible-prometheus-efk-base
nodeset: kolla-ansible-centos8
vars:
base_distro: centos
install_type: source
- job:
name: kolla-ansible-ubuntu-source-prometheus-efk
parent: kolla-ansible-prometheus-efk-base
nodeset: kolla-ansible-bionic
vars:
base_distro: ubuntu
install_type: source

View File

@ -41,6 +41,8 @@
- kolla-ansible-ubuntu-source-linuxbridge
- kolla-ansible-centos8-source-ovn
- kolla-ansible-ubuntu-source-ovn
- kolla-ansible-centos8-source-prometheus-efk
- kolla-ansible-ubuntu-source-prometheus-efk
check-arm64:
jobs:
- kolla-ansible-debian-source-aarch64