CI: add prometheus-efk scenario
Tests prometheus, grafana, and centralised logging. The tests could be improved in future by querying logs in elasticsearch, and metrics in prometheus. Change-Id: Iabad035d583d291169f23be3d71931cb260e87ae
This commit is contained in:
parent
5142a2bec4
commit
f44876c406
@ -10,7 +10,7 @@
|
||||
- name: set facts for commonly used variables
|
||||
vars:
|
||||
# NOTE(yoctozepto): needed here to use in other facts too
|
||||
openstack_core_enabled: "{{ scenario not in ['bifrost', 'mariadb'] }}"
|
||||
openstack_core_enabled: "{{ scenario not in ['bifrost', 'mariadb', 'prometheus-efk'] }}"
|
||||
set_fact:
|
||||
kolla_inventory_path: "/etc/kolla/inventory"
|
||||
logs_dir: "/tmp/logs"
|
||||
@ -408,6 +408,14 @@
|
||||
executable: /bin/bash
|
||||
chdir: "{{ kolla_ansible_src_dir }}"
|
||||
when: scenario == "mariadb"
|
||||
|
||||
- name: Run test-prometheus-efk.sh script
|
||||
script:
|
||||
cmd: test-prometheus-efk.sh
|
||||
executable: /bin/bash
|
||||
chdir: "{{ kolla_ansible_src_dir }}"
|
||||
when: scenario == "prometheus-efk"
|
||||
|
||||
when: scenario != "bifrost"
|
||||
|
||||
# NOTE(yoctozepto): each host checks itself
|
||||
|
@ -137,3 +137,11 @@ neutron_plugin_agent: "linuxbridge"
|
||||
{% if scenario == "ovn" %}
|
||||
neutron_plugin_agent: "ovn"
|
||||
{% endif %}
|
||||
|
||||
{% if scenario == "prometheus-efk" %}
|
||||
enable_chrony: "no"
|
||||
enable_central_logging: "yes"
|
||||
enable_grafana: "yes"
|
||||
enable_prometheus: "yes"
|
||||
enable_prometheus_openstack_exporter: "no"
|
||||
{% endif %}
|
||||
|
150
tests/test-prometheus-efk.sh
Executable file
150
tests/test-prometheus-efk.sh
Executable file
@ -0,0 +1,150 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -o xtrace
|
||||
set -o errexit
|
||||
set -o pipefail
|
||||
|
||||
# Enable unbuffered output
|
||||
export PYTHONUNBUFFERED=1
|
||||
|
||||
function check_kibana {
|
||||
# Query kibana, and check that the returned page looks like a kibana page.
|
||||
KIBANA_URL=${OS_AUTH_URL%:*}:5601
|
||||
output_path=$1
|
||||
kibana_password=$(awk '$1 == "kibana_password:" { print $2 }' /etc/kolla/passwords.yml)
|
||||
args=(
|
||||
--include
|
||||
--location
|
||||
--fail
|
||||
--user
|
||||
kibana:$kibana_password
|
||||
)
|
||||
if [[ "$TLS_ENABLED" = "True" ]]; then
|
||||
args+=(--cacert $OS_CACERT)
|
||||
fi
|
||||
if ! curl "${args[@]}" $KIBANA_URL > $output_path; then
|
||||
return 1
|
||||
fi
|
||||
if ! grep '<title>Kibana</title>' $output_path >/dev/null; then
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
function check_grafana {
|
||||
# Query grafana, and check that the returned page looks like a grafana page.
|
||||
GRAFANA_URL=${OS_AUTH_URL%:*}:3000
|
||||
output_path=$1
|
||||
grafana_password=$(awk '$1 == "grafana_admin_password:" { print $2 }' /etc/kolla/passwords.yml)
|
||||
args=(
|
||||
--include
|
||||
--location
|
||||
--fail
|
||||
--user
|
||||
admin:$grafana_password
|
||||
)
|
||||
if [[ "$TLS_ENABLED" = "True" ]]; then
|
||||
args+=(--cacert $OS_CACERT)
|
||||
fi
|
||||
if ! curl "${args[@]}" $GRAFANA_URL > $output_path; then
|
||||
return 1
|
||||
fi
|
||||
if ! grep '<title>Grafana</title>' $output_path >/dev/null; then
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
function check_prometheus {
|
||||
# Query prometheus graph, and check that the returned page looks like a
|
||||
# prometheus page.
|
||||
PROMETHEUS_URL=${OS_AUTH_URL%:*}:9091/graph
|
||||
output_path=$1
|
||||
args=(
|
||||
--include
|
||||
--location
|
||||
--fail
|
||||
)
|
||||
if [[ "$TLS_ENABLED" = "True" ]]; then
|
||||
args+=(--cacert $OS_CACERT)
|
||||
fi
|
||||
if ! curl "${args[@]}" $PROMETHEUS_URL > $output_path; then
|
||||
return 1
|
||||
fi
|
||||
if ! grep '<title>Prometheus' $output_path >/dev/null; then
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
function test_kibana {
|
||||
# TODO(mgoddard): Query elasticsearch for logs.
|
||||
echo "TESTING: Kibana"
|
||||
output_path=$(mktemp)
|
||||
attempt=1
|
||||
while ! check_kibana $output_path; do
|
||||
echo "Kibana not accessible yet"
|
||||
attempt=$((attempt+1))
|
||||
if [[ $attempt -eq 12 ]]; then
|
||||
echo "FAILED: Kibana did not become accessible. Response:"
|
||||
cat $output_path
|
||||
return 1
|
||||
fi
|
||||
sleep 10
|
||||
done
|
||||
echo "SUCCESS: Kibana"
|
||||
}
|
||||
|
||||
function test_grafana {
|
||||
echo "TESTING: Grafana"
|
||||
output_path=$(mktemp)
|
||||
attempt=1
|
||||
while ! check_grafana $output_path; do
|
||||
echo "Grafana not accessible yet"
|
||||
attempt=$((attempt+1))
|
||||
if [[ $attempt -eq 12 ]]; then
|
||||
echo "FAILED: Grafana did not become accessible. Response:"
|
||||
cat $output_path
|
||||
return 1
|
||||
fi
|
||||
sleep 10
|
||||
done
|
||||
echo "SUCCESS: Grafana"
|
||||
}
|
||||
|
||||
function test_prometheus {
|
||||
# TODO(mgoddard): Query metrics.
|
||||
echo "TESTING: Prometheus"
|
||||
output_path=$(mktemp)
|
||||
attempt=1
|
||||
while ! check_prometheus $output_path; do
|
||||
echo "Prometheus not accessible yet"
|
||||
attempt=$((attempt+1))
|
||||
if [[ $attempt -eq 12 ]]; then
|
||||
echo "FAILED: Prometheus did not become accessible. Response:"
|
||||
cat $output_path
|
||||
return 1
|
||||
fi
|
||||
sleep 10
|
||||
done
|
||||
echo "SUCCESS: Prometheus"
|
||||
}
|
||||
|
||||
function test_prometheus_efk_logged {
|
||||
. /etc/kolla/admin-openrc.sh
|
||||
|
||||
test_kibana
|
||||
test_grafana
|
||||
test_prometheus
|
||||
}
|
||||
|
||||
function test_prometheus_efk {
|
||||
echo "Testing prometheus and EFK"
|
||||
test_prometheus_efk_logged > /tmp/logs/ansible/test-prometheus-efk 2>&1
|
||||
result=$?
|
||||
if [[ $result != 0 ]]; then
|
||||
echo "Testing prometheus and EFK failed. See ansible/test-prometheus-efk for details"
|
||||
else
|
||||
echo "Successfully tested prometheus and EFK. See ansible/test-prometheus-efk for details"
|
||||
fi
|
||||
return $result
|
||||
}
|
||||
|
||||
test_prometheus_efk
|
@ -75,6 +75,10 @@ function prepare_images {
|
||||
GATE_IMAGES="^cron,^fluentd,^haproxy,^keepalived,^kolla-toolbox,^mariadb"
|
||||
fi
|
||||
|
||||
if [[ $SCENARIO == "prometheus-efk" ]]; then
|
||||
GATE_IMAGES="^cron,^elasticsearch,^fluentd,^grafana,^haproxy,^keepalived,^kibana,^kolla-toolbox,^mariadb,^memcached,^prometheus,^rabbitmq"
|
||||
fi
|
||||
|
||||
# NOTE(yoctozepto): we cannot build and push at the same time on debian
|
||||
# buster see https://github.com/docker/for-linux/issues/711.
|
||||
PUSH="true"
|
||||
|
@ -173,3 +173,12 @@
|
||||
- ^tests/test-core-openstack.sh
|
||||
vars:
|
||||
scenario: ovn
|
||||
|
||||
- job:
|
||||
name: kolla-ansible-prometheus-efk-base
|
||||
parent: kolla-ansible-base
|
||||
voting: false
|
||||
files:
|
||||
- ^ansible/roles/(common|elasticsearch|grafana|kibana|prometheus)/
|
||||
vars:
|
||||
scenario: prometheus-efk
|
||||
|
@ -280,3 +280,19 @@
|
||||
vars:
|
||||
base_distro: ubuntu
|
||||
install_type: source
|
||||
|
||||
- job:
|
||||
name: kolla-ansible-centos8-source-prometheus-efk
|
||||
parent: kolla-ansible-prometheus-efk-base
|
||||
nodeset: kolla-ansible-centos8
|
||||
vars:
|
||||
base_distro: centos
|
||||
install_type: source
|
||||
|
||||
- job:
|
||||
name: kolla-ansible-ubuntu-source-prometheus-efk
|
||||
parent: kolla-ansible-prometheus-efk-base
|
||||
nodeset: kolla-ansible-bionic
|
||||
vars:
|
||||
base_distro: ubuntu
|
||||
install_type: source
|
||||
|
@ -41,6 +41,8 @@
|
||||
- kolla-ansible-ubuntu-source-linuxbridge
|
||||
- kolla-ansible-centos8-source-ovn
|
||||
- kolla-ansible-ubuntu-source-ovn
|
||||
- kolla-ansible-centos8-source-prometheus-efk
|
||||
- kolla-ansible-ubuntu-source-prometheus-efk
|
||||
check-arm64:
|
||||
jobs:
|
||||
- kolla-ansible-debian-source-aarch64
|
||||
|
Loading…
Reference in New Issue
Block a user