diff --git a/tests/run.yml b/tests/run.yml index 9c8ddb6e64..5260228269 100644 --- a/tests/run.yml +++ b/tests/run.yml @@ -10,7 +10,7 @@ - name: set facts for commonly used variables vars: # NOTE(yoctozepto): needed here to use in other facts too - openstack_core_enabled: "{{ scenario not in ['bifrost', 'mariadb'] }}" + openstack_core_enabled: "{{ scenario not in ['bifrost', 'mariadb', 'prometheus-efk'] }}" set_fact: kolla_inventory_path: "/etc/kolla/inventory" logs_dir: "/tmp/logs" @@ -408,6 +408,14 @@ executable: /bin/bash chdir: "{{ kolla_ansible_src_dir }}" when: scenario == "mariadb" + + - name: Run test-prometheus-efk.sh script + script: + cmd: test-prometheus-efk.sh + executable: /bin/bash + chdir: "{{ kolla_ansible_src_dir }}" + when: scenario == "prometheus-efk" + when: scenario != "bifrost" # NOTE(yoctozepto): each host checks itself diff --git a/tests/templates/globals-default.j2 b/tests/templates/globals-default.j2 index 4d41ec3c4d..31d96bc619 100644 --- a/tests/templates/globals-default.j2 +++ b/tests/templates/globals-default.j2 @@ -137,3 +137,11 @@ neutron_plugin_agent: "linuxbridge" {% if scenario == "ovn" %} neutron_plugin_agent: "ovn" {% endif %} + +{% if scenario == "prometheus-efk" %} +enable_chrony: "no" +enable_central_logging: "yes" +enable_grafana: "yes" +enable_prometheus: "yes" +enable_prometheus_openstack_exporter: "no" +{% endif %} diff --git a/tests/test-prometheus-efk.sh b/tests/test-prometheus-efk.sh new file mode 100755 index 0000000000..1828ea6107 --- /dev/null +++ b/tests/test-prometheus-efk.sh @@ -0,0 +1,150 @@ +#!/bin/bash + +set -o xtrace +set -o errexit +set -o pipefail + +# Enable unbuffered output +export PYTHONUNBUFFERED=1 + +function check_kibana { + # Query kibana, and check that the returned page looks like a kibana page. + KIBANA_URL=${OS_AUTH_URL%:*}:5601 + output_path=$1 + kibana_password=$(awk '$1 == "kibana_password:" { print $2 }' /etc/kolla/passwords.yml) + args=( + --include + --location + --fail + --user + kibana:$kibana_password + ) + if [[ "$TLS_ENABLED" = "True" ]]; then + args+=(--cacert $OS_CACERT) + fi + if ! curl "${args[@]}" $KIBANA_URL > $output_path; then + return 1 + fi + if ! grep 'Kibana' $output_path >/dev/null; then + return 1 + fi +} + +function check_grafana { + # Query grafana, and check that the returned page looks like a grafana page. + GRAFANA_URL=${OS_AUTH_URL%:*}:3000 + output_path=$1 + grafana_password=$(awk '$1 == "grafana_admin_password:" { print $2 }' /etc/kolla/passwords.yml) + args=( + --include + --location + --fail + --user + admin:$grafana_password + ) + if [[ "$TLS_ENABLED" = "True" ]]; then + args+=(--cacert $OS_CACERT) + fi + if ! curl "${args[@]}" $GRAFANA_URL > $output_path; then + return 1 + fi + if ! grep 'Grafana' $output_path >/dev/null; then + return 1 + fi +} + +function check_prometheus { + # Query prometheus graph, and check that the returned page looks like a + # prometheus page. + PROMETHEUS_URL=${OS_AUTH_URL%:*}:9091/graph + output_path=$1 + args=( + --include + --location + --fail + ) + if [[ "$TLS_ENABLED" = "True" ]]; then + args+=(--cacert $OS_CACERT) + fi + if ! curl "${args[@]}" $PROMETHEUS_URL > $output_path; then + return 1 + fi + if ! grep 'Prometheus' $output_path >/dev/null; then + return 1 + fi +} + +function test_kibana { + # TODO(mgoddard): Query elasticsearch for logs. + echo "TESTING: Kibana" + output_path=$(mktemp) + attempt=1 + while ! check_kibana $output_path; do + echo "Kibana not accessible yet" + attempt=$((attempt+1)) + if [[ $attempt -eq 12 ]]; then + echo "FAILED: Kibana did not become accessible. Response:" + cat $output_path + return 1 + fi + sleep 10 + done + echo "SUCCESS: Kibana" +} + +function test_grafana { + echo "TESTING: Grafana" + output_path=$(mktemp) + attempt=1 + while ! check_grafana $output_path; do + echo "Grafana not accessible yet" + attempt=$((attempt+1)) + if [[ $attempt -eq 12 ]]; then + echo "FAILED: Grafana did not become accessible. Response:" + cat $output_path + return 1 + fi + sleep 10 + done + echo "SUCCESS: Grafana" +} + +function test_prometheus { + # TODO(mgoddard): Query metrics. + echo "TESTING: Prometheus" + output_path=$(mktemp) + attempt=1 + while ! check_prometheus $output_path; do + echo "Prometheus not accessible yet" + attempt=$((attempt+1)) + if [[ $attempt -eq 12 ]]; then + echo "FAILED: Prometheus did not become accessible. Response:" + cat $output_path + return 1 + fi + sleep 10 + done + echo "SUCCESS: Prometheus" +} + +function test_prometheus_efk_logged { + . /etc/kolla/admin-openrc.sh + + test_kibana + test_grafana + test_prometheus +} + +function test_prometheus_efk { + echo "Testing prometheus and EFK" + test_prometheus_efk_logged > /tmp/logs/ansible/test-prometheus-efk 2>&1 + result=$? + if [[ $result != 0 ]]; then + echo "Testing prometheus and EFK failed. See ansible/test-prometheus-efk for details" + else + echo "Successfully tested prometheus and EFK. See ansible/test-prometheus-efk for details" + fi + return $result +} + +test_prometheus_efk diff --git a/tools/setup_gate.sh b/tools/setup_gate.sh index d2052cbcab..5fcea9c4f0 100755 --- a/tools/setup_gate.sh +++ b/tools/setup_gate.sh @@ -75,6 +75,10 @@ function prepare_images { GATE_IMAGES="^cron,^fluentd,^haproxy,^keepalived,^kolla-toolbox,^mariadb" fi + if [[ $SCENARIO == "prometheus-efk" ]]; then + GATE_IMAGES="^cron,^elasticsearch,^fluentd,^grafana,^haproxy,^keepalived,^kibana,^kolla-toolbox,^mariadb,^memcached,^prometheus,^rabbitmq" + fi + # NOTE(yoctozepto): we cannot build and push at the same time on debian # buster see https://github.com/docker/for-linux/issues/711. PUSH="true" diff --git a/zuul.d/base.yaml b/zuul.d/base.yaml index 100cb7a330..c59141e57b 100644 --- a/zuul.d/base.yaml +++ b/zuul.d/base.yaml @@ -173,3 +173,12 @@ - ^tests/test-core-openstack.sh vars: scenario: ovn + +- job: + name: kolla-ansible-prometheus-efk-base + parent: kolla-ansible-base + voting: false + files: + - ^ansible/roles/(common|elasticsearch|grafana|kibana|prometheus)/ + vars: + scenario: prometheus-efk diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 8f7e21e15c..c7e9f3f220 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -280,3 +280,19 @@ vars: base_distro: ubuntu install_type: source + +- job: + name: kolla-ansible-centos8-source-prometheus-efk + parent: kolla-ansible-prometheus-efk-base + nodeset: kolla-ansible-centos8 + vars: + base_distro: centos + install_type: source + +- job: + name: kolla-ansible-ubuntu-source-prometheus-efk + parent: kolla-ansible-prometheus-efk-base + nodeset: kolla-ansible-bionic + vars: + base_distro: ubuntu + install_type: source diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 86278b9b31..dfb23cda50 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -41,6 +41,8 @@ - kolla-ansible-ubuntu-source-linuxbridge - kolla-ansible-centos8-source-ovn - kolla-ansible-ubuntu-source-ovn + - kolla-ansible-centos8-source-prometheus-efk + - kolla-ansible-ubuntu-source-prometheus-efk check-arm64: jobs: - kolla-ansible-debian-source-aarch64