CI: add prometheus-efk scenario

Tests prometheus, grafana, and centralised logging.

The tests could be improved in future by querying logs in elasticsearch,
and metrics in prometheus.

Change-Id: Iabad035d583d291169f23be3d71931cb260e87ae

tests/run.yml

@@ -10,7 +10,7 @@
     - name: set facts for commonly used variables
       vars:
         # NOTE(yoctozepto): needed here to use in other facts too
-        openstack_core_enabled: "{{ scenario not in ['bifrost', 'mariadb'] }}"
+        openstack_core_enabled: "{{ scenario not in ['bifrost', 'mariadb', 'prometheus-efk'] }}"
       set_fact:
         kolla_inventory_path: "/etc/kolla/inventory"
         logs_dir: "/tmp/logs"
@@ -408,6 +408,14 @@
             executable: /bin/bash
             chdir: "{{ kolla_ansible_src_dir }}"
           when: scenario == "mariadb"
+
+        - name: Run test-prometheus-efk.sh script
+          script:
+            cmd: test-prometheus-efk.sh
+            executable: /bin/bash
+            chdir: "{{ kolla_ansible_src_dir }}"
+          when: scenario == "prometheus-efk"
+
       when: scenario != "bifrost"
 
 # NOTE(yoctozepto): each host checks itself

tests/templates/globals-default.j2

@@ -137,3 +137,11 @@ neutron_plugin_agent: "linuxbridge"
 {% if scenario == "ovn" %}
 neutron_plugin_agent: "ovn"
 {% endif %}
+
+{% if scenario == "prometheus-efk" %}
+enable_chrony: "no"
+enable_central_logging: "yes"
+enable_grafana: "yes"
+enable_prometheus: "yes"
+enable_prometheus_openstack_exporter: "no"
+{% endif %}

tests/test-prometheus-efk.sh

@@ -0,0 +1,150 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+set -o pipefail
+
+# Enable unbuffered output
+export PYTHONUNBUFFERED=1
+
+function check_kibana {
+    # Query kibana, and check that the returned page looks like a kibana page.
+    KIBANA_URL=${OS_AUTH_URL%:*}:5601
+    output_path=$1
+    kibana_password=$(awk '$1 == "kibana_password:" { print $2 }' /etc/kolla/passwords.yml)
+    args=(
+        --include
+        --location
+        --fail
+        --user
+        kibana:$kibana_password
+    )
+    if [[ "$TLS_ENABLED" = "True" ]]; then
+        args+=(--cacert $OS_CACERT)
+    fi
+    if ! curl "${args[@]}" $KIBANA_URL > $output_path; then
+        return 1
+    fi
+    if ! grep '<title>Kibana</title>' $output_path >/dev/null; then
+        return 1
+    fi
+}
+
+function check_grafana {
+    # Query grafana, and check that the returned page looks like a grafana page.
+    GRAFANA_URL=${OS_AUTH_URL%:*}:3000
+    output_path=$1
+    grafana_password=$(awk '$1 == "grafana_admin_password:" { print $2 }' /etc/kolla/passwords.yml)
+    args=(
+        --include
+        --location
+        --fail
+        --user
+        admin:$grafana_password
+    )
+    if [[ "$TLS_ENABLED" = "True" ]]; then
+        args+=(--cacert $OS_CACERT)
+    fi
+    if ! curl "${args[@]}" $GRAFANA_URL > $output_path; then
+        return 1
+    fi
+    if ! grep '<title>Grafana</title>' $output_path >/dev/null; then
+        return 1
+    fi
+}
+
+function check_prometheus {
+    # Query prometheus graph, and check that the returned page looks like a
+    # prometheus page.
+    PROMETHEUS_URL=${OS_AUTH_URL%:*}:9091/graph
+    output_path=$1
+    args=(
+        --include
+        --location
+        --fail
+    )
+    if [[ "$TLS_ENABLED" = "True" ]]; then
+        args+=(--cacert $OS_CACERT)
+    fi
+    if ! curl "${args[@]}" $PROMETHEUS_URL > $output_path; then
+        return 1
+    fi
+    if ! grep '<title>Prometheus' $output_path >/dev/null; then
+        return 1
+    fi
+}
+
+function test_kibana {
+    # TODO(mgoddard): Query elasticsearch for logs.
+    echo "TESTING: Kibana"
+    output_path=$(mktemp)
+    attempt=1
+    while ! check_kibana $output_path; do
+        echo "Kibana not accessible yet"
+        attempt=$((attempt+1))
+        if [[ $attempt -eq 12 ]]; then
+            echo "FAILED: Kibana did not become accessible. Response:"
+            cat $output_path
+            return 1
+        fi
+        sleep 10
+    done
+    echo "SUCCESS: Kibana"
+}
+
+function test_grafana {
+    echo "TESTING: Grafana"
+    output_path=$(mktemp)
+    attempt=1
+    while ! check_grafana $output_path; do
+        echo "Grafana not accessible yet"
+        attempt=$((attempt+1))
+        if [[ $attempt -eq 12 ]]; then
+            echo "FAILED: Grafana did not become accessible. Response:"
+            cat $output_path
+            return 1
+        fi
+        sleep 10
+    done
+    echo "SUCCESS: Grafana"
+}
+
+function test_prometheus {
+    # TODO(mgoddard): Query metrics.
+    echo "TESTING: Prometheus"
+    output_path=$(mktemp)
+    attempt=1
+    while ! check_prometheus $output_path; do
+        echo "Prometheus not accessible yet"
+        attempt=$((attempt+1))
+        if [[ $attempt -eq 12 ]]; then
+            echo "FAILED: Prometheus did not become accessible. Response:"
+            cat $output_path
+            return 1
+        fi
+        sleep 10
+    done
+    echo "SUCCESS: Prometheus"
+}
+
+function test_prometheus_efk_logged {
+    . /etc/kolla/admin-openrc.sh
+
+    test_kibana
+    test_grafana
+    test_prometheus
+}
+
+function test_prometheus_efk {
+    echo "Testing prometheus and EFK"
+    test_prometheus_efk_logged > /tmp/logs/ansible/test-prometheus-efk 2>&1
+    result=$?
+    if [[ $result != 0 ]]; then
+        echo "Testing prometheus and EFK failed. See ansible/test-prometheus-efk for details"
+    else
+        echo "Successfully tested prometheus and EFK. See ansible/test-prometheus-efk for details"
+    fi
+    return $result
+}
+
+test_prometheus_efk

tools/setup_gate.sh

@@ -75,6 +75,10 @@ function prepare_images {
         GATE_IMAGES="^cron,^fluentd,^haproxy,^keepalived,^kolla-toolbox,^mariadb"
     fi
 
+    if [[ $SCENARIO == "prometheus-efk" ]]; then
+        GATE_IMAGES="^cron,^elasticsearch,^fluentd,^grafana,^haproxy,^keepalived,^kibana,^kolla-toolbox,^mariadb,^memcached,^prometheus,^rabbitmq"
+    fi
+
     # NOTE(yoctozepto): we cannot build and push at the same time on debian
     # buster see https://github.com/docker/for-linux/issues/711.
     PUSH="true"

zuul.d/base.yaml

@@ -173,3 +173,12 @@
       - ^tests/test-core-openstack.sh
     vars:
       scenario: ovn
+
+- job:
+    name: kolla-ansible-prometheus-efk-base
+    parent: kolla-ansible-base
+    voting: false
+    files:
+      - ^ansible/roles/(common|elasticsearch|grafana|kibana|prometheus)/
+    vars:
+      scenario: prometheus-efk

zuul.d/jobs.yaml

@@ -280,3 +280,19 @@
     vars:
       base_distro: ubuntu
       install_type: source
+
+- job:
+    name: kolla-ansible-centos8-source-prometheus-efk
+    parent: kolla-ansible-prometheus-efk-base
+    nodeset: kolla-ansible-centos8
+    vars:
+      base_distro: centos
+      install_type: source
+
+- job:
+    name: kolla-ansible-ubuntu-source-prometheus-efk
+    parent: kolla-ansible-prometheus-efk-base
+    nodeset: kolla-ansible-bionic
+    vars:
+      base_distro: ubuntu
+      install_type: source

zuul.d/project.yaml

@@ -41,6 +41,8 @@
         - kolla-ansible-ubuntu-source-linuxbridge
         - kolla-ansible-centos8-source-ovn
         - kolla-ansible-ubuntu-source-ovn
+        - kolla-ansible-centos8-source-prometheus-efk
+        - kolla-ansible-ubuntu-source-prometheus-efk
     check-arm64:
       jobs:
         - kolla-ansible-debian-source-aarch64