CI: Add monasca scenario

Adds the following new Zuul job for testing deployment of Monasca and
associated services:

* kolla-ansible-centos8-source-monasca

All core OpenStack services except for Keystone are disabled to ensure
enough memory is available.

A follow up patch will replace the basic tests here with Tempest.

Co-Authored-By: Doug Szumski <doug@stackhpc.com>

Change-Id: I5d33fd3d7b69798ba0aa23509f7b809065f61c19

tests/run.yml

@@ -10,7 +10,7 @@
     - name: set facts for commonly used variables
       vars:
         # NOTE(yoctozepto): needed here to use in other facts too
-        openstack_core_enabled: "{{ scenario not in ['bifrost', 'mariadb', 'prometheus-efk'] }}"
+        openstack_core_enabled: "{{ scenario not in ['bifrost', 'mariadb', 'prometheus-efk', 'monasca'] }}"
       set_fact:
         kolla_inventory_path: "/etc/kolla/inventory"
         logs_dir: "/tmp/logs"
@@ -22,7 +22,7 @@
         build_image_tag: "change_{{ zuul.change | default('none') }}"
         openstack_core_enabled: "{{ openstack_core_enabled }}"
         openstack_core_tested: "{{ scenario in ['core', 'ceph-ansible', 'zun', 'cells', 'swift', 'linuxbridge', 'ovn'] }}"
-        dashboard_enabled: "{{ openstack_core_enabled }}"
+        dashboard_enabled: "{{ openstack_core_enabled or scenario in ['monasca'] }}"
         upper_constraints_file: "{{ ansible_env.HOME }}/src/opendev.org/openstack/requirements/upper-constraints.txt"
         docker_image_tag_suffix: "{{ '-aarch64' if ansible_architecture == 'aarch64' else '' }}"
 
@@ -427,6 +427,13 @@
             chdir: "{{ kolla_ansible_src_dir }}"
           when: scenario == "magnum"
 
+        - name: Run test-monasca.sh script
+          script:
+            cmd: test-monasca.sh
+            executable: /bin/bash
+            chdir: "{{ kolla_ansible_src_dir }}"
+          when: scenario == "monasca"
+
         - name: Run test-masakari.sh script
           script:
             cmd: test-masakari.sh

tests/setup_gate.sh

@@ -26,6 +26,9 @@ function setup_openstack_clients {
     if [[ $SCENARIO == scenario_nfv ]]; then
         packages+=(python-tackerclient python-barbicanclient python-mistralclient)
     fi
+    if [[ $SCENARIO == monasca ]]; then
+        packages+=(python-monascaclient)
+    fi
     if [[ "debian" == $BASE_DISTRO ]]; then
         sudo apt -y install python3-venv
     fi
@@ -85,6 +88,10 @@ function prepare_images {
         GATE_IMAGES="^cron,^elasticsearch,^fluentd,^grafana,^haproxy,^keepalived,^kibana,^kolla-toolbox,^mariadb,^memcached,^prometheus,^rabbitmq"
     fi
 
+    if [[ $SCENARIO == "monasca" ]]; then
+        # FIXME(mgoddard): No need for OpenStack core images.
+        GATE_IMAGES+=",^elasticsearch,^influxdb,^kafka,^kibana,^logstash,^monasca,^storm,^zookeeper"
+    fi
     sudo tee -a /etc/kolla/kolla-build.conf <<EOF
 [profiles]
 gate = ${GATE_IMAGES}

tests/templates/globals-default.j2

@@ -157,3 +157,9 @@ enable_magnum: "yes"
 enable_trove: "yes"
 enable_octavia: "yes"
 {% endif %}
+
+{% if scenario == "monasca" %}
+enable_keystone: "yes"
+enable_monasca: "yes"
+enable_rabbitmq: "no"
+{% endif %}

tests/templates/inventory.j2

@@ -1,14 +1,25 @@
 # These initial groups are the only groups required to be modified. The
 # additional groups are for more control of the environment.
 [control]
+{% if scenario == 'monasca' %}
+{% for host in hostvars if host in ['primary', 'secondary1', 'secondary2'] %}
+{{ host }} ansible_host={{ hostvars[host]['ansible_host'] }} ansible_user=kolla ansible_ssh_private_key_file={{ ansible_env.HOME ~ '/.ssh/id_rsa_kolla' }}
+{% endfor %}
+{% else %}
 {% for host in hostvars %}
 {{ host }} ansible_host={{ hostvars[host]['ansible_host'] }} ansible_user=kolla ansible_ssh_private_key_file={{ ansible_env.HOME ~ '/.ssh/id_rsa_kolla' }}
 {% endfor %}
+{% endif %}
 
+{% if scenario == 'monasca' %}
+[network:children]
+control
+{% else %}
 [network]
 {% for host in hostvars %}
 {{ host }} ansible_host={{ hostvars[host]['ansible_host'] }} ansible_user=kolla ansible_ssh_private_key_file={{ ansible_env.HOME ~ '/.ssh/id_rsa_kolla' }}
 {% endfor %}
+{% endif %}
 
 [compute]
 {% for host in hostvars %}
@@ -21,9 +32,15 @@
 {% endfor %}
 
 [monitoring]
+{% if scenario == 'monasca' %}
+{% for host in hostvars if host in ['secondary3', 'secondary4', 'secondary5'] %}
+{{ host }} ansible_host={{ hostvars[host]['ansible_host'] }} ansible_user=kolla ansible_ssh_private_key_file={{ ansible_env.HOME ~ '/.ssh/id_rsa_kolla' }}
+{% endfor %}
+{% else %}
 {% for host in hostvars %}
 {{ host }} ansible_host={{ hostvars[host]['ansible_host'] }} ansible_user=kolla ansible_ssh_private_key_file={{ ansible_env.HOME ~ '/.ssh/id_rsa_kolla' }}
 {% endfor %}
+{% endif %}
 
 [deployment]
 {% for host in hostvars %}
@@ -85,8 +102,9 @@ monitoring
 [etcd:children]
 control
 
-[influxdb:children]
+[influxdb]
-monitoring
+# NOTE(dszumski): This doesn't support clustering
+primary
 
 [prometheus:children]
 monitoring

tests/test-monasca.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+set -o nounset
+set -o pipefail
+
+function test_monasca_metrics {
+    # Check that the monitoring endpoints are registered
+    openstack endpoint list -f value --service monitoring --interface internal -c URL
+    openstack endpoint list -f value --service monitoring --interface public -c URL
+
+    # Run some CLI commands
+    MONASCA_PROJECT_ID=$(openstack project list --user monasca-agent -f value -c ID)
+    monasca metric-list --tenant-id "$MONASCA_PROJECT_ID"
+    monasca alarm-list
+    monasca notification-list
+
+    # Test the metric pipeline by waiting for some metrics to arrive from the
+    # Monasca Agent. If the metric doesn't yet exist, nothing is returned.
+    METRIC_STATS_CMD="monasca metric-statistics mem.free_mb --tenant-id $MONASCA_PROJECT_ID COUNT -300 --merge_metrics"
+    for i in {1..60}; do
+        if [[ $($METRIC_STATS_CMD) == *'mem.free_mb'* ]]; then
+            return 0
+        fi
+        sleep 1
+    done
+    return 1
+}
+
+function test_monasca_logs {
+    # Check that the logging endpoints are registered
+    openstack endpoint list -f value --service logging --interface internal -c URL
+    openstack endpoint list -f value --service logging --interface public -c URL
+
+    # Test the logging pipeline by waiting for some logs to arrive from
+    # Fluentd into the Monasca Elasticsearch index
+    # TODO: Use index name set in config
+
+    # NOTE(dszumski): When querying logs via the Monasca Log API *is*
+    # supported, we can replace this in favour of calling querying the Log API.
+    ELASTICSEARCH_URL=${OS_AUTH_URL%:*}:9200
+    for i in {1..60}; do
+        if [[ $(curl -s -X GET $ELASTICSEARCH_URL/_cat/indices?v) == *"monasca-"* ]]; then
+            return 0
+        fi
+        sleep 1
+    done
+    return 1
+}
+
+function test_monasca_logged {
+    . /etc/kolla/admin-openrc.sh
+    # Activate virtualenv to access Monasca client
+    . ~/openstackclient-venv/bin/activate
+
+    test_monasca_metrics
+    result=$?
+    if [[ $result != 0 ]]; then
+        echo "Failed testing metrics pipeline"
+        return $result
+    fi
+
+    test_monasca_logs
+    result=$?
+    if [[ $result != 0 ]]; then
+        echo "Failed testing logging pipeline"
+        return $result
+    fi
+}
+
+function test_monasca {
+    echo "Testing Monasca"
+    test_monasca_logged > /tmp/logs/ansible/test-monasca 2>&1
+    result=$?
+    if [[ $result != 0 ]]; then
+        echo "Monasca test failed. See ansible/test-monasca for details"
+    else
+        echo "Successfully tested Monasca. See ansible/test-monasca for details"
+    fi
+    return $result
+}
+
+test_monasca

zuul.d/base.yaml

@@ -148,6 +148,17 @@
     vars:
       scenario: masakari
 
+- job:
+    name: kolla-ansible-monasca-base
+    parent: kolla-ansible-base
+    voting: false
+    files:
+      - ^ansible/roles/(elasticsearch|influxdb|kafka|kibana|monasca|storm|zookeeper)/
+      - ^tests/test-monasca.sh
+      - ^tests/test-dashboard.sh
+    vars:
+      scenario: monasca
+
 - job:
     name: kolla-ansible-mariadb-base
     parent: kolla-ansible-base

zuul.d/jobs.yaml

@@ -292,6 +292,14 @@
       base_distro: centos
       install_type: source
 
+- job:
+    name: kolla-ansible-centos8-source-monasca
+    parent: kolla-ansible-monasca-base
+    nodeset: kolla-ansible-centos8-multi-monasca
+    vars:
+      base_distro: centos
+      install_type: source
+
 - job:
     name: kolla-ansible-centos8-source-cells
     parent: kolla-ansible-cells-base

zuul.d/nodesets.yaml

@@ -70,3 +70,30 @@
     nodes:
       - name: primary
         label: nested-virt-centos-8
+
+- nodeset:
+    name: kolla-ansible-centos8-multi-monasca
+    nodes:
+      - name: primary
+        label: centos-8
+      - name: secondary1
+        label: centos-8
+      - name: secondary2
+        label: centos-8
+      - name: secondary3
+        label: centos-8
+      - name: secondary4
+        label: centos-8
+      - name: secondary5
+        label: centos-8
+    groups:
+      - name: switch
+        nodes:
+          - primary
+      - name: peers
+        nodes:
+          - secondary1
+          - secondary2
+          - secondary3
+          - secondary4
+          - secondary5

zuul.d/project.yaml

@@ -48,6 +48,7 @@
         - kolla-ansible-ubuntu-source-ovn
         - kolla-ansible-centos8-source-prometheus-efk
         - kolla-ansible-ubuntu-source-prometheus-efk
+        - kolla-ansible-centos8-source-monasca
     check-arm64:
       jobs:
         - kolla-ansible-debian-source-aarch64