Adding Prometheus blackbox exporter

This commit follows up the work in Kolla to provide deploy and configure the
Prometheus blackbox exporter.

An example blackbox-exporter module has been added (disabled by default)
called os_endpoint. This allows for the probing of endpoints over HTTP
and HTTPS. This can be used to monitor that OpenStack endpoints return a status
code of either 200 or 300, and the word 'versions' in the payload.

This change introduces a new variable `prometheus_blackbox_exporter_endpoints`.
Currently no defaults are specified because the configuration is heavily
dependent on the deployment.

Co-authored-by: Jack Heskett <Jack.Heskett@gresearch.co.uk>
Change-Id: I36ad4961078d90e2fd70c9a3368f5157d6fd89cd

ansible/group_vars/all.yml

@@ -387,6 +387,9 @@ prometheus_alertmanager_cluster_port: "9094"
 prometheus_openstack_exporter_port: "9198"
 prometheus_elasticsearch_exporter_port: "9108"
 
+# Prometheus blackbox-exporter ports
+prometheus_blackbox_exporter_port: "9115"
+
 qdrouterd_port: "31459"
 
 qinling_api_port: "7070"
@@ -1059,7 +1062,7 @@ enable_prometheus_alertmanager: "{{ enable_prometheus | bool }}"
 enable_prometheus_ceph_mgr_exporter: "{{ enable_ceph | bool and enable_prometheus | bool }}"
 enable_prometheus_openstack_exporter: "{{ enable_prometheus | bool }}"
 enable_prometheus_elasticsearch_exporter: "{{ enable_prometheus | bool and enable_elasticsearch | bool }}"
-
+enable_prometheus_blackbox_exporter: "{{ enable_prometheus | bool }}"
 prometheus_alertmanager_user: "admin"
 prometheus_openstack_exporter_interval: "60s"
 prometheus_elasticsearch_exporter_interval: "60s"

ansible/inventory/all-in-one

@@ -757,6 +757,9 @@ monitoring
 [prometheus-elasticsearch-exporter:children]
 elasticsearch
 
+[prometheus-blackbox-exporter:children]
+monitoring
+
 [masakari-api:children]
 control
 

ansible/inventory/multinode

@@ -776,6 +776,9 @@ monitoring
 [prometheus-elasticsearch-exporter:children]
 elasticsearch
 
+[prometheus-blackbox-exporter:children]
+monitoring
+
 [masakari-api:children]
 control
 

ansible/roles/prometheus/defaults/main.yml

@@ -87,12 +87,32 @@ prometheus_services:
     image: "{{ prometheus_elasticsearch_exporter_image_full }}"
     volumes: "{{ prometheus_elasticsearch_exporter_default_volumes + prometheus_elasticsearch_exporter_extra_volumes }}"
     dimensions: "{{ prometheus_elasticsearch_exporter_dimensions }}"
+  prometheus-blackbox-exporter:
+    container_name: "prometheus_blackbox_exporter"
+    group: "prometheus-blackbox-exporter"
+    enabled: "{{ enable_prometheus_blackbox_exporter | bool }}"
+    image: "{{ prometheus_blackbox_exporter_image_full }}"
+    volumes: "{{ prometheus_blackbox_exporter_default_volumes + prometheus_blackbox_exporter_extra_volumes }}"
+    dimensions: "{{ prometheus_blackbox_exporter_dimensions }}"
 
 ####################
 # Database
 ####################
 prometheus_mysql_exporter_database_user: "{% if use_preconfigured_databases | bool and use_common_mariadb_user | bool %}{{ database_user }}{% else %}prometheus{% endif %}"
 
+####################
+# Blackbox
+####################
+
+# A list of endpoints to monitor. Each target is in the format:
+# 'service_name:blackbox_exporter_module:endpoint' for example:
+#
+# prometheus_blackbox_exporter_targets:
+#   - 'glance:os_endpoint:http://{{ kolla_external_vip_address }}:{{ glance_api_port}}'
+#
+# For a list of modules see the alertmanager config.
+prometheus_blackbox_exporter_endpoints: []
+
 ####################
 # Docker
 ####################
@@ -136,6 +156,10 @@ prometheus_elasticsearch_exporter_image: "{{ docker_registry ~ '/' if docker_reg
 prometheus_elasticsearch_exporter_tag: "{{ prometheus_tag }}"
 prometheus_elasticsearch_exporter_image_full: "{{ prometheus_elasticsearch_exporter_image }}:{{ prometheus_elasticsearch_exporter_tag }}"
 
+prometheus_blackbox_exporter_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ prometheus_install_type }}-prometheus-blackbox-exporter"
+prometheus_blackbox_exporter_tag: "{{ prometheus_tag }}"
+prometheus_blackbox_exporter_image_full: "{{ prometheus_blackbox_exporter_image }}:{{ prometheus_blackbox_exporter_tag }}"
+
 prometheus_server_dimensions: "{{ default_container_dimensions }}"
 prometheus_haproxy_exporter_dimensions: "{{ default_container_dimensions }}"
 prometheus_mysqld_exporter_dimensions: "{{ default_container_dimensions }}"
@@ -145,6 +169,7 @@ prometheus_cadvisor_dimensions: "{{ default_container_dimensions }}"
 prometheus_alertmanager_dimensions: "{{ default_container_dimensions }}"
 prometheus_openstack_exporter_dimensions: "{{ default_container_dimensions }}"
 prometheus_elasticsearch_exporter_dimensions: "{{ default_container_dimensions }}"
+prometheus_blackbox_exporter_dimensions: "{{ default_container_dimensions }}"
 
 prometheus_server_default_volumes:
   - "{{ node_config_directory }}/prometheus-server/:{{ container_config_directory }}/:ro"
@@ -193,6 +218,10 @@ prometheus_elasticsearch_exporter_default_volumes:
   - "{{ node_config_directory }}/prometheus-elasticsearch-exporter/:{{ container_config_directory }}/:ro"
   - "/etc/localtime:/etc/localtime:ro"
   - "kolla_logs:/var/log/kolla/"
+prometheus_blackbox_exporter_default_volumes:
+  - "{{ node_config_directory }}/prometheus-blackbox-exporter/:{{ container_config_directory }}/:ro"
+  - "/etc/localtime:/etc/localtime:ro"
+  - "kolla_logs:/var/log/kolla/"
 
 prometheus_extra_volumes: "{{ default_extra_volumes }}"
 prometheus_server_extra_volumes: "{{ prometheus_extra_volumes }}"
@@ -204,3 +233,4 @@ prometheus_cadvisor_extra_volumes: "{{ prometheus_extra_volumes }}"
 prometheus_alertmanager_extra_volumes: "{{ prometheus_extra_volumes }}"
 prometheus_openstack_exporter_extra_volumes: "{{ prometheus_extra_volumes }}"
 prometheus_elasticsearch_exporter_extra_volumes: "{{ prometheus_extra_volumes }}"
+prometheus_blackbox_exporter_extra_volumes: "{{ prometheus_extra_volumes }}"

ansible/roles/prometheus/handlers/main.yml

@@ -134,3 +134,18 @@
     dimensions: "{{ service.dimensions }}"
   when:
     - kolla_action != "config"
+
+- name: Restart prometheus-blackbox-exporter container
+  vars:
+    service_name: "prometheus-blackbox-exporter"
+    service: "{{ prometheus_services[service_name] }}"
+  become: true
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes }}"
+    dimensions: "{{ service.dimensions }}"
+  when:
+    - kolla_action != "config"

ansible/roles/prometheus/tasks/config.yml

@@ -123,6 +123,24 @@
   notify:
     - Restart prometheus-openstack-exporter container
 
+- name: Copying config file for blackbox exporter
+  become: true
+  vars:
+    service: "{{ prometheus_services['prometheus-blackbox-exporter'] }}"
+  template:
+    src: "{{ item }}"
+    dest: "{{ node_config_directory }}/prometheus-blackbox-exporter/prometheus-blackbox-exporter.yml"
+    mode: "0660"
+  when:
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+  with_first_found:
+    - "{{ node_custom_config }}/prometheus/{{ inventory_hostname }}/prometheus-blackbox-exporter.yml"
+    - "{{ node_custom_config }}/prometheus/prometheus-blackbox-exporter.yml"
+    - "{{ role_path }}/templates/prometheus-blackbox-exporter.yml.j2"
+  notify:
+    - Restart prometheus-blackbox-exporter container
+
 - name: Check prometheus containers
   become: true
   kolla_docker:

ansible/roles/prometheus/tasks/precheck.yml

@@ -12,6 +12,7 @@
       - prometheus_alertmanager
       - prometheus_openstack_exporter
       - prometheus_elasticsearch_exporter
+      - prometheus_blackbox_exporter
   register: container_facts
 
 - name: Checking free port for Prometheus server
@@ -127,3 +128,17 @@
     - enable_prometheus_elasticsearch_exporter | bool
   with_items:
     - "{{ prometheus_elasticsearch_exporter_port }}"
+
+- name: Checking free ports for Prometheus blackbox-exporter
+  wait_for:
+    host: "{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}"
+    port: "{{ item }}"
+    connect_timeout: 1
+    timeout: 1
+    state: stopped
+  when:
+    - container_facts['prometheus_blackbox_exporter'] is not defined
+    - inventory_hostname in groups['prometheus-blackbox-exporter']
+    - enable_prometheus_blackbox_exporter | bool
+  with_items:
+    - "{{ prometheus_blackbox_exporter_port }}"

ansible/roles/prometheus/templates/prometheus-blackbox-exporter.json.j2

@@ -0,0 +1,18 @@
+{
+    "command": "/opt/blackbox_exporter/blackbox_exporter --config.file=/etc/prometheus/blackbox.yml --web.listen-address={{ api_interface_address }}:{{ prometheus_blackbox_exporter_port }} --log.level=info",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/prometheus-blackbox-exporter.yml",
+            "dest": "/etc/prometheus/blackbox.yml",
+            "owner": "prometheus",
+            "perm": "0600"
+        }
+    ],
+    "permissions": [
+        {
+            "path": "/var/log/kolla/prometheus",
+            "owner": "prometheus:kolla",
+            "recurse": true
+        }
+    ]
+}

ansible/roles/prometheus/templates/prometheus-blackbox-exporter.yml.j2

@@ -0,0 +1,24 @@
+modules:
+  http_2xx:
+    prober: http
+  os_endpoint:
+    prober: http
+    http:
+      valid_status_codes: [200,300]
+      headers:
+        Content-Type: application/json
+      fail_if_body_not_matches_regexp:
+        - "versions"
+  http_post_2xx:
+    prober: http
+    http:
+      method: POST
+  tcp_connect:
+    prober: tcp
+  ssh_banner:
+    prober: tcp
+    tcp:
+      query_response:
+      - expect: "^SSH-2.0-"
+  icmp:
+    prober: icmp

ansible/roles/prometheus/templates/prometheus.yml.j2

@@ -96,6 +96,36 @@ scrape_configs:
 {% endfor %}
 {% endif %}
 
+{% if enable_prometheus_blackbox_exporter | bool and prometheus_blackbox_exporter_endpoints | length > 0 | bool %}
+  - job_name: blackbox_exporter
+    metrics_path: /probe
+    honor_labels: true
+    static_configs:
+      - targets:
+{% for target in prometheus_blackbox_exporter_endpoints %}
+        - '{{ target }}'
+{% endfor %}
+    relabel_configs:
+      - source_labels: [__address__]
+        regex: (\w+):(\w+):(.+)
+        target_label: service
+        replacement: ${1}
+      - source_labels: [__address__]
+        regex: (\w+):(\w+):(.+)
+        target_label: __param_module
+        replacement: ${2}
+      - source_labels: [__param_module]
+        target_label: module
+      - source_labels: [__address__]
+        regex: (\w+):(\w+):(.+)
+        target_label: __param_target
+        replacement: ${3}
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: '{{ api_interface_address }}:{{ prometheus_blackbox_exporter_port }}'
+{% endif %}
+
 {% if enable_prometheus_alertmanager | bool %}
 alerting:
   alertmanagers:

ansible/site.yml

@@ -490,6 +490,7 @@
     - prometheus-alertmanager
     - prometheus-openstack-exporter
     - prometheus-elasticsearch-exporter
+    - prometheus-blackbox-exporter
     - '&enable_prometheus_True'
   serial: '{{ kolla_serial|default("0") }}'
   roles:

etc/kolla/globals.yml

@@ -582,6 +582,7 @@
 #enable_prometheus_ceph_mgr_exporter: "{{ enable_prometheus | bool and enable_ceph | bool }}"
 #enable_prometheus_openstack_exporter: "{{ enable_prometheus | bool }}"
 #enable_prometheus_elasticsearch_exporter: "{{ enable_prometheus | bool and enable_elasticsearch | bool }}"
+#enable_prometheus_blackbox_exporter: "{{ enable_prometheus | bool }}"
 
 #########
 # Freezer

releasenotes/notes/prometheus-blackbox-exporter-ee096376656c3751.yaml

@@ -0,0 +1,9 @@
+---
+features:
+  - |
+    Prometheus blackbox exporter
+
+    An example blackbox-exporter module has been added (disabled by default)
+    called os_endpoint. This allows for the probing of endpoints over HTTP
+    and HTTPS. This can be used to monitor that OpenStack endpoints return a status
+    code of either 200 or 300, and the word 'versions' in the payload.

tests/templates/inventory.j2

@@ -696,3 +696,6 @@ monitoring
 
 [prometheus-elasticsearch-exporter:children]
 elasticsearch
+
+[prometheus-blackbox-exporter:children]
+monitoring