Use friendly prometheus instance labels

Replaces the instance label on prometheus metrics with the inventory
hostname as opposed to the ip address. The ip address is still used as
the target address which means that there is no issue of the hostname
being unresolvable. Can be optionally enabled or set to FQDNs by
changing the prometheus_instance_label variable as mentioned in the
release notes.

Co-Authored-By: Will Szumski <will@stackhpc.com>
Change-Id: I387c9d8f5c01baf6054381834ecf4e554d0fff35

ansible/group_vars/all.yml

@@ -497,6 +497,9 @@ prometheus_elasticsearch_exporter_port: "9108"
 # Prometheus blackbox-exporter ports
 prometheus_blackbox_exporter_port: "9115"
 
+# Prometheus instance label to use for metrics
+prometheus_instance_label:
+
 proxysql_admin_port: "6032"
 
 rabbitmq_port: "{{ '5671' if rabbitmq_enable_tls | bool else '5672' }}"

ansible/roles/prometheus/templates/prometheus.yml.j2

@@ -19,71 +19,103 @@ rule_files:
 scrape_configs:
   - job_name: prometheus
     static_configs:
-      - targets:
 {% for host in groups['prometheus'] %}
+      - targets:
         - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ prometheus_port }}'
+{% if hostvars[host].prometheus_instance_label | default(false, true) %}
+        labels:
+          instance: "{{ hostvars[host].prometheus_instance_label }}"
+{% endif %}
 {% endfor %}
 
 {% if enable_prometheus_node_exporter | bool %}
   - job_name: node
     static_configs:
-      - targets:
 {% for host in groups['prometheus-node-exporter'] %}
+      - targets:
         - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_node_exporter_port'] }}'
+{% if hostvars[host].prometheus_instance_label | default(false, true) %}
+        labels:
+          instance: "{{ hostvars[host].prometheus_instance_label }}"
+{% endif %}
 {% endfor %}
 {% endif %}
 
 {% if enable_prometheus_mysqld_exporter | bool %}
   - job_name: mysqld
     static_configs:
-      - targets:
 {% for host in groups['prometheus-mysqld-exporter'] %}
+      - targets:
         - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_mysqld_exporter_port'] }}'
+{% if hostvars[host].prometheus_instance_label | default(false, true) %}
+        labels:
+          instance: "{{ hostvars[host].prometheus_instance_label }}"
+{% endif %}
 {% endfor %}
 {% endif %}
 
 {% if enable_prometheus_haproxy_exporter | bool %}
   - job_name: haproxy
     static_configs:
-      - targets:
 {% for host in groups['prometheus-haproxy-exporter'] %}
+      - targets:
         - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_haproxy_exporter_port'] }}'
+{% if hostvars[host].prometheus_instance_label | default(false, true) %}
+        labels:
+          instance: "{{ hostvars[host].prometheus_instance_label }}"
+{% endif %}
 {% endfor %}
 {% endif %}
 
 {% if enable_prometheus_rabbitmq_exporter | bool %}
   - job_name: rabbitmq
     static_configs:
-      - targets:
 {% for host in groups['rabbitmq'] %}
+      - targets:
         - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_rabbitmq_exporter_port'] }}'
+{% if hostvars[host].prometheus_instance_label | default(false, true) %}
+        labels:
+          instance: "{{ hostvars[host].prometheus_instance_label }}"
+{% endif %}
 {% endfor %}
 {% endif %}
 
 {% if enable_prometheus_memcached_exporter | bool %}
   - job_name: memcached
     static_configs:
-      - targets:
 {% for host in groups['prometheus-memcached-exporter'] %}
+      - targets:
         - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_memcached_exporter_port'] }}'
+{% if hostvars[host].prometheus_instance_label | default(false, true) %}
+        labels:
+          instance: "{{ hostvars[host].prometheus_instance_label }}"
+{% endif %}
 {% endfor %}
 {% endif %}
 
 {% if enable_prometheus_cadvisor | bool %}
   - job_name: cadvisor
     static_configs:
-      - targets:
 {% for host in groups["prometheus-cadvisor"] %}
+      - targets:
         - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_cadvisor_port'] }}'
+{% if hostvars[host].prometheus_instance_label | default(false, true) %}
+        labels:
+          instance: "{{ hostvars[host].prometheus_instance_label }}"
+{% endif %}
 {% endfor %}
 {% endif %}
 
 {% if enable_prometheus_fluentd_integration | bool %}
   - job_name: fluentd
     static_configs:
-      - targets:
 {% for host in groups['fluentd'] %}
+      - targets:
         - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_fluentd_integration_port'] }}'
+{% if hostvars[host].prometheus_instance_label | default(false, true) %}
+        labels:
+          instance: "{{ hostvars[host].prometheus_instance_label }}"
+{% endif %}
 {% endfor %}
 {% endif %}
 
@@ -114,9 +146,13 @@ scrape_configs:
   - job_name: elasticsearch_exporter
     scrape_interval: {{ prometheus_elasticsearch_exporter_interval }}
     static_configs:
-      - targets:
 {% for host in groups["prometheus-elasticsearch-exporter"] %}
+      - targets:
         - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_elasticsearch_exporter_port'] }}'
+{% if hostvars[host].prometheus_instance_label | default(false, true) %}
+        labels:
+          instance: "{{ hostvars[host].prometheus_instance_label }}"
+{% endif %}
 {% endfor %}
 {% endif %}
 
@@ -155,27 +191,39 @@ scrape_configs:
     scrape_interval: {{ prometheus_libvirt_exporter_interval }}
     honor_labels: true
     static_configs:
-      - targets:
 {% for host in groups["prometheus-libvirt-exporter"] %}
+      - targets:
         - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_libvirt_exporter_port'] }}'
+{% if hostvars[host].prometheus_instance_label | default(false, true) %}
+        labels:
+          instance: "{{ hostvars[host].prometheus_instance_label }}"
+{% endif %}
 {% endfor %}
 {% endif %}
 
 {% if enable_prometheus_etcd_integration | bool %}
   - job_name: etcd
     static_configs:
-      - targets:
 {% for host in groups["etcd"] %}
+      - targets:
         - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_etcd_integration_port'] }}'
+{% if hostvars[host].prometheus_instance_label | default(false, true) %}
+        labels:
+          instance: "{{ hostvars[host].prometheus_instance_label }}"
+{% endif %}
 {% endfor %}
 {% endif %}
 
 {% if enable_prometheus_alertmanager | bool %}
   - job_name: alertmanager
     static_configs:
-      - targets:
 {% for host in groups['prometheus-alertmanager'] %}
+      - targets:
         - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_alertmanager_port'] }}'
+{% if hostvars[host].prometheus_instance_label | default(false, true) %}
+        labels:
+          instance: "{{ hostvars[host].prometheus_instance_label }}"
+{% endif %}
 {% endfor %}
 
 alerting:

doc/source/reference/logging-and-monitoring/prometheus-guide.rst

@@ -156,3 +156,40 @@ files:
         - 192.168.1.1
       labels:
           job: ipmi_exporter
+
+Metric Instance labels
+~~~~~~~~~~~~~~~~~~~~~~
+
+Previously, Prometheus metrics used to label instances based on their IP
+addresses. This behaviour can now be changed such that instances can be
+labelled based on their inventory hostname instead. The IP address remains as
+the target address, therefore, even if the hostname is unresolvable, it doesn't
+pose an issue.
+
+The default behavior still labels instances with their IP addresses. However,
+this can be adjusted by changing the ``prometheus_instance_label`` variable.
+This variable accepts the following values:
+
+* ``None``: Instance labels will be IP addresses (default)
+* ``{{ ansible_facts.hostname }}``: Instance labels will be hostnames
+* ``{{ ansible_facts.nodename }}``: Instance labels will FQDNs
+
+To implement this feature, modify the configuration file
+``/etc/kolla/globals.yml`` and update the ``prometheus_instance_label``
+variable accordingly. Remember, changing this variable will cause Prometheus to
+scrape metrics with new names for a short period. This will result in duplicate
+metrics until all metrics are replaced with their new labels.
+
+.. code-block:: yaml
+
+   prometheus_instance_label: "{{ ansible_facts.hostname }}"
+
+This metric labeling feature may become the default setting in future releases.
+Therefore, if you wish to retain the current default (IP address labels), make
+sure to set the ``prometheus_instance_label`` variable to ``None``.
+
+.. note::
+
+   This feature may generate duplicate metrics temporarily while Prometheus
+   updates the metric labels. Please be aware of this while analyzing metrics
+   during the transition period.

releasenotes/notes/friendly-hostnames-1bb1254b2b434a11.yaml

@@ -0,0 +1,10 @@
+---
+features:
+  - |
+    Adds the ability for the instance label on prometheus metrics to be
+    replaced with the inventory hostname as opposed to using the ip address as
+    the metric label. The ip address is still used as the target address which
+    means that there is no issue of the hostname being unresolvable.
+
+    More information on how to use this feature can be found in the
+    reference documentation for logging and monitoring.