From eef3ff30845fed3971572e3f426d42f23734940b Mon Sep 17 00:00:00 2001 From: Dawud Date: Fri, 3 Mar 2023 13:31:54 +0000 Subject: [PATCH] Use friendly prometheus instance labels Replaces the instance label on prometheus metrics with the inventory hostname as opposed to the ip address. The ip address is still used as the target address which means that there is no issue of the hostname being unresolvable. Can be optionally enabled or set to FQDNs by changing the prometheus_instance_label variable as mentioned in the release notes. Co-Authored-By: Will Szumski Change-Id: I387c9d8f5c01baf6054381834ecf4e554d0fff35 --- ansible/group_vars/all.yml | 3 + .../prometheus/templates/prometheus.yml.j2 | 72 +++++++++++++++---- .../prometheus-guide.rst | 37 ++++++++++ .../friendly-hostnames-1bb1254b2b434a11.yaml | 10 +++ 4 files changed, 110 insertions(+), 12 deletions(-) create mode 100644 releasenotes/notes/friendly-hostnames-1bb1254b2b434a11.yaml diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 8ddb843396..a3466e49a4 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -497,6 +497,9 @@ prometheus_elasticsearch_exporter_port: "9108" # Prometheus blackbox-exporter ports prometheus_blackbox_exporter_port: "9115" +# Prometheus instance label to use for metrics +prometheus_instance_label: + proxysql_admin_port: "6032" rabbitmq_port: "{{ '5671' if rabbitmq_enable_tls | bool else '5672' }}" diff --git a/ansible/roles/prometheus/templates/prometheus.yml.j2 b/ansible/roles/prometheus/templates/prometheus.yml.j2 index 85483cc918..43d76bada7 100644 --- a/ansible/roles/prometheus/templates/prometheus.yml.j2 +++ b/ansible/roles/prometheus/templates/prometheus.yml.j2 @@ -19,71 +19,103 @@ rule_files: scrape_configs: - job_name: prometheus static_configs: - - targets: {% for host in groups['prometheus'] %} + - targets: - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ prometheus_port }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} {% endfor %} {% if enable_prometheus_node_exporter | bool %} - job_name: node static_configs: - - targets: {% for host in groups['prometheus-node-exporter'] %} + - targets: - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_node_exporter_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} {% endfor %} {% endif %} {% if enable_prometheus_mysqld_exporter | bool %} - job_name: mysqld static_configs: - - targets: {% for host in groups['prometheus-mysqld-exporter'] %} + - targets: - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_mysqld_exporter_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} {% endfor %} {% endif %} {% if enable_prometheus_haproxy_exporter | bool %} - job_name: haproxy static_configs: - - targets: {% for host in groups['prometheus-haproxy-exporter'] %} + - targets: - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_haproxy_exporter_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} {% endfor %} {% endif %} {% if enable_prometheus_rabbitmq_exporter | bool %} - job_name: rabbitmq static_configs: - - targets: {% for host in groups['rabbitmq'] %} + - targets: - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_rabbitmq_exporter_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} {% endfor %} {% endif %} {% if enable_prometheus_memcached_exporter | bool %} - job_name: memcached static_configs: - - targets: {% for host in groups['prometheus-memcached-exporter'] %} + - targets: - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_memcached_exporter_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} {% endfor %} {% endif %} {% if enable_prometheus_cadvisor | bool %} - job_name: cadvisor static_configs: - - targets: {% for host in groups["prometheus-cadvisor"] %} + - targets: - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_cadvisor_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} {% endfor %} {% endif %} {% if enable_prometheus_fluentd_integration | bool %} - job_name: fluentd static_configs: - - targets: {% for host in groups['fluentd'] %} + - targets: - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_fluentd_integration_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} {% endfor %} {% endif %} @@ -114,9 +146,13 @@ scrape_configs: - job_name: elasticsearch_exporter scrape_interval: {{ prometheus_elasticsearch_exporter_interval }} static_configs: - - targets: {% for host in groups["prometheus-elasticsearch-exporter"] %} + - targets: - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_elasticsearch_exporter_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} {% endfor %} {% endif %} @@ -155,27 +191,39 @@ scrape_configs: scrape_interval: {{ prometheus_libvirt_exporter_interval }} honor_labels: true static_configs: - - targets: {% for host in groups["prometheus-libvirt-exporter"] %} + - targets: - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_libvirt_exporter_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} {% endfor %} {% endif %} {% if enable_prometheus_etcd_integration | bool %} - job_name: etcd static_configs: - - targets: {% for host in groups["etcd"] %} + - targets: - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_etcd_integration_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} {% endfor %} {% endif %} {% if enable_prometheus_alertmanager | bool %} - job_name: alertmanager static_configs: - - targets: {% for host in groups['prometheus-alertmanager'] %} + - targets: - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_alertmanager_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} {% endfor %} alerting: diff --git a/doc/source/reference/logging-and-monitoring/prometheus-guide.rst b/doc/source/reference/logging-and-monitoring/prometheus-guide.rst index bf9103ca76..1f814107a5 100644 --- a/doc/source/reference/logging-and-monitoring/prometheus-guide.rst +++ b/doc/source/reference/logging-and-monitoring/prometheus-guide.rst @@ -156,3 +156,40 @@ files: - 192.168.1.1 labels: job: ipmi_exporter + +Metric Instance labels +~~~~~~~~~~~~~~~~~~~~~~ + +Previously, Prometheus metrics used to label instances based on their IP +addresses. This behaviour can now be changed such that instances can be +labelled based on their inventory hostname instead. The IP address remains as +the target address, therefore, even if the hostname is unresolvable, it doesn't +pose an issue. + +The default behavior still labels instances with their IP addresses. However, +this can be adjusted by changing the ``prometheus_instance_label`` variable. +This variable accepts the following values: + +* ``None``: Instance labels will be IP addresses (default) +* ``{{ ansible_facts.hostname }}``: Instance labels will be hostnames +* ``{{ ansible_facts.nodename }}``: Instance labels will FQDNs + +To implement this feature, modify the configuration file +``/etc/kolla/globals.yml`` and update the ``prometheus_instance_label`` +variable accordingly. Remember, changing this variable will cause Prometheus to +scrape metrics with new names for a short period. This will result in duplicate +metrics until all metrics are replaced with their new labels. + +.. code-block:: yaml + + prometheus_instance_label: "{{ ansible_facts.hostname }}" + +This metric labeling feature may become the default setting in future releases. +Therefore, if you wish to retain the current default (IP address labels), make +sure to set the ``prometheus_instance_label`` variable to ``None``. + +.. note:: + + This feature may generate duplicate metrics temporarily while Prometheus + updates the metric labels. Please be aware of this while analyzing metrics + during the transition period. diff --git a/releasenotes/notes/friendly-hostnames-1bb1254b2b434a11.yaml b/releasenotes/notes/friendly-hostnames-1bb1254b2b434a11.yaml new file mode 100644 index 0000000000..153b8c4652 --- /dev/null +++ b/releasenotes/notes/friendly-hostnames-1bb1254b2b434a11.yaml @@ -0,0 +1,10 @@ +--- +features: + - | + Adds the ability for the instance label on prometheus metrics to be + replaced with the inventory hostname as opposed to using the ip address as + the metric label. The ip address is still used as the target address which + means that there is no issue of the hostname being unresolvable. + + More information on how to use this feature can be found in the + reference documentation for logging and monitoring.