diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 3094c830fd..cbbb4c8b14 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -500,6 +500,9 @@ prometheus_elasticsearch_exporter_port: "9108" # Prometheus blackbox-exporter ports prometheus_blackbox_exporter_port: "9115" +# Prometheus instance label to use for metrics +prometheus_instance_label: + proxysql_admin_port: "6032" rabbitmq_port: "{{ '5671' if rabbitmq_enable_tls | bool else '5672' }}" diff --git a/ansible/roles/prometheus/templates/prometheus.yml.j2 b/ansible/roles/prometheus/templates/prometheus.yml.j2 index 85483cc918..43d76bada7 100644 --- a/ansible/roles/prometheus/templates/prometheus.yml.j2 +++ b/ansible/roles/prometheus/templates/prometheus.yml.j2 @@ -19,71 +19,103 @@ rule_files: scrape_configs: - job_name: prometheus static_configs: - - targets: {% for host in groups['prometheus'] %} + - targets: - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ prometheus_port }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} {% endfor %} {% if enable_prometheus_node_exporter | bool %} - job_name: node static_configs: - - targets: {% for host in groups['prometheus-node-exporter'] %} + - targets: - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_node_exporter_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} {% endfor %} {% endif %} {% if enable_prometheus_mysqld_exporter | bool %} - job_name: mysqld static_configs: - - targets: {% for host in groups['prometheus-mysqld-exporter'] %} + - targets: - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_mysqld_exporter_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} {% endfor %} {% endif %} {% if enable_prometheus_haproxy_exporter | bool %} - job_name: haproxy static_configs: - - targets: {% for host in groups['prometheus-haproxy-exporter'] %} + - targets: - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_haproxy_exporter_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} {% endfor %} {% endif %} {% if enable_prometheus_rabbitmq_exporter | bool %} - job_name: rabbitmq static_configs: - - targets: {% for host in groups['rabbitmq'] %} + - targets: - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_rabbitmq_exporter_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} {% endfor %} {% endif %} {% if enable_prometheus_memcached_exporter | bool %} - job_name: memcached static_configs: - - targets: {% for host in groups['prometheus-memcached-exporter'] %} + - targets: - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_memcached_exporter_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} {% endfor %} {% endif %} {% if enable_prometheus_cadvisor | bool %} - job_name: cadvisor static_configs: - - targets: {% for host in groups["prometheus-cadvisor"] %} + - targets: - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_cadvisor_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} {% endfor %} {% endif %} {% if enable_prometheus_fluentd_integration | bool %} - job_name: fluentd static_configs: - - targets: {% for host in groups['fluentd'] %} + - targets: - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_fluentd_integration_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} {% endfor %} {% endif %} @@ -114,9 +146,13 @@ scrape_configs: - job_name: elasticsearch_exporter scrape_interval: {{ prometheus_elasticsearch_exporter_interval }} static_configs: - - targets: {% for host in groups["prometheus-elasticsearch-exporter"] %} + - targets: - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_elasticsearch_exporter_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} {% endfor %} {% endif %} @@ -155,27 +191,39 @@ scrape_configs: scrape_interval: {{ prometheus_libvirt_exporter_interval }} honor_labels: true static_configs: - - targets: {% for host in groups["prometheus-libvirt-exporter"] %} + - targets: - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_libvirt_exporter_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} {% endfor %} {% endif %} {% if enable_prometheus_etcd_integration | bool %} - job_name: etcd static_configs: - - targets: {% for host in groups["etcd"] %} + - targets: - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_etcd_integration_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} {% endfor %} {% endif %} {% if enable_prometheus_alertmanager | bool %} - job_name: alertmanager static_configs: - - targets: {% for host in groups['prometheus-alertmanager'] %} + - targets: - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_alertmanager_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} {% endfor %} alerting: diff --git a/doc/source/reference/logging-and-monitoring/prometheus-guide.rst b/doc/source/reference/logging-and-monitoring/prometheus-guide.rst index bf9103ca76..1f814107a5 100644 --- a/doc/source/reference/logging-and-monitoring/prometheus-guide.rst +++ b/doc/source/reference/logging-and-monitoring/prometheus-guide.rst @@ -156,3 +156,40 @@ files: - 192.168.1.1 labels: job: ipmi_exporter + +Metric Instance labels +~~~~~~~~~~~~~~~~~~~~~~ + +Previously, Prometheus metrics used to label instances based on their IP +addresses. This behaviour can now be changed such that instances can be +labelled based on their inventory hostname instead. The IP address remains as +the target address, therefore, even if the hostname is unresolvable, it doesn't +pose an issue. + +The default behavior still labels instances with their IP addresses. However, +this can be adjusted by changing the ``prometheus_instance_label`` variable. +This variable accepts the following values: + +* ``None``: Instance labels will be IP addresses (default) +* ``{{ ansible_facts.hostname }}``: Instance labels will be hostnames +* ``{{ ansible_facts.nodename }}``: Instance labels will FQDNs + +To implement this feature, modify the configuration file +``/etc/kolla/globals.yml`` and update the ``prometheus_instance_label`` +variable accordingly. Remember, changing this variable will cause Prometheus to +scrape metrics with new names for a short period. This will result in duplicate +metrics until all metrics are replaced with their new labels. + +.. code-block:: yaml + + prometheus_instance_label: "{{ ansible_facts.hostname }}" + +This metric labeling feature may become the default setting in future releases. +Therefore, if you wish to retain the current default (IP address labels), make +sure to set the ``prometheus_instance_label`` variable to ``None``. + +.. note:: + + This feature may generate duplicate metrics temporarily while Prometheus + updates the metric labels. Please be aware of this while analyzing metrics + during the transition period. diff --git a/releasenotes/notes/friendly-hostnames-1bb1254b2b434a11.yaml b/releasenotes/notes/friendly-hostnames-1bb1254b2b434a11.yaml new file mode 100644 index 0000000000..153b8c4652 --- /dev/null +++ b/releasenotes/notes/friendly-hostnames-1bb1254b2b434a11.yaml @@ -0,0 +1,10 @@ +--- +features: + - | + Adds the ability for the instance label on prometheus metrics to be + replaced with the inventory hostname as opposed to using the ip address as + the metric label. The ip address is still used as the target address which + means that there is no issue of the hostname being unresolvable. + + More information on how to use this feature can be found in the + reference documentation for logging and monitoring.