From 656f6cdb083c4b40e9c6e12863631b80b8f6d1b5 Mon Sep 17 00:00:00 2001
From: k-s-dean <kyle@stackhpc.com>
Date: Tue, 10 May 2022 16:36:59 +0100
Subject: [PATCH] Put openstack exporter behind HAproxy so only one is queried
 at a time

Closes-Bug: #1972818

Change-Id: I9e36b9169b6725bf6db953e464fc099087747778
---
 ansible/roles/prometheus/defaults/main.yml         |  6 ++++++
 .../roles/prometheus/templates/prometheus.yml.j2   |  4 +---
 .../openstack-exporter-hammering-os-apis.yaml      | 14 ++++++++++++++
 3 files changed, 21 insertions(+), 3 deletions(-)
 create mode 100644 releasenotes/notes/openstack-exporter-hammering-os-apis.yaml

diff --git a/ansible/roles/prometheus/defaults/main.yml b/ansible/roles/prometheus/defaults/main.yml
index ad8d00b922..9ef0ef2f1f 100644
--- a/ansible/roles/prometheus/defaults/main.yml
+++ b/ansible/roles/prometheus/defaults/main.yml
@@ -79,6 +79,12 @@ prometheus_services:
     image: "{{ prometheus_openstack_exporter_image_full }}"
     volumes: "{{ prometheus_openstack_exporter_default_volumes + prometheus_openstack_exporter_extra_volumes }}"
     dimensions: "{{ prometheus_openstack_exporter_dimensions }}"
+    haproxy:
+      prometheus_openstack_exporter:
+        enabled: "{{ enable_prometheus_openstack_exporter | bool }}"
+        mode: "http"
+        external: false
+        port: "{{ prometheus_openstack_exporter_port }}"
   prometheus-elasticsearch-exporter:
     container_name: prometheus_elasticsearch_exporter
     group: prometheus-elasticsearch-exporter
diff --git a/ansible/roles/prometheus/templates/prometheus.yml.j2 b/ansible/roles/prometheus/templates/prometheus.yml.j2
index 09ab077e1c..cd36d72880 100644
--- a/ansible/roles/prometheus/templates/prometheus.yml.j2
+++ b/ansible/roles/prometheus/templates/prometheus.yml.j2
@@ -100,9 +100,7 @@ scrape_configs:
     honor_labels: true
     static_configs:
       - targets:
-{% for host in groups["prometheus-openstack-exporter"] %}
-        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_openstack_exporter_port'] }}'
-{% endfor %}
+        - '{{ kolla_internal_vip_address | put_address_in_context('url') }}:{{ prometheus_openstack_exporter_port }}'
 {% endif %}
 
 {% if enable_prometheus_elasticsearch_exporter | bool %}
diff --git a/releasenotes/notes/openstack-exporter-hammering-os-apis.yaml b/releasenotes/notes/openstack-exporter-hammering-os-apis.yaml
new file mode 100644
index 0000000000..78ef561c9d
--- /dev/null
+++ b/releasenotes/notes/openstack-exporter-hammering-os-apis.yaml
@@ -0,0 +1,14 @@
+---
+fixes:
+  - |
+    The prometheus openstack exporters are now behind haproxy,
+    providing a unique time series in the prometheus database.
+    Also ensures that only one exporter queries
+    the openstack APIs at any given time interval.
+    With the previous behavior each openstack exporter
+    was scraped at the same time.
+    This caused each exporter to query the openstack APIs
+    simultaneously introducing unneccesary load and duplicate
+    time series in the prometheus database due to the instance
+    label being unique for each exporter.
+    `LP#1972818 <https://bugs.launchpad.net/kolla-ansible/+bug/1972818>`__