Add extras directory to prometheus config

This provides a generic mechanism to include extra files
that you can reference in prometheus.yml, for example:

scrape_targets:
  - job_name: ipmi
    params:
      module: default
    scrape_interval: 1m
    scrape_timeout: 30s
    metrics_path: /ipmi
    scheme: http
    file_sd_configs:
    - files:
      - /etc/prometheus/extras/file_sd/ipmi-exporter-targets.yml
      refresh_interval: 5m

Change-Id: Ie2f085204b71725b901a179ee51541f1f383c6fa
Related: blueprint custom-prometheus-targets

ansible/roles/prometheus/tasks/config.yml

@@ -158,5 +158,49 @@
   notify:
     - Restart prometheus-blackbox-exporter container
 
+- block:
+    - name: Find extra prometheus server config files
+      find:
+        paths: "{{ node_custom_config }}/prometheus/extras/"
+        patterns: "*"
+        recurse: true
+      delegate_to: localhost
+      register: prometheus_config_extras_result
+      run_once: true
+
+    - name: Create subdirectories for extra config files
+      become: true
+      vars:
+        dirs: >-
+          {{ prometheus_config_extras_result.files | default([])
+          | map(attribute='path') | map('dirname') | unique
+          | map('relpath', base) | list }}
+      file:
+        path: "{{ node_config_directory }}/prometheus-server/{{ item }}"
+        state: "directory"
+        owner: "{{ config_owner_user }}"
+        group: "{{ config_owner_group }}"
+        mode: "0770"
+        recurse: true
+      with_items: "{{ dirs }}"
+
+    - name: Template extra prometheus server config files
+      become: true
+      vars:
+        relpath: "{{ item | relpath(base) }}"
+      template:
+        src: "{{ item }}"
+        dest: "{{ node_config_directory }}/prometheus-server/{{ relpath }}"
+        mode: "0660"
+      with_items: "{{ prometheus_config_extras_result.files | default([]) | map(attribute='path') | list }}"
+      notify:
+        - Restart prometheus-server container
+  vars:
+    base: "{{ node_custom_config }}/prometheus/"
+    service: "{{ prometheus_services['prometheus-server']}}"
+  when:
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+
 - include_tasks: check-containers.yml
   when: kolla_action != "config"

ansible/roles/prometheus/templates/prometheus-server.json.j2

@@ -6,6 +6,12 @@
             "dest": "/etc/prometheus/prometheus.yml",
             "owner": "prometheus",
             "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/extras/*",
+            "dest": "/etc/prometheus/extras/",
+            "preserve_properties": true,
+            "optional": true
         }
 {% if enable_prometheus_alertmanager %}
         ,{
@@ -27,6 +33,11 @@
             "path": "/var/log/kolla/prometheus",
             "owner": "prometheus:kolla",
             "recurse": true
+        },
+        {
+            "path": "/etc/prometheus/extras/",
+            "owner": "prometheus:kolla",
+            "recurse": true
         }
     ]
 }

doc/source/reference/logging-and-monitoring/prometheus-guide.rst

@@ -66,3 +66,64 @@ will be merged with any files in ``{{ node_custom_config }}/prometheus/prometheu
 so in order to override a list value instead of extending it, you will need to make
 sure that no files in ``{{ node_custom_config }}/prometheus/prometheus.yml.d``
 set a key with an equivalent hierarchical path.
+
+Extra files
+~~~~~~~~~~~
+
+Sometimes it is necessary to reference additional files from within
+``prometheus.yml``, for example, when defining file service discovery
+configuration. To enable you to do this, kolla-ansible will resursively
+discover any files in ``{{ node_custom_config }}/prometheus/extras`` and
+template them. The templated output is then copied to
+``/etc/prometheus/extras`` within the container on startup. For example to
+configure `ipmi_exporter <https://github.com/soundcloud/ipmi_exporter>`_, using
+the default value for ``node_custom_config``, you could create the following
+files:
+
+- ``/etc/kolla/config/prometheus/prometheus.yml.d/ipmi-exporter.yml``:
+
+    .. code-block:: jinja
+
+        ---
+        scrape_configs:
+        - job_name: ipmi
+          params:
+            module: ["default"]
+            scrape_interval: 1m
+            scrape_timeout: 30s
+            metrics_path: /ipmi
+            scheme: http
+            file_sd_configs:
+              - files:
+                  - /etc/prometheus/extras/file_sd/ipmi-exporter-targets.yml
+            refresh_interval: 5m
+            relabel_configs:
+              - source_labels: [__address__]
+                separator: ;
+                regex: (.*)
+                target_label: __param_target
+                replacement: ${1}
+                action: replace
+              - source_labels: [__param_target]
+                separator: ;
+                regex: (.*)
+                target_label: instance
+                replacement: ${1}
+                action: replace
+              - separator: ;
+                regex: .*
+                target_label: __address__
+                replacement: "{{ ipmi_exporter_listen_address }}:9290"
+                action: replace
+
+  where ``ipmi_exporter_listen_address`` is a variable containing the IP address of
+  the node where the exporter is running.
+
+-  ``/etc/kolla/config/prometheus/extras/file_sd/ipmi-exporter-targets.yml``:
+    .. code-block:: yaml
+
+        ---
+        - targets:
+          - 192.168.1.1
+        labels:
+            job: ipmi_exporter

releasenotes/notes/add-prometheus-extras-57bd14856afc893d.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    Adds a mechanism to copy user defined files via the ``extras`` directory
+    of prometheus config. This can can be useful for certain prometheus config
+    customizations that reference additional files. An example is setting up
+    `file based service discovery <https://prometheus.io/docs/guides/file-sd/>`_.