Add support for PROMETHEUS_CUSTOM_SCRAPE_TARGETS

sg-core devstack plugin used to provide passing custom scrape
targets to prometheus config.

This cr brings back that functionality to d-p-p.

Change-Id: I05c6ad0d0cd84e9e6a2b0d3be21cce48fbd11478
Signed-off-by: Chandan Kumar (raukadah) <chkumar@redhat.com>

.zuul.yaml

@@ -4,6 +4,7 @@
         - openstack-tox-linters
         - watcher-prometheus-integration
         - devstack-prometheus
+        - devstack-prometheus-custom-target
     gate:
       jobs:
         - openstack-tox-linters
@@ -40,3 +41,17 @@
       devstack_localrc:
         # Avoid collecting cpu resources.
         NODE_EXPORTER_COLLECTOR_EXCLUDE: "cpu"
+
+- job:
+    name: devstack-prometheus-custom-target
+    parent: devstack-prometheus-base
+    description: |
+      A Devstack job to install/configure prometheus and node exporter
+      on a single node and generate custom prometheus config.
+    vars:
+      devstack_services:
+        prometheus: true
+        node_exporter: true
+      devstack_localrc:
+        # custom target
+        PROMETHEUS_CUSTOM_SCRAPE_TARGETS: "localhost:9090,localhost:9100"

devstack/files/custom_prometheus.yml

@@ -0,0 +1,4 @@
+global:
+  scrape_interval:     15s # By default, scrape targets every 15 seconds.
+
+scrape_configs:

devstack/lib/prometheus

@@ -41,8 +41,15 @@ function configure_prometheus {
     sudo mkdir -p /etc/prometheus
     sudo chown $(whoami):$(whoami) /etc/prometheus
 
-    # Copy User's prometheus config
+    if [[ ${PROMETHEUS_CUSTOM_SCRAPE_TARGETS} != "" ]]; then
-    sudo cp ${PROMETHEUS_CONFIG_FILE} /etc/prometheus
+
+        #  Generate custom config
+        generate_custom_prometheus_config
+    else
+
+        # Copy User's prometheus config
+        sudo cp ${PROMETHEUS_CONFIG_FILE} /etc/prometheus
+    fi
 
     # Change the permission of prometheus config
     sudo chmod 0644 /etc/prometheus/$(basename ${PROMETHEUS_CONFIG_FILE})
@@ -55,6 +62,36 @@ function configure_prometheus {
     sudo chown $(whoami):$(whoami) ${PROMETHEUS_DATA_DIRECTORY}
 }
 
+function generate_custom_prometheus_config {
+    # Generate custome prometheus config
+
+    # local base config
+    local BASE_CONFIG_FILE=${PROMETHEUS_DEVSTACK_DIR}/custom_prometheus.yml
+
+    # Temprory place to hold config file
+    local RESULT_CONFIG_FILE=${PROMETHEUS_DEVSTACK_DIR}/tmp_prometheus.yml
+
+    cat ${BASE_CONFIG_FILE} > ${RESULT_CONFIG_FILE}
+
+    # Generate custom config
+    if [[ ${PROMETHEUS_CUSTOM_SCRAPE_TARGETS} != "" ]]; then
+        echo "  - job_name: 'custom'" >> ${RESULT_CONFIG_FILE}
+        echo "    static_configs:" >> ${RESULT_CONFIG_FILE}
+            TARGETS=$(echo ${PROMETHEUS_CUSTOM_SCRAPE_TARGETS} | tr "," "\n")
+            for TARGET in ${TARGETS[@]}
+            do
+            echo "      - targets: [$TARGET]" >> $RESULT_CONFIG_FILE
+            done
+    fi
+
+    # Copy custom generated
+    sudo cp ${RESULT_CONFIG_FILE} /etc/prometheus/prometheus.yml
+
+    # Drop tmp_prometheus.yml
+    rm ${PROMETHEUS_DEVSTACK_DIR}/tmp_prometheus.yml
+
+}
+
 function init_prometheus {
     prometheus_command="/usr/local/bin/prometheus"
     prometheus_command+=" --config.file=/etc/prometheus/prometheus.yml"

devstack/settings

@@ -9,6 +9,10 @@ PROMETHEUS_PROJECT_DIR=$DEST/devstack-plugin-prometheus
 PROMETHEUS_DEVSTACK_DIR=${PROMETHEUS_PROJECT_DIR}/devstack/files
 PROMETHEUS_CONFIG_FILE=${PROMETHEUS_CONFIG_FILE:-${PROMETHEUS_DEVSTACK_DIR}/prometheus.yml}
 
+# List of "," delimited targets. Each target is <ip>:<port>
+PROMETHEUS_CUSTOM_SCRAPE_TARGETS=${PROMETHEUS_CUSTOM_SCRAPE_TARGETS:-""}
+
+
 ## Variables related to node exporter
 NODE_EXPORTER_VERSION=1.8.2
 # List of "," excluded collector names