From 2cf4df0850931fed8549780d7abebe2592d389e1 Mon Sep 17 00:00:00 2001
From: Lingxian Kong <anlin.kong@gmail.com>
Date: Fri, 1 Mar 2019 14:16:31 +1300
Subject: [PATCH] Fix prometheus installation script

- Fix the indent in the file.
- Use 'kubectl apply' instead of 'kubectl create' for more robust
  service restart.
- Do not retry infinitely when Prometheus datasource already injected
  into Grafana

Story: #2005117
Task: #29765

Change-Id: I5857fe62f922d27860946fd318296950834a8797
---
 .../fragments/enable-prometheus-monitoring.sh | 194 +++++++++---------
 1 file changed, 99 insertions(+), 95 deletions(-)

diff --git a/magnum/drivers/common/templates/kubernetes/fragments/enable-prometheus-monitoring.sh b/magnum/drivers/common/templates/kubernetes/fragments/enable-prometheus-monitoring.sh
index 633ee8fbaa..4dc0cbc6cf 100644
--- a/magnum/drivers/common/templates/kubernetes/fragments/enable-prometheus-monitoring.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-prometheus-monitoring.sh
@@ -449,111 +449,115 @@ if [ "$(echo $PROMETHEUS_MONITORING | tr '[:upper:]' '[:lower:]')" = "true" ]; t
 
     # Write the binary for enable-monitoring
     KUBE_MON_BIN_CONTENT='''#!/bin/sh
-    until  [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
-    do
-        echo "Waiting for Kubernetes API..."
-        sleep 5
-    done
+until  [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
+do
+    echo "Waiting for Kubernetes API..."
+    sleep 5
+done
 
-    # Check if prometheus-monitoring namespace exist already before creating the namespace
-    kubectl get namespace prometheus-monitoring
-    if [ "$?" != "0" ] && \
-            [ -f "'''${PROMETHEUS_MON_BASE_DIR}'''/prometheusNamespace.yaml" ]; then
-        kubectl create -f  '''${PROMETHEUS_MON_BASE_DIR}'''/prometheusNamespace.yaml
+# Check if prometheus-monitoring namespace exist already before creating the namespace
+kubectl get namespace prometheus-monitoring
+if [ "$?" != "0" ] && \
+        [ -f "'''${PROMETHEUS_MON_BASE_DIR}'''/prometheusNamespace.yaml" ]; then
+    kubectl apply -f  '''${PROMETHEUS_MON_BASE_DIR}'''/prometheusNamespace.yaml
+fi
+
+# Check if all resources exist already before creating them
+# Check if configmap Prometheus exists
+kubectl get configmap prometheus -n prometheus-monitoring
+if [ "$?" != "0" ] && \
+        [ -f "'''${PROMETHEUS_MON_BASE_DIR}'''/prometheusConfigMap.yaml" ]; then
+    kubectl apply -f '''${PROMETHEUS_MON_BASE_DIR}'''/prometheusConfigMap.yaml
+fi
+
+# Check if deployment and service Prometheus exist
+kubectl get service prometheus -n prometheus-monitoring | kubectl get deployment prometheus -n prometheus-monitoring
+if [ "${PIPESTATUS[0]}" != "0" ] && [ "${PIPESTATUS[1]}" != "0" ] && \
+        [ -f "'''${PROMETHEUS_MON_BASE_DIR}'''/prometheusService.yaml" ]; then
+    kubectl apply -f '''${PROMETHEUS_MON_BASE_DIR}'''/prometheusService.yaml
+fi
+
+# Check if node exporter daemonset exist
+kubectl get daemonset node-exporter -n prometheus-monitoring
+if [ "$?" != "0" ] && \
+        [ -f "'''${PROMETHEUS_MON_BASE_DIR}'''/nodeExporter.yaml" ]; then
+    kubectl apply -f '''${PROMETHEUS_MON_BASE_DIR}'''/nodeExporter.yaml
+fi
+
+# Check if configmap graf-dash exists
+kubectl get configmap graf-dash -n prometheus-monitoring
+if [ "$?" != "0" ] && \
+        [ -f '''$GRAFANA_DEF_DASHBOARD_FILE''' ]; then
+    kubectl create configmap graf-dash --from-file='''$GRAFANA_DEF_DASHBOARD_FILE''' -n prometheus-monitoring
+fi
+
+# Check if deployment and service Grafana exist
+kubectl get service grafana -n prometheus-monitoring | kubectl get deployment grafana -n prometheus-monitoring
+if [ "${PIPESTATUS[0]}" != "0" ] && [ "${PIPESTATUS[1]}" != "0" ] && \
+        [ -f "'''${PROMETHEUS_MON_BASE_DIR}'''/grafanaService.yaml" ]; then
+    kubectl apply -f '''${PROMETHEUS_MON_BASE_DIR}'''/grafanaService.yaml
+fi
+
+# Wait for Grafana pod and then inject data source
+while true
+do
+    echo "Waiting for Grafana pod to be up and Running"
+    if [ "$(kubectl get po -n prometheus-monitoring -l name=grafana -o jsonpath={..phase})" = "Running" ]; then
+        break
+    fi
+    sleep 2
+done
+
+# Which node is running Grafana
+NODE_IP=`kubectl get po -n prometheus-monitoring -o jsonpath={.items[0].status.hostIP} -l name=grafana`
+PROM_SERVICE_IP=`kubectl get svc prometheus --namespace prometheus-monitoring -o jsonpath={..clusterIP}`
+GRAFANA_SERVICE_IP=`kubectl get svc grafana --namespace prometheus-monitoring -o jsonpath={..clusterIP}`
+
+# The Grafana pod might be running but the app might still be initiating
+echo "Check if Grafana is ready..."
+curl -sS --user admin:$ADMIN_PASSWD -X GET http://$GRAFANA_SERVICE_IP:3000/api/datasources/1
+until [ $? -eq 0 ]
+do
+    sleep 2
+    curl -sS --user admin:$ADMIN_PASSWD -X GET http://$GRAFANA_SERVICE_IP:3000/api/datasources/1
+done
+
+# Inject Prometheus datasource into Grafana
+while true
+do
+    INJECT=`curl -sS --user admin:$ADMIN_PASSWD -X POST  \
+        -H "Content-Type: application/json;charset=UTF-8" \
+        --data-binary '''"'"'''{"name":"k8sPrometheus","isDefault":true,
+            "type":"prometheus","url":"http://'''"'"'''$PROM_SERVICE_IP'''"'"''':9090","access":"proxy"}'''"'"'''\
+        "http://$GRAFANA_SERVICE_IP:3000/api/datasources/"`
+
+    if [[ "$INJECT" = *"Datasource added"* ]]; then
+        echo "Prometheus datasource injected into Grafana"
+        break
+    elif [[ "$INJECT" = *"Data source with same name already exists"* ]]; then
+        echo "Prometheus datasource already injected into Grafana"
+        break
     fi
 
-    # Check if all resources exist already before creating them
-    # Check if configmap Prometheus exists
-    kubectl get configmap prometheus -n prometheus-monitoring
-    if [ "$?" != "0" ] && \
-            [ -f "'''${PROMETHEUS_MON_BASE_DIR}'''/prometheusConfigMap.yaml" ]; then
-        kubectl create -f '''${PROMETHEUS_MON_BASE_DIR}'''/prometheusConfigMap.yaml
-    fi
-
-    # Check if deployment and service Prometheus exist
-    kubectl get service prometheus -n prometheus-monitoring | kubectl get deployment prometheus -n prometheus-monitoring
-    if [ "${PIPESTATUS[0]}" != "0" ] && [ "${PIPESTATUS[1]}" != "0" ] && \
-            [ -f "'''${PROMETHEUS_MON_BASE_DIR}'''/prometheusService.yaml" ]; then
-        kubectl create -f '''${PROMETHEUS_MON_BASE_DIR}'''/prometheusService.yaml
-    fi
-
-    # Check if node exporter daemonset exist
-    kubectl get daemonset node-exporter -n prometheus-monitoring
-    if [ "$?" != "0" ] && \
-            [ -f "'''${PROMETHEUS_MON_BASE_DIR}'''/nodeExporter.yaml" ]; then
-        kubectl create -f '''${PROMETHEUS_MON_BASE_DIR}'''/nodeExporter.yaml
-    fi
-
-    # Check if configmap graf-dash exists
-    kubectl get configmap graf-dash -n prometheus-monitoring
-    if [ "$?" != "0" ] && \
-            [ -f '''$GRAFANA_DEF_DASHBOARD_FILE''' ]; then
-        kubectl create configmap graf-dash --from-file='''$GRAFANA_DEF_DASHBOARD_FILE''' -n prometheus-monitoring
-    fi
-
-    # Check if deployment and service Grafana exist
-    kubectl get service grafana -n prometheus-monitoring | kubectl get deployment grafana -n prometheus-monitoring
-    if [ "${PIPESTATUS[0]}" != "0" ] && [ "${PIPESTATUS[1]}" != "0" ] && \
-            [ -f "'''${PROMETHEUS_MON_BASE_DIR}'''/grafanaService.yaml" ]; then
-        kubectl create -f '''${PROMETHEUS_MON_BASE_DIR}'''/grafanaService.yaml
-    fi
-
-    # Wait for Grafana pod and then inject data source
-    while true
-    do
-        echo "Waiting for Grafana pod to be up and Running"
-        if [ "$(kubectl get po -n prometheus-monitoring -l name=grafana -o jsonpath={..phase})" = "Running" ]; then
-            break
-        fi
-        sleep 2
-    done
-
-    # Which node is running Grafana
-    NODE_IP=`kubectl get po -n prometheus-monitoring -o jsonpath={.items[0].status.hostIP} -l name=grafana`
-    PROM_SERVICE_IP=`kubectl get svc prometheus --namespace prometheus-monitoring -o jsonpath={..clusterIP}`
-    GRAFANA_SERVICE_IP=`kubectl get svc grafana --namespace prometheus-monitoring -o jsonpath={..clusterIP}`
-
-    # The Grafana pod might be running but the app might still be initiating
-    echo "Check if Grafana is ready..."
-    curl --user admin:$ADMIN_PASSWD -X GET http://$GRAFANA_SERVICE_IP:3000/api/datasources/1
-    until [ $? -eq 0 ]
-    do
-        sleep 2
-        curl --user admin:$ADMIN_PASSWD -X GET http://$GRAFANA_SERVICE_IP:3000/api/datasources/1
-    done
-
-    # Inject Prometheus datasource into Grafana
-    while true
-    do
-        INJECT=`curl --user admin:$ADMIN_PASSWD -X POST  \
-            -H "Content-Type: application/json;charset=UTF-8" \
-            --data-binary '''"'"'''{"name":"k8sPrometheus","isDefault":true,
-                "type":"prometheus","url":"http://'''"'"'''$PROM_SERVICE_IP'''"'"''':9090","access":"proxy"}'''"'"'''\
-            "http://$GRAFANA_SERVICE_IP:3000/api/datasources/"`
-
-        if [[ "$INJECT" = *"Datasource added"* ]]; then
-            echo "Prometheus datasource injected into Grafana"
-            break
-        fi
-        echo "Trying to inject Prometheus datasource into Grafana - "$INJECT
-    done
-    '''
+    echo "Trying to inject Prometheus datasource into Grafana - "$INJECT
+done
+'''
     writeFile $KUBE_MON_BIN "$KUBE_MON_BIN_CONTENT"
 
 
     # Write the monitoring service
     KUBE_MON_SERVICE_CONTENT='''[Unit]
-    Description=Enable Prometheus monitoring stack
+Description=Enable Prometheus monitoring stack
 
-    [Service]
-    Type=oneshot
-    Environment=HOME=/root
-    EnvironmentFile=-/etc/kubernetes/config
-    ExecStart='''${KUBE_MON_BIN}'''
+[Service]
+Type=oneshot
+Environment=HOME=/root
+EnvironmentFile=-/etc/kubernetes/config
+ExecStart='''${KUBE_MON_BIN}'''
 
-    [Install]
-    WantedBy=multi-user.target
-    '''
+[Install]
+WantedBy=multi-user.target
+'''
     writeFile $KUBE_MON_SERVICE "$KUBE_MON_SERVICE_CONTENT"
 
     chown root:root ${KUBE_MON_BIN}