Fix prometheus installation script
- Fix the indent in the file. - Use 'kubectl apply' instead of 'kubectl create' for more robust service restart. - Do not retry infinitely when Prometheus datasource already injected into Grafana Story: #2005117 Task: #29765 Change-Id: I5857fe62f922d27860946fd318296950834a8797
This commit is contained in:
parent
3695536085
commit
2cf4df0850
|
@ -449,111 +449,115 @@ if [ "$(echo $PROMETHEUS_MONITORING | tr '[:upper:]' '[:lower:]')" = "true" ]; t
|
||||||
|
|
||||||
# Write the binary for enable-monitoring
|
# Write the binary for enable-monitoring
|
||||||
KUBE_MON_BIN_CONTENT='''#!/bin/sh
|
KUBE_MON_BIN_CONTENT='''#!/bin/sh
|
||||||
until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
|
until [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
|
||||||
do
|
do
|
||||||
echo "Waiting for Kubernetes API..."
|
echo "Waiting for Kubernetes API..."
|
||||||
sleep 5
|
sleep 5
|
||||||
done
|
done
|
||||||
|
|
||||||
# Check if prometheus-monitoring namespace exist already before creating the namespace
|
# Check if prometheus-monitoring namespace exist already before creating the namespace
|
||||||
kubectl get namespace prometheus-monitoring
|
kubectl get namespace prometheus-monitoring
|
||||||
if [ "$?" != "0" ] && \
|
if [ "$?" != "0" ] && \
|
||||||
[ -f "'''${PROMETHEUS_MON_BASE_DIR}'''/prometheusNamespace.yaml" ]; then
|
[ -f "'''${PROMETHEUS_MON_BASE_DIR}'''/prometheusNamespace.yaml" ]; then
|
||||||
kubectl create -f '''${PROMETHEUS_MON_BASE_DIR}'''/prometheusNamespace.yaml
|
kubectl apply -f '''${PROMETHEUS_MON_BASE_DIR}'''/prometheusNamespace.yaml
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check if all resources exist already before creating them
|
||||||
|
# Check if configmap Prometheus exists
|
||||||
|
kubectl get configmap prometheus -n prometheus-monitoring
|
||||||
|
if [ "$?" != "0" ] && \
|
||||||
|
[ -f "'''${PROMETHEUS_MON_BASE_DIR}'''/prometheusConfigMap.yaml" ]; then
|
||||||
|
kubectl apply -f '''${PROMETHEUS_MON_BASE_DIR}'''/prometheusConfigMap.yaml
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check if deployment and service Prometheus exist
|
||||||
|
kubectl get service prometheus -n prometheus-monitoring | kubectl get deployment prometheus -n prometheus-monitoring
|
||||||
|
if [ "${PIPESTATUS[0]}" != "0" ] && [ "${PIPESTATUS[1]}" != "0" ] && \
|
||||||
|
[ -f "'''${PROMETHEUS_MON_BASE_DIR}'''/prometheusService.yaml" ]; then
|
||||||
|
kubectl apply -f '''${PROMETHEUS_MON_BASE_DIR}'''/prometheusService.yaml
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check if node exporter daemonset exist
|
||||||
|
kubectl get daemonset node-exporter -n prometheus-monitoring
|
||||||
|
if [ "$?" != "0" ] && \
|
||||||
|
[ -f "'''${PROMETHEUS_MON_BASE_DIR}'''/nodeExporter.yaml" ]; then
|
||||||
|
kubectl apply -f '''${PROMETHEUS_MON_BASE_DIR}'''/nodeExporter.yaml
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check if configmap graf-dash exists
|
||||||
|
kubectl get configmap graf-dash -n prometheus-monitoring
|
||||||
|
if [ "$?" != "0" ] && \
|
||||||
|
[ -f '''$GRAFANA_DEF_DASHBOARD_FILE''' ]; then
|
||||||
|
kubectl create configmap graf-dash --from-file='''$GRAFANA_DEF_DASHBOARD_FILE''' -n prometheus-monitoring
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check if deployment and service Grafana exist
|
||||||
|
kubectl get service grafana -n prometheus-monitoring | kubectl get deployment grafana -n prometheus-monitoring
|
||||||
|
if [ "${PIPESTATUS[0]}" != "0" ] && [ "${PIPESTATUS[1]}" != "0" ] && \
|
||||||
|
[ -f "'''${PROMETHEUS_MON_BASE_DIR}'''/grafanaService.yaml" ]; then
|
||||||
|
kubectl apply -f '''${PROMETHEUS_MON_BASE_DIR}'''/grafanaService.yaml
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Wait for Grafana pod and then inject data source
|
||||||
|
while true
|
||||||
|
do
|
||||||
|
echo "Waiting for Grafana pod to be up and Running"
|
||||||
|
if [ "$(kubectl get po -n prometheus-monitoring -l name=grafana -o jsonpath={..phase})" = "Running" ]; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
|
||||||
|
# Which node is running Grafana
|
||||||
|
NODE_IP=`kubectl get po -n prometheus-monitoring -o jsonpath={.items[0].status.hostIP} -l name=grafana`
|
||||||
|
PROM_SERVICE_IP=`kubectl get svc prometheus --namespace prometheus-monitoring -o jsonpath={..clusterIP}`
|
||||||
|
GRAFANA_SERVICE_IP=`kubectl get svc grafana --namespace prometheus-monitoring -o jsonpath={..clusterIP}`
|
||||||
|
|
||||||
|
# The Grafana pod might be running but the app might still be initiating
|
||||||
|
echo "Check if Grafana is ready..."
|
||||||
|
curl -sS --user admin:$ADMIN_PASSWD -X GET http://$GRAFANA_SERVICE_IP:3000/api/datasources/1
|
||||||
|
until [ $? -eq 0 ]
|
||||||
|
do
|
||||||
|
sleep 2
|
||||||
|
curl -sS --user admin:$ADMIN_PASSWD -X GET http://$GRAFANA_SERVICE_IP:3000/api/datasources/1
|
||||||
|
done
|
||||||
|
|
||||||
|
# Inject Prometheus datasource into Grafana
|
||||||
|
while true
|
||||||
|
do
|
||||||
|
INJECT=`curl -sS --user admin:$ADMIN_PASSWD -X POST \
|
||||||
|
-H "Content-Type: application/json;charset=UTF-8" \
|
||||||
|
--data-binary '''"'"'''{"name":"k8sPrometheus","isDefault":true,
|
||||||
|
"type":"prometheus","url":"http://'''"'"'''$PROM_SERVICE_IP'''"'"''':9090","access":"proxy"}'''"'"'''\
|
||||||
|
"http://$GRAFANA_SERVICE_IP:3000/api/datasources/"`
|
||||||
|
|
||||||
|
if [[ "$INJECT" = *"Datasource added"* ]]; then
|
||||||
|
echo "Prometheus datasource injected into Grafana"
|
||||||
|
break
|
||||||
|
elif [[ "$INJECT" = *"Data source with same name already exists"* ]]; then
|
||||||
|
echo "Prometheus datasource already injected into Grafana"
|
||||||
|
break
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Check if all resources exist already before creating them
|
echo "Trying to inject Prometheus datasource into Grafana - "$INJECT
|
||||||
# Check if configmap Prometheus exists
|
done
|
||||||
kubectl get configmap prometheus -n prometheus-monitoring
|
'''
|
||||||
if [ "$?" != "0" ] && \
|
|
||||||
[ -f "'''${PROMETHEUS_MON_BASE_DIR}'''/prometheusConfigMap.yaml" ]; then
|
|
||||||
kubectl create -f '''${PROMETHEUS_MON_BASE_DIR}'''/prometheusConfigMap.yaml
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Check if deployment and service Prometheus exist
|
|
||||||
kubectl get service prometheus -n prometheus-monitoring | kubectl get deployment prometheus -n prometheus-monitoring
|
|
||||||
if [ "${PIPESTATUS[0]}" != "0" ] && [ "${PIPESTATUS[1]}" != "0" ] && \
|
|
||||||
[ -f "'''${PROMETHEUS_MON_BASE_DIR}'''/prometheusService.yaml" ]; then
|
|
||||||
kubectl create -f '''${PROMETHEUS_MON_BASE_DIR}'''/prometheusService.yaml
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Check if node exporter daemonset exist
|
|
||||||
kubectl get daemonset node-exporter -n prometheus-monitoring
|
|
||||||
if [ "$?" != "0" ] && \
|
|
||||||
[ -f "'''${PROMETHEUS_MON_BASE_DIR}'''/nodeExporter.yaml" ]; then
|
|
||||||
kubectl create -f '''${PROMETHEUS_MON_BASE_DIR}'''/nodeExporter.yaml
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Check if configmap graf-dash exists
|
|
||||||
kubectl get configmap graf-dash -n prometheus-monitoring
|
|
||||||
if [ "$?" != "0" ] && \
|
|
||||||
[ -f '''$GRAFANA_DEF_DASHBOARD_FILE''' ]; then
|
|
||||||
kubectl create configmap graf-dash --from-file='''$GRAFANA_DEF_DASHBOARD_FILE''' -n prometheus-monitoring
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Check if deployment and service Grafana exist
|
|
||||||
kubectl get service grafana -n prometheus-monitoring | kubectl get deployment grafana -n prometheus-monitoring
|
|
||||||
if [ "${PIPESTATUS[0]}" != "0" ] && [ "${PIPESTATUS[1]}" != "0" ] && \
|
|
||||||
[ -f "'''${PROMETHEUS_MON_BASE_DIR}'''/grafanaService.yaml" ]; then
|
|
||||||
kubectl create -f '''${PROMETHEUS_MON_BASE_DIR}'''/grafanaService.yaml
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Wait for Grafana pod and then inject data source
|
|
||||||
while true
|
|
||||||
do
|
|
||||||
echo "Waiting for Grafana pod to be up and Running"
|
|
||||||
if [ "$(kubectl get po -n prometheus-monitoring -l name=grafana -o jsonpath={..phase})" = "Running" ]; then
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
sleep 2
|
|
||||||
done
|
|
||||||
|
|
||||||
# Which node is running Grafana
|
|
||||||
NODE_IP=`kubectl get po -n prometheus-monitoring -o jsonpath={.items[0].status.hostIP} -l name=grafana`
|
|
||||||
PROM_SERVICE_IP=`kubectl get svc prometheus --namespace prometheus-monitoring -o jsonpath={..clusterIP}`
|
|
||||||
GRAFANA_SERVICE_IP=`kubectl get svc grafana --namespace prometheus-monitoring -o jsonpath={..clusterIP}`
|
|
||||||
|
|
||||||
# The Grafana pod might be running but the app might still be initiating
|
|
||||||
echo "Check if Grafana is ready..."
|
|
||||||
curl --user admin:$ADMIN_PASSWD -X GET http://$GRAFANA_SERVICE_IP:3000/api/datasources/1
|
|
||||||
until [ $? -eq 0 ]
|
|
||||||
do
|
|
||||||
sleep 2
|
|
||||||
curl --user admin:$ADMIN_PASSWD -X GET http://$GRAFANA_SERVICE_IP:3000/api/datasources/1
|
|
||||||
done
|
|
||||||
|
|
||||||
# Inject Prometheus datasource into Grafana
|
|
||||||
while true
|
|
||||||
do
|
|
||||||
INJECT=`curl --user admin:$ADMIN_PASSWD -X POST \
|
|
||||||
-H "Content-Type: application/json;charset=UTF-8" \
|
|
||||||
--data-binary '''"'"'''{"name":"k8sPrometheus","isDefault":true,
|
|
||||||
"type":"prometheus","url":"http://'''"'"'''$PROM_SERVICE_IP'''"'"''':9090","access":"proxy"}'''"'"'''\
|
|
||||||
"http://$GRAFANA_SERVICE_IP:3000/api/datasources/"`
|
|
||||||
|
|
||||||
if [[ "$INJECT" = *"Datasource added"* ]]; then
|
|
||||||
echo "Prometheus datasource injected into Grafana"
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
echo "Trying to inject Prometheus datasource into Grafana - "$INJECT
|
|
||||||
done
|
|
||||||
'''
|
|
||||||
writeFile $KUBE_MON_BIN "$KUBE_MON_BIN_CONTENT"
|
writeFile $KUBE_MON_BIN "$KUBE_MON_BIN_CONTENT"
|
||||||
|
|
||||||
|
|
||||||
# Write the monitoring service
|
# Write the monitoring service
|
||||||
KUBE_MON_SERVICE_CONTENT='''[Unit]
|
KUBE_MON_SERVICE_CONTENT='''[Unit]
|
||||||
Description=Enable Prometheus monitoring stack
|
Description=Enable Prometheus monitoring stack
|
||||||
|
|
||||||
[Service]
|
[Service]
|
||||||
Type=oneshot
|
Type=oneshot
|
||||||
Environment=HOME=/root
|
Environment=HOME=/root
|
||||||
EnvironmentFile=-/etc/kubernetes/config
|
EnvironmentFile=-/etc/kubernetes/config
|
||||||
ExecStart='''${KUBE_MON_BIN}'''
|
ExecStart='''${KUBE_MON_BIN}'''
|
||||||
|
|
||||||
[Install]
|
[Install]
|
||||||
WantedBy=multi-user.target
|
WantedBy=multi-user.target
|
||||||
'''
|
'''
|
||||||
writeFile $KUBE_MON_SERVICE "$KUBE_MON_SERVICE_CONTENT"
|
writeFile $KUBE_MON_SERVICE "$KUBE_MON_SERVICE_CONTENT"
|
||||||
|
|
||||||
chown root:root ${KUBE_MON_BIN}
|
chown root:root ${KUBE_MON_BIN}
|
||||||
|
|
Loading…
Reference in New Issue