@ -1,4 +1,7 @@
#!/bin/bash
#!/bin/sh
step = "enable-prometheus-monitoring"
printf " Starting to run ${ step } \n "
. /etc/sysconfig/heat-params
@ -361,144 +364,136 @@ writeFile $grafanaService_file "$grafanaService_content"
. /etc/sysconfig/heat-params
# NOTE(flwang): Let's keep the same addons yaml file on all masters,
# but if it's not the primary/bootstrapping master, don't try to
# create those resources to avoid race condition issue until the
# kubectl issue https://github.com/kubernetes/kubernetes/issues/44165
# fixed.
if [ " $MASTER_INDEX " != "0" ] ; then
exit 0
fi
if [ " $( echo $PROMETHEUS_MONITORING | tr '[:upper:]' '[:lower:]' ) " = "false" ] ; then
exit 0
fi
PROMETHEUS_MON_BASE_DIR = "/srv/magnum/kubernetes/monitoring"
KUBE_MON_BIN = ${ PROMETHEUS_MON_BASE_DIR } "/bin/kube-enable-monitoring"
KUBE_MON_SERVICE = "/etc/systemd/system/kube-enable-monitoring.service"
GRAFANA_DEF_DASHBOARDS = ${ PROMETHEUS_MON_BASE_DIR } "/dashboards"
GRAFANA_DEF_DASHBOARD_FILE = $GRAFANA_DEF_DASHBOARDS "/default.json"
# Write the binary for enable-monitoring
KUBE_MON_BIN_CONTENT = '' ' #!/bin/sh
until [ "ok" = " $( curl --silent http://127.0.0.1:8080/healthz) " ]
do
echo "Waiting for Kubernetes API..."
sleep 5
done
# Check if prometheus-monitoring namespace exist already before creating the namespace
kubectl get namespace prometheus-monitoring
if [ " $? " != "0" ] && \
[ -f " ''' ${ PROMETHEUS_MON_BASE_DIR } '''/prometheusNamespace.yaml " ] ; then
kubectl create -f '' '${PROMETHEUS_MON_BASE_DIR}' '' /prometheusNamespace.yaml
fi
# Check if all resources exist already before creating them
# Check if configmap Prometheus exists
kubectl get configmap prometheus -n prometheus-monitoring
if [ " $? " != "0" ] && \
[ -f " ''' ${ PROMETHEUS_MON_BASE_DIR } '''/prometheusConfigMap.yaml " ] ; then
kubectl create -f '' '${PROMETHEUS_MON_BASE_DIR}' '' /prometheusConfigMap.yaml
fi
if [ " $( echo $PROMETHEUS_MONITORING | tr '[:upper:]' '[:lower:]' ) " = "true" ] ; then
PROMETHEUS_MON_BASE_DIR = "/srv/magnum/kubernetes/monitoring"
KUBE_MON_BIN = ${ PROMETHEUS_MON_BASE_DIR } "/bin/kube-enable-monitoring"
KUBE_MON_SERVICE = "/etc/systemd/system/kube-enable-monitoring.service"
GRAFANA_DEF_DASHBOARDS = ${ PROMETHEUS_MON_BASE_DIR } "/dashboards"
GRAFANA_DEF_DASHBOARD_FILE = $GRAFANA_DEF_DASHBOARDS "/default.json"
# Write the binary for enable-monitoring
KUBE_MON_BIN_CONTENT = '' ' #!/bin/sh
until [ "ok" = " $( curl --silent http://127.0.0.1:8080/healthz) " ]
do
echo "Waiting for Kubernetes API..."
sleep 5
done
# Check if prometheus-monitoring namespace exist already before creating the namespace
kubectl get namespace prometheus-monitoring
if [ " $? " != "0" ] && \
[ -f " ''' ${ PROMETHEUS_MON_BASE_DIR } '''/prometheusNamespace.yaml " ] ; then
kubectl create -f '' '${PROMETHEUS_MON_BASE_DIR}' '' /prometheusNamespace.yaml
fi
# Check if deployment and service Prometheus exist
kubectl get service prometheus -n prometheus-monitoring | kubectl get deployment prometheus -n prometheus-monitoring
if [ " ${ PIPESTATUS [0] } " != "0" ] && [ " ${ PIPESTATUS [1] } " != "0" ] && \
[ -f " ''' ${ PROMETHEUS_MON_BASE_DIR } '''/prometheusService.yaml " ] ; then
kubectl create -f '' '${PROMETHEUS_MON_BASE_DIR}' '' /prometheusService.yaml
fi
# Check if all resources exist already before creating them
# Check if configmap Prometheus exists
kubectl get configmap prometheus -n prometheus-monitoring
if [ " $? " != "0" ] && \
[ -f " ''' ${ PROMETHEUS_MON_BASE_DIR } '''/prometheusConfigMap.yaml " ] ; then
kubectl create -f '' '${PROMETHEUS_MON_BASE_DIR}' '' /prometheusConfigMap.yaml
fi
# Check if configmap graf-dash exists
kubectl get configmap graf-dash -n prometheus-monitoring
if [ " $? " != "0" ] && \
[ -f '' '$GRAFANA_DEF_DASHBOARD_FILE' '' ] ; then
kubectl create configmap graf-dash --from-file= '' '$GRAFANA_DEF_DASHBOARD_FILE' '' -n prometheus-monitoring
fi
# Check if deployment and service Prometheus exist
kubectl get service prometheus -n prometheus-monitoring | kubectl get deployment prometheus -n prometheus-monitoring
if [ " ${ PIPESTATUS [0] } " != "0" ] && [ " ${ PIPESTATUS [1] } " != "0" ] && \
[ -f " ''' ${ PROMETHEUS_MON_BASE_DIR } '''/prometheusService.yaml " ] ; then
kubectl create -f '' '${PROMETHEUS_MON_BASE_DIR}' '' /prometheusService.yaml
fi
# Check if deployment and service Grafana exist
kubectl get service grafana -n prometheus-monitoring | kubectl get deployment grafana -n prometheus-monitoring
if [ " ${ PIPESTATUS [0] } " != "0" ] && [ " ${ PIPESTATUS [1] } " != "0" ] && \
[ -f " ''' ${ PROMETHEUS_MON_BASE_DIR } '''/grafanaService.yaml " ] ; then
kubectl create -f '' '${PROMETHEUS_MON_BASE_DIR}' '' /grafanaService.yaml
fi
# Check if configmap graf-dash exists
kubectl get configmap graf-dash -n prometheus-monitoring
if [ " $? " != "0" ] && \
[ -f '' '$GRAFANA_DEF_DASHBOARD_FILE' '' ] ; then
kubectl create configmap graf-dash --from-file= '' '$GRAFANA_DEF_DASHBOARD_FILE' '' -n prometheus-monitoring
fi
# Wait for Grafana pod and then inject data source
while true
do
echo "Waiting for Grafana pod to be up and Running"
if [ " $( kubectl get po -n prometheus-monitoring -l name = grafana -o jsonpath = { ..phase} ) " = "Running" ] ; then
break
# Check if deployment and service Grafana exist
kubectl get service grafana -n prometheus-monitoring | kubectl get deployment grafana -n prometheus-monitoring
if [ " ${ PIPESTATUS [0] } " != "0" ] && [ " ${ PIPESTATUS [1] } " != "0" ] && \
[ -f " ''' ${ PROMETHEUS_MON_BASE_DIR } '''/grafanaService.yaml " ] ; then
kubectl create -f '' '${PROMETHEUS_MON_BASE_DIR}' '' /grafanaService.yaml
fi
sleep 2
done
# Which node is running Grafana
NODE_IP = ` kubectl get po -n prometheus-monitoring -o jsonpath = { .items[ 0] .status.hostIP} -l name = grafana`
PROM_SERVICE_IP = ` kubectl get svc prometheus --namespace prometheus-monitoring -o jsonpath = { ..clusterIP} `
# The Grafana pod might be running but the app might still be initiating
echo "Check if Grafana is ready..."
curl --user admin:$ADMIN_PASSWD -X GET http://$NODE_IP :3000/api/datasources/1
until [ $? -eq 0 ]
do
sleep 2
# Wait for Grafana pod and then inject data source
while true
do
echo "Waiting for Grafana pod to be up and Running"
if [ " $( kubectl get po -n prometheus-monitoring -l name = grafana -o jsonpath = { ..phase} ) " = "Running" ] ; then
break
fi
sleep 2
done
# Which node is running Grafana
NODE_IP = ` kubectl get po -n prometheus-monitoring -o jsonpath = { .items[ 0] .status.hostIP} -l name = grafana`
PROM_SERVICE_IP = ` kubectl get svc prometheus --namespace prometheus-monitoring -o jsonpath = { ..clusterIP} `
# The Grafana pod might be running but the app might still be initiating
echo "Check if Grafana is ready..."
curl --user admin:$ADMIN_PASSWD -X GET http://$NODE_IP :3000/api/datasources/1
done
# Inject Prometheus datasource into Grafana
while true
do
INJECT = ` curl --user admin:$ADMIN_PASSWD -X POST \
-H "Content-Type: application/json;charset=UTF-8" \
--data-binary '' '"' "'''{" name":" k8sPrometheus"," isDefault" :true,
"type" :"prometheus" ,"url" :"http://'''" '"' '' $PROM_SERVICE_IP '' '"' "''':9090" ,"access" :"proxy" } '' '"' " ''' \
" http:// $NODE_IP :3000/api/datasources/ " `
if [ [ " $INJECT " = *"Datasource added" * ] ] ; then
echo "Prometheus datasource injected into Grafana"
break
until [ $? -eq 0 ]
do
sleep 2
curl --user admin:$ADMIN_PASSWD -X GET http://$NODE_IP :3000/api/datasources/1
done
# Inject Prometheus datasource into Grafana
while true
do
INJECT = ` curl --user admin:$ADMIN_PASSWD -X POST \
-H "Content-Type: application/json;charset=UTF-8" \
--data-binary '' '"' "'''{" name":" k8sPrometheus"," isDefault" :true,
"type" :"prometheus" ,"url" :"http://'''" '"' '' $PROM_SERVICE_IP '' '"' "''':9090" ,"access" :"proxy" } '' '"' " ''' \
" http:// $NODE_IP :3000/api/datasources/ " `
if [ [ " $INJECT " = *"Datasource added" * ] ] ; then
echo "Prometheus datasource injected into Grafana"
break
fi
echo "Trying to inject Prometheus datasource into Grafana - " $INJECT
done
'' '
writeFile $KUBE_MON_BIN " $KUBE_MON_BIN_CONTENT "
# Write the monitoring service
KUBE_MON_SERVICE_CONTENT = '' ' [ Unit]
Description = Enable Prometheus monitoring stack
[ Service]
Type = oneshot
Environment = HOME = /root
EnvironmentFile = -/etc/kubernetes/config
ExecStart = '' '${KUBE_MON_BIN}' ''
[ Install]
WantedBy = multi-user.target
'' '
writeFile $KUBE_MON_SERVICE " $KUBE_MON_SERVICE_CONTENT "
chown root:root ${ KUBE_MON_BIN }
chmod 0755 ${ KUBE_MON_BIN }
chown root:root ${ KUBE_MON_SERVICE }
chmod 0644 ${ KUBE_MON_SERVICE }
# Download the default JSON Grafana dashboard
# Not a crucial step, so allow it to fail
# TODO: this JSON should be passed into the minions as gzip in cloud-init
GRAFANA_DASHB_URL = "https://grafana.net/api/dashboards/1621/revisions/1/download"
mkdir -p $GRAFANA_DEF_DASHBOARDS
curl $GRAFANA_DASHB_URL -o $GRAFANA_DEF_DASHBOARD_FILE || echo "Failed to fetch default Grafana dashboard"
if [ -f $GRAFANA_DEF_DASHBOARD_FILE ] ; then
sed -i -- 's|${DS_PROMETHEUS}|k8sPrometheus|g' $GRAFANA_DEF_DASHBOARD_FILE
fi
echo "Trying to inject Prometheus datasource into Grafana - " $INJECT
done
'' '
writeFile $KUBE_MON_BIN " $KUBE_MON_BIN_CONTENT "
# Write the monitoring service
KUBE_MON_SERVICE_CONTENT = '' ' [ Unit]
Description = Enable Prometheus monitoring stack
[ Service]
Type = oneshot
Environment = HOME = /root
EnvironmentFile = -/etc/kubernetes/config
ExecStart = '' '${KUBE_MON_BIN}' ''
[ Install]
WantedBy = multi-user.target
'' '
writeFile $KUBE_MON_SERVICE " $KUBE_MON_SERVICE_CONTENT "
chown root:root ${ KUBE_MON_BIN }
chmod 0755 ${ KUBE_MON_BIN }
chown root:root ${ KUBE_MON_SERVICE }
chmod 0644 ${ KUBE_MON_SERVICE }
# Download the default JSON Grafana dashboard
# Not a crucial step, so allow it to fail
# TODO: this JSON should be passed into the minions as gzip in cloud-init
GRAFANA_DASHB_URL = "https://grafana.net/api/dashboards/1621/revisions/1/download"
mkdir -p $GRAFANA_DEF_DASHBOARDS
curl $GRAFANA_DASHB_URL -o $GRAFANA_DEF_DASHBOARD_FILE || echo "Failed to fetch default Grafana dashboard"
if [ -f $GRAFANA_DEF_DASHBOARD_FILE ] ; then
sed -i -- 's|${DS_PROMETHEUS}|k8sPrometheus|g' $GRAFANA_DEF_DASHBOARD_FILE
# Launch the monitoring service
set -x
systemctl daemon-reload
systemctl enable kube-enable-monitoring.service
systemctl start --no-block kube-enable-monitoring.service
fi
# Launch the monitoring service
set -x
systemctl daemon-reload
systemctl enable kube-enable-monitoring.service
systemctl start --no-block kube-enable-monitoring.service
printf " Finished running ${ step } \n "
Lingxian Kong
5 years ago