Grafana: Support multiple datasources

This updates the Grafana chart to support the definition of
multiple datasources. This moves to defining a template in the
chart's values.yaml file that allows for inline gotpl for
defining an arbitrary number of datasources. This also updates the
grafana dashboards to include a selector for the Prometheus
datasource to use via a drop down selector. This is vetted out in
the federated monitoring job

Change-Id: I55171fed5c2b343130d135d0b42bc96ff11c4712
Signed-off-by: Steve Wilkerson <sw5822@att.com>

grafana/templates/configmap-etc.yaml

@@ -35,9 +35,9 @@ metadata:
   name: grafana-etc
 type: Opaque
 data:
-  datasources.yaml: {{ include "grafana.utils.generate_datasources" (dict "envAll" $envAll "datasources" .Values.conf.provisioning.datasources) | b64enc }}
   dashboards.yaml: {{ toYaml .Values.conf.provisioning.dashboards | b64enc }}
   grafana.ini: {{ include "helm-toolkit.utils.to_ini" .Values.conf.grafana | b64enc }}
+{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.provisioning.datasources.template "key" "datasources.yaml" "format" "Secret") | indent 2 }}
 {{ if not (empty .Values.conf.ldap) }}
 {{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.ldap.template "key" "ldap.toml" "format" "Secret") | indent 2 }}
 {{ end }}

grafana/templates/utils/_generate_datasources.tpl

@@ -1,45 +0,0 @@
-{{/*
-Copyright 2017 The Openstack-Helm Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/}}
-
-# This function generates the required datasource configuration for grafana.
-# This allows us to generate an arbitrary number of datasources for grafana
-
-{{- define "grafana.utils.generate_datasources" -}}
-{{- $envAll := index . "envAll" -}}
-{{- $datasources := index . "datasources" -}}
-{{- $_ := set $envAll.Values "__datasources" ( list ) }}
-{{- range $datasource, $config := $datasources -}}
-{{- if empty $config.url -}}
-{{- $datasource_url := tuple $datasource "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}
-{{- $_ := set $config "url" $datasource_url }}
-{{- end }}
-{{- if and ($config.basicAuth) (empty $config.basicAuthUser) -}}
-{{- $datasource_endpoint := index $envAll.Values.endpoints $datasource -}}
-{{- $datasource_user :=  $datasource_endpoint.auth.user.username -}}
-{{- $_ := set $config "basicAuthUser" $datasource_user -}}
-{{- end }}
-{{- if and ($config.basicAuth) (empty $config.basicAuthPassword) -}}
-{{- $datasource_endpoint := index $envAll.Values.endpoints $datasource -}}
-{{- $datasource_password :=  $datasource_endpoint.auth.user.password -}}
-{{- $_ := set $config "basicAuthPassword" $datasource_password -}}
-{{- end }}
-{{- $__datasources := append $envAll.Values.__datasources $config }}
-{{- $_ := set $envAll.Values "__datasources" $__datasources }}
-{{- end }}
-apiVersion: 1
-datasources:
-{{ toYaml $envAll.Values.__datasources }}
-{{- end -}}

tools/deployment/federated-monitoring/080-mariadb.sh

@@ -0,0 +1 @@
+../osh-infra-monitoring/045-mariadb.sh

tools/deployment/federated-monitoring/090-grafana.sh

@@ -0,0 +1,167 @@
+#!/bin/bash
+
+# Copyright 2017 The Openstack-Helm Authors.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+set -xe
+
+#NOTE: Lint and package chart
+make grafana
+
+tee /tmp/grafana.yaml << EOF
+endpoints:
+  monitoring_one:
+    name: prometheus-one
+    namespace: osh-infra
+    auth:
+      user:
+        username: admin
+        password: changeme
+    hosts:
+      default: prom-metrics-one
+      public: prometheus-one
+    host_fqdn_override:
+      default: null
+    path:
+      default: null
+    scheme:
+      default: http
+    port:
+      api:
+        default: 80
+        public: 80
+  monitoring_two:
+    name: prometheus-two
+    namespace: osh-infra
+    auth:
+      user:
+        username: admin
+        password: changeme
+    hosts:
+      default: prom-metrics-two
+      public: prometheus-two
+    host_fqdn_override:
+      default: null
+    path:
+      default: null
+    scheme:
+      default: http
+    port:
+      api:
+        default: 80
+        public: 80
+  monitoring_three:
+    name: prometheus-three
+    namespace: osh-infra
+    auth:
+      user:
+        username: admin
+        password: changeme
+    hosts:
+      default: prom-metrics-three
+      public: prometheus-three
+    host_fqdn_override:
+      default: null
+    path:
+      default: null
+    scheme:
+      default: http
+    port:
+      api:
+        default: 80
+        public: 80
+  monitoring_federated:
+    name: prometheus-federate
+    namespace: osh-infra
+    auth:
+      user:
+        username: admin
+        password: changeme
+    hosts:
+      default: prom-metrics-federate
+      public: prometheus-federate
+    host_fqdn_override:
+      default: null
+    path:
+      default: null
+    scheme:
+      default: http
+    port:
+      api:
+        default: 80
+        public: 80
+conf:
+  provisioning:
+    datasources:
+      template: |
+        apiVersion: 1
+        datasources:
+        - name: prometheus-one
+          type: prometheus
+          access: proxy
+          orgId: 1
+          editable: false
+          basicAuth: true
+          basicAuthUser: admin
+          secureJsonData:
+            basicAuthPassword: changeme
+          url: {{ tuple "monitoring_one" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}
+        - name: prometheus-two
+          type: prometheus
+          access: proxy
+          orgId: 1
+          editable: false
+          basicAuth: true
+          basicAuthUser: admin
+          secureJsonData:
+            basicAuthPassword: changeme
+          url: {{ tuple "monitoring_two" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}
+        - name: prometheus-three
+          type: prometheus
+          access: proxy
+          orgId: 1
+          editable: false
+          basicAuth: true
+          basicAuthUser: admin
+          secureJsonData:
+            basicAuthPassword: changeme
+          url: {{ tuple "monitoring_three" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}
+        - name: prometheus-federated
+          type: prometheus
+          access: proxy
+          orgId: 1
+          editable: false
+          basicAuth: true
+          basicAuthUser: admin
+          secureJsonData:
+            basicAuthPassword: changeme
+          url: {{ tuple "monitoring_federated" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}
+
+EOF
+
+#NOTE: Deploy command
+helm upgrade --install grafana ./grafana \
+    --namespace=osh-infra \
+    --values=/tmp/grafana.yaml
+
+#NOTE: Wait for deploy
+./tools/deployment/common/wait-for-pods.sh osh-infra
+
+#NOTE: Validate Deployment info
+helm status grafana
+
+helm test grafana
+
+echo "Get list of all configured datasources in Grafana"
+curl -u admin:password http://grafana.osh-infra.svc.cluster.local/api/datasources | jq -r .

zuul.d/jobs.yaml

@@ -190,6 +190,8 @@
         - ./tools/deployment/federated-monitoring/050-node-exporter.sh
         - ./tools/deployment/federated-monitoring/060-prometheus.sh
         - ./tools/deployment/federated-monitoring/070-federated-prometheus.sh
+        - ./tools/deployment/federated-monitoring/080-mariadb.sh
+        - ./tools/deployment/federated-monitoring/090-grafana.sh
         - ./tools/deployment/federated-monitoring/100-prometheus-selenium.sh || true
 
 - job: