Add run migrator job prior running grafana pods

During the first run, grafana will run migrator job, which populates
necessary fields in database. Previously, if there are two or more
grafana pods which start up simultaneously, it causes the race condition
for database access and finally one of the pods will fail (in some
cases both of them), leaving the grafana database in incomplete state.

Signed-off-by: Ruslan Aliev <raliev@mirantis.com>
Change-Id: I5a7993b3cad2d48af3f73218d6c61c216520e1c5
This commit is contained in:
Ruslan Aliev 2022-06-14 15:34:34 -05:00
parent fce7ca38ae
commit e99dfc1c84
6 changed files with 200 additions and 4 deletions

View File

@ -15,7 +15,7 @@ apiVersion: v1
appVersion: v7.4.5
description: OpenStack-Helm Grafana
name: grafana
version: 0.1.13
version: 0.1.14
home: https://grafana.com/
sources:
- https://github.com/grafana/grafana

View File

@ -13,15 +13,29 @@ See the License for the specific language governing permissions and
limitations under the License.
*/}}
set -ex
set -exo pipefail
COMMAND="${@:-start}"
PORT={{ tuple "grafana" "internal" "grafana" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
PIDFILE=/tmp/pid
function start () {
exec /usr/share/grafana/bin/grafana-server -homepath=/usr/share/grafana -config=/etc/grafana/grafana.ini
exec /usr/share/grafana/bin/grafana-server -homepath=/usr/share/grafana -config=/etc/grafana/grafana.ini --pidfile="$PIDFILE"
}
function run_migrator () {
start &
timeout 60 bash -c "until timeout 5 bash -c '</dev/tcp/127.0.0.1/${PORT}'; do sleep 1; done"
stop
}
function stop () {
kill -TERM 1
if [ -f "$PIDFILE" ]; then
echo -e "Found pidfile, killing running grafana-server"
kill -9 `cat $PIDFILE`
rm $PIDFILE
else
kill -TERM 1
fi
}
$COMMAND

View File

@ -0,0 +1,156 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.deployment }}
{{- $envAll := . }}
{{- $mounts_grafana := .Values.pod.mounts.grafana.grafana }}
{{- $serviceAccountName := "grafana-run-migrator" }}
{{ tuple $envAll "run_migrator" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
---
apiVersion: batch/v1
kind: Job
metadata:
name: grafana-run-migrator
annotations:
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
labels:
{{ tuple $envAll "grafana" "run-migrator" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
spec:
template:
metadata:
labels:
{{ tuple $envAll "grafana" "run-migrator" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
annotations:
configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
{{ dict "envAll" $envAll "podName" "grafana-run-migrator" "containerNames" (list "grafana-run-migrator" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
spec:
{{ dict "envAll" $envAll "application" "run_migrator" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
serviceAccountName: {{ $serviceAccountName }}
restartPolicy: OnFailure
nodeSelector:
{{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value | quote }}
initContainers:
{{ tuple $envAll "run_migrator" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
containers:
- name: grafana-run-migrator
{{ tuple $envAll "grafana" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.run_migrator | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
{{ dict "envAll" $envAll "application" "run_migrator" "container" "grafana_run_migrator" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
command:
- /tmp/grafana.sh
- run_migrator
ports:
- name: dashboard
containerPort: {{ tuple "grafana" "internal" "grafana" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
readinessProbe:
httpGet:
path: /login
port: {{ tuple "grafana" "internal" "grafana" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
initialDelaySeconds: 30
timeoutSeconds: 30
env:
- name: GF_SECURITY_ADMIN_USER
valueFrom:
secretKeyRef:
name: grafana-admin-creds
key: GRAFANA_ADMIN_USERNAME
- name: GF_SECURITY_ADMIN_PASSWORD
valueFrom:
secretKeyRef:
name: grafana-admin-creds
key: GRAFANA_ADMIN_PASSWORD
- name: PROMETHEUS_URL
value: {{ tuple "monitoring" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}
{{- if .Values.manifests.certificates }}
- name: CACERT
valueFrom:
secretKeyRef:
key: ca.crt
name: prometheus-tls-api
{{- end }}
{{- if .Values.pod.env.grafana }}
{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.grafana | indent 12 }}
{{- end }}
volumeMounts:
- name: pod-tmp
mountPath: /tmp
- name: pod-etc-grafana
mountPath: /etc/grafana
- name: pod-screenshots-grafana
mountPath: /var/lib/grafana/png
- name: pod-dashboards-grafana
mountPath: /etc/grafana/dashboards
- name: pod-provisioning-grafana
mountPath: {{ .Values.conf.grafana.paths.provisioning }}
- name: grafana-bin
mountPath: /tmp/grafana.sh
subPath: grafana.sh
readOnly: true
- name: grafana-etc
mountPath: {{ .Values.conf.grafana.paths.provisioning }}/dashboards/dashboards.yaml
subPath: dashboards.yaml
- name: grafana-etc
mountPath: {{ .Values.conf.grafana.paths.provisioning }}/datasources/datasources.yaml
subPath: datasources.yaml
- name: grafana-etc
mountPath: /etc/grafana/grafana.ini
subPath: grafana.ini
- name: grafana-etc
mountPath: /etc/grafana/ldap.toml
subPath: ldap.toml
- name: data
mountPath: /var/lib/grafana/data
{{- range $group, $dashboards := .Values.conf.dashboards }}
{{- range $key, $value := $dashboards }}
- name: grafana-dashboards-{{$group}}
mountPath: /etc/grafana/dashboards/{{$key}}.json
subPath: {{$key}}.json
{{- end }}
{{- end }}
{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.endpoints.oslo_db.auth.admin.secret.tls.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
{{ if $mounts_grafana.volumeMounts }}{{ toYaml $mounts_grafana.volumeMounts | indent 12 }}{{ end }}
volumes:
- name: pod-tmp
emptyDir: {}
- name: pod-etc-grafana
emptyDir: {}
- name: pod-screenshots-grafana
emptyDir: {}
- name: pod-dashboards-grafana
emptyDir: {}
- name: pod-provisioning-grafana
emptyDir: {}
- name: grafana-bin
configMap:
name: grafana-bin
defaultMode: 0555
- name: grafana-etc
secret:
secretName: grafana-etc
defaultMode: 0444
{{- range $group, $dashboards := .Values.conf.dashboards }}
- name: grafana-dashboards-{{$group}}
configMap:
name: grafana-dashboards-{{$group}}
defaultMode: 0555
{{- end }}
- name: data
emptyDir: {}
{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.endpoints.oslo_db.auth.admin.secret.tls.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
{{ if $mounts_grafana.volumes }}{{ toYaml $mounts_grafana.volumes | indent 8 }}{{ end }}
{{- end }}

View File

@ -76,6 +76,13 @@ pod:
grafana_set_admin_password:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
run_migrator:
pod:
runAsUser: 104
container:
grafana_set_admin_password:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
test:
pod:
runAsUser: 104
@ -153,6 +160,13 @@ pod:
limits:
memory: "1024Mi"
cpu: "2000m"
run_migrator:
requests:
memory: "128Mi"
cpu: "100m"
limits:
memory: "1024Mi"
cpu: "2000m"
tests:
requests:
memory: "128Mi"
@ -310,6 +324,7 @@ dependencies:
- grafana-db-init
- grafana-db-session-sync
- grafana-set-admin-user
- grafana-run-migrator
services:
- endpoint: internal
service: oslo_db
@ -323,6 +338,12 @@ dependencies:
services:
- endpoint: internal
service: oslo_db
run_migrator:
jobs:
- grafana-set-admin-user
services:
- endpoint: internal
service: oslo_db
tests:
services:
- endpoint: internal
@ -375,6 +396,7 @@ manifests:
job_db_session_sync: true
job_image_repo_sync: true
job_set_admin_user: true
job_run_migrator: true
network_policy: false
secret_db: true
secret_db_session: true

View File

@ -17,6 +17,9 @@ pod:
grafana-set-admin-user:
grafana-set-admin-password: runtime/default
init: runtime/default
grafana-run-migrator:
grafana-run-migrator: runtime/default
init: runtime/default
grafana-test:
init: runtime/default
grafana-selenium-tests: runtime/default

View File

@ -14,4 +14,5 @@ grafana:
- 0.1.11 Update htk requirements
- 0.1.12 Add iDRAC dashboard to Grafana
- 0.1.13 Update prometheus metric name
- 0.1.14 Add run migrator job
...