Browse Source

[k8s] Install prometheus monitoring with helm

The Kubernetes Helm repository includes in its stable distribution
a prometheus-operator Chart.
This stable/prometheus-operator chart can be used to install all the
dependencies and some default configurations to use prometheus.
The installed extra charts are:
  * stable/prometheus-node-exporter (data scraping)
  * stable/prometheus (prometheus and alertmanager server)
  * stable/grafana (visualization dashboard)
  * stable/prometheus-operator (supervision and simple configuration)

The prometheus-operator is installed by using the label
monitoring_enabled=True. Also, the label grafana_admin_passwd can be
used to set the admin password for access to the grafana dashboard

This patch allows for transferral of prometheus monitoring maintenance
work to be done by the kubernetes/helm team.

Task: 28544
Story: 2004623
depends_on: I99d3a78085ba10030200f12bbfe58a72964e2326
Change-Id: I80d590785bf30f9d634debeaf51c0d4cce0aeb93
Signed-off-by: Diogo Guerra <dy090.guerra@gmail.com>
tags/8.0.0.0rc1^0
Diogo Guerra 4 months ago
parent
commit
a46d2ffc91

+ 14
- 0
doc/source/user/index.rst View File

@@ -304,6 +304,9 @@ the table are linked to more details elsewhere in the user guide.
304 304
 +---------------------------------------+--------------------+---------------+
305 305
 | `mesos_slave_executor_env_variables`_ | (file name)        | ""            |
306 306
 +---------------------------------------+--------------------+---------------+
307
+| `monitoring_enabled`_                 | - true             | false         |
308
+|                                       | - false            |               |
309
++---------------------------------------+--------------------+---------------+
307 310
 | `swarm_strategy`_                     | - spread           | spread        |
308 311
 |                                       | - binpack          |               |
309 312
 |                                       | - random           |               |
@@ -1108,6 +1111,12 @@ _`container_infra_prefix`
1108 1111
   * docker.io/prom/prometheus:latest
1109 1112
   * gcr.io/google_containers/kubernetes-dashboard-amd64:v1.5.1
1110 1113
   * gcr.io/google_containers/pause:3.0
1114
+  * gcr.io/google-containers/hyperkube:v1.12.1
1115
+  * quay.io/prometheus/alertmanager:v0.15.3
1116
+  * quay.io/coreos/prometheus-operator:v0.15.3
1117
+  * quay.io/coreos/configmap-reload:v0.0.1
1118
+  * quay.io/coreos/prometheus-config-reloader:v0.26.0
1119
+  * quay.io/prometheus/prometheus:v2.5.0
1111 1120
 
1112 1121
 _`kube_tag`
1113 1122
   This label allows users to select `a specific Kubernetes release,
@@ -1221,6 +1230,11 @@ _`k8s_keystone_auth_tag`
1221 1230
   <https://hub.docker.com/r/k8scloudprovider/k8s-keystone-auth/tags/>`_.
1222 1231
   Stein-default: 1.13.0
1223 1232
 
1233
+_`monitoring_enabled`
1234
+  Enable installation of cluster monitoring solution provided by the
1235
+  stable/prometheus-operator helm chart.
1236
+  Default: false
1237
+
1224 1238
 _`tiller_enabled`
1225 1239
   If set to true, tiller will be deployed in the kube-system namespace.
1226 1240
   Defaults to false.

+ 1
- 0
magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml View File

@@ -36,6 +36,7 @@ write_files:
36 36
       VERIFY_CA="$VERIFY_CA"
37 37
       CLUSTER_UUID="$CLUSTER_UUID"
38 38
       MAGNUM_URL="$MAGNUM_URL"
39
+      MONITORING_ENABLED="$MONITORING_ENABLED"
39 40
       VOLUME_DRIVER="$VOLUME_DRIVER"
40 41
       REGION_NAME="$REGION_NAME"
41 42
       HTTP_PROXY="$HTTP_PROXY"

+ 165
- 0
magnum/drivers/common/templates/kubernetes/helm/prometheus-operator.sh View File

@@ -0,0 +1,165 @@
1
+#!/bin/bash
2
+
3
+. /etc/sysconfig/heat-params
4
+
5
+set -ex
6
+
7
+step="prometheus-operator"
8
+printf "Starting to run ${step}\n"
9
+
10
+### Configuration
11
+###############################################################################
12
+CHART_NAME="prometheus-operator"
13
+CHART_VERSION="0.1.31"
14
+
15
+if [ "$(echo ${MONITORING_ENABLED} | tr '[:upper:]' '[:lower:]')" = "true" ]; then
16
+
17
+    # Validate if communication node <-> master is secure or insecure
18
+    PROTOCOL="https"
19
+    INSECURE_SKIP_VERIFY="False"
20
+    if [ "$TLS_DISABLED" = "True" ]; then
21
+        PROTOCOL="http"
22
+        INSECURE_SKIP_VERIFY="True"
23
+    fi
24
+
25
+    if [ "$(echo ${VERIFY_CA} | tr '[:upper:]' '[:lower:]')" == "false" ]; then
26
+        INSECURE_SKIP_VERIFY="True"
27
+    fi
28
+
29
+    HELM_MODULE_CONFIG_FILE="/srv/magnum/kubernetes/helm/${CHART_NAME}.yaml"
30
+    [ -f ${HELM_MODULE_CONFIG_FILE} ] || {
31
+        echo "Writing File: ${HELM_MODULE_CONFIG_FILE}"
32
+        mkdir -p $(dirname ${HELM_MODULE_CONFIG_FILE})
33
+        cat << EOF > ${HELM_MODULE_CONFIG_FILE}
34
+---
35
+kind: ConfigMap
36
+apiVersion: v1
37
+metadata:
38
+  name: ${CHART_NAME}-config
39
+  namespace: magnum-tiller
40
+  labels:
41
+    app: helm
42
+data:
43
+  install-${CHART_NAME}.sh: |
44
+    #!/bin/bash
45
+    set -ex
46
+    mkdir -p \${HELM_HOME}
47
+    cp /etc/helm/* \${HELM_HOME}
48
+
49
+    # HACK - Force wait because of bug https://github.com/helm/helm/issues/5170
50
+    until helm init --client-only --wait
51
+    do
52
+        sleep 5s
53
+    done
54
+    helm repo update
55
+
56
+    if [[ \$(helm history prometheus-operator | grep prometheus-operator) ]]; then
57
+        echo "${CHART_NAME} already installed on server. Continue..."
58
+        exit 0
59
+    else
60
+        helm install stable/${CHART_NAME} --namespace monitoring --name ${CHART_NAME} --version v${CHART_VERSION} --values /opt/magnum/install-${CHART_NAME}-values.yaml
61
+    fi
62
+
63
+  install-${CHART_NAME}-values.yaml:  |
64
+    nameOverride: prometheus
65
+    fullnameOverride: prometheus
66
+
67
+    alertmanager:
68
+      alertmanagerSpec:
69
+        image:
70
+          repository: ${CONTAINER_INFRA_PREFIX:-quay.io/}prometheus/alertmanager
71
+
72
+    # Dashboard
73
+    grafana:
74
+      #enabled: ${ENABLE_GRAFANA}
75
+      adminPassword: ${ADMIN_PASSWD}
76
+
77
+    kubeApiServer:
78
+      tlsConfig:
79
+        insecureSkipVerify: ${INSECURE_SKIP_VERIFY}
80
+
81
+    kubelet:
82
+      serviceMonitor:
83
+        https: ${PROTOCOL}
84
+
85
+    coreDns:
86
+      enabled: true
87
+      service:
88
+        port: 9153
89
+        targetPort: 9153
90
+        selector:
91
+          k8s-app: coredns
92
+
93
+    kubeEtcd:
94
+      service:
95
+        port: 4001
96
+        targetPort: 4001
97
+        selector:
98
+          k8s-app: etcd-server
99
+      serviceMonitor:
100
+        scheme: ${PROTOCOL}
101
+        insecureSkipVerify: ${INSECURE_SKIP_VERIFY}
102
+        ##  If Protocol is http this files should be neglected
103
+        caFile: ${CERT_DIR}/ca.crt
104
+        certFile: ${CERT_DIR}/kubelet.crt
105
+        keyFile: ${CERT_DIR}/kubelet.key
106
+
107
+    prometheusOperator:
108
+      image:
109
+        repository: ${CONTAINER_INFRA_PREFIX:-quay.io/}coreos/prometheus-operator
110
+      configmapReloadImage:
111
+        repository: ${CONTAINER_INFRA_PREFIX:-quay.io/}coreos/configmap-reload
112
+      prometheusConfigReloaderImage:
113
+        repository: ${CONTAINER_INFRA_PREFIX:-quay.io/}coreos/prometheus-config-reloader
114
+      hyperkubeImage:
115
+        repository: ${CONTAINER_INFRA_PREFIX:-gcr.io/google-containers/}hyperkube
116
+
117
+    prometheus:
118
+      prometheusSpec:
119
+        image:
120
+          repository: ${CONTAINER_INFRA_PREFIX:-quay.io/}prometheus/prometheus
121
+        retention: 14d
122
+---
123
+apiVersion: batch/v1
124
+kind: Job
125
+metadata:
126
+  name: install-${CHART_NAME}-job
127
+  namespace: magnum-tiller
128
+spec:
129
+  backoffLimit: 5
130
+  template:
131
+    spec:
132
+      serviceAccountName: tiller
133
+      containers:
134
+      - name: config-helm
135
+        image: docker.io/openstackmagnum/helm-client:dev
136
+        command:
137
+        - bash
138
+        args:
139
+        - /opt/magnum/install-${CHART_NAME}.sh
140
+        env:
141
+        - name: HELM_HOME
142
+          value: /helm_home
143
+        - name: TILLER_NAMESPACE
144
+          value: magnum-tiller
145
+        - name: HELM_TLS_ENABLE
146
+          value: "true"
147
+        volumeMounts:
148
+        - name: install-${CHART_NAME}-config
149
+          mountPath: /opt/magnum/
150
+        - mountPath: /etc/helm
151
+          name: helm-client-certs
152
+      restartPolicy: Never
153
+      volumes:
154
+      - name: install-${CHART_NAME}-config
155
+        configMap:
156
+          name: ${CHART_NAME}-config
157
+      - name: helm-client-certs
158
+        secret:
159
+          secretName: helm-client-secret
160
+EOF
161
+    }
162
+
163
+fi
164
+
165
+printf "Finished running ${step}\n"

+ 1
- 0
magnum/drivers/heat/k8s_fedora_template_def.py View File

@@ -116,6 +116,7 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition):
116 116
                       'grafana_tag',
117 117
                       'heat_container_agent_tag',
118 118
                       'keystone_auth_enabled', 'k8s_keystone_auth_tag',
119
+                      'monitoring_enabled',
119 120
                       'tiller_enabled',
120 121
                       'tiller_tag',
121 122
                       'tiller_namespace',

+ 13
- 2
magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml View File

@@ -540,6 +540,11 @@ parameters:
540 540
     description: tag of the k8s_keystone_auth container
541 541
     default: 1.13.0
542 542
 
543
+  monitoring_enabled:
544
+    type: boolean
545
+    description: Enable or disable prometheus-operator monitoring solution.
546
+    default: false
547
+
543 548
   project_id:
544 549
     type: string
545 550
     description: >
@@ -824,6 +829,7 @@ resources:
824 829
           heat_container_agent_tag: {get_param: heat_container_agent_tag}
825 830
           keystone_auth_enabled: {get_param: keystone_auth_enabled}
826 831
           k8s_keystone_auth_tag: {get_param: k8s_keystone_auth_tag}
832
+          monitoring_enabled: {get_param: monitoring_enabled}
827 833
           project_id: {get_param: project_id}
828 834
           tiller_enabled: {get_param: tiller_enabled}
829 835
           tiller_tag: {get_param: tiller_tag}
@@ -847,8 +853,6 @@ resources:
847 853
             - get_file: ../../common/templates/kubernetes/fragments/calico-service.sh
848 854
             - get_file: ../../common/templates/kubernetes/fragments/flannel-service.sh
849 855
             - get_file: ../../common/templates/kubernetes/fragments/enable-helm-tiller.sh
850
-            - get_file: ../../common/templates/kubernetes/helm/metrics-server.sh
851
-            - get_file: ../../common/templates/kubernetes/fragments/install-helm-modules.sh
852 856
             - str_replace:
853 857
                 template: {get_file: ../../common/templates/kubernetes/fragments/enable-prometheus-monitoring.sh}
854 858
                 params:
@@ -861,6 +865,13 @@ resources:
861 865
             - get_file: ../../common/templates/kubernetes/fragments/kube-dashboard-service.sh
862 866
             - get_file: ../../common/templates/kubernetes/fragments/enable-keystone-auth.sh
863 867
             - get_file: ../../common/templates/kubernetes/fragments/enable-auto-healing.sh
868
+            # Helm Based Installation Configuration Scripts
869
+            - get_file: ../../common/templates/kubernetes/helm/metrics-server.sh
870
+            - str_replace:
871
+                template: {get_file:  ../../common/templates/kubernetes/helm/prometheus-operator.sh}
872
+                params:
873
+                  "${ADMIN_PASSWD}": {get_param: grafana_admin_passwd}
874
+            - get_file: ../../common/templates/kubernetes/fragments/install-helm-modules.sh
864 875
 
865 876
   kube_cluster_deploy:
866 877
     type: OS::Heat::SoftwareDeployment

+ 6
- 0
magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml View File

@@ -417,6 +417,11 @@ parameters:
417 417
     type: string
418 418
     description: tag of the k8s_keystone_auth container
419 419
 
420
+  monitoring_enabled:
421
+    type: boolean
422
+    description: Enable or disable prometheus-operator monitoring solution.
423
+    default: false
424
+
420 425
   project_id:
421 426
     type: string
422 427
     description: >
@@ -543,6 +548,7 @@ resources:
543 548
             "$HEAT_CONTAINER_AGENT_TAG": {get_param: heat_container_agent_tag}
544 549
             "$KEYSTONE_AUTH_ENABLED": {get_param: keystone_auth_enabled}
545 550
             "$K8S_KEYSTONE_AUTH_TAG": {get_param: k8s_keystone_auth_tag}
551
+            "$MONITORING_ENABLED": {get_param: monitoring_enabled}
546 552
             "$PROJECT_ID": {get_param: project_id}
547 553
             "$EXTERNAL_NETWORK_ID": {get_param: external_network}
548 554
             "$TILLER_ENABLED": {get_param: tiller_enabled}

+ 6
- 0
magnum/tests/unit/drivers/test_template_definition.py View File

@@ -485,6 +485,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
485 485
             'keystone_auth_enabled')
486 486
         k8s_keystone_auth_tag = mock_cluster.labels.get(
487 487
             'k8s_keystone_auth_tag')
488
+        monitoring_enabled = mock_cluster.labels.get(
489
+            'monitoring_enabled')
488 490
         project_id = mock_cluster.project_id
489 491
         tiller_enabled = mock_cluster.labels.get(
490 492
             'tiller_enabled')
@@ -549,6 +551,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
549 551
             'heat_container_agent_tag': heat_container_agent_tag,
550 552
             'keystone_auth_enabled': keystone_auth_enabled,
551 553
             'k8s_keystone_auth_tag': k8s_keystone_auth_tag,
554
+            'monitoring_enabled': monitoring_enabled,
552 555
             'project_id': project_id,
553 556
             'external_network': external_network_id,
554 557
             'tiller_enabled': tiller_enabled,
@@ -855,6 +858,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
855 858
             'keystone_auth_enabled')
856 859
         k8s_keystone_auth_tag = mock_cluster.labels.get(
857 860
             'k8s_keystone_auth_tag')
861
+        monitoring_enabled = mock_cluster.labels.get(
862
+            'monitoring_enabled')
858 863
         project_id = mock_cluster.project_id
859 864
         tiller_enabled = mock_cluster.labels.get(
860 865
             'tiller_enabled')
@@ -921,6 +926,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
921 926
             'heat_container_agent_tag': heat_container_agent_tag,
922 927
             'keystone_auth_enabled': keystone_auth_enabled,
923 928
             'k8s_keystone_auth_tag': k8s_keystone_auth_tag,
929
+            'monitoring_enabled': monitoring_enabled,
924 930
             'project_id': project_id,
925 931
             'external_network': external_network_id,
926 932
             'tiller_enabled': tiller_enabled,

+ 8
- 0
releasenotes/notes/helm-install-prometheus-operator-ea87752bc57a0945.yaml View File

@@ -0,0 +1,8 @@
1
+---
2
+features:
3
+  - |
4
+    Added monitoring_enabled to install prometheus-operator monitoring
5
+    solution by means of helm stable/prometheus-operator public chart.
6
+    Defaults to false. grafana_admin_passwd label can be used to set
7
+    grafana dashboard admin access password. If grafana_admin_passwd
8
+    is not set the password defaults to prom_operator.

Loading…
Cancel
Save