[k8s] Install prometheus monitoring with helm
The Kubernetes Helm repository includes in its stable distribution a prometheus-operator Chart. This stable/prometheus-operator chart can be used to install all the dependencies and some default configurations to use prometheus. The installed extra charts are: * stable/prometheus-node-exporter (data scraping) * stable/prometheus (prometheus and alertmanager server) * stable/grafana (visualization dashboard) * stable/prometheus-operator (supervision and simple configuration) The prometheus-operator is installed by using the label monitoring_enabled=True. Also, the label grafana_admin_passwd can be used to set the admin password for access to the grafana dashboard This patch allows for transferral of prometheus monitoring maintenance work to be done by the kubernetes/helm team. Task: 28544 Story: 2004623 depends_on: I99d3a78085ba10030200f12bbfe58a72964e2326 Change-Id: I80d590785bf30f9d634debeaf51c0d4cce0aeb93 Signed-off-by: Diogo Guerra <dy090.guerra@gmail.com>
This commit is contained in:
parent
d1957c71dc
commit
a46d2ffc91
|
@ -304,6 +304,9 @@ the table are linked to more details elsewhere in the user guide.
|
||||||
+---------------------------------------+--------------------+---------------+
|
+---------------------------------------+--------------------+---------------+
|
||||||
| `mesos_slave_executor_env_variables`_ | (file name) | "" |
|
| `mesos_slave_executor_env_variables`_ | (file name) | "" |
|
||||||
+---------------------------------------+--------------------+---------------+
|
+---------------------------------------+--------------------+---------------+
|
||||||
|
| `monitoring_enabled`_ | - true | false |
|
||||||
|
| | - false | |
|
||||||
|
+---------------------------------------+--------------------+---------------+
|
||||||
| `swarm_strategy`_ | - spread | spread |
|
| `swarm_strategy`_ | - spread | spread |
|
||||||
| | - binpack | |
|
| | - binpack | |
|
||||||
| | - random | |
|
| | - random | |
|
||||||
|
@ -1108,6 +1111,12 @@ _`container_infra_prefix`
|
||||||
* docker.io/prom/prometheus:latest
|
* docker.io/prom/prometheus:latest
|
||||||
* gcr.io/google_containers/kubernetes-dashboard-amd64:v1.5.1
|
* gcr.io/google_containers/kubernetes-dashboard-amd64:v1.5.1
|
||||||
* gcr.io/google_containers/pause:3.0
|
* gcr.io/google_containers/pause:3.0
|
||||||
|
* gcr.io/google-containers/hyperkube:v1.12.1
|
||||||
|
* quay.io/prometheus/alertmanager:v0.15.3
|
||||||
|
* quay.io/coreos/prometheus-operator:v0.15.3
|
||||||
|
* quay.io/coreos/configmap-reload:v0.0.1
|
||||||
|
* quay.io/coreos/prometheus-config-reloader:v0.26.0
|
||||||
|
* quay.io/prometheus/prometheus:v2.5.0
|
||||||
|
|
||||||
_`kube_tag`
|
_`kube_tag`
|
||||||
This label allows users to select `a specific Kubernetes release,
|
This label allows users to select `a specific Kubernetes release,
|
||||||
|
@ -1221,6 +1230,11 @@ _`k8s_keystone_auth_tag`
|
||||||
<https://hub.docker.com/r/k8scloudprovider/k8s-keystone-auth/tags/>`_.
|
<https://hub.docker.com/r/k8scloudprovider/k8s-keystone-auth/tags/>`_.
|
||||||
Stein-default: 1.13.0
|
Stein-default: 1.13.0
|
||||||
|
|
||||||
|
_`monitoring_enabled`
|
||||||
|
Enable installation of cluster monitoring solution provided by the
|
||||||
|
stable/prometheus-operator helm chart.
|
||||||
|
Default: false
|
||||||
|
|
||||||
_`tiller_enabled`
|
_`tiller_enabled`
|
||||||
If set to true, tiller will be deployed in the kube-system namespace.
|
If set to true, tiller will be deployed in the kube-system namespace.
|
||||||
Defaults to false.
|
Defaults to false.
|
||||||
|
|
|
@ -36,6 +36,7 @@ write_files:
|
||||||
VERIFY_CA="$VERIFY_CA"
|
VERIFY_CA="$VERIFY_CA"
|
||||||
CLUSTER_UUID="$CLUSTER_UUID"
|
CLUSTER_UUID="$CLUSTER_UUID"
|
||||||
MAGNUM_URL="$MAGNUM_URL"
|
MAGNUM_URL="$MAGNUM_URL"
|
||||||
|
MONITORING_ENABLED="$MONITORING_ENABLED"
|
||||||
VOLUME_DRIVER="$VOLUME_DRIVER"
|
VOLUME_DRIVER="$VOLUME_DRIVER"
|
||||||
REGION_NAME="$REGION_NAME"
|
REGION_NAME="$REGION_NAME"
|
||||||
HTTP_PROXY="$HTTP_PROXY"
|
HTTP_PROXY="$HTTP_PROXY"
|
||||||
|
|
|
@ -0,0 +1,165 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
. /etc/sysconfig/heat-params
|
||||||
|
|
||||||
|
set -ex
|
||||||
|
|
||||||
|
step="prometheus-operator"
|
||||||
|
printf "Starting to run ${step}\n"
|
||||||
|
|
||||||
|
### Configuration
|
||||||
|
###############################################################################
|
||||||
|
CHART_NAME="prometheus-operator"
|
||||||
|
CHART_VERSION="0.1.31"
|
||||||
|
|
||||||
|
if [ "$(echo ${MONITORING_ENABLED} | tr '[:upper:]' '[:lower:]')" = "true" ]; then
|
||||||
|
|
||||||
|
# Validate if communication node <-> master is secure or insecure
|
||||||
|
PROTOCOL="https"
|
||||||
|
INSECURE_SKIP_VERIFY="False"
|
||||||
|
if [ "$TLS_DISABLED" = "True" ]; then
|
||||||
|
PROTOCOL="http"
|
||||||
|
INSECURE_SKIP_VERIFY="True"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$(echo ${VERIFY_CA} | tr '[:upper:]' '[:lower:]')" == "false" ]; then
|
||||||
|
INSECURE_SKIP_VERIFY="True"
|
||||||
|
fi
|
||||||
|
|
||||||
|
HELM_MODULE_CONFIG_FILE="/srv/magnum/kubernetes/helm/${CHART_NAME}.yaml"
|
||||||
|
[ -f ${HELM_MODULE_CONFIG_FILE} ] || {
|
||||||
|
echo "Writing File: ${HELM_MODULE_CONFIG_FILE}"
|
||||||
|
mkdir -p $(dirname ${HELM_MODULE_CONFIG_FILE})
|
||||||
|
cat << EOF > ${HELM_MODULE_CONFIG_FILE}
|
||||||
|
---
|
||||||
|
kind: ConfigMap
|
||||||
|
apiVersion: v1
|
||||||
|
metadata:
|
||||||
|
name: ${CHART_NAME}-config
|
||||||
|
namespace: magnum-tiller
|
||||||
|
labels:
|
||||||
|
app: helm
|
||||||
|
data:
|
||||||
|
install-${CHART_NAME}.sh: |
|
||||||
|
#!/bin/bash
|
||||||
|
set -ex
|
||||||
|
mkdir -p \${HELM_HOME}
|
||||||
|
cp /etc/helm/* \${HELM_HOME}
|
||||||
|
|
||||||
|
# HACK - Force wait because of bug https://github.com/helm/helm/issues/5170
|
||||||
|
until helm init --client-only --wait
|
||||||
|
do
|
||||||
|
sleep 5s
|
||||||
|
done
|
||||||
|
helm repo update
|
||||||
|
|
||||||
|
if [[ \$(helm history prometheus-operator | grep prometheus-operator) ]]; then
|
||||||
|
echo "${CHART_NAME} already installed on server. Continue..."
|
||||||
|
exit 0
|
||||||
|
else
|
||||||
|
helm install stable/${CHART_NAME} --namespace monitoring --name ${CHART_NAME} --version v${CHART_VERSION} --values /opt/magnum/install-${CHART_NAME}-values.yaml
|
||||||
|
fi
|
||||||
|
|
||||||
|
install-${CHART_NAME}-values.yaml: |
|
||||||
|
nameOverride: prometheus
|
||||||
|
fullnameOverride: prometheus
|
||||||
|
|
||||||
|
alertmanager:
|
||||||
|
alertmanagerSpec:
|
||||||
|
image:
|
||||||
|
repository: ${CONTAINER_INFRA_PREFIX:-quay.io/}prometheus/alertmanager
|
||||||
|
|
||||||
|
# Dashboard
|
||||||
|
grafana:
|
||||||
|
#enabled: ${ENABLE_GRAFANA}
|
||||||
|
adminPassword: ${ADMIN_PASSWD}
|
||||||
|
|
||||||
|
kubeApiServer:
|
||||||
|
tlsConfig:
|
||||||
|
insecureSkipVerify: ${INSECURE_SKIP_VERIFY}
|
||||||
|
|
||||||
|
kubelet:
|
||||||
|
serviceMonitor:
|
||||||
|
https: ${PROTOCOL}
|
||||||
|
|
||||||
|
coreDns:
|
||||||
|
enabled: true
|
||||||
|
service:
|
||||||
|
port: 9153
|
||||||
|
targetPort: 9153
|
||||||
|
selector:
|
||||||
|
k8s-app: coredns
|
||||||
|
|
||||||
|
kubeEtcd:
|
||||||
|
service:
|
||||||
|
port: 4001
|
||||||
|
targetPort: 4001
|
||||||
|
selector:
|
||||||
|
k8s-app: etcd-server
|
||||||
|
serviceMonitor:
|
||||||
|
scheme: ${PROTOCOL}
|
||||||
|
insecureSkipVerify: ${INSECURE_SKIP_VERIFY}
|
||||||
|
## If Protocol is http this files should be neglected
|
||||||
|
caFile: ${CERT_DIR}/ca.crt
|
||||||
|
certFile: ${CERT_DIR}/kubelet.crt
|
||||||
|
keyFile: ${CERT_DIR}/kubelet.key
|
||||||
|
|
||||||
|
prometheusOperator:
|
||||||
|
image:
|
||||||
|
repository: ${CONTAINER_INFRA_PREFIX:-quay.io/}coreos/prometheus-operator
|
||||||
|
configmapReloadImage:
|
||||||
|
repository: ${CONTAINER_INFRA_PREFIX:-quay.io/}coreos/configmap-reload
|
||||||
|
prometheusConfigReloaderImage:
|
||||||
|
repository: ${CONTAINER_INFRA_PREFIX:-quay.io/}coreos/prometheus-config-reloader
|
||||||
|
hyperkubeImage:
|
||||||
|
repository: ${CONTAINER_INFRA_PREFIX:-gcr.io/google-containers/}hyperkube
|
||||||
|
|
||||||
|
prometheus:
|
||||||
|
prometheusSpec:
|
||||||
|
image:
|
||||||
|
repository: ${CONTAINER_INFRA_PREFIX:-quay.io/}prometheus/prometheus
|
||||||
|
retention: 14d
|
||||||
|
---
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: Job
|
||||||
|
metadata:
|
||||||
|
name: install-${CHART_NAME}-job
|
||||||
|
namespace: magnum-tiller
|
||||||
|
spec:
|
||||||
|
backoffLimit: 5
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
serviceAccountName: tiller
|
||||||
|
containers:
|
||||||
|
- name: config-helm
|
||||||
|
image: docker.io/openstackmagnum/helm-client:dev
|
||||||
|
command:
|
||||||
|
- bash
|
||||||
|
args:
|
||||||
|
- /opt/magnum/install-${CHART_NAME}.sh
|
||||||
|
env:
|
||||||
|
- name: HELM_HOME
|
||||||
|
value: /helm_home
|
||||||
|
- name: TILLER_NAMESPACE
|
||||||
|
value: magnum-tiller
|
||||||
|
- name: HELM_TLS_ENABLE
|
||||||
|
value: "true"
|
||||||
|
volumeMounts:
|
||||||
|
- name: install-${CHART_NAME}-config
|
||||||
|
mountPath: /opt/magnum/
|
||||||
|
- mountPath: /etc/helm
|
||||||
|
name: helm-client-certs
|
||||||
|
restartPolicy: Never
|
||||||
|
volumes:
|
||||||
|
- name: install-${CHART_NAME}-config
|
||||||
|
configMap:
|
||||||
|
name: ${CHART_NAME}-config
|
||||||
|
- name: helm-client-certs
|
||||||
|
secret:
|
||||||
|
secretName: helm-client-secret
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf "Finished running ${step}\n"
|
|
@ -116,6 +116,7 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition):
|
||||||
'grafana_tag',
|
'grafana_tag',
|
||||||
'heat_container_agent_tag',
|
'heat_container_agent_tag',
|
||||||
'keystone_auth_enabled', 'k8s_keystone_auth_tag',
|
'keystone_auth_enabled', 'k8s_keystone_auth_tag',
|
||||||
|
'monitoring_enabled',
|
||||||
'tiller_enabled',
|
'tiller_enabled',
|
||||||
'tiller_tag',
|
'tiller_tag',
|
||||||
'tiller_namespace',
|
'tiller_namespace',
|
||||||
|
|
|
@ -540,6 +540,11 @@ parameters:
|
||||||
description: tag of the k8s_keystone_auth container
|
description: tag of the k8s_keystone_auth container
|
||||||
default: 1.13.0
|
default: 1.13.0
|
||||||
|
|
||||||
|
monitoring_enabled:
|
||||||
|
type: boolean
|
||||||
|
description: Enable or disable prometheus-operator monitoring solution.
|
||||||
|
default: false
|
||||||
|
|
||||||
project_id:
|
project_id:
|
||||||
type: string
|
type: string
|
||||||
description: >
|
description: >
|
||||||
|
@ -824,6 +829,7 @@ resources:
|
||||||
heat_container_agent_tag: {get_param: heat_container_agent_tag}
|
heat_container_agent_tag: {get_param: heat_container_agent_tag}
|
||||||
keystone_auth_enabled: {get_param: keystone_auth_enabled}
|
keystone_auth_enabled: {get_param: keystone_auth_enabled}
|
||||||
k8s_keystone_auth_tag: {get_param: k8s_keystone_auth_tag}
|
k8s_keystone_auth_tag: {get_param: k8s_keystone_auth_tag}
|
||||||
|
monitoring_enabled: {get_param: monitoring_enabled}
|
||||||
project_id: {get_param: project_id}
|
project_id: {get_param: project_id}
|
||||||
tiller_enabled: {get_param: tiller_enabled}
|
tiller_enabled: {get_param: tiller_enabled}
|
||||||
tiller_tag: {get_param: tiller_tag}
|
tiller_tag: {get_param: tiller_tag}
|
||||||
|
@ -847,8 +853,6 @@ resources:
|
||||||
- get_file: ../../common/templates/kubernetes/fragments/calico-service.sh
|
- get_file: ../../common/templates/kubernetes/fragments/calico-service.sh
|
||||||
- get_file: ../../common/templates/kubernetes/fragments/flannel-service.sh
|
- get_file: ../../common/templates/kubernetes/fragments/flannel-service.sh
|
||||||
- get_file: ../../common/templates/kubernetes/fragments/enable-helm-tiller.sh
|
- get_file: ../../common/templates/kubernetes/fragments/enable-helm-tiller.sh
|
||||||
- get_file: ../../common/templates/kubernetes/helm/metrics-server.sh
|
|
||||||
- get_file: ../../common/templates/kubernetes/fragments/install-helm-modules.sh
|
|
||||||
- str_replace:
|
- str_replace:
|
||||||
template: {get_file: ../../common/templates/kubernetes/fragments/enable-prometheus-monitoring.sh}
|
template: {get_file: ../../common/templates/kubernetes/fragments/enable-prometheus-monitoring.sh}
|
||||||
params:
|
params:
|
||||||
|
@ -861,6 +865,13 @@ resources:
|
||||||
- get_file: ../../common/templates/kubernetes/fragments/kube-dashboard-service.sh
|
- get_file: ../../common/templates/kubernetes/fragments/kube-dashboard-service.sh
|
||||||
- get_file: ../../common/templates/kubernetes/fragments/enable-keystone-auth.sh
|
- get_file: ../../common/templates/kubernetes/fragments/enable-keystone-auth.sh
|
||||||
- get_file: ../../common/templates/kubernetes/fragments/enable-auto-healing.sh
|
- get_file: ../../common/templates/kubernetes/fragments/enable-auto-healing.sh
|
||||||
|
# Helm Based Installation Configuration Scripts
|
||||||
|
- get_file: ../../common/templates/kubernetes/helm/metrics-server.sh
|
||||||
|
- str_replace:
|
||||||
|
template: {get_file: ../../common/templates/kubernetes/helm/prometheus-operator.sh}
|
||||||
|
params:
|
||||||
|
"${ADMIN_PASSWD}": {get_param: grafana_admin_passwd}
|
||||||
|
- get_file: ../../common/templates/kubernetes/fragments/install-helm-modules.sh
|
||||||
|
|
||||||
kube_cluster_deploy:
|
kube_cluster_deploy:
|
||||||
type: OS::Heat::SoftwareDeployment
|
type: OS::Heat::SoftwareDeployment
|
||||||
|
|
|
@ -417,6 +417,11 @@ parameters:
|
||||||
type: string
|
type: string
|
||||||
description: tag of the k8s_keystone_auth container
|
description: tag of the k8s_keystone_auth container
|
||||||
|
|
||||||
|
monitoring_enabled:
|
||||||
|
type: boolean
|
||||||
|
description: Enable or disable prometheus-operator monitoring solution.
|
||||||
|
default: false
|
||||||
|
|
||||||
project_id:
|
project_id:
|
||||||
type: string
|
type: string
|
||||||
description: >
|
description: >
|
||||||
|
@ -543,6 +548,7 @@ resources:
|
||||||
"$HEAT_CONTAINER_AGENT_TAG": {get_param: heat_container_agent_tag}
|
"$HEAT_CONTAINER_AGENT_TAG": {get_param: heat_container_agent_tag}
|
||||||
"$KEYSTONE_AUTH_ENABLED": {get_param: keystone_auth_enabled}
|
"$KEYSTONE_AUTH_ENABLED": {get_param: keystone_auth_enabled}
|
||||||
"$K8S_KEYSTONE_AUTH_TAG": {get_param: k8s_keystone_auth_tag}
|
"$K8S_KEYSTONE_AUTH_TAG": {get_param: k8s_keystone_auth_tag}
|
||||||
|
"$MONITORING_ENABLED": {get_param: monitoring_enabled}
|
||||||
"$PROJECT_ID": {get_param: project_id}
|
"$PROJECT_ID": {get_param: project_id}
|
||||||
"$EXTERNAL_NETWORK_ID": {get_param: external_network}
|
"$EXTERNAL_NETWORK_ID": {get_param: external_network}
|
||||||
"$TILLER_ENABLED": {get_param: tiller_enabled}
|
"$TILLER_ENABLED": {get_param: tiller_enabled}
|
||||||
|
|
|
@ -485,6 +485,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
|
||||||
'keystone_auth_enabled')
|
'keystone_auth_enabled')
|
||||||
k8s_keystone_auth_tag = mock_cluster.labels.get(
|
k8s_keystone_auth_tag = mock_cluster.labels.get(
|
||||||
'k8s_keystone_auth_tag')
|
'k8s_keystone_auth_tag')
|
||||||
|
monitoring_enabled = mock_cluster.labels.get(
|
||||||
|
'monitoring_enabled')
|
||||||
project_id = mock_cluster.project_id
|
project_id = mock_cluster.project_id
|
||||||
tiller_enabled = mock_cluster.labels.get(
|
tiller_enabled = mock_cluster.labels.get(
|
||||||
'tiller_enabled')
|
'tiller_enabled')
|
||||||
|
@ -549,6 +551,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
|
||||||
'heat_container_agent_tag': heat_container_agent_tag,
|
'heat_container_agent_tag': heat_container_agent_tag,
|
||||||
'keystone_auth_enabled': keystone_auth_enabled,
|
'keystone_auth_enabled': keystone_auth_enabled,
|
||||||
'k8s_keystone_auth_tag': k8s_keystone_auth_tag,
|
'k8s_keystone_auth_tag': k8s_keystone_auth_tag,
|
||||||
|
'monitoring_enabled': monitoring_enabled,
|
||||||
'project_id': project_id,
|
'project_id': project_id,
|
||||||
'external_network': external_network_id,
|
'external_network': external_network_id,
|
||||||
'tiller_enabled': tiller_enabled,
|
'tiller_enabled': tiller_enabled,
|
||||||
|
@ -855,6 +858,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
|
||||||
'keystone_auth_enabled')
|
'keystone_auth_enabled')
|
||||||
k8s_keystone_auth_tag = mock_cluster.labels.get(
|
k8s_keystone_auth_tag = mock_cluster.labels.get(
|
||||||
'k8s_keystone_auth_tag')
|
'k8s_keystone_auth_tag')
|
||||||
|
monitoring_enabled = mock_cluster.labels.get(
|
||||||
|
'monitoring_enabled')
|
||||||
project_id = mock_cluster.project_id
|
project_id = mock_cluster.project_id
|
||||||
tiller_enabled = mock_cluster.labels.get(
|
tiller_enabled = mock_cluster.labels.get(
|
||||||
'tiller_enabled')
|
'tiller_enabled')
|
||||||
|
@ -921,6 +926,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
|
||||||
'heat_container_agent_tag': heat_container_agent_tag,
|
'heat_container_agent_tag': heat_container_agent_tag,
|
||||||
'keystone_auth_enabled': keystone_auth_enabled,
|
'keystone_auth_enabled': keystone_auth_enabled,
|
||||||
'k8s_keystone_auth_tag': k8s_keystone_auth_tag,
|
'k8s_keystone_auth_tag': k8s_keystone_auth_tag,
|
||||||
|
'monitoring_enabled': monitoring_enabled,
|
||||||
'project_id': project_id,
|
'project_id': project_id,
|
||||||
'external_network': external_network_id,
|
'external_network': external_network_id,
|
||||||
'tiller_enabled': tiller_enabled,
|
'tiller_enabled': tiller_enabled,
|
||||||
|
|
|
@ -0,0 +1,8 @@
|
||||||
|
---
|
||||||
|
features:
|
||||||
|
- |
|
||||||
|
Added monitoring_enabled to install prometheus-operator monitoring
|
||||||
|
solution by means of helm stable/prometheus-operator public chart.
|
||||||
|
Defaults to false. grafana_admin_passwd label can be used to set
|
||||||
|
grafana dashboard admin access password. If grafana_admin_passwd
|
||||||
|
is not set the password defaults to prom_operator.
|
Loading…
Reference in New Issue