From 4aade351b07e04791d6ac9aa6a1abc8bc3a8f98e Mon Sep 17 00:00:00 2001 From: Spyros Trigazis Date: Tue, 26 Sep 2017 09:17:17 +0000 Subject: [PATCH] Fix prometheus scrape configuration Make prometheus configuration compatible with 1.7.3 or greater. The new config matches the one in the pormetheus kubernetes example [1] minus the configuration for ingress that we don't deploy. [1] https://github.com/prometheus/prometheus/blob/master/documentation/examples/prometheus-kubernetes.yml Closes-Bug: #1719187 Change-Id: Ied8b8d8fb610e5ecba18e37f98dd1a4c9bddbf30 --- .../fragments/write-prometheus-configmap.yaml | 135 ++++++++++++++---- 1 file changed, 108 insertions(+), 27 deletions(-) diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-prometheus-configmap.yaml b/magnum/drivers/common/templates/kubernetes/fragments/write-prometheus-configmap.yaml index 7b28aac242..0098024d09 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/write-prometheus-configmap.yaml +++ b/magnum/drivers/common/templates/kubernetes/fragments/write-prometheus-configmap.yaml @@ -18,7 +18,22 @@ write_files: evaluation_interval: 10s scrape_configs: - - job_name: 'kubernetes-nodes-cadvisor' + - job_name: 'kubernetes-apiservers' + + kubernetes_sd_configs: + - role: endpoints + + scheme: https + + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + relabel_configs: + - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: default;kubernetes;https + - job_name: 'kubernetes-nodes' + scheme: https tls_config: ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token @@ -27,42 +42,108 @@ write_files: relabel_configs: - action: labelmap regex: __meta_kubernetes_node_label_(.+) - - source_labels: [__meta_kubernetes_role] - action: replace - target_label: kubernetes_role - - source_labels: [__address__] - regex: '(.*):10250' - replacement: '${1}:10255' - target_label: __address__ - metric_relabel_configs: - - action: replace - source_labels: [id] - regex: '^/machine\.slice/machine-rkt\\x2d([^\\]+)\\.+/([^/]+)\.service$' - target_label: rkt_container_name - replacement: '${2}-${1}' - - action: replace - source_labels: [id] - regex: '^/system\.slice/(.+)\.service$' - target_label: systemd_service_name - replacement: '${1}' + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics - - job_name: 'kubernetes-apiserver-cadvisor' + - job_name: 'kubernetes-cadvisor' + scheme: https tls_config: - insecure_skip_verify: true ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token kubernetes_sd_configs: - - role: endpoints + - role: node relabel_configs: - action: labelmap regex: __meta_kubernetes_node_label_(.+) - - source_labels: [__meta_kubernetes_role] + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor + + - job_name: 'kubernetes-service-endpoints' + + kubernetes_sd_configs: + - role: endpoints + + relabel_configs: + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] + action: replace + target_label: __scheme__ + regex: (https?) + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] action: replace - target_label: kubernetes_role - - source_labels: [__address__] - regex: '(.*):10250' - replacement: '${1}:10255' target_label: __address__ + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_service_name] + action: replace + target_label: kubernetes_name + + - job_name: 'kubernetes-services' + metrics_path: /probe + params: + module: [http_2xx] + kubernetes_sd_configs: + - role: service + relabel_configs: + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe] + action: keep + regex: true + - source_labels: [__address__] + target_label: __param_target + - target_label: __address__ + replacement: blackbox + - source_labels: [__param_target] + target_label: instance + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_service_name] + target_label: kubernetes_name + + - job_name: 'kubernetes-pods' + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: kubernetes_pod_name - job_name: 'kubernetes-node-exporter' tls_config: