Browse Source

Scrape internal kubernetes components

* Scrape metrics from kube-{controller-manager,scheduler}
* Disable PrometheusRule for etcd
* Extra configurations for future https updates

task: 38702
story: 2006765

Change-Id: Idace486e0e173e7beea0d5376e9edfd461377021
Signed-off-by: Diogo Guerra <diogo.filipe.tomas.guerra@cern.ch>
changes/84/721284/5
Diogo Guerra 2 years ago
parent
commit
7ab504d1e2
  1. 85
      magnum/drivers/common/templates/kubernetes/helm/prometheus-operator.sh
  2. 1
      magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml
  3. 1
      magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml
  4. 5
      releasenotes/notes/monitoring_scrape_internal-6697e50f091b0c9c.yaml

85
magnum/drivers/common/templates/kubernetes/helm/prometheus-operator.sh

@ -26,6 +26,8 @@ if [ "$(echo ${MONITORING_ENABLED} | tr '[:upper:]' '[:lower:]')" = "true" ]; th
PROTOCOL="http"
INSECURE_SKIP_VERIFY="True"
fi
# FIXME: Force protocol to http as we don't want to use the cluster certs
USE_HTTPS="False"
if [ "$(echo ${VERIFY_CA} | tr '[:upper:]' '[:lower:]')" == "false" ]; then
INSECURE_SKIP_VERIFY="True"
@ -112,6 +114,11 @@ data:
nameOverride: prometheus
fullnameOverride: prometheus
defaultRules:
rules:
#TODO: To enable this we need firstly take care of exposing certs
etcd: false
alertmanager:
alertmanagerSpec:
image:
@ -135,11 +142,29 @@ data:
kubeApiServer:
tlsConfig:
insecureSkipVerify: ${INSECURE_SKIP_VERIFY}
insecureSkipVerify: "False"
kubelet:
serviceMonitor:
https: ${PROTOCOL}
https: "True"
kubeControllerManager:
## If your kube controller manager is not deployed as a pod, specify IPs it can be found on
endpoints: ${KUBE_MASTERS_PRIVATE}
## If using kubeControllerManager.endpoints only the port and targetPort are used
service:
port: 10252
targetPort: 10252
# selector:
# component: kube-controller-manager
serviceMonitor:
## Enable scraping kube-controller-manager over https.
## Requires proper certs (not self-signed) and delegated authentication/authorization checks
https: ${USE_HTTPS}
# Skip TLS certificate validation when scraping
insecureSkipVerify: null
# Name of the server to use when validating TLS certificate
serverName: null
coreDns:
enabled: true
@ -150,14 +175,49 @@ data:
k8s-app: kube-dns
kubeEtcd:
## If your etcd is not deployed as a pod, specify IPs it can be found on
endpoints: ${KUBE_MASTERS_PRIVATE}
## Etcd service. If using kubeEtcd.endpoints only the port and targetPort are used
service:
port: 2379
targetPort: 2379
# selector:
# component: etcd
## Configure secure access to the etcd cluster by loading a secret into prometheus and
## specifying security configuration below. For example, with a secret named etcd-client-cert
serviceMonitor:
scheme: ${PROTOCOL}
scheme: https
insecureSkipVerify: true
## If Protocol is http this files should be neglected
caFile: /etc/prometheus/secrets/etcd-certificates/ca.crt
certFile: /etc/prometheus/secrets/etcd-certificates/kubelet.crt
keyFile: /etc/prometheus/secrets/etcd-certificates/kubelet.key
kubeScheduler:
## If your kube scheduler is not deployed as a pod, specify IPs it can be found on
endpoints: ${KUBE_MASTERS_PRIVATE}
## If using kubeScheduler.endpoints only the port and targetPort are used
service:
port: 10251
targetPort: 10251
# selector:
# component: kube-scheduler
serviceMonitor:
## Enable scraping kube-scheduler over https.
## Requires proper certs (not self-signed) and delegated authentication/authorization checks
https: ${USE_HTTPS}
## Skip TLS certificate validation when scraping
insecureSkipVerify: null
## Name of the server to use when validating TLS certificate
serverName: null
# kubeProxy:
# ## If your kube proxy is not deployed as a pod, specify IPs it can be found on
# endpoints: [] # masters + minions
# serviceMonitor:
# ## Enable scraping kube-proxy over https.
# ## Requires proper certs (not self-signed) and delegated authentication/authorization checks
# https: ${USE_HTTPS}
kube-state-metrics:
priorityClassName: "system-cluster-critical"
resources:
@ -173,6 +233,12 @@ data:
limits:
cpu: 20m
memory: 20M
extraArgs:
- --collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+)($|/)
- --collector.filesystem.ignored-fs-types=^(autofs|binfmt_misc|cgroup|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|mqueue|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|sysfs|tracefs)$
sidecars: []
## - name: nvidia-dcgm-exporter
## image: nvidia/dcgm-exporter:1.4.3
prometheusOperator:
priorityClassName: "system-cluster-critical"
@ -207,12 +273,19 @@ data:
retention: 14d
externalLabels:
cluster_uuid: ${CLUSTER_UUID}
## Secrets is a list of Secrets in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods.
## The Secrets are mounted into /etc/prometheus/secrets/. Secrets changes after initial creation of a Prometheus object are not
## reflected in the running Pods. To change the secrets mounted into the Prometheus Pods, the object must be deleted and recreated
## with the new list of secrets.
# secrets:
# - etcd-certificates
# - kube-controller-manager-certificates
# - kube-scheduler-certificates
# - kube-proxy-manager-certificates
resources:
requests:
cpu: ${PROMETHEUS_SERVER_CPU}m
memory: ${PROMETHEUS_SERVER_RAM}M
# secrets:
# - etcd-certificates
priorityClassName: "system-cluster-critical"
${APP_ADDITIONAL_SERVICE_MONITORS}
${APP_ADDITIONAL_POD_MONITORS}

1
magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml

@ -1289,6 +1289,7 @@ resources:
template: {get_file: ../../common/templates/kubernetes/helm/prometheus-operator.sh}
params:
"${ADMIN_PASSWD}": {get_param: grafana_admin_passwd}
"${KUBE_MASTERS_PRIVATE}": {get_attr: [kube_masters, kube_master_external_ip]}
- get_file: ../../common/templates/kubernetes/helm/prometheus-adapter.sh
- get_file: ../../common/templates/kubernetes/helm/ingress-nginx.sh
- get_file: ../../common/templates/kubernetes/fragments/install-helm-modules.sh

1
magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml

@ -1325,6 +1325,7 @@ resources:
template: {get_file: ../../common/templates/kubernetes/helm/prometheus-operator.sh}
params:
"${ADMIN_PASSWD}": {get_param: grafana_admin_passwd}
"${KUBE_MASTERS_PRIVATE}": {get_attr: [kube_masters, kube_master_external_ip]}
- get_file: ../../common/templates/kubernetes/helm/prometheus-adapter.sh
- get_file: ../../common/templates/kubernetes/helm/ingress-nginx.sh
- get_file: ../../common/templates/kubernetes/fragments/install-helm-modules.sh

5
releasenotes/notes/monitoring_scrape_internal-6697e50f091b0c9c.yaml

@ -0,0 +1,5 @@
---
fixes:
- |
Scrape metrics from kube-{controller-manager,scheduler}.
Disable PrometheusRule for etcd.
Loading…
Cancel
Save