openstack/magnum
Code Issues Proposed changes

3. Configure monitoring apps path based endpoints

Browse Source 
* Add monitoring_ingress_enabled magnum label to set up ingress with
path based routing for all the configured services
{alertmanager,grafana,prometheus}. When using this,
cluster_root_domain_name magnum label must be used to setup base path
where this services are available.
* Add cluster_basic_auth_secret magnum label to configure basic auth
on unprotected services {alertmanager and  prometheus}. This is only
in effect when app access is routed by ingress.
* Set services logFormat to json to enable easier machine log parsing.

task: 39477
story: 2006765

Depends-On: Ieb90605182626869528349a7fdeed65061914bcb
Change-Id: Ie0e7000e0d94b2037f2c398fa67a2a2b7e256bc3
Signed-off-by: Diogo Guerra <diogo.filipe.tomas.guerra@cern.ch>
This commit is contained in:
Diogo Guerra 2020-04-20 19:31:01 +09:00 committed by Diogo Guerra
parent aec5d469be
commit ea64468ab3
10 changed files with 266 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
doc/source/user/index.rst
View File

@ -328,6 +328,13 @@ the table are linked to more details elsewhere in the user guide.
+---------------------------------------+--------------------+---------------+
| `monitoring_interval_seconds`_ | see below | see below |
+---------------------------------------+--------------------+---------------+
| `monitoring_ingress_enabled`_ | - true | false |
| | - false | |
+---------------------------------------+--------------------+---------------+
| `cluster_basic_auth_secret`_ | see below | see below |
+---------------------------------------+--------------------+---------------+
| `cluster_root_domain_name`_ | see below | see below |
+---------------------------------------+--------------------+---------------+
| `prometheus_operator_chart_tag`_ | see below | see below |
+---------------------------------------+--------------------+---------------+
| `prometheus_adapter_enabled`_ | - true | true |
@ -1505,6 +1512,23 @@ _`monitoring_storage_class_name`
monitoring_retention_size and one for grafana which is fixed at 1Gi.
Default: ""
_`monitoring_ingress_enabled`
Enable configuration of ingresses for the enabled monitoring services
{alertmanager,grafana,prometheus}.
Default: false
_`cluster_basic_auth_secret`
The kubernetes secret to use for the proxy basic auth username and password
for the unprotected services {alertmanager,prometheus}. Basic auth is only
set up if this file is specified.
The secret must be in the same namespace as the used proxy (kube-system).
Default: ""
_`cluster_root_domain_name`
The root domain name to use for the cluster automatically set up
applications.
Default: "localhost"
_`prometheus_adapter_enabled`
Enable installation of cluster custom metrics provided by the
stable/prometheus-adapter helm chart. This service depends on

3
magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh
View File

@ -63,6 +63,9 @@ MONITORING_RETENTION_DAYS="$MONITORING_RETENTION_DAYS"
MONITORING_RETENTION_SIZE="$MONITORING_RETENTION_SIZE"
MONITORING_INTERVAL_SECONDS="$MONITORING_INTERVAL_SECONDS"
MONITORING_STORAGE_CLASS_NAME="$MONITORING_STORAGE_CLASS_NAME"
MONITORING_INGRESS_ENABLED="$MONITORING_INGRESS_ENABLED"
CLUSTER_BASIC_AUTH_SECRET="$CLUSTER_BASIC_AUTH_SECRET"
CLUSTER_ROOT_DOMAIN_NAME="$CLUSTER_ROOT_DOMAIN_NAME"
PROMETHEUS_OPERATOR_CHART_TAG="$PROMETHEUS_OPERATOR_CHART_TAG"
PROMETHEUS_ADAPTER_ENABLED="$PROMETHEUS_ADAPTER_ENABLED"
PROMETHEUS_ADAPTER_CHART_TAG="$PROMETHEUS_ADAPTER_CHART_TAG"

131
magnum/drivers/common/templates/kubernetes/helm/prometheus-operator.sh
View File

@ -31,6 +31,48 @@ EOF
APP_GRAFANA_PERSISTENT_STORAGE="true"
fi
# Create services for grafana/prometheus/alermanager
APP_INGRESS_PATH_APPEND=""
APP_INGRESS_ANNOTATIONS=""
APP_INGRESS_ROUTE_ANNOTATIONS=""
APP_INGRESS_BASIC_AUTH_ANNOTATIONS=""
if [ "${INGRESS_CONTROLLER}" == "nginx" ]; then
APP_INGRESS_PATH_APPEND="(/|$)(.*)"
APP_INGRESS_ANNOTATIONS=$(cat << EOF
nginx.ingress.kubernetes.io/ssl-redirect: "true"
nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
EOF
)
APP_INGRESS_ROUTE_ANNOTATIONS=$(cat << 'EOF'
nginx.ingress.kubernetes.io/rewrite-target: /$2
EOF
)
if [ "${CLUSTER_BASIC_AUTH_SECRET}" != "" ]; then
APP_INGRESS_BASIC_AUTH_ANNOTATIONS=$(cat << EOF
nginx.ingress.kubernetes.io/auth-type: basic
nginx.ingress.kubernetes.io/auth-secret: ${CLUSTER_BASIC_AUTH_SECRET}
EOF
)
fi #END BASIC AUTH
elif [ "${INGRESS_CONTROLLER}" == "traefik" ]; then
APP_INGRESS_ANNOTATIONS=$(cat << EOF
traefik.ingress.kubernetes.io/frontend-entry-points: https
traefik.ingress.kubernetes.io/protocol: http
EOF
)
APP_INGRESS_ROUTE_ANNOTATIONS=$(cat << EOF
traefik.ingress.kubernetes.io/rule-type: PathPrefixStrip
EOF
)
if [ "${CLUSTER_BASIC_AUTH_SECRET}" != "" ]; then
APP_INGRESS_BASIC_AUTH_ANNOTATIONS=$(cat << EOF
ingress.kubernetes.io/auth-type: basic
ingress.kubernetes.io/auth-secret: ${CLUSTER_BASIC_AUTH_SECRET}
EOF
)
fi #END BASIC AUTH
fi
# Validate if communication node <-> master is secure or insecure
PROTOCOL="https"
INSECURE_SKIP_VERIFY="False"
@ -54,17 +96,36 @@ prometheus-operator:
etcd: false
alertmanager:
ingress:
enabled: ${MONITORING_INGRESS_ENABLED}
annotations:
kubernetes.io/ingress.class: ${INGRESS_CONTROLLER}
${APP_INGRESS_ANNOTATIONS}
${APP_INGRESS_ROUTE_ANNOTATIONS}
${APP_INGRESS_BASIC_AUTH_ANNOTATIONS}
## Hosts must be provided if Ingress is enabled.
hosts:
- ${CLUSTER_ROOT_DOMAIN_NAME}
paths:
- /alertmanager${APP_INGRESS_PATH_APPEND}
## TLS configuration for Alertmanager Ingress
## Secret must be manually created in the namespace
tls: []
# - secretName: alertmanager-general-tls
# hosts:
# - alertmanager.example.com
alertmanagerSpec:
image:
repository: ${CONTAINER_INFRA_PREFIX:-quay.io/prometheus/}alertmanager
# # Needs testing
logFormat: json
externalUrl: https://${CLUSTER_ROOT_DOMAIN_NAME}/alertmanager
# routePrefix: /alertmanager
# resources:
# requests:
# cpu: 100m
# memory: 256Mi
priorityClassName: "system-cluster-critical"
# Dashboard
grafana:
image:
repository: ${CONTAINER_INFRA_PREFIX:-grafana/}grafana
@ -76,10 +137,46 @@ prometheus-operator:
cpu: 100m
memory: 128Mi
adminPassword: ${GRAFANA_ADMIN_PASSWD}
ingress:
enabled: ${MONITORING_INGRESS_ENABLED}
annotations:
kubernetes.io/ingress.class: ${INGRESS_CONTROLLER}
${APP_INGRESS_ANNOTATIONS}
## Hostnames.
## Must be provided if Ingress is enable.
hosts:
- ${CLUSTER_ROOT_DOMAIN_NAME}
path: /grafana${APP_INGRESS_PATH_APPEND}
## TLS configuration for grafana Ingress
## Secret must be manually created in the namespace
tls: []
# - secretName: grafana-general-tls
# hosts:
# - grafana.example.com
persistence:
enabled: ${APP_GRAFANA_PERSISTENT_STORAGE}
storageClassName: ${MONITORING_STORAGE_CLASS_NAME}
size: 1Gi
grafana.ini:
server:
domain: ${CLUSTER_ROOT_DOMAIN_NAME}
root_url: https://${CLUSTER_ROOT_DOMAIN_NAME}/grafana
serve_from_sub_path: true
paths:
data: /var/lib/grafana/data
logs: /var/log/grafana
plugins: /var/lib/grafana/plugins
provisioning: /etc/grafana/provisioning
analytics:
check_for_updates: true
log:
mode: console
log.console:
format: json
grafana_net:
url: https://grafana.net
plugins:
- grafana-piechart-panel
kubeApiServer:
tlsConfig:
@ -196,6 +293,7 @@ prometheus-operator:
# requests:
# cpu: 5m
# memory: 10Mi
logFormat: json
image:
repository: ${CONTAINER_INFRA_PREFIX:-quay.io/coreos/}prometheus-operator
configmapReloadImage:
@ -206,6 +304,32 @@ prometheus-operator:
repository: ${CONTAINER_INFRA_PREFIX:-k8s.gcr.io/}hyperkube
prometheus:
ingress:
enabled: ${MONITORING_INGRESS_ENABLED}
annotations:
kubernetes.io/ingress.class: ${INGRESS_CONTROLLER}
${APP_INGRESS_ANNOTATIONS}
${APP_INGRESS_ROUTE_ANNOTATIONS}
${APP_INGRESS_BASIC_AUTH_ANNOTATIONS}
## Hostnames.
## Must be provided if Ingress is enabled.
hosts:
- ${CLUSTER_ROOT_DOMAIN_NAME}
paths:
- /prometheus${APP_INGRESS_PATH_APPEND}
## TLS configuration for Prometheus Ingress
## Secret must be manually created in the namespace
tls: []
# - secretName: prometheus-general-tls
# hosts:
# - prometheus.example.com
serviceMonitor:
## scheme: HTTP scheme to use for scraping. Can be used with tlsConfig for example if using istio mTLS.
scheme: ""
## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS.
## Of type: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#tlsconfig
tlsConfig: {}
bearerTokenFile:
prometheusSpec:
scrapeInterval: ${MONITORING_INTERVAL_SECONDS}s
scrapeInterval: 30s
@ -215,6 +339,7 @@ prometheus-operator:
retention: 14d
externalLabels:
cluster_uuid: ${CLUSTER_UUID}
externalUrl: https://${CLUSTER_ROOT_DOMAIN_NAME}/prometheus
## Secrets is a list of Secrets in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods.
## The Secrets are mounted into /etc/prometheus/secrets/. Secrets changes after initial creation of a Prometheus object are not
## reflected in the running Pods. To change the secrets mounted into the Prometheus Pods, the object must be deleted and recreated
@ -226,6 +351,8 @@ prometheus-operator:
# - kube-proxy-manager-certificates
retention: ${MONITORING_RETENTION_DAYS}d
retentionSize: ${MONITORING_RETENTION_SIZE_GB}GB
logFormat: json
#routePrefix: /prometheus
resources:
requests:
cpu: ${PROMETHEUS_SERVER_CPU}m

3
magnum/drivers/heat/k8s_fedora_template_def.py
View File

@ -102,6 +102,9 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition):
'monitoring_retention_size',
'monitoring_interval_seconds',
'monitoring_storage_class_name',
'monitoring_ingress_enabled',
'cluster_basic_auth_secret',
'cluster_root_domain_name',
'prometheus_operator_chart_tag',
'prometheus_adapter_enabled',
'prometheus_adapter_chart_tag',

20
magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml
View File

@ -716,6 +716,23 @@ parameters:
description: The kubernetes storage class name to use for the prometheus pvc.
default: ""
monitoring_ingress_enabled:
type: boolean
description: Enable/disable configuration of ingresses for the monitoring services.
default: false
cluster_basic_auth_secret:
type: string
description: >
The kubernetes secret name to use for the proxy basic auth username and password.
default: ""
cluster_root_domain_name:
type: string
description: >
The root domain name to use for the cluster automatically set up applications.
default: "localhost"
prometheus_operator_chart_tag:
type: string
description: The stable/prometheus-operator chart version to use.
@ -1254,6 +1271,9 @@ resources:
monitoring_retention_size: {get_param: monitoring_retention_size}
monitoring_interval_seconds: {get_param: monitoring_interval_seconds}
monitoring_storage_class_name: {get_param: monitoring_storage_class_name}
monitoring_ingress_enabled: {get_param: monitoring_ingress_enabled}
cluster_basic_auth_secret: {get_param: cluster_basic_auth_secret}
cluster_root_domain_name: {get_param: cluster_root_domain_name}
prometheus_operator_chart_tag: {get_param: prometheus_operator_chart_tag}
prometheus_adapter_enabled: {get_param: prometheus_adapter_enabled}
prometheus_adapter_chart_tag: {get_param: prometheus_adapter_chart_tag}

17
magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml
View File

@ -481,6 +481,20 @@ parameters:
type: string
description: The kubernetes storage class name to use for the prometheus pvc.
monitoring_ingress_enabled:
type: boolean
description: Enable/disable configuration of ingresses for the monitoring services.
cluster_basic_auth_secret:
type: string
description: >
The kubernetes secret name to use for the proxy basic auth username and password.
cluster_root_domain_name:
type: string
description: >
The root domain name to use for the cluster automatically set up applications.
prometheus_operator_chart_tag:
type: string
description: The stable/prometheus-operator chart version to use.
@ -817,6 +831,9 @@ resources:
"$MONITORING_RETENTION_SIZE": {get_param: monitoring_retention_size}
"$MONITORING_INTERVAL_SECONDS": {get_param: monitoring_interval_seconds}
"$MONITORING_STORAGE_CLASS_NAME": {get_param: monitoring_storage_class_name}
"$MONITORING_INGRESS_ENABLED": {get_param: monitoring_ingress_enabled}
"$CLUSTER_BASIC_AUTH_SECRET": {get_param: cluster_basic_auth_secret}
"$CLUSTER_ROOT_DOMAIN_NAME": {get_param: cluster_root_domain_name}
"$PROMETHEUS_OPERATOR_CHART_TAG": {get_param: prometheus_operator_chart_tag}
"$PROMETHEUS_ADAPTER_ENABLED": {get_param: prometheus_adapter_enabled}
"$PROMETHEUS_ADAPTER_CHART_TAG": {get_param: prometheus_adapter_chart_tag}

20
magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml
View File

@ -726,6 +726,23 @@ parameters:
description: The kubernetes storage class name to use for the prometheus pvc.
default: ""
monitoring_ingress_enabled:
type: boolean
description: Enable/disable configuration of ingresses for the monitoring services.
default: false
cluster_basic_auth_secret:
type: string
description: >
The kubernetes secret name to use for the proxy basic auth username and password.
default: ""
cluster_root_domain_name:
type: string
description: >
The root domain name to use for the cluster automatically set up applications.
default: "localhost"
prometheus_operator_chart_tag:
type: string
description: The stable/prometheus-operator chart version to use.
@ -1282,6 +1299,9 @@ resources:
monitoring_retention_size: {get_param: monitoring_retention_size}
monitoring_interval_seconds: {get_param: monitoring_interval_seconds}
monitoring_storage_class_name: {get_param: monitoring_storage_class_name}
monitoring_ingress_enabled: {get_param: monitoring_ingress_enabled}
cluster_basic_auth_secret: {get_param: cluster_basic_auth_secret}
cluster_root_domain_name: {get_param: cluster_root_domain_name}
prometheus_operator_chart_tag: {get_param: prometheus_operator_chart_tag}
prometheus_adapter_enabled: {get_param: prometheus_adapter_enabled}
prometheus_adapter_chart_tag: {get_param: prometheus_adapter_chart_tag}

17
magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml
View File

@ -485,6 +485,20 @@ parameters:
type: string
description: The kubernetes storage class name to use for the prometheus pvc.
monitoring_ingress_enabled:
type: boolean
description: Enable/disable configuration of ingresses for the monitoring services.
cluster_basic_auth_secret:
type: string
description: >
The kubernetes secret name to use for the proxy basic auth username and password.
cluster_root_domain_name:
type: string
description: >
The root domain name to use for the cluster automatically set up applications.
prometheus_operator_chart_tag:
type: string
description: The stable/prometheus-operator chart version to use.
@ -836,6 +850,9 @@ resources:
"$MONITORING_RETENTION_SIZE": {get_param: monitoring_retention_size}
"$MONITORING_INTERVAL_SECONDS": {get_param: monitoring_interval_seconds}
"$MONITORING_STORAGE_CLASS_NAME": {get_param: monitoring_storage_class_name}
"$MONITORING_INGRESS_ENABLED": {get_param: monitoring_ingress_enabled}
"$CLUSTER_BASIC_AUTH_SECRET": {get_param: cluster_basic_auth_secret}
"$CLUSTER_ROOT_DOMAIN_NAME": {get_param: cluster_root_domain_name}
"$PROMETHEUS_OPERATOR_CHART_TAG": {get_param: prometheus_operator_chart_tag}
"$PROMETHEUS_ADAPTER_ENABLED": {get_param: prometheus_adapter_enabled}
"$PROMETHEUS_ADAPTER_CHART_TAG": {get_param: prometheus_adapter_chart_tag}

18
magnum/tests/unit/drivers/test_template_definition.py
View File

@ -547,6 +547,12 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
'monitoring_interval_seconds')
monitoring_storage_class_name = mock_cluster.labels.get(
'monitoring_storage_class_name')
monitoring_ingress_enabled = mock_cluster.labels.get(
'monitoring_ingress_enabled')
cluster_basic_auth_secret = mock_cluster.labels.get(
'cluster_basic_auth_secret')
cluster_root_domain_name = mock_cluster.labels.get(
'cluster_root_domain_name')
prometheus_operator_chart_tag = mock_cluster.labels.get(
'prometheus_operator_chart_tag')
prometheus_adapter_enabled = mock_cluster.labels.get(
@ -686,6 +692,9 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
'monitoring_retention_size': monitoring_retention_size,
'monitoring_interval_seconds': monitoring_interval_seconds,
'monitoring_storage_class_name': monitoring_storage_class_name,
'monitoring_ingress_enabled': monitoring_ingress_enabled,
'cluster_basic_auth_secret': cluster_basic_auth_secret,
'cluster_root_domain_name': cluster_root_domain_name,
'prometheus_operator_chart_tag': prometheus_operator_chart_tag,
'prometheus_adapter_enabled': prometheus_adapter_enabled,
'prometheus_adapter_chart_tag': prometheus_adapter_chart_tag,
@ -1090,6 +1099,12 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
'monitoring_interval_seconds')
monitoring_storage_class_name = mock_cluster.labels.get(
'monitoring_storage_class_name')
monitoring_ingress_enabled = mock_cluster.labels.get(
'monitoring_ingress_enabled')
cluster_basic_auth_secret = mock_cluster.labels.get(
'cluster_basic_auth_secret')
cluster_root_domain_name = mock_cluster.labels.get(
'cluster_root_domain_name')
prometheus_operator_chart_tag = mock_cluster.labels.get(
'prometheus_operator_chart_tag')
prometheus_adapter_enabled = mock_cluster.labels.get(
@ -1232,6 +1247,9 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
'monitoring_retention_size': monitoring_retention_size,
'monitoring_interval_seconds': monitoring_interval_seconds,
'monitoring_storage_class_name': monitoring_storage_class_name,
'monitoring_ingress_enabled': monitoring_ingress_enabled,
'cluster_basic_auth_secret': cluster_basic_auth_secret,
'cluster_root_domain_name': cluster_root_domain_name,
'prometheus_operator_chart_tag': prometheus_operator_chart_tag,
'prometheus_adapter_enabled': prometheus_adapter_enabled,
'prometheus_adapter_chart_tag': prometheus_adapter_chart_tag,

15
releasenotes/notes/configure_monitoring_app_endpoints-f00600c244a76cf4.yaml Normal file
View File

@ -0,0 +1,15 @@
---
features:
- |
Added monitoring_ingress_enabled magnum label to set up ingress with
path based routing for all the configured services
{alertmanager,grafana,prometheus}. When using this,
cluster_root_domain_name magnum label must be used to setup base path
where this services are available.
Added cluster_basic_auth_secret magnum label to configure basic auth
on unprotected services {alertmanager and prometheus}. This is only
in effect when app access is routed by ingress.
upgrade:
- |
Configured {alertmanager,grafana,prometheus} services logFormat to
json to enable easier machine log parsing.