3. Configure monitoring apps path based endpoints

* Add monitoring_ingress_enabled magnum label to set up ingress with
path based routing for all the configured services
{alertmanager,grafana,prometheus}. When using this,
cluster_root_domain_name magnum label must be used to setup base path
where this services are available.
* Add cluster_basic_auth_secret magnum label to configure basic auth
on unprotected services {alertmanager and  prometheus}. This is only
in effect when app access is routed by ingress.
* Set services logFormat to json to enable easier machine log parsing.

task: 39477
story: 2006765

Depends-On: Ieb90605182626869528349a7fdeed65061914bcb
Change-Id: Ie0e7000e0d94b2037f2c398fa67a2a2b7e256bc3
Signed-off-by: Diogo Guerra <diogo.filipe.tomas.guerra@cern.ch>

doc/source/user/index.rst

@@ -328,6 +328,13 @@ the table are linked to more details elsewhere in the user guide.
 +---------------------------------------+--------------------+---------------+
 | `monitoring_interval_seconds`_        | see below          | see below     |
 +---------------------------------------+--------------------+---------------+
+| `monitoring_ingress_enabled`_         | - true             | false         |
+|                                       | - false            |               |
++---------------------------------------+--------------------+---------------+
+| `cluster_basic_auth_secret`_          | see below          | see below     |
++---------------------------------------+--------------------+---------------+
+| `cluster_root_domain_name`_           | see below          | see below     |
++---------------------------------------+--------------------+---------------+
 | `prometheus_operator_chart_tag`_      | see below          | see below     |
 +---------------------------------------+--------------------+---------------+
 | `prometheus_adapter_enabled`_         | - true             | true          |
@@ -1505,6 +1512,23 @@ _`monitoring_storage_class_name`
   monitoring_retention_size and one for grafana which is fixed at 1Gi.
   Default: ""
 
+_`monitoring_ingress_enabled`
+  Enable configuration of ingresses for the enabled monitoring services
+  {alertmanager,grafana,prometheus}.
+  Default: false
+
+_`cluster_basic_auth_secret`
+  The kubernetes secret to use for the proxy basic auth username and password
+  for the unprotected services {alertmanager,prometheus}. Basic auth is only
+  set up if this file is specified.
+  The secret must be in the same namespace as the used proxy (kube-system).
+  Default: ""
+
+_`cluster_root_domain_name`
+  The root domain name to use for the cluster automatically set up
+  applications.
+  Default: "localhost"
+
 _`prometheus_adapter_enabled`
   Enable installation of cluster custom metrics provided by the
   stable/prometheus-adapter helm chart. This service depends on

magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh

@@ -63,6 +63,9 @@ MONITORING_RETENTION_DAYS="$MONITORING_RETENTION_DAYS"
 MONITORING_RETENTION_SIZE="$MONITORING_RETENTION_SIZE"
 MONITORING_INTERVAL_SECONDS="$MONITORING_INTERVAL_SECONDS"
 MONITORING_STORAGE_CLASS_NAME="$MONITORING_STORAGE_CLASS_NAME"
+MONITORING_INGRESS_ENABLED="$MONITORING_INGRESS_ENABLED"
+CLUSTER_BASIC_AUTH_SECRET="$CLUSTER_BASIC_AUTH_SECRET"
+CLUSTER_ROOT_DOMAIN_NAME="$CLUSTER_ROOT_DOMAIN_NAME"
 PROMETHEUS_OPERATOR_CHART_TAG="$PROMETHEUS_OPERATOR_CHART_TAG"
 PROMETHEUS_ADAPTER_ENABLED="$PROMETHEUS_ADAPTER_ENABLED"
 PROMETHEUS_ADAPTER_CHART_TAG="$PROMETHEUS_ADAPTER_CHART_TAG"

magnum/drivers/common/templates/kubernetes/helm/prometheus-operator.sh

@@ -31,6 +31,48 @@ EOF
         APP_GRAFANA_PERSISTENT_STORAGE="true"
     fi
 
+    # Create services for grafana/prometheus/alermanager
+    APP_INGRESS_PATH_APPEND=""
+    APP_INGRESS_ANNOTATIONS=""
+    APP_INGRESS_ROUTE_ANNOTATIONS=""
+    APP_INGRESS_BASIC_AUTH_ANNOTATIONS=""
+    if [ "${INGRESS_CONTROLLER}" == "nginx" ]; then
+        APP_INGRESS_PATH_APPEND="(/|$)(.*)"
+        APP_INGRESS_ANNOTATIONS=$(cat << EOF
+        nginx.ingress.kubernetes.io/ssl-redirect: "true"
+        nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
+EOF
+        )
+        APP_INGRESS_ROUTE_ANNOTATIONS=$(cat << 'EOF'
+        nginx.ingress.kubernetes.io/rewrite-target: /$2
+EOF
+        )
+        if [ "${CLUSTER_BASIC_AUTH_SECRET}" != "" ]; then
+            APP_INGRESS_BASIC_AUTH_ANNOTATIONS=$(cat << EOF
+        nginx.ingress.kubernetes.io/auth-type: basic
+        nginx.ingress.kubernetes.io/auth-secret: ${CLUSTER_BASIC_AUTH_SECRET}
+EOF
+            )
+        fi #END BASIC AUTH
+    elif [ "${INGRESS_CONTROLLER}" == "traefik" ]; then
+        APP_INGRESS_ANNOTATIONS=$(cat << EOF
+        traefik.ingress.kubernetes.io/frontend-entry-points: https
+        traefik.ingress.kubernetes.io/protocol: http
+EOF
+        )
+        APP_INGRESS_ROUTE_ANNOTATIONS=$(cat << EOF
+        traefik.ingress.kubernetes.io/rule-type: PathPrefixStrip
+EOF
+        )
+        if [ "${CLUSTER_BASIC_AUTH_SECRET}" != "" ]; then
+            APP_INGRESS_BASIC_AUTH_ANNOTATIONS=$(cat << EOF
+        ingress.kubernetes.io/auth-type: basic
+        ingress.kubernetes.io/auth-secret: ${CLUSTER_BASIC_AUTH_SECRET}
+EOF
+            )
+        fi #END BASIC AUTH
+    fi
+
     # Validate if communication node <-> master is secure or insecure
     PROTOCOL="https"
     INSECURE_SKIP_VERIFY="False"
@@ -54,17 +96,36 @@ prometheus-operator:
       etcd: false
 
   alertmanager:
+    ingress:
+      enabled: ${MONITORING_INGRESS_ENABLED}
+      annotations:
+        kubernetes.io/ingress.class: ${INGRESS_CONTROLLER}
+${APP_INGRESS_ANNOTATIONS}
+${APP_INGRESS_ROUTE_ANNOTATIONS}
+${APP_INGRESS_BASIC_AUTH_ANNOTATIONS}
+      ## Hosts must be provided if Ingress is enabled.
+      hosts:
+      - ${CLUSTER_ROOT_DOMAIN_NAME}
+      paths:
+      - /alertmanager${APP_INGRESS_PATH_APPEND}
+      ## TLS configuration for Alertmanager Ingress
+      ## Secret must be manually created in the namespace
+      tls: []
+      # - secretName: alertmanager-general-tls
+      #   hosts:
+      #   - alertmanager.example.com
     alertmanagerSpec:
       image:
         repository: ${CONTAINER_INFRA_PREFIX:-quay.io/prometheus/}alertmanager
-      # # Needs testing
+      logFormat: json
+      externalUrl: https://${CLUSTER_ROOT_DOMAIN_NAME}/alertmanager
+      # routePrefix: /alertmanager
       # resources:
       #   requests:
       #     cpu: 100m
       #     memory: 256Mi
       priorityClassName: "system-cluster-critical"
 
-  # Dashboard
   grafana:
     image:
       repository: ${CONTAINER_INFRA_PREFIX:-grafana/}grafana
@@ -76,10 +137,46 @@ prometheus-operator:
         cpu: 100m
         memory: 128Mi
     adminPassword: ${GRAFANA_ADMIN_PASSWD}
+    ingress:
+      enabled: ${MONITORING_INGRESS_ENABLED}
+      annotations:
+        kubernetes.io/ingress.class: ${INGRESS_CONTROLLER}
+${APP_INGRESS_ANNOTATIONS}
+      ## Hostnames.
+      ## Must be provided if Ingress is enable.
+      hosts:
+      - ${CLUSTER_ROOT_DOMAIN_NAME}
+      path: /grafana${APP_INGRESS_PATH_APPEND}
+      ## TLS configuration for grafana Ingress
+      ## Secret must be manually created in the namespace
+      tls: []
+      # - secretName: grafana-general-tls
+      #   hosts:
+      #   - grafana.example.com
     persistence:
       enabled: ${APP_GRAFANA_PERSISTENT_STORAGE}
       storageClassName: ${MONITORING_STORAGE_CLASS_NAME}
       size: 1Gi
+    grafana.ini:
+      server:
+        domain: ${CLUSTER_ROOT_DOMAIN_NAME}
+        root_url: https://${CLUSTER_ROOT_DOMAIN_NAME}/grafana
+        serve_from_sub_path: true
+      paths:
+        data: /var/lib/grafana/data
+        logs: /var/log/grafana
+        plugins: /var/lib/grafana/plugins
+        provisioning: /etc/grafana/provisioning
+      analytics:
+        check_for_updates: true
+      log:
+        mode: console
+      log.console:
+        format: json
+      grafana_net:
+        url: https://grafana.net
+    plugins:
+    - grafana-piechart-panel
 
   kubeApiServer:
     tlsConfig:
@@ -196,6 +293,7 @@ prometheus-operator:
     # requests:
     #   cpu: 5m
     #   memory: 10Mi
+    logFormat: json
     image:
       repository: ${CONTAINER_INFRA_PREFIX:-quay.io/coreos/}prometheus-operator
     configmapReloadImage:
@@ -206,6 +304,32 @@ prometheus-operator:
       repository: ${CONTAINER_INFRA_PREFIX:-k8s.gcr.io/}hyperkube
 
   prometheus:
+    ingress:
+      enabled: ${MONITORING_INGRESS_ENABLED}
+      annotations:
+        kubernetes.io/ingress.class: ${INGRESS_CONTROLLER}
+${APP_INGRESS_ANNOTATIONS}
+${APP_INGRESS_ROUTE_ANNOTATIONS}
+${APP_INGRESS_BASIC_AUTH_ANNOTATIONS}
+      ## Hostnames.
+      ## Must be provided if Ingress is enabled.
+      hosts:
+      - ${CLUSTER_ROOT_DOMAIN_NAME}
+      paths:
+      - /prometheus${APP_INGRESS_PATH_APPEND}
+      ## TLS configuration for Prometheus Ingress
+      ## Secret must be manually created in the namespace
+      tls: []
+        # - secretName: prometheus-general-tls
+        #   hosts:
+        #     - prometheus.example.com
+    serviceMonitor:
+      ## scheme: HTTP scheme to use for scraping. Can be used with tlsConfig for example if using istio mTLS.
+      scheme: ""
+      ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS.
+      ## Of type: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#tlsconfig
+      tlsConfig: {}
+      bearerTokenFile:
     prometheusSpec:
       scrapeInterval: ${MONITORING_INTERVAL_SECONDS}s
       scrapeInterval: 30s
@@ -215,6 +339,7 @@ prometheus-operator:
       retention: 14d
       externalLabels:
         cluster_uuid: ${CLUSTER_UUID}
+      externalUrl: https://${CLUSTER_ROOT_DOMAIN_NAME}/prometheus
       ## Secrets is a list of Secrets in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods.
       ## The Secrets are mounted into /etc/prometheus/secrets/. Secrets changes after initial creation of a Prometheus object are not
       ## reflected in the running Pods. To change the secrets mounted into the Prometheus Pods, the object must be deleted and recreated
@@ -226,6 +351,8 @@ prometheus-operator:
       # - kube-proxy-manager-certificates
       retention: ${MONITORING_RETENTION_DAYS}d
       retentionSize: ${MONITORING_RETENTION_SIZE_GB}GB
+      logFormat: json
+      #routePrefix: /prometheus
       resources:
         requests:
           cpu: ${PROMETHEUS_SERVER_CPU}m

magnum/drivers/heat/k8s_fedora_template_def.py

@@ -102,6 +102,9 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition):
                       'monitoring_retention_size',
                       'monitoring_interval_seconds',
                       'monitoring_storage_class_name',
+                      'monitoring_ingress_enabled',
+                      'cluster_basic_auth_secret',
+                      'cluster_root_domain_name',
                       'prometheus_operator_chart_tag',
                       'prometheus_adapter_enabled',
                       'prometheus_adapter_chart_tag',

magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml

@@ -716,6 +716,23 @@ parameters:
     description: The kubernetes storage class name to use for the prometheus pvc.
     default: ""
 
+  monitoring_ingress_enabled:
+    type: boolean
+    description: Enable/disable configuration of ingresses for the monitoring services.
+    default: false
+
+  cluster_basic_auth_secret:
+    type: string
+    description: >
+      The kubernetes secret name to use for the proxy basic auth username and password.
+    default: ""
+
+  cluster_root_domain_name:
+    type: string
+    description: >
+      The root domain name to use for the cluster automatically set up applications.
+    default: "localhost"
+
   prometheus_operator_chart_tag:
     type: string
     description: The stable/prometheus-operator chart version to use.
@@ -1254,6 +1271,9 @@ resources:
           monitoring_retention_size: {get_param: monitoring_retention_size}
           monitoring_interval_seconds: {get_param: monitoring_interval_seconds}
           monitoring_storage_class_name: {get_param: monitoring_storage_class_name}
+          monitoring_ingress_enabled: {get_param: monitoring_ingress_enabled}
+          cluster_basic_auth_secret: {get_param: cluster_basic_auth_secret}
+          cluster_root_domain_name: {get_param: cluster_root_domain_name}
           prometheus_operator_chart_tag: {get_param: prometheus_operator_chart_tag}
           prometheus_adapter_enabled: {get_param: prometheus_adapter_enabled}
           prometheus_adapter_chart_tag: {get_param: prometheus_adapter_chart_tag}

magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml

@@ -481,6 +481,20 @@ parameters:
     type: string
     description: The kubernetes storage class name to use for the prometheus pvc.
 
+  monitoring_ingress_enabled:
+    type: boolean
+    description: Enable/disable configuration of ingresses for the monitoring services.
+
+  cluster_basic_auth_secret:
+    type: string
+    description: >
+      The kubernetes secret name to use for the proxy basic auth username and password.
+
+  cluster_root_domain_name:
+    type: string
+    description: >
+      The root domain name to use for the cluster automatically set up applications.
+
   prometheus_operator_chart_tag:
     type: string
     description: The stable/prometheus-operator chart version to use.
@@ -817,6 +831,9 @@ resources:
                   "$MONITORING_RETENTION_SIZE": {get_param: monitoring_retention_size}
                   "$MONITORING_INTERVAL_SECONDS": {get_param: monitoring_interval_seconds}
                   "$MONITORING_STORAGE_CLASS_NAME": {get_param: monitoring_storage_class_name}
+                  "$MONITORING_INGRESS_ENABLED": {get_param: monitoring_ingress_enabled}
+                  "$CLUSTER_BASIC_AUTH_SECRET": {get_param: cluster_basic_auth_secret}
+                  "$CLUSTER_ROOT_DOMAIN_NAME": {get_param: cluster_root_domain_name}
                   "$PROMETHEUS_OPERATOR_CHART_TAG": {get_param: prometheus_operator_chart_tag}
                   "$PROMETHEUS_ADAPTER_ENABLED": {get_param: prometheus_adapter_enabled}
                   "$PROMETHEUS_ADAPTER_CHART_TAG": {get_param: prometheus_adapter_chart_tag}

magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml

@@ -726,6 +726,23 @@ parameters:
     description: The kubernetes storage class name to use for the prometheus pvc.
     default: ""
 
+  monitoring_ingress_enabled:
+    type: boolean
+    description: Enable/disable configuration of ingresses for the monitoring services.
+    default: false
+
+  cluster_basic_auth_secret:
+    type: string
+    description: >
+      The kubernetes secret name to use for the proxy basic auth username and password.
+    default: ""
+
+  cluster_root_domain_name:
+    type: string
+    description: >
+      The root domain name to use for the cluster automatically set up applications.
+    default: "localhost"
+
   prometheus_operator_chart_tag:
     type: string
     description: The stable/prometheus-operator chart version to use.
@@ -1282,6 +1299,9 @@ resources:
           monitoring_retention_size: {get_param: monitoring_retention_size}
           monitoring_interval_seconds: {get_param: monitoring_interval_seconds}
           monitoring_storage_class_name: {get_param: monitoring_storage_class_name}
+          monitoring_ingress_enabled: {get_param: monitoring_ingress_enabled}
+          cluster_basic_auth_secret: {get_param: cluster_basic_auth_secret}
+          cluster_root_domain_name: {get_param: cluster_root_domain_name}
           prometheus_operator_chart_tag: {get_param: prometheus_operator_chart_tag}
           prometheus_adapter_enabled: {get_param: prometheus_adapter_enabled}
           prometheus_adapter_chart_tag: {get_param: prometheus_adapter_chart_tag}

magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml

@@ -485,6 +485,20 @@ parameters:
     type: string
     description: The kubernetes storage class name to use for the prometheus pvc.
 
+  monitoring_ingress_enabled:
+    type: boolean
+    description: Enable/disable configuration of ingresses for the monitoring services.
+
+  cluster_basic_auth_secret:
+    type: string
+    description: >
+      The kubernetes secret name to use for the proxy basic auth username and password.
+
+  cluster_root_domain_name:
+    type: string
+    description: >
+      The root domain name to use for the cluster automatically set up applications.
+
   prometheus_operator_chart_tag:
     type: string
     description: The stable/prometheus-operator chart version to use.
@@ -836,6 +850,9 @@ resources:
                   "$MONITORING_RETENTION_SIZE": {get_param: monitoring_retention_size}
                   "$MONITORING_INTERVAL_SECONDS": {get_param: monitoring_interval_seconds}
                   "$MONITORING_STORAGE_CLASS_NAME": {get_param: monitoring_storage_class_name}
+                  "$MONITORING_INGRESS_ENABLED": {get_param: monitoring_ingress_enabled}
+                  "$CLUSTER_BASIC_AUTH_SECRET": {get_param: cluster_basic_auth_secret}
+                  "$CLUSTER_ROOT_DOMAIN_NAME": {get_param: cluster_root_domain_name}
                   "$PROMETHEUS_OPERATOR_CHART_TAG": {get_param: prometheus_operator_chart_tag}
                   "$PROMETHEUS_ADAPTER_ENABLED": {get_param: prometheus_adapter_enabled}
                   "$PROMETHEUS_ADAPTER_CHART_TAG": {get_param: prometheus_adapter_chart_tag}

magnum/tests/unit/drivers/test_template_definition.py

@@ -547,6 +547,12 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
             'monitoring_interval_seconds')
         monitoring_storage_class_name = mock_cluster.labels.get(
             'monitoring_storage_class_name')
+        monitoring_ingress_enabled = mock_cluster.labels.get(
+            'monitoring_ingress_enabled')
+        cluster_basic_auth_secret = mock_cluster.labels.get(
+            'cluster_basic_auth_secret')
+        cluster_root_domain_name = mock_cluster.labels.get(
+            'cluster_root_domain_name')
         prometheus_operator_chart_tag = mock_cluster.labels.get(
             'prometheus_operator_chart_tag')
         prometheus_adapter_enabled = mock_cluster.labels.get(
@@ -686,6 +692,9 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
             'monitoring_retention_size': monitoring_retention_size,
             'monitoring_interval_seconds': monitoring_interval_seconds,
             'monitoring_storage_class_name': monitoring_storage_class_name,
+            'monitoring_ingress_enabled': monitoring_ingress_enabled,
+            'cluster_basic_auth_secret': cluster_basic_auth_secret,
+            'cluster_root_domain_name': cluster_root_domain_name,
             'prometheus_operator_chart_tag': prometheus_operator_chart_tag,
             'prometheus_adapter_enabled': prometheus_adapter_enabled,
             'prometheus_adapter_chart_tag': prometheus_adapter_chart_tag,
@@ -1090,6 +1099,12 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
             'monitoring_interval_seconds')
         monitoring_storage_class_name = mock_cluster.labels.get(
             'monitoring_storage_class_name')
+        monitoring_ingress_enabled = mock_cluster.labels.get(
+            'monitoring_ingress_enabled')
+        cluster_basic_auth_secret = mock_cluster.labels.get(
+            'cluster_basic_auth_secret')
+        cluster_root_domain_name = mock_cluster.labels.get(
+            'cluster_root_domain_name')
         prometheus_operator_chart_tag = mock_cluster.labels.get(
             'prometheus_operator_chart_tag')
         prometheus_adapter_enabled = mock_cluster.labels.get(
@@ -1232,6 +1247,9 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
             'monitoring_retention_size': monitoring_retention_size,
             'monitoring_interval_seconds': monitoring_interval_seconds,
             'monitoring_storage_class_name': monitoring_storage_class_name,
+            'monitoring_ingress_enabled': monitoring_ingress_enabled,
+            'cluster_basic_auth_secret': cluster_basic_auth_secret,
+            'cluster_root_domain_name': cluster_root_domain_name,
             'prometheus_operator_chart_tag': prometheus_operator_chart_tag,
             'prometheus_adapter_enabled': prometheus_adapter_enabled,
             'prometheus_adapter_chart_tag': prometheus_adapter_chart_tag,

releasenotes/notes/configure_monitoring_app_endpoints-f00600c244a76cf4.yaml

@@ -0,0 +1,15 @@
+---
+features:
+  - |
+    Added monitoring_ingress_enabled magnum label to set up ingress with
+    path based routing for all the configured services
+    {alertmanager,grafana,prometheus}. When using this,
+    cluster_root_domain_name magnum label must be used to setup base path
+    where this services are available.
+    Added cluster_basic_auth_secret magnum label to configure basic auth
+    on unprotected services {alertmanager and  prometheus}. This is only
+    in effect when app access is routed by ingress.
+upgrade:
+  - |
+    Configured {alertmanager,grafana,prometheus} services logFormat to
+    json to enable easier machine log parsing.