From 125ffd723787eaee2ece8e72bb60235994725edd Mon Sep 17 00:00:00 2001 From: Steven Fitzpatrick Date: Mon, 12 Jul 2021 22:08:14 +0000 Subject: [PATCH] LMA Configuration Updates This change fixes some configurations of LMA CRs and values, such that logs and metrics are properly visible on kibana and grafana after workload and workload-config phase. Also: - Added dashboards to grafana - Added a container to kibana which sets up the discover page - Added the DependsOn directive was added to the workload-lma prometheus-elasticsearch-exporter HR - Added NodePort bindings to LMA services in multi-tenant/lma subcluster Change-Id: I14338f5cab0808654173a09494f5fde540397dcc --- .../patches/kube-prometheus-stack.yaml | 4 +-- .../lma-infra/patches/logging-operator.yaml | 2 +- .../lma-infra/patches/logging-spec.yaml | 5 ++-- .../monitoring-stack/patches/grafana.yaml | 15 +++++++++- .../patches/thanos-operator.yaml | 2 +- manifests/function/kibana/helmrelease.yaml | 29 +++++++++++++++++- .../kibana/replacements/versions.yaml | 20 +++++++++++++ .../logging/flows/cluster-flow-all.yaml | 13 +------- .../outputs/cluster-output-elasticsearch.yaml | 6 ++-- .../monitoring/thanos/lma-infra-thanos.yaml | 2 +- .../storeendpoints/lma-infra-prometheus.yaml | 2 +- .../lma/workload-config/kustomization.yaml | 3 ++ .../catalogues/subcluster-networking.yaml | 8 +++++ .../lma/workload-config/kustomization.yaml | 4 +++ .../workload-config/patches/thanos-query.yaml | 11 +++++++ .../replacements/kustomization.yaml | 2 ++ .../replacements/lma-service-ports.yaml | 19 ++++++++++++ .../lma/workload/services/kustomization.yaml | 20 ++++++++++--- .../workload/services/patches/grafana.yaml | 8 +++++ .../patches/kube-prometheus-stack.yaml | 12 ++++++++ .../replacements/lma-service-ports.yaml | 30 +++++++++++++++++++ .../lma-elasticsearch-output.yaml | 2 +- 22 files changed, 188 insertions(+), 31 deletions(-) create mode 100644 manifests/type/multi-tenant/sub-clusters/lma/workload-config/patches/thanos-query.yaml create mode 100644 manifests/type/multi-tenant/sub-clusters/lma/workload-config/replacements/kustomization.yaml create mode 100644 manifests/type/multi-tenant/sub-clusters/lma/workload-config/replacements/lma-service-ports.yaml create mode 100644 manifests/type/multi-tenant/sub-clusters/lma/workload/services/patches/grafana.yaml create mode 100644 manifests/type/multi-tenant/sub-clusters/lma/workload/services/patches/kube-prometheus-stack.yaml diff --git a/manifests/composite/lma-infra/patches/kube-prometheus-stack.yaml b/manifests/composite/lma-infra/patches/kube-prometheus-stack.yaml index 6663137e3..dce57eee9 100644 --- a/manifests/composite/lma-infra/patches/kube-prometheus-stack.yaml +++ b/manifests/composite/lma-infra/patches/kube-prometheus-stack.yaml @@ -3,9 +3,9 @@ kind: HelmRelease metadata: name: kube-prometheus-stack spec: - releaseName: lma-infra-monitoring + releaseName: monitoring values: - fullnameOverride: lma-infra-monitoring + fullnameOverride: monitoring grafana: enabled: false prometheus: diff --git a/manifests/composite/lma-infra/patches/logging-operator.yaml b/manifests/composite/lma-infra/patches/logging-operator.yaml index a0625eba9..28b47a254 100644 --- a/manifests/composite/lma-infra/patches/logging-operator.yaml +++ b/manifests/composite/lma-infra/patches/logging-operator.yaml @@ -6,7 +6,7 @@ spec: dependsOn: - name: kube-prometheus-stack values: - fullnameOverride: lma-infra-logging-operator + fullnameOverride: logging-operator monitoring: serviceMonitor: enabled: true diff --git a/manifests/composite/lma-infra/patches/logging-spec.yaml b/manifests/composite/lma-infra/patches/logging-spec.yaml index b573f69a3..c8c7cf05b 100644 --- a/manifests/composite/lma-infra/patches/logging-spec.yaml +++ b/manifests/composite/lma-infra/patches/logging-spec.yaml @@ -10,9 +10,8 @@ spec: # https://github.com/banzaicloud/logging-operator/blob/master/charts/logging-operator-logging/values.yaml values: controlNamespace: lma - loggingRef: lma-infra-logging - nameOverride: lma-infra-logging - fullnameOverride: lma-infra-logging + nameOverride: logging + fullnameOverride: logging fluentbit: tolerations: - key: node-role.kubernetes.io/master diff --git a/manifests/composite/monitoring-stack/patches/grafana.yaml b/manifests/composite/monitoring-stack/patches/grafana.yaml index bde57a785..2a78d9bd1 100644 --- a/manifests/composite/monitoring-stack/patches/grafana.yaml +++ b/manifests/composite/monitoring-stack/patches/grafana.yaml @@ -13,7 +13,8 @@ spec: datasources: - name: lma-infra-thanos type: prometheus - url: lma-infra-thanos-query.lma.svc.cluster.local:10902 + url: http://monitoring-thanos-query.lma.svc.cluster.local:10902 + access: server dashboardProviders: dashboardproviders.yaml: apiVersion: 1 @@ -26,6 +27,18 @@ spec: path: /var/lib/grafana/dashboards/default dashboards: default: + node-exporter: + gnetId: 7249 + revision: 1 + datasource: lma-infra-thanos + elasticsearch: + gnetId: 2322 + revision: 4 + datasource: lma-infra-thanos + kube-state-metrics: + gnetId: 13332 + revision: 6 + datasource: lma-infra-thanos logging: gnetId: 7752 revision: 4 diff --git a/manifests/composite/monitoring-stack/patches/thanos-operator.yaml b/manifests/composite/monitoring-stack/patches/thanos-operator.yaml index 656bb731f..568845817 100644 --- a/manifests/composite/monitoring-stack/patches/thanos-operator.yaml +++ b/manifests/composite/monitoring-stack/patches/thanos-operator.yaml @@ -4,4 +4,4 @@ metadata: name: thanos-operator data: values: - nameOverride: lma-infra-thanos + nameOverride: monitoring-thanos diff --git a/manifests/function/kibana/helmrelease.yaml b/manifests/function/kibana/helmrelease.yaml index 8a1bf0669..0bae3d972 100644 --- a/manifests/function/kibana/helmrelease.yaml +++ b/manifests/function/kibana/helmrelease.yaml @@ -13,4 +13,31 @@ spec: name: collator interval: 1m timeout: 5m - values: {} + values: + extraContainers: + - name: index-patterns + image: IMAGE:TAG + command: + - bash + - -c + - | + #!/bin/bash + set -ex + KIBANA_URL=http://localhost:5601 + + while [[ "$(curl -s -o /dev/null -w '%{http_code}\n' $KIBANA_URL/app/kibana)" != "200" ]]; do + echo "Waiting for API" + sleep 1; + done + + curl -XPOST $KIBANA_URL/api/index_patterns/index_pattern \ + -H 'kbn-xsrf: true' \ + -H 'Content-Type: application/json' \ + -d '{ "index_pattern": { "title": "logstash-*", "timeFieldName":"@timestamp" }}' + + curl -XPOST $KIBANA_URL/api/kibana/settings/defaultIndex \ + -H 'Content-Type: application/json' \ + -H 'kbn-xsrf: true' \ + -d '{"value" : "logstash-*"}' + + tail -f /dev/null diff --git a/manifests/function/kibana/replacements/versions.yaml b/manifests/function/kibana/replacements/versions.yaml index b9ce5e612..19c73394c 100644 --- a/manifests/function/kibana/replacements/versions.yaml +++ b/manifests/function/kibana/replacements/versions.yaml @@ -47,3 +47,23 @@ replacements: kind: HelmRelease name: kibana fieldrefs: ["{.spec.values.imageTag}"] +- source: + objref: + kind: VersionsCatalogue + name: versions-treasuremap + fieldref: "{.spec.image_components.kibana.kibana.repository}" + target: + objref: + kind: HelmRelease + name: kibana + fieldrefs: ["{.spec.values.extraContainers[?(@.name == 'index-patterns')].image}%IMAGE%"] +- source: + objref: + kind: VersionsCatalogue + name: versions-treasuremap + fieldref: "{.spec.image_components.kibana.kibana.tag}" + target: + objref: + kind: HelmRelease + name: kibana + fieldrefs: ["{.spec.values.extraContainers[?(@.name == 'index-patterns')].image}%TAG%"] diff --git a/manifests/function/lma-configs/logging/flows/cluster-flow-all.yaml b/manifests/function/lma-configs/logging/flows/cluster-flow-all.yaml index 5c54a6813..ebcf612cc 100644 --- a/manifests/function/lma-configs/logging/flows/cluster-flow-all.yaml +++ b/manifests/function/lma-configs/logging/flows/cluster-flow-all.yaml @@ -5,17 +5,6 @@ metadata: spec: match: - select: {} - filters: - - enhanceK8s: {} - - tag_normaliser: - format: ${namespace_name}.${pod_name}.${container_name} - - parser: - reserve_data: true - parse: - type: multi-format - patterns: - - format: nginx - - format: json - - format: multiline + filters: [] globalOutputRefs: - elasticsearch diff --git a/manifests/function/lma-configs/logging/outputs/cluster-output-elasticsearch.yaml b/manifests/function/lma-configs/logging/outputs/cluster-output-elasticsearch.yaml index a00fd5277..22d6f11f7 100644 --- a/manifests/function/lma-configs/logging/outputs/cluster-output-elasticsearch.yaml +++ b/manifests/function/lma-configs/logging/outputs/cluster-output-elasticsearch.yaml @@ -3,12 +3,12 @@ kind: ClusterOutput metadata: name: elasticsearch spec: - loggingRef: lma-infra-logging elasticsearch: - # TODO: Replace this with the real endpoint host: elasticsearch-ingest.lma.svc.cluster.local port: 9200 - scheme: https + scheme: http + logstash_format: true + include_timestamp: true ssl_verify: false ssl_version: TLSv1_2 buffer: diff --git a/manifests/function/lma-configs/monitoring/thanos/lma-infra-thanos.yaml b/manifests/function/lma-configs/monitoring/thanos/lma-infra-thanos.yaml index 34aa8eac6..4be8495b4 100644 --- a/manifests/function/lma-configs/monitoring/thanos/lma-infra-thanos.yaml +++ b/manifests/function/lma-configs/monitoring/thanos/lma-infra-thanos.yaml @@ -1,7 +1,7 @@ apiVersion: monitoring.banzaicloud.io/v1alpha1 kind: Thanos metadata: - name: lma-infra-thanos + name: monitoring-thanos spec: queryDiscovery: true query: {} diff --git a/manifests/function/lma-configs/monitoring/thanos/storeendpoints/lma-infra-prometheus.yaml b/manifests/function/lma-configs/monitoring/thanos/storeendpoints/lma-infra-prometheus.yaml index ef8b60e3c..894558124 100644 --- a/manifests/function/lma-configs/monitoring/thanos/storeendpoints/lma-infra-prometheus.yaml +++ b/manifests/function/lma-configs/monitoring/thanos/storeendpoints/lma-infra-prometheus.yaml @@ -3,7 +3,7 @@ kind: StoreEndpoint metadata: name: lma-infra-prometheus spec: - thanos: lma-infra-thanos + thanos: monitoring-thanos selector: labels: app: prometheus diff --git a/manifests/site/reference-multi-tenant/sub-clusters/lma/workload-config/kustomization.yaml b/manifests/site/reference-multi-tenant/sub-clusters/lma/workload-config/kustomization.yaml index 4fdd9c4c0..86e003145 100644 --- a/manifests/site/reference-multi-tenant/sub-clusters/lma/workload-config/kustomization.yaml +++ b/manifests/site/reference-multi-tenant/sub-clusters/lma/workload-config/kustomization.yaml @@ -1,3 +1,6 @@ resources: - ../../../../../type/multi-tenant/sub-clusters/lma/workload-config - ../catalogues + +transformers: + - ../../../../../type/multi-tenant/sub-clusters/lma/workload-config/replacements diff --git a/manifests/type/multi-tenant/shared/catalogues/subcluster-networking.yaml b/manifests/type/multi-tenant/shared/catalogues/subcluster-networking.yaml index 982ce76c4..4731797ef 100644 --- a/manifests/type/multi-tenant/shared/catalogues/subcluster-networking.yaml +++ b/manifests/type/multi-tenant/shared/catalogues/subcluster-networking.yaml @@ -36,6 +36,14 @@ spec: nodePort: 30003 - name: kibana nodePort: 30004 + - name: grafana + nodePort: 30005 + - name: prometheus + nodePort: 30006 + - name: alertmanager + nodePort: 30007 + - name: thanos-query + nodePort: 30008 # Potential ports that can be used by sub-cluster services. - name: loadBalancerWorker nodePortRange: diff --git a/manifests/type/multi-tenant/sub-clusters/lma/workload-config/kustomization.yaml b/manifests/type/multi-tenant/sub-clusters/lma/workload-config/kustomization.yaml index a327714bc..f98ba5ead 100644 --- a/manifests/type/multi-tenant/sub-clusters/lma/workload-config/kustomization.yaml +++ b/manifests/type/multi-tenant/sub-clusters/lma/workload-config/kustomization.yaml @@ -6,3 +6,7 @@ patches: target: kind: ClusterFlow name: all-pods + - path: patches/thanos-query.yaml + target: + kind: Thanos + name: monitoring-thanos diff --git a/manifests/type/multi-tenant/sub-clusters/lma/workload-config/patches/thanos-query.yaml b/manifests/type/multi-tenant/sub-clusters/lma/workload-config/patches/thanos-query.yaml new file mode 100644 index 000000000..97d5669fe --- /dev/null +++ b/manifests/type/multi-tenant/sub-clusters/lma/workload-config/patches/thanos-query.yaml @@ -0,0 +1,11 @@ +apiVersion: monitoring.banzaicloud.io/v1alpha1 +kind: Thanos +metadata: + name: monitoring-thanos +spec: + query: + serviceOverrides: + type: NodePort + ports: + - name: http + port: 10902 \ No newline at end of file diff --git a/manifests/type/multi-tenant/sub-clusters/lma/workload-config/replacements/kustomization.yaml b/manifests/type/multi-tenant/sub-clusters/lma/workload-config/replacements/kustomization.yaml new file mode 100644 index 000000000..8b64321c4 --- /dev/null +++ b/manifests/type/multi-tenant/sub-clusters/lma/workload-config/replacements/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - lma-service-ports.yaml diff --git a/manifests/type/multi-tenant/sub-clusters/lma/workload-config/replacements/lma-service-ports.yaml b/manifests/type/multi-tenant/sub-clusters/lma/workload-config/replacements/lma-service-ports.yaml new file mode 100644 index 000000000..abbf6d93a --- /dev/null +++ b/manifests/type/multi-tenant/sub-clusters/lma/workload-config/replacements/lma-service-ports.yaml @@ -0,0 +1,19 @@ +apiVersion: airshipit.org/v1alpha1 +kind: ReplacementTransformer +metadata: + name: lma-service-ports + annotations: + config.kubernetes.io/function: |- + container: + image: localhost/replacement-transformer +replacements: +- source: + objref: + kind: VariableCatalogue + name: subcluster-networking + fieldref: "{.spec.lma.exposed_services[?(.name == 'thanos-query')].nodePort}" + target: + objref: + kind: Thanos + name: monitoring-thanos + fieldrefs: ["{.spec.query.serviceOverrides.ports[?(.name == 'http')].nodePort}"] diff --git a/manifests/type/multi-tenant/sub-clusters/lma/workload/services/kustomization.yaml b/manifests/type/multi-tenant/sub-clusters/lma/workload/services/kustomization.yaml index fd0dccdd9..f420381e7 100644 --- a/manifests/type/multi-tenant/sub-clusters/lma/workload/services/kustomization.yaml +++ b/manifests/type/multi-tenant/sub-clusters/lma/workload/services/kustomization.yaml @@ -3,10 +3,6 @@ resources: - ../../../../../../composite/elastic-stack - ../../../../../../composite/monitoring-stack -# This kustomization exists be able to set the namespace here without -# affecting the namespaces of anything else in the parent directory -namespace: lma - patches: - path: patches/elasticsearch-ingest.yaml target: @@ -20,3 +16,19 @@ patches: target: kind: HelmRelease name: kibana + - path: patches/grafana.yaml + target: + kind: HelmRelease + name: grafana + - path: patches/kube-prometheus-stack.yaml + target: + kind: HelmRelease + name: kube-prometheus-stack + - patch: |- + - op: add + path: /spec/dependsOn + value: + - name: kube-prometheus-stack + target: + kind: HelmRelease + name: prometheus-elasticsearch-exporter diff --git a/manifests/type/multi-tenant/sub-clusters/lma/workload/services/patches/grafana.yaml b/manifests/type/multi-tenant/sub-clusters/lma/workload/services/patches/grafana.yaml new file mode 100644 index 000000000..d555553b0 --- /dev/null +++ b/manifests/type/multi-tenant/sub-clusters/lma/workload/services/patches/grafana.yaml @@ -0,0 +1,8 @@ +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: grafana +spec: + values: + service: + type: NodePort diff --git a/manifests/type/multi-tenant/sub-clusters/lma/workload/services/patches/kube-prometheus-stack.yaml b/manifests/type/multi-tenant/sub-clusters/lma/workload/services/patches/kube-prometheus-stack.yaml new file mode 100644 index 000000000..75da4c522 --- /dev/null +++ b/manifests/type/multi-tenant/sub-clusters/lma/workload/services/patches/kube-prometheus-stack.yaml @@ -0,0 +1,12 @@ +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: kube-prometheus-stack +spec: + values: + alertmanager: + service: + type: NodePort + prometheus: + service: + type: NodePort diff --git a/manifests/type/multi-tenant/sub-clusters/lma/workload/services/replacements/lma-service-ports.yaml b/manifests/type/multi-tenant/sub-clusters/lma/workload/services/replacements/lma-service-ports.yaml index d8ee6b740..f67e6639a 100644 --- a/manifests/type/multi-tenant/sub-clusters/lma/workload/services/replacements/lma-service-ports.yaml +++ b/manifests/type/multi-tenant/sub-clusters/lma/workload/services/replacements/lma-service-ports.yaml @@ -27,3 +27,33 @@ replacements: kind: HelmRelease name: kibana fieldrefs: ["{.spec.values.service.nodePort}"] +- source: + objref: + kind: VariableCatalogue + name: subcluster-networking + fieldref: "{.spec.lma.exposed_services[?(.name == 'grafana')].nodePort}" + target: + objref: + kind: HelmRelease + name: grafana + fieldrefs: ["{.spec.values.service.nodePort}"] +- source: + objref: + kind: VariableCatalogue + name: subcluster-networking + fieldref: "{.spec.lma.exposed_services[?(.name == 'prometheus')].nodePort}" + target: + objref: + kind: HelmRelease + name: kube-prometheus-stack + fieldrefs: ["{.spec.values.prometheus.service.nodePort}"] +- source: + objref: + kind: VariableCatalogue + name: subcluster-networking + fieldref: "{.spec.lma.exposed_services[?(.name == 'alertmanager')].nodePort}" + target: + objref: + kind: HelmRelease + name: kube-prometheus-stack + fieldrefs: ["{.spec.values.alertmanager.service.nodePort}"] diff --git a/manifests/type/multi-tenant/target/workload-config/replacements/lma-elasticsearch-output.yaml b/manifests/type/multi-tenant/target/workload-config/replacements/lma-elasticsearch-output.yaml index 0ea32bbe7..481477023 100644 --- a/manifests/type/multi-tenant/target/workload-config/replacements/lma-elasticsearch-output.yaml +++ b/manifests/type/multi-tenant/target/workload-config/replacements/lma-elasticsearch-output.yaml @@ -21,7 +21,7 @@ replacements: objref: kind: VariableCatalogue name: subcluster-networking - fieldref: "{.spec.lma.exposed_services[?(@.name == 'elasticsearch-ingest')].nodePort}" + fieldref: "{.spec.lma.exposed_services[?(.name == 'elasticsearch-ingest')].nodePort}" target: objref: kind: ClusterOutput