From 8ad827159ab4b443aff1eb24f7db5eb7ba4ceccc Mon Sep 17 00:00:00 2001 From: Steven Fitzpatrick Date: Wed, 26 May 2021 16:10:58 +0000 Subject: [PATCH] Remove Minio Components from airship-core lma Work on this minio chart has been halted, so we need an alternative S3 provider in the long term (#111). Removing references to this function and associated CRs from airship-core deployments and the lma-configs composite Also, taking the opportunity to move our cluster flow & output definitions out of the logging-operator-logging HelmRelease and into proper CR documents Relates-To: #111 Relates-To: #150 Change-Id: Id4b4b8d07fb6b38ba033e76a015d8703efca727a --- manifests/composite/lma-infra/namespace.yaml | 2 +- .../lma-infra/patches/logging-spec.yaml | 18 ------------- .../monitoring-stack/patches/grafana.yaml | 2 +- .../logging/flows/cluster-flow-all.yaml | 20 +++++++++++++++ .../logging/flows/kustomization.yaml | 2 +- .../lma-configs/logging/flows/minio.yaml | 12 --------- .../outputs/cluster-output-elasticsearch.yaml | 17 +++++++++++++ .../logging/outputs/kustomization.yaml | 2 +- .../lma-configs/logging/outputs/minio.yaml | 25 ------------------- .../monitoring/prometheus/kustomization.yaml | 3 +-- .../servicemonitors/lma-infra-minio.yaml | 13 ---------- .../monitoring/thanos/kustomization.yaml | 1 - .../thanos/objectstores/kustomization.yaml | 3 +-- .../objectstores/lma-infra-thanos-minio.yaml | 12 --------- .../thanos/storeendpoints/kustomization.yaml | 1 - .../storeendpoints/lma-infra-minio.yaml | 11 -------- .../lma-infra/lma-infra-object-store.yaml | 17 ------------- .../target/lma-stack/minio-admin-secret.yaml | 8 ------ .../target/lma-stack/patches/minio.yaml | 17 ------------- 19 files changed, 43 insertions(+), 143 deletions(-) create mode 100644 manifests/function/lma-configs/logging/flows/cluster-flow-all.yaml delete mode 100644 manifests/function/lma-configs/logging/flows/minio.yaml create mode 100644 manifests/function/lma-configs/logging/outputs/cluster-output-elasticsearch.yaml delete mode 100644 manifests/function/lma-configs/logging/outputs/minio.yaml delete mode 100644 manifests/function/lma-configs/monitoring/prometheus/servicemonitors/lma-infra-minio.yaml delete mode 100644 manifests/function/lma-configs/monitoring/thanos/objectstores/lma-infra-thanos-minio.yaml delete mode 100644 manifests/function/lma-configs/monitoring/thanos/storeendpoints/lma-infra-minio.yaml delete mode 100644 manifests/site/reference-airship-core/target/lma-infra/lma-infra-object-store.yaml delete mode 100644 manifests/site/reference-airship-core/target/lma-stack/minio-admin-secret.yaml delete mode 100644 manifests/site/reference-airship-core/target/lma-stack/patches/minio.yaml diff --git a/manifests/composite/lma-infra/namespace.yaml b/manifests/composite/lma-infra/namespace.yaml index 22df02fd9..d29c1328f 100644 --- a/manifests/composite/lma-infra/namespace.yaml +++ b/manifests/composite/lma-infra/namespace.yaml @@ -1,4 +1,4 @@ apiVersion: v1 kind: Namespace metadata: - name: lma-infra + name: lma diff --git a/manifests/composite/lma-infra/patches/logging-spec.yaml b/manifests/composite/lma-infra/patches/logging-spec.yaml index 2366ef915..b573f69a3 100644 --- a/manifests/composite/lma-infra/patches/logging-spec.yaml +++ b/manifests/composite/lma-infra/patches/logging-spec.yaml @@ -27,21 +27,3 @@ spec: tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule - clusterFlows: - - name: all-pods - spec: - globalOutputRefs: - - elasticsearch - clusterOutputs: - - name: elasticsearch - spec: - elasticsearch: - host: elasticsearch-ingest.lma-infra.svc.cluster.local - port: 9200 - scheme: https - ssl_verify: false - ssl_version: TLSv1_2 - buffer: - timekey: 1m - timekey_wait: 30s - timekey_use_utc: true diff --git a/manifests/composite/monitoring-stack/patches/grafana.yaml b/manifests/composite/monitoring-stack/patches/grafana.yaml index 269fe4c6d..bde57a785 100644 --- a/manifests/composite/monitoring-stack/patches/grafana.yaml +++ b/manifests/composite/monitoring-stack/patches/grafana.yaml @@ -13,7 +13,7 @@ spec: datasources: - name: lma-infra-thanos type: prometheus - url: lma-infra-thanos-query.lma-infra.svc.cluster.local:10902 + url: lma-infra-thanos-query.lma.svc.cluster.local:10902 dashboardProviders: dashboardproviders.yaml: apiVersion: 1 diff --git a/manifests/function/lma-configs/logging/flows/cluster-flow-all.yaml b/manifests/function/lma-configs/logging/flows/cluster-flow-all.yaml new file mode 100644 index 000000000..e419279f5 --- /dev/null +++ b/manifests/function/lma-configs/logging/flows/cluster-flow-all.yaml @@ -0,0 +1,20 @@ +apiVersion: logging.banzaicloud.io/v1beta1 +kind: ClusterFlow +metadata: + name: all-pods +spec: + match: + - select: {} + filters: + - enhanceK8s: {} + - tag_normaliser: + format: ${namespace_name}.${pod_name}.${container_name} + - parser: + reserve_data: true + parse: + type: multi-format + patterns: + - format: nginx + - format: json + - format: multiline + globalOutputRefs: elasticsearch diff --git a/manifests/function/lma-configs/logging/flows/kustomization.yaml b/manifests/function/lma-configs/logging/flows/kustomization.yaml index 5ddd3d45b..3907dd7b8 100644 --- a/manifests/function/lma-configs/logging/flows/kustomization.yaml +++ b/manifests/function/lma-configs/logging/flows/kustomization.yaml @@ -1,2 +1,2 @@ resources: - - minio.yaml \ No newline at end of file + - cluster-flow-all.yaml diff --git a/manifests/function/lma-configs/logging/flows/minio.yaml b/manifests/function/lma-configs/logging/flows/minio.yaml deleted file mode 100644 index 2c6cde52d..000000000 --- a/manifests/function/lma-configs/logging/flows/minio.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: logging.banzaicloud.io/v1beta1 -kind: Flow -metadata: - name: lma-infra-minio -spec: - loggingRef: lma-infra-logging - match: - - select: - labels: - app: minio - localOutputRefs: - - lma-infra-minio \ No newline at end of file diff --git a/manifests/function/lma-configs/logging/outputs/cluster-output-elasticsearch.yaml b/manifests/function/lma-configs/logging/outputs/cluster-output-elasticsearch.yaml new file mode 100644 index 000000000..a00fd5277 --- /dev/null +++ b/manifests/function/lma-configs/logging/outputs/cluster-output-elasticsearch.yaml @@ -0,0 +1,17 @@ +apiVersion: logging.banzaicloud.io/v1beta1 +kind: ClusterOutput +metadata: + name: elasticsearch +spec: + loggingRef: lma-infra-logging + elasticsearch: + # TODO: Replace this with the real endpoint + host: elasticsearch-ingest.lma.svc.cluster.local + port: 9200 + scheme: https + ssl_verify: false + ssl_version: TLSv1_2 + buffer: + timekey: 1m + timekey_wait: 30s + timekey_use_utc: true diff --git a/manifests/function/lma-configs/logging/outputs/kustomization.yaml b/manifests/function/lma-configs/logging/outputs/kustomization.yaml index ef48e8a6d..fe863316b 100644 --- a/manifests/function/lma-configs/logging/outputs/kustomization.yaml +++ b/manifests/function/lma-configs/logging/outputs/kustomization.yaml @@ -1,2 +1,2 @@ resources: - - minio.yaml + - cluster-output-elasticsearch.yaml diff --git a/manifests/function/lma-configs/logging/outputs/minio.yaml b/manifests/function/lma-configs/logging/outputs/minio.yaml deleted file mode 100644 index eed1a80a3..000000000 --- a/manifests/function/lma-configs/logging/outputs/minio.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: logging.banzaicloud.io/v1beta1 -kind: Output -metadata: - name: lma-infra-minio -spec: - loggingRef: lma-infra-logging - s3: - aws_key_id: - valueFrom: - secretKeyRef: - key: fluentd-accesskey - name: lma-infra-object-store - aws_sec_key: - valueFrom: - secretKeyRef: - key: fluentd-secretkey - name: lma-infra-object-store - buffer: - timekey: 10s - timekey_use_utc: true - timekey_wait: 0s - force_path_style: "true" - path: ${tag}/%Y/%m/%d/ - s3_bucket: logs - s3_endpoint: http://minio.lma-infra.svc.cluster.local:9000 diff --git a/manifests/function/lma-configs/monitoring/prometheus/kustomization.yaml b/manifests/function/lma-configs/monitoring/prometheus/kustomization.yaml index 675060687..e584213cf 100644 --- a/manifests/function/lma-configs/monitoring/prometheus/kustomization.yaml +++ b/manifests/function/lma-configs/monitoring/prometheus/kustomization.yaml @@ -1,2 +1 @@ -resources: - - servicemonitors/lma-infra-minio.yaml +resources: [] diff --git a/manifests/function/lma-configs/monitoring/prometheus/servicemonitors/lma-infra-minio.yaml b/manifests/function/lma-configs/monitoring/prometheus/servicemonitors/lma-infra-minio.yaml deleted file mode 100644 index ab17284c5..000000000 --- a/manifests/function/lma-configs/monitoring/prometheus/servicemonitors/lma-infra-minio.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: lma-infra-minio - labels: - app: minio -spec: - selector: - matchLabels: - app: minio - endpoints: - - port: metrics - path: /minio/prometheus/metrics diff --git a/manifests/function/lma-configs/monitoring/thanos/kustomization.yaml b/manifests/function/lma-configs/monitoring/thanos/kustomization.yaml index 2e5c1d074..85c0b2d80 100644 --- a/manifests/function/lma-configs/monitoring/thanos/kustomization.yaml +++ b/manifests/function/lma-configs/monitoring/thanos/kustomization.yaml @@ -1,4 +1,3 @@ resources: - lma-infra-thanos.yaml - storeendpoints - - objectstores diff --git a/manifests/function/lma-configs/monitoring/thanos/objectstores/kustomization.yaml b/manifests/function/lma-configs/monitoring/thanos/objectstores/kustomization.yaml index 484a021e6..b7398e9e5 100644 --- a/manifests/function/lma-configs/monitoring/thanos/objectstores/kustomization.yaml +++ b/manifests/function/lma-configs/monitoring/thanos/objectstores/kustomization.yaml @@ -1,2 +1 @@ -resources: - - lma-infra-thanos-minio.yaml \ No newline at end of file +resources: [] \ No newline at end of file diff --git a/manifests/function/lma-configs/monitoring/thanos/objectstores/lma-infra-thanos-minio.yaml b/manifests/function/lma-configs/monitoring/thanos/objectstores/lma-infra-thanos-minio.yaml deleted file mode 100644 index 2a5de0ee9..000000000 --- a/manifests/function/lma-configs/monitoring/thanos/objectstores/lma-infra-thanos-minio.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: monitoring.banzaicloud.io/v1alpha1 -kind: ObjectStore -metadata: - name: lma-infra-thanos-minio -spec: - config: - mountFrom: - secretKeyRef: - name: lma-infra-object-store - key: thanos-config.yaml - bucketWeb: {} - compactor: {} diff --git a/manifests/function/lma-configs/monitoring/thanos/storeendpoints/kustomization.yaml b/manifests/function/lma-configs/monitoring/thanos/storeendpoints/kustomization.yaml index b8669ea83..5ccd7bf2b 100644 --- a/manifests/function/lma-configs/monitoring/thanos/storeendpoints/kustomization.yaml +++ b/manifests/function/lma-configs/monitoring/thanos/storeendpoints/kustomization.yaml @@ -1,3 +1,2 @@ resources: - lma-infra-prometheus.yaml - - lma-infra-minio.yaml diff --git a/manifests/function/lma-configs/monitoring/thanos/storeendpoints/lma-infra-minio.yaml b/manifests/function/lma-configs/monitoring/thanos/storeendpoints/lma-infra-minio.yaml deleted file mode 100644 index cb7b9c044..000000000 --- a/manifests/function/lma-configs/monitoring/thanos/storeendpoints/lma-infra-minio.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: monitoring.banzaicloud.io/v1alpha1 -kind: StoreEndpoint -metadata: - name: lma-infra-minio -spec: - thanos: lma-infra-thanos - config: - mountFrom: - secretKeyRef: - name: lma-infra-object-store - key: thanos-config.yaml diff --git a/manifests/site/reference-airship-core/target/lma-infra/lma-infra-object-store.yaml b/manifests/site/reference-airship-core/target/lma-infra/lma-infra-object-store.yaml deleted file mode 100644 index 95ad88e9e..000000000 --- a/manifests/site/reference-airship-core/target/lma-infra/lma-infra-object-store.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: lma-infra-object-store -type: Opaque -stringData: - fluentd-accesskey: admin - fluentd-secretkey: changeme - thanos-config.yaml: | - type: s3 - config: - insecure: true - endpoint: minio.lma-infra.svc.cluster.local:9000 - bucket: metrics - region: lma-infra - access_key: admin - secret_key: changeme diff --git a/manifests/site/reference-airship-core/target/lma-stack/minio-admin-secret.yaml b/manifests/site/reference-airship-core/target/lma-stack/minio-admin-secret.yaml deleted file mode 100644 index 1538e9304..000000000 --- a/manifests/site/reference-airship-core/target/lma-stack/minio-admin-secret.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: minio-admin-secret -type: Opaque -stringData: - accesskey: admin - secretkey: changeme diff --git a/manifests/site/reference-airship-core/target/lma-stack/patches/minio.yaml b/manifests/site/reference-airship-core/target/lma-stack/patches/minio.yaml deleted file mode 100644 index 97f0631ae..000000000 --- a/manifests/site/reference-airship-core/target/lma-stack/patches/minio.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: "helm.toolkit.fluxcd.io/v2beta1" -kind: HelmRelease -metadata: - name: minio -spec: - values: - replicas: 1 - persistence: - enabled: false - existingSecret: minio-admin-secret - buckets: - - name: logs - policy: none - purge: false - - name: metrics - policy: none - purge: false