Add LMA to multi-tenant type and reference site

This change introduces LMA components to the multi-tenant type:

- Deploy lma-infra, elastic-stack and monitoring stack in lma/workload phase
- Added lma config to lma/workload-config for local-subcluster logging
- Added Node Ports for Elasticsearch & Kibana services in the lma subcluster
- Spec Elasticsearch resources for subcluster hosts

To deploy this in the reference-multi-tenant site, various phase
entrypoints have been added to the sub-clusters/lma directory

Relates-To: #150
Change-Id: I4d3b0d6ed9643e702c92760c1849e10320196ff8

manifests/site/reference-multi-tenant/sub-clusters/lma/catalogues/kustomization.yaml

@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  # This pulls in general site catalog information which is valid across clusters
+  # It also pulls in undercloud-specific values, which will be replaced below
+  - ../../../target/catalogues/
+
+transformers:
+  # This replaces lma-specific network data from the lma stanza
+  # of the subcluster-networking catalogue into the standard networking catalogue
+  - ../../../../../type/multi-tenant/sub-clusters/lma/catalogue-replacements

manifests/site/reference-multi-tenant/sub-clusters/lma/controlplane/kustomization.yaml

@@ -0,0 +1,6 @@
+resources:
+  - ../../../../../type/multi-tenant/sub-clusters/lma/controlplane
+  - ../catalogues
+
+transformers:
+  - ../../../../../type/multi-tenant/sub-clusters/lma/controlplane/replacements

manifests/site/reference-multi-tenant/sub-clusters/lma/initinfra-networking/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+  - ../../../../../type/multi-tenant/sub-clusters/lma/initinfra-networking

manifests/site/reference-multi-tenant/sub-clusters/lma/initinfra/kustomization.yaml

@@ -0,0 +1,6 @@
+resources:
+  - ../../../../../type/multi-tenant/sub-clusters/lma/initinfra
+  - ../catalogues
+
+transformers:
+  - ../../../../../type/multi-tenant/sub-clusters/lma/initinfra/replacements

manifests/site/reference-multi-tenant/sub-clusters/lma/provide-infra/kustomization.yaml

@@ -0,0 +1,6 @@
+resources:
+  - ../../../../../type/multi-tenant/sub-clusters/lma/provide-infra
+  - ../catalogues
+
+transformers:
+  - ../../../../../type/multi-tenant/sub-clusters/lma/provide-infra/replacements

manifests/site/reference-multi-tenant/sub-clusters/lma/workers/kustomization.yaml

@@ -0,0 +1,7 @@
+resources:
+  - ../../../../../type/multi-tenant/sub-clusters/lma/workers
+  - ../catalogues
+  - metal3machinetemplate.yaml
+
+transformers:
+  - ../../../../../type/multi-tenant/sub-clusters/lma/workers/replacements

manifests/site/reference-multi-tenant/sub-clusters/lma/workers/metal3machinetemplate.yaml

@@ -0,0 +1,10 @@
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
+kind: Metal3MachineTemplate
+metadata:
+  name: worker-1
+spec:
+  template:
+    spec:
+      image:
+        url: http://10.23.24.102:80/images/data-plane.qcow2
+        checksum: http://10.23.24.102:80/images/data-plane.qcow2.md5sum

manifests/site/reference-multi-tenant/sub-clusters/lma/workload-config/kustomization.yaml

@@ -0,0 +1,3 @@
+resources:
+  - ../../../../../type/multi-tenant/sub-clusters/lma/workload-config
+  - ../catalogues

manifests/site/reference-multi-tenant/sub-clusters/lma/workload/kustomization.yaml

@@ -0,0 +1,6 @@
+resources:
+  - ../../../../../type/multi-tenant/sub-clusters/lma/workload
+  - ../catalogues
+
+transformers:
+  - ../../../../../type/multi-tenant/sub-clusters/lma/workload/replacements

manifests/type/multi-tenant/shared/catalogues/subcluster-networking.yaml

@@ -32,6 +32,10 @@ spec:
         nodePort: 30001
       - name: loadBalancerControlPlane
         nodePort: 30002
+      - name: elasticsearch-ingest
+        nodePort: 30003
+      - name: kibana
+        nodePort: 30004
       # Potential ports that can be used by sub-cluster services.
       - name: loadBalancerWorker
         nodePortRange:

manifests/type/multi-tenant/sub-clusters/lma/lma-configs/kustomization.yaml

@@ -1,4 +0,0 @@
-resources:
-  - ../../../../../function/lma-configs
-
-namespace: lma

manifests/type/multi-tenant/sub-clusters/lma/phases/kustomization.yaml

@@ -7,7 +7,6 @@ nameSuffix: "-lma"
 
 resources:
   - ../../../../sub-cluster/phases
-  - lma_phases.yaml
 
 patchesStrategicMerge:
   - phases_patch.yaml

manifests/type/multi-tenant/sub-clusters/lma/phases/lma_phases.yaml

@@ -1,12 +0,0 @@
----
-apiVersion: airshipit.org/v1alpha1
-kind: Phase
-metadata:
-  name: lma-configs
-  clusterName: lma
-config:
-  executorRef:
-    apiVersion: airshipit.org/v1alpha1
-    kind: KubernetesApply
-    name: kubernetes-apply
-  documentEntryPoint: sub-clusters/lma/lma-configs

manifests/type/multi-tenant/sub-clusters/lma/phases/phases_patch.yaml

@@ -45,3 +45,11 @@ metadata:
   clusterName: lma
 config:
   documentEntryPoint: sub-clusters/lma/workload
+---
+apiVersion: airshipit.org/v1alpha1
+kind: Phase
+metadata:
+  name: workload-config
+  clusterName: lma
+config:
+  documentEntryPoint: sub-clusters/lma/workload-config

manifests/type/multi-tenant/sub-clusters/lma/workload-config/kustomization.yaml

@@ -0,0 +1,8 @@
+resources:
+  - ../../../../../function/lma-configs
+
+patches:
+  - path: patches/cluster-flow.yaml
+    target:
+      kind: ClusterFlow
+      name: all-pods

manifests/type/multi-tenant/sub-clusters/lma/workload-config/patches/cluster-flow.yaml

@@ -0,0 +1,6 @@
+- op: add
+  path: /spec/filters/-
+  value:
+    record_modifier:
+      records:
+        - airship-cluster: lma

manifests/type/multi-tenant/sub-clusters/lma/workload/kustomization.yaml

@@ -1,9 +1,20 @@
 resources:
-  - ../../../../sub-cluster/workload
+  - ../../../../../composite/lma-infra
+  - ../../../../../composite/elastic-stack
   - ../../../../../composite/monitoring-stack
-  - ../../../../../function/minio
 
 namespace: lma
 
 patches:
-  - path: patches/minio.yaml
+  - path: patches/elasticsearch-ingest.yaml
+    target:
+      kind: HelmRelease
+      name: elasticsearch-ingest
+  - path: patches/elasticsearch-data.yaml
+    target:
+      kind: HelmRelease
+      name: elasticsearch-data
+  - path: patches/kibana.yaml
+    target:
+      kind: HelmRelease
+      name: kibana

manifests/type/multi-tenant/sub-clusters/lma/workload/patches/elasticsearch-data.yaml

@@ -0,0 +1,21 @@
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: elasticsearch-data
+spec:
+  values:
+    persistence:
+      enabled: true
+    replicas: 3
+    resources:
+      requests:
+        cpu: "4000m"
+        memory: "16Gi"
+      limits:
+        cpu: "8000m"
+        memory: "32Gi"
+    volumeClaimTemplate:
+      resouces:
+        requests:
+          storage: 20Ti
+      storageclass: block-storage-sc

manifests/type/multi-tenant/sub-clusters/lma/workload/patches/elasticsearch-ingest.yaml

@@ -0,0 +1,23 @@
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: elasticsearch-ingest
+spec:
+  values:
+    persistence:
+      enabled: true
+    replicas: 3
+    resources:
+      requests:
+        cpu: "4000m"
+        memory: "16Gi"
+      limits:
+        cpu: "8000m"
+        memory: "24Gi"
+    volumeClaimTemplate:
+      resouces:
+        requests:
+          storage: 10Gi
+      storageclass: block-storage-sc
+    service:
+      type: NodePort

manifests/type/multi-tenant/sub-clusters/lma/workload/patches/kibana.yaml

@@ -0,0 +1,8 @@
+apiVersion: "helm.toolkit.fluxcd.io/v2beta1"
+kind: HelmRelease
+metadata:
+  name: kibana
+spec:
+  values:
+    service:
+      type: NodePort

manifests/type/multi-tenant/sub-clusters/lma/workload/patches/minio.yaml

@@ -1,17 +0,0 @@
-apiVersion: "helm.toolkit.fluxcd.io/v2beta1"
-kind: HelmRelease
-metadata:
-  name: minio
-spec:
-  values:
-    replicas: 1
-    persistence:
-      enabled: false
-    existingSecret: minio-admin-secret
-    buckets:
-      - name: logs
-        policy: none
-        purge: false
-      - name: metrics
-        policy: none
-        purge: false

manifests/type/multi-tenant/sub-clusters/lma/workload/replacements/kustomization.yaml

@@ -1,3 +1,5 @@
 resources:
+  - ../../../../../../composite/lma-infra/replacements
+  - ../../../../../../composite/elastic-stack/replacements
   - ../../../../../../composite/monitoring-stack/replacements
-  - ../../../../../../function/minio/replacements
+  - lma-service-ports.yaml

manifests/type/multi-tenant/sub-clusters/lma/workload/replacements/lma-service-ports.yaml

@@ -0,0 +1,29 @@
+apiVersion: airshipit.org/v1alpha1
+kind: ReplacementTransformer
+metadata:
+  name: lma-service-ports
+  annotations:
+    config.kubernetes.io/function: |-
+      container:
+        image: quay.io/airshipit/replacement-transformer:v2
+replacements:
+- source:
+    objref:
+      kind: VariableCatalogue
+      name: subcluster-networking
+    fieldref: "{.spec.lma.exposed_services[(@.name == 'elasticsearch-ingest')].nodePort}"
+  target:
+    objref:
+      kind: HelmRepository
+      name: elasticsearch-ingest
+    fieldrefs: ["{.spec.values.service.nodePort}"]
+- source:
+    objref:
+      kind: VariableCatalogue
+      name: subcluster-networking
+    fieldref: "{.spec.lma.exposed_services[(@.name == 'kibana')].nodePort}"
+  target:
+    objref:
+      kind: HelmRepository
+      name: kibana
+    fieldrefs: ["{.spec.values.service.nodePort}"]

manifests/type/sub-cluster/phases/phases.yaml

@@ -72,3 +72,15 @@ config:
     kind: KubernetesApply
     name: kubernetes-apply
   documentEntryPoint: sub-clusters/sub-cluster/workload
+---
+apiVersion: airshipit.org/v1alpha1
+kind: Phase
+metadata:
+  name: workload-config
+  clusterName: sub-cluster
+config:
+  executorRef:
+    apiVersion: airshipit.org/v1alpha1
+    kind: KubernetesApply
+    name: kubernetes-apply
+  documentEntryPoint: sub-clusters/sub-cluster/workload-config

manifests/type/sub-cluster/workload-config/kustomization.yaml

@@ -0,0 +1,3 @@
+# NOTE: This directory should not be inherited; it should be redefined within the
+# type that defines the actual sub-cluster.
+resources: []