Adopt networking catalogues to Treasuremap

This adds network deduplication via variable catalogues and replacement.
This was added to airshipctl under the change:
https://review.opendev.org/#/c/749611/

Change-Id: Ib6a50c58e3b62de4f74c5def58b94ab6dbd1e949
Closes: https://github.com/airshipit/airshipctl/issues/315

manifests/site/test-site/ephemeral/bootstrap/hostgenerator/kustomization.yaml

@@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ../../../../../../../airshipctl/manifests/function/hostgenerator-m3
-  - ../../../shared/catalogues/
+  - ../../catalogues/
   - host-generation.yaml
 
 transformers:

manifests/site/test-site/ephemeral/bootstrap/kustomization.yaml

@@ -2,9 +2,13 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ../../../../type/airship-core/ephemeral/bootstrap
+  - ../catalogues
 
 generators:
   - hostgenerator
 
 patchesStrategicMerge:
   - baremetalhost.yaml
+
+transformers:
+  - ../../../../type/airship-core/ephemeral/bootstrap/replacements

manifests/site/test-site/ephemeral/catalogues/README.md

@@ -0,0 +1,4 @@
+# Catalogue Definitions for Target Cluster
+
+This inherits Site-level catalogues from the neighboring target cluster's
+`catalogues` kustomization, and tweaks a few values for the ephemeral cluster.

manifests/site/test-site/ephemeral/catalogues/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../../target/catalogues
+patchesStrategicMerge:
+  - networking.yaml

manifests/site/test-site/ephemeral/catalogues/networking.yaml

@@ -0,0 +1,19 @@
+# This makes a couple small networking tweaks that are specific to the
+# ephemeral cluster, on top of the target cluster networking definition.
+# These values can be overridden at the site, type, etc levels as appropriate.
+apiVersion: airshipit.org/v1alpha1
+kind: VariableCatalogue
+metadata:
+  name: networking
+
+# The catalogue should be overridden as appropriate for different kubernetes
+# clusters, e.g. ephemeral vs target vs tenant
+kubernetes:
+  podCidr: "192.168.0.0/24"
+  controlPlaneEndpoint:
+    host: "10.23.25.101"
+  apiserverCertSANs: "[10.23.25.101, 10.23.24.101]"
+
+ironic:
+  provisioningIp: "10.23.24.101"
+  dhcpRange: "10.23.24.200,10.23.24.250"

manifests/site/test-site/ephemeral/controlplane/hostgenerator/kustomization.yaml

@@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ../../../../../../../airshipctl/manifests/function/hostgenerator-m3
-  - ../../../shared/catalogues/
+  - ../../catalogues/
   - host-generation.yaml
 
 transformers:

manifests/site/test-site/ephemeral/controlplane/kustomization.yaml

@@ -2,11 +2,9 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ../../../../type/airship-core/ephemeral/controlplane
-  - ../../shared/catalogues
+  - ../catalogues
   # TODO (dukov) It's recocommended to upload BareMetalHost objects separately
   # otherwise nodes will hang in 'registering' state for quite a long time
   - nodes
-patchesStrategicMerge:
-  - versions-catalogue-patch.yaml
 transformers:
-  - ../../../../type/airship-core/ephemeral/controlplane/replacements
+  - ../../../../type/airship-core/ephemeral/controlplane/replacements

manifests/site/test-site/ephemeral/initinfra/kustomization.yaml

@@ -1,7 +1,5 @@
 resources:
   - ../../../../type/airship-core/ephemeral/initinfra
-  - ../../shared/catalogues
+  - ../catalogues
-patchesStrategicMerge:
-  - patch_bmo_config.yaml
 transformers:
-  - ../../../../type/airship-core/ephemeral/initinfra/replacements
+  - ../../../../type/airship-core/ephemeral/initinfra/replacements

manifests/site/test-site/ephemeral/initinfra/patch_bmo_config.yaml

@@ -1,8 +0,0 @@
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: ironic-vars
-data:
-  PROVISIONING_IP: "10.23.24.101"
-  DHCP_RANGE: "10.23.24.200,10.23.24.250"

manifests/site/test-site/target/catalogues/README.md

@@ -0,0 +1,5 @@
+# Catalogue Definitions for Target Cluster
+
+This inherits Type-level catalogues, and adds in Site-specific values.
+The neighboring ephemeral cluster's `catalogues` entrypoint applies further
+customizations on top of this for ephemeral use.

manifests/site/test-site/target/catalogues/hosts.yaml

@@ -3,12 +3,14 @@
 apiVersion: airshipit.org/v1alpha1
 kind: VariableCatalogue
 metadata:
+  name: host-catalogue
   labels:
     airshipit.org/deploy-k8s: "false"
-  name: host-catalogue
+
 hosts:
   m3:
     node01:
+      bootMode: UEFI
       macAddress: 52:54:00:b6:ed:31
       bmcAddress: redfish+http://10.23.25.1:8000/redfish/v1/Systems/air-target-1
       bmcUsername: root
@@ -19,9 +21,9 @@ hosts:
       macAddresses:
         oam: 52:54:00:9b:27:4c
         pxe: 52:54:00:b6:ed:31
-      bootMode: legacy
+      hardwareProfile: default # defined in the hostgenerator-m3 function
-      hardwareProfile: default  # defined in the hostgenerator-m3 function
     node02:
+      bootMode: UEFI
       macAddress: 52:54:00:b6:ed:02
       bmcAddress: redfish+http://10.23.25.2:8000/redfish/v1/Systems/air-target-2
       bmcUsername: username
@@ -32,4 +34,17 @@ hosts:
       macAddresses:
         oam: 52:54:00:9b:27:02
         pxe: 52:54:00:b6:ed:02
-      hardwareProfile: default  # defined in the hostgenerator-m3 function
+      hardwareProfile: example # defined in the hardwareprofile-example function
+    node03:
+      bootMode: UEFI
+      macAddress: 52:54:00:b6:ed:23
+      bmcAddress: redfish+http://10.23.25.1:8000/redfish/v1/Systems/air-worker-1
+      bmcUsername: username
+      bmcPassword: password
+      ipAddresses:
+        oam-ipv4: 10.23.25.103
+        pxe-ipv4: 10.23.24.103
+      macAddresses:
+        oam: 52:54:00:9b:27:07
+        pxe: 52:54:00:b6:ed:23
+      hardwareProfile: default # defined in the hardwareprofile-example function

manifests/site/test-site/target/catalogues/kustomization.yaml

@@ -1,5 +1,10 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
+
 resources:
   - ../../../../type/airship-core/shared/catalogues
   - hosts.yaml
+
+patchesStrategicMerge:
+  - versions-airshipctl.yaml
+  - networking.yaml

manifests/site/test-site/target/catalogues/networking.yaml

@@ -0,0 +1,19 @@
+# This makes a couple small networking tweaks that are specific to the
+# ephemeral cluster, on top of the target cluster networking definition.
+# These values can be overridden at the site, type, etc levels as appropriate.
+apiVersion: airshipit.org/v1alpha1
+kind: VariableCatalogue
+metadata:
+  name: networking
+
+# The catalogue should be overridden as appropriate for different kubernetes
+# clusters, e.g. ephemeral vs target vs tenant
+kubernetes:
+  controlPlaneEndpoint:
+    host: "10.23.25.102"
+  apiserverCertSANs: "[10.23.25.102, 10.23.24.102]"
+
+ironic:
+  provisioningIp: "10.23.24.102"
+  dhcpRange: "10.23.24.200,10.23.24.250"
+

manifests/site/test-site/target/catalogues/versions-airshipctl.yaml

@@ -1,11 +1,12 @@
-# Patch the versions catalogue to use the site-specific local image URL
+# Override default controlplane image location
-# TODO: patch this in from a site-networking catalogue in the future
 apiVersion: airshipit.org/v1alpha1
 kind: VariableCatalogue
 metadata:
   name: versions-airshipctl
+
 files:
   k8scontrol:
+    # Host the image in a locally served location for CI
     cluster_controlplane_image:
       url: http://10.23.24.1:8099/target-image.qcow2
       checksum: http://10.23.24.1:8099/target-image.qcow2.md5sum

manifests/site/test-site/target/initinfra/kustomization.yaml

@@ -1,8 +1,6 @@
 resources:
   - ../../../../type/airship-core/target/initinfra
-  - ../../shared/catalogues
   - ../../../../function/hostconfig-operator
-patchesStrategicMerge:
+  - ../catalogues
-  - patch_bmo_config.yaml
 transformers:
   - ../../../../type/airship-core/target/initinfra/replacements

manifests/site/test-site/target/initinfra/patch_bmo_config.yaml

@@ -1,8 +0,0 @@
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: ironic-vars
-data:
-  PROVISIONING_IP: "10.23.25.102"
-  DHCP_RANGE: "10.23.25.200,10.23.25.250"

manifests/site/test-site/target/workers/hostgenerator/kustomization.yaml

@@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ../../../../../../../airshipctl/manifests/function/hostgenerator-m3
-  - ../../../shared/catalogues/
+  - ../../catalogues/
   - host-generation.yaml
 
 transformers:

manifests/site/virtual-network-cloud/ephemeral/bootstrap/hostgenerator/kustomization.yaml

@@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ../../../../../../../airshipctl/manifests/function/hostgenerator-m3
-  - ../../../shared/catalogues/
+  - ../../catalogues/
   - host-generation.yaml
 
 transformers:

manifests/site/virtual-network-cloud/ephemeral/bootstrap/kustomization.yaml

@@ -2,9 +2,13 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ../../../../type/network-cloud/ephemeral/bootstrap
+  - ../catalogues
 
 generators:
   - hostgenerator
 
 patchesStrategicMerge:
   - baremetalhost.yaml
+
+transformers:
+  - ../../../../type/network-cloud/ephemeral/bootstrap/replacements

manifests/site/virtual-network-cloud/ephemeral/catalogues/README.md

@@ -0,0 +1,4 @@
+# Catalogue Definitions for Target Cluster
+
+This inherits Site-level catalogues from the neighboring target cluster's
+`catalogues` kustomization, and tweaks a few values for the ephemeral cluster.

manifests/site/virtual-network-cloud/ephemeral/catalogues/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../../target/catalogues
+patchesStrategicMerge:
+  - networking.yaml

manifests/site/virtual-network-cloud/ephemeral/catalogues/networking.yaml

@@ -0,0 +1,20 @@
+# This makes a couple small networking tweaks that are specific to the
+# ephemeral cluster, on top of the target cluster networking definition.
+# These values can be overridden at the site, type, etc levels as appropriate.
+apiVersion: airshipit.org/v1alpha1
+kind: VariableCatalogue
+metadata:
+  name: networking-catalogue
+
+# The catalogue should be overridden as appropriate for different kubernetes
+# clusters, e.g. ephemeral vs target vs tenant
+kubernetes:
+  podCidr: "192.168.0.0/24"
+  controlPlaneEndpoint:
+    host: "10.23.25.101"
+  apiserverCertSANs: "[10.23.25.101, 10.23.24.101]"
+
+ironic:
+  provisioningIp: "10.23.24.101"
+  dhcpRange: "10.23.24.200,10.23.24.250"
+

manifests/site/virtual-network-cloud/ephemeral/controlplane/hostgenerator/kustomization.yaml

@@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ../../../../../../../airshipctl/manifests/function/hostgenerator-m3
-  - ../../../shared/catalogues/
+  - ../../catalogues/
   - host-generation.yaml
 
 transformers:

manifests/site/virtual-network-cloud/ephemeral/controlplane/kustomization.yaml

@@ -2,11 +2,9 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ../../../../type/network-cloud/ephemeral/controlplane
-  - ../../shared/catalogues
+  - ../catalogues
   # TODO (dukov) It's recocommended to upload BareMetalHost objects separately
   # otherwise nodes will hang in 'registering' state for quite a long time
   - nodes
-patchesStrategicMerge:
-  - versions-catalogue-patch.yaml
 transformers:
   - ../../../../type/network-cloud/ephemeral/controlplane/replacements

manifests/site/virtual-network-cloud/ephemeral/initinfra/kustomization.yaml

@@ -1,7 +1,5 @@
 resources:
   - ../../../../type/network-cloud/ephemeral/initinfra
-  - ../../shared/catalogues
+  - ../catalogues
-patchesStrategicMerge:
-  - patch_bmo_config.yaml
 transformers:
-  - ../../../../type/airship-core/ephemeral/initinfra/replacements
+  - ../../../../type/airship-core/ephemeral/initinfra/replacements

manifests/site/virtual-network-cloud/ephemeral/initinfra/patch_bmo_config.yaml

@@ -1,9 +0,0 @@
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  labels:
-  name: ironic-vars
-data:
-  PROVISIONING_IP: "10.23.24.101"
-  DHCP_RANGE: "10.23.24.200,10.23.24.250"

manifests/site/virtual-network-cloud/target/catalogues/README.md

@@ -0,0 +1,5 @@
+# Catalogue Definitions for Target Cluster
+
+This inherits Type-level catalogues, and adds in Site-specific values.
+The neighboring ephemeral cluster's `catalogues` entrypoint applies further
+customizations on top of this for ephemeral use.

manifests/site/virtual-network-cloud/target/catalogues/hosts.yaml

@@ -3,9 +3,10 @@
 apiVersion: airshipit.org/v1alpha1
 kind: VariableCatalogue
 metadata:
+  name: host-catalogue
   labels:
     airshipit.org/deploy-k8s: "false"
-  name: host-catalogue
+
 hosts:
   m3:
     node01:

manifests/site/virtual-network-cloud/target/catalogues/kustomization.yaml

@@ -1,5 +1,10 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
+
 resources:
   - ../../../../type/network-cloud/shared/catalogues
   - hosts.yaml
+
+patchesStrategicMerge:
+  - versions-airshipctl.yaml
+  - networking.yaml

manifests/site/virtual-network-cloud/target/catalogues/networking.yaml

@@ -0,0 +1,19 @@
+# This makes a couple small networking tweaks that are specific to the
+# ephemeral cluster, on top of the target cluster networking definition.
+# These values can be overridden at the site, type, etc levels as appropriate.
+apiVersion: airshipit.org/v1alpha1
+kind: VariableCatalogue
+metadata:
+  name: networking-catalogue
+
+# The catalogue should be overridden as appropriate for different kubernetes
+# clusters, e.g. ephemeral vs target vs tenant
+kubernetes:
+  controlPlaneEndpoint:
+    host: "10.23.25.102"
+  apiserverCertSANs: "[10.23.25.102, 10.23.24.102]"
+
+ironic:
+  provisioningIp: "10.23.24.102"
+  dhcpRange: "10.23.24.200,10.23.24.250"
+

manifests/site/virtual-network-cloud/target/catalogues/versions-airshipctl.yaml

@@ -0,0 +1,12 @@
+# Override default controlplane image location
+apiVersion: airshipit.org/v1alpha1
+kind: VariableCatalogue
+metadata:
+  name: versions-airshipctl
+
+files:
+  k8scontrol:
+    # Host the image in a locally served location for CI
+    cluster_controlplane_image:
+      url: http://10.23.24.1:8099/target-image.qcow2
+      checksum: http://10.23.24.1:8099/target-image.qcow2.md5sum

manifests/type/airship-core/ephemeral/bootstrap/replacements/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../../../../../../../airshipctl/manifests/function/ephemeral/replacements

manifests/type/airship-core/ephemeral/initinfra/replacements/kustomization.yaml

@@ -1,2 +1,3 @@
 resources:
   - ../../../../../../../airshipctl/manifests/function/baremetal-operator/replacements
+  - ../../../../../../../airshipctl/manifests/composite/infra/replacements

manifests/type/airship-core/shared/catalogues/kustomization.yaml

@@ -1,8 +1,9 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - ../../../../../../airshipctl/manifests/function/airshipctl-catalogues/
+  - ../../../../../../airshipctl/manifests/function/airshipctl-base-catalogues
-  - common-networking.yaml
+patchesStrategicMerge:
+  - networking.yaml
 # Uncomment to patch versions catalogue once needed
 # patchesStrategicMerge:
 #   - versions.yaml

manifests/type/airship-core/shared/catalogues/networking.yaml

@@ -1,12 +1,17 @@
-# Type-level networking catalogue.  This info feeds the Templater
+# The default networking catalogue for functions hosted in the airshipctl project.
-# kustomize plugin config in the hostgenerator-m3 function.
+# These values can be overridden at the site, type, etc levels as appropriate.
 apiVersion: airshipit.org/v1alpha1
 kind: VariableCatalogue
 metadata:
   labels:
     airshipit.org/deploy-k8s: "false"
-  name: common-networking-catalogue
+  name: networking
-commonNetworking:
+
+# This section is only relevant when using Metal3 BareMetalHosts, and
+# is consumed by the `hostgenerator-m3` function.
+# It defines host-level networking that is common across all BMHs in a site,
+# and will typically be fully overridden at the Type and Site level.
+commonHostNetworking:
   links:
     - id: oam
       name: oam
@@ -38,4 +43,3 @@ commonNetworking:
       type: dns
     - address: 8.8.4.4
       type: dns
-

manifests/type/airship-core/target/initinfra/replacements/kustomization.yaml

@@ -1,2 +1,4 @@
 resources:
   - ../../../../../../../airshipctl/manifests/function/baremetal-operator/replacements
+  - ../../../../../../../airshipctl/manifests/function/helm-operator/replacements
+  - ../../../../../../../airshipctl/manifests/composite/infra/replacements

manifests/type/network-cloud/ephemeral/bootstrap/replacements/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../../../../airship-core/ephemeral/bootstrap/replacements

manifests/type/network-cloud/shared/catalogues/kustomization.yaml

@@ -3,7 +3,7 @@ kind: Kustomization
 resources:
   - ../../../airship-core/shared/catalogues
 
-# NOTE(drewwalters96): Uncomment to patch common-networking catalogue.
+# NOTE(drewwalters96): Uncomment to patch networking catalogue.
 # patchesStrategicMerge:
 #   - common-networking.yaml
 # Uncomment to patch versions catalogue once needed