Add version catalogues

This takes advantage of version catalogue replacement, which was added to
airshipctl in the changes below.  It also refactors the kustomize structure
a bit to support that well.

This adds a treasuremap clusterctl function, which kustomizes the same function
from airshipctl to correct for paths.  This replaces the previous
site-level clusterctl entrypoints.

This gives each type and site a catalogues entrypoint, which are wired together
(site inheriting from type, and network-cloud from airship-core) for
inheritance.  Each phase that needs catalogue-based replacement now
pulls in its' site's catalogues definition, and then applies any
phase specific overrides only when necessary.

https://review.opendev.org/#/c/746608
https://review.opendev.org/#/c/746764/

Change-Id: I18364083f72628f9519af3d2fab0d6a4879d2d14
Relates-To: https://github.com/airshipit/airshipctl/issues/316

manifests/function/clusterctl/README.md

@@ -0,0 +1,4 @@
+# Treasuremap Clusterctl
+
+This patches the airshipctl `clusterctl` function to correct for
+relative paths back to airshipctl's CAPI functions.

manifests/function/clusterctl/clusterctl-patch.yaml

@@ -0,0 +1,13 @@
+- op: replace
+  path: /providers/0/versions/v0.3.2
+  value:  ../airshipctl/manifests/function/capm3/v0.3.2
+- op: replace
+  path: /providers/1/versions/v0.3.7
+  value:  ../airshipctl/manifests/function/cabpk/v0.3.7
+- op: replace
+  path: /providers/2/versions/v0.3.7
+  value:  ../airshipctl/manifests/function/capi/v0.3.7
+- op: replace
+  path: /providers/3/versions/v0.3.7
+  value:  ../airshipctl/manifests/function/cacpk/v0.3.7
+

manifests/function/clusterctl/kustomization.yaml

@@ -0,0 +1,9 @@
+resources:
+  - ../../../../airshipctl/manifests/function/clusterctl
+patchesJson6902:
+  - target:
+      group: airshipit.org
+      version: v1alpha1
+      kind: Clusterctl
+      name: clusterctl-v1
+    path: clusterctl-patch.yaml

manifests/function/clusterctl/replacements/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../../../../../airshipctl/manifests/function/clusterctl/replacements

manifests/site/test-site/ephemeral/controlplane/before.yaml

@@ -1,128 +0,0 @@
-apiVersion: v1
-data:
-  password: cjAwdG1l
-  username: cm9vdA==
-kind: Secret
-metadata:
-  labels:
-    airshipit.org/k8s-role: controlplane-host
-  name: node01-bmc-secret
-type: Opaque
----
-apiVersion: v1
-kind: Secret
-metadata:
-  labels:
-    airshipit.org/k8s-role: controlplane-host
-  name: node01-network-data
-stringData:
-  networkData: "links:\n-\n  id: oam\n  mtu: \"1500\"\n  name: enp0s3\n  type: phy\n
-    \ \n  ethernet_mac_address: 52:54:00:9b:27:4c\n-\n  id: pxe\n  mtu: \"1500\"\n
-    \ name: enp0s4\n  type: phy\n  \n  ethernet_mac_address: 52:54:00:b6:ed:31\nnetworks:\n-\n
-    \ id: oam-ipv4\n  link: oam\n  netmask: 255.255.255.0\n  routes:\n  - gateway:
-    10.23.25.1\n    netmask: 0.0.0.0\n    network: 0.0.0.0\n  type: ipv4\n  \n  ip_address:
-    10.23.25.102\n-\n  id: pxe-ipv4\n  link: pxe\n  netmask: 255.255.255.0\n  type:
-    ipv4\n  \n  ip_address: 10.23.24.102\nservices:\n  - address: 8.8.8.8\n    type:
-    dns\n  - address: 8.8.4.4\n    type: dns\n  \n"
-type: Opaque
----
-apiVersion: cluster.x-k8s.io/v1alpha3
-kind: Cluster
-metadata:
-  name: target-cluster
-spec:
-  clusterNetwork:
-    pods:
-      cidrBlocks:
-      - 192.168.0.0/18
-    serviceDomain: cluster.local
-    services:
-      cidrBlocks:
-      - 10.96.0.0/12
-  controlPlaneRef:
-    apiVersion: controlplane.cluster.x-k8s.io/v1alpha3
-    kind: KubeadmControlPlane
-    name: cluster-controlplane
-  infrastructureRef:
-    apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
-    kind: Metal3Cluster
-    name: target-cluster
----
-apiVersion: controlplane.cluster.x-k8s.io/v1alpha3
-kind: KubeadmControlPlane
-metadata:
-  name: cluster-controlplane
-spec:
-  infrastructureTemplate:
-    apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
-    kind: Metal3MachineTemplate
-    name: cluster-controlplane
-  kubeadmConfigSpec:
-    files:
-    - content: |
-        kubernetes:
-          version: 1.17.3
-        container_runtime:
-          type: docker
-      owner: root:root
-      path: /konfigadm.yml
-      permissions: "0640"
-    initConfiguration:
-      nodeRegistration:
-        kubeletExtraArgs:
-          node-labels: metal3.io/uuid={{ ds.meta_data.uuid }}
-        name: '{{ ds.meta_data.local_hostname }}'
-    joinConfiguration:
-      controlPlane: {}
-      nodeRegistration:
-        kubeletExtraArgs:
-          node-labels: metal3.io/uuid={{ ds.meta_data.uuid }}
-        name: '{{ ds.meta_data.local_hostname }}'
-    postKubeadmCommands:
-    - kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f https://docs.projectcalico.org/v3.9/manifests/calico.yaml
-    preKubeadmCommands:
-    - wget -O "/usr/bin/konfigadm" https://github.com/flanksource/konfigadm/releases/download/v0.5.3/konfigadm
-      && chmod +x "/usr/bin/konfigadm"
-    - konfigadm apply --config=/konfigadm.yml
-  replicas: 1
-  version: v1.17.0
----
-apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
-kind: Metal3Cluster
-metadata:
-  name: target-cluster
-spec:
-  controlPlaneEndpoint:
-    host: 10.23.25.102
-    port: 6443
-  noCloudProvider: true
----
-apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
-kind: Metal3MachineTemplate
-metadata:
-  name: cluster-controlplane
-spec:
-  template:
-    spec:
-      hostSelector:
-        matchLabels:
-          airshipit.org/k8s-role: controlplane-host
-      image:
-        checksum: http://10.23.24.1:8099/target-image.qcow2.md5sum
-        url: http://10.23.24.1:8099/target-image.qcow2
----
-apiVersion: metal3.io/v1alpha1
-kind: BareMetalHost
-metadata:
-  labels:
-    airshipit.org/k8s-role: controlplane-host
-  name: node01
-spec:
-  bmc:
-    address: redfish+http://10.23.25.1:8000/redfish/v1/Systems/air-target-1
-    credentialsName: node01-bmc-secret
-  bootMACAddress: 52:54:00:b6:ed:31
-  networkData:
-    name: node01-network-data
-    namespace: default
-  online: false

manifests/site/test-site/ephemeral/controlplane/control-machine-template-patch.yaml

@@ -1,11 +0,0 @@
----
-apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
-kind: Metal3MachineTemplate
-metadata:
-  name: cluster-controlplane
-spec:
-  template:
-    spec:
-      image:
-        url: http://10.23.24.1:8099/target-image.qcow2
-        checksum: http://10.23.24.1:8099/target-image.qcow2.md5sum

manifests/site/test-site/ephemeral/controlplane/kustomization.yaml

@@ -2,8 +2,11 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ../../../../type/airship-core/ephemeral/controlplane
+  - ../../shared/catalogues
   # TODO (dukov) It's recocommended to upload BareMetalHost objects separately
   # otherwise nodes will hang in 'registering' state for quite a long time
   - nodes
 patchesStrategicMerge:
-  - control-machine-template-patch.yaml
+  - versions-catalogue-patch.yaml
+transformers:
+  - ../../../../type/airship-core/ephemeral/controlplane/replacements

manifests/site/test-site/ephemeral/controlplane/versions-catalogue-patch.yaml

@@ -0,0 +1,11 @@
+# Patch the versions catalogue to use the site-specific local image URL
+# TODO: patch this in from a site-networking catalogue in the future
+apiVersion: airshipit.org/v1alpha1
+kind: VariableCatalogue
+metadata:
+  name: versions-airshipctl
+files:
+  k8scontrol:
+    cluster_controlplane_image:
+      url: http://10.23.24.1:8099/target-image.qcow2
+      checksum: http://10.23.24.1:8099/target-image.qcow2.md5sum

manifests/site/test-site/ephemeral/initinfra/kustomization.yaml

@@ -1,4 +1,7 @@
 resources:
   - ../../../../type/airship-core/ephemeral/initinfra
+  - ../../shared/catalogues
 patchesStrategicMerge:
   - patch_bmo_config.yaml
+transformers:
+  - ../../../../type/airship-core/ephemeral/initinfra/replacements

manifests/site/test-site/shared/catalogues/hosts.yaml

@@ -3,6 +3,8 @@
 apiVersion: airshipit.org/v1alpha1
 kind: VariableCatalogue
 metadata:
+  labels:
+    airshipit.org/deploy-k8s: "false"
   name: host-catalogue
 hosts:
   m3:

manifests/site/test-site/target/initinfra/kustomization.yaml

@@ -1,4 +1,7 @@
 resources:
   - ../../../../type/airship-core/target/initinfra
+  - ../../shared/catalogues
 patchesStrategicMerge:
   - patch_bmo_config.yaml
+transformers:
+  - ../../../../type/airship-core/target/initinfra/replacements

manifests/site/virtual-network-cloud/ephemeral/controlplane/control-machine-template-patch.yaml

@@ -1,11 +0,0 @@
----
-apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
-kind: Metal3MachineTemplate
-metadata:
-  name: cluster-controlplane
-spec:
-  template:
-    spec:
-      image:
-        url: http://10.23.24.1:8099/target-image.qcow2
-        checksum: http://10.23.24.1:8099/target-image.qcow2.md5sum

manifests/site/virtual-network-cloud/ephemeral/controlplane/kustomization.yaml

@@ -2,8 +2,11 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ../../../../type/network-cloud/ephemeral/controlplane
+  - ../../shared/catalogues
   # TODO (dukov) It's recocommended to upload BareMetalHost objects separately
   # otherwise nodes will hang in 'registering' state for quite a long time
   - nodes
 patchesStrategicMerge:
-  - control-machine-template-patch.yaml
+  - versions-catalogue-patch.yaml
+transformers:
+  - ../../../../type/network-cloud/ephemeral/controlplane/replacements

manifests/site/virtual-network-cloud/ephemeral/controlplane/versions-catalogue-patch.yaml

@@ -0,0 +1,11 @@
+# Patch the versions catalogue to use the site-specific local image URL
+# TODO: patch this in from a site-networking catalogue in the future
+apiVersion: airshipit.org/v1alpha1
+kind: VariableCatalogue
+metadata:
+  name: versions-airshipctl
+files:
+  k8scontrol:
+    cluster_controlplane_image:
+      url: http://10.23.24.1:8099/target-image.qcow2
+      checksum: http://10.23.24.1:8099/target-image.qcow2.md5sum

manifests/site/virtual-network-cloud/ephemeral/initinfra/kustomization.yaml

@@ -1,4 +1,7 @@
 resources:
   - ../../../../type/network-cloud/ephemeral/initinfra
+  - ../../shared/catalogues
 patchesStrategicMerge:
   - patch_bmo_config.yaml
+transformers:
+  - ../../../../type/airship-core/ephemeral/initinfra/replacements

manifests/site/virtual-network-cloud/shared/catalogues/hosts.yaml

@@ -3,6 +3,8 @@
 apiVersion: airshipit.org/v1alpha1
 kind: VariableCatalogue
 metadata:
+  labels:
+    airshipit.org/deploy-k8s: "false"
   name: host-catalogue
 hosts:
   m3:

manifests/type/airship-core/ephemeral/controlplane/replacements/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../../../../../../../airshipctl/manifests/function/k8scontrol/replacements

manifests/type/airship-core/ephemeral/initinfra/kustomization.yaml

@@ -1,5 +1,5 @@
 resources:
-  - ../../shared/clusterctl
+  - ../../../../function/clusterctl
   - ../../../../../../airshipctl/manifests/composite/infra/
   - ../../../../../../airshipctl/manifests/function/baremetal-operator/
 commonLabels:

manifests/type/airship-core/ephemeral/initinfra/replacements/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+  - ../../../../../../../airshipctl/manifests/function/baremetal-operator/replacements

manifests/type/airship-core/shared/catalogues/common-networking.yaml

@@ -3,6 +3,8 @@
 apiVersion: airshipit.org/v1alpha1
 kind: VariableCatalogue
 metadata:
+  labels:
+    airshipit.org/deploy-k8s: "false"
   name: common-networking-catalogue
 commonNetworking:
   links:

manifests/type/airship-core/shared/catalogues/kustomization.yaml

@@ -1,4 +1,9 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
+  - ../../../../../../airshipctl/manifests/function/airshipctl-catalogues/
   - common-networking.yaml
+# Uncomment to patch versions catalogue once needed
+# patchesStrategicMerge:
+#   - versions.yaml
+

manifests/type/airship-core/shared/catalogues/versions.yaml

@@ -0,0 +1,2 @@
+# NOTE(drewwalters96): This patch file is applied to override the
+# airship-core versions catalogue.

manifests/type/airship-core/shared/clusterctl/clusterctl.yaml

@@ -1,31 +0,0 @@
-apiVersion: airshipit.org/v1alpha1
-kind: Clusterctl
-metadata:
-  labels:
-    airshipit.org/deploy-k8s: "false"
-  name: clusterctl-v1
-init-options:
-  core-provider: "cluster-api:v0.3.3"
-  bootstrap-providers:
-    - "kubeadm:v0.3.3"
-  infrastructure-providers:
-    - "metal3:v0.3.1"
-  control-plane-providers:
-    - "kubeadm:v0.3.3"
-providers:
-  - name: "metal3"
-    type: "InfrastructureProvider"
-    versions:
-      v0.3.1: ../airshipctl/manifests/function/capm3/v0.3.1
-  - name: "kubeadm"
-    type: "BootstrapProvider"
-    versions:
-      v0.3.3: ../airshipctl/manifests/function/cabpk/v0.3.3
-  - name: "cluster-api"
-    type: "CoreProvider"
-    versions:
-      v0.3.3: ../airshipctl/manifests/function/capi/v0.3.3
-  - name: "kubeadm"
-    type: "ControlPlaneProvider"
-    versions:
-      v0.3.3: ../airshipctl/manifests/function/cacpk/v0.3.3

manifests/type/airship-core/shared/clusterctl/kustomization.yaml

@@ -1,2 +0,0 @@
-resources:
-  - clusterctl.yaml

manifests/type/airship-core/target/initinfra/kustomization.yaml

@@ -1,7 +1,8 @@
 resources:
-  - ../../shared/clusterctl
+  - ../../../../function/clusterctl
   - ../../../../../../airshipctl/manifests/composite/infra/
   - ../../../../../../airshipctl/manifests/function/baremetal-operator/
   - ../../../../../../airshipctl/manifests/function/helm-operator/
 commonLabels:
   airshipit.org/stage: initinfra
+

manifests/type/airship-core/target/initinfra/replacements/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+  - ../../../../../../../airshipctl/manifests/function/baremetal-operator/replacements

manifests/type/network-cloud/ephemeral/controlplane/replacements/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../../../../airship-core/ephemeral/controlplane/replacements

manifests/type/network-cloud/ephemeral/initinfra/replacements/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+  - ../../../../airship-core/ephemeral/initinfra/replacements

manifests/type/network-cloud/shared/catalogues/kustomization.yaml

@@ -6,3 +6,7 @@ resources:
 # NOTE(drewwalters96): Uncomment to patch common-networking catalogue.
 # patchesStrategicMerge:
 #   - common-networking.yaml
+# Uncomment to patch versions catalogue once needed
+# patchesStrategicMerge:
+#   - versions.yaml
+

manifests/type/network-cloud/shared/catalogues/versions.yaml

@@ -0,0 +1,2 @@
+# NOTE(drewwalters96): This patch file is applied to override the
+# airship-core versions catalogue.