Support for vm-infra-bridge

- Added vm-infra-bridge/workers-capm3 function for supporting kubeadmConfigTemplate - Added k8scontrol-vm-bridge function dir to house vm-infra-bridge (controlplane) - Added a new VariableCatalogue to support vm networking in multi-tenant/shared. - Added a new systemd service vm-infra-bridge to handle persistence of bridge and NATing info. Relates-To: #122 Closes: #122 Change-Id: I5585b05be9e68976e402d025ddc8578870c137f1
2021-04-16 07:39:06 +00:00 · 2021-04-16 07:39:06 +00:00 · 64bc219f9a
parent 2940a34610
commit 64bc219f9a
31 changed files with 447 additions and 174 deletions
--- a/manifests/function/k8scontrol-vm-infra/kustomization.yaml
+++ b/manifests/function/k8scontrol-vm-infra/kustomization.yaml
@ -0,0 +1,13 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- ../k8scontrol-ha
+
+patchesJson6902:
+- target:
+    group: controlplane.cluster.x-k8s.io
+    version: v1alpha3
+    kind: KubeadmControlPlane
+    name: cluster-controlplane
+  path:  vm-infra-bridge.yaml
--- a/manifests/function/k8scontrol-vm-infra/replacements/kustomization.yaml
+++ b/manifests/function/k8scontrol-vm-infra/replacements/kustomization.yaml
@ -0,0 +1,2 @@
+resources:
+  - vm-infra-bridge.yaml
--- a/manifests/function/k8scontrol-vm-infra/replacements/vm-infra-bridge.yaml
+++ b/manifests/function/k8scontrol-vm-infra/replacements/vm-infra-bridge.yaml
@ -0,0 +1,49 @@
+apiVersion: airshipit.org/v1alpha1
+kind: ReplacementTransformer
+metadata:
+  name: vm-infra-bridge-replacements
+  annotations:
+    config.kubernetes.io/function: |-
+      container:
+        image: quay.io/airshipit/replacement-transformer:v2
+replacements:
+- source:
+    objref:
+      kind: VariableCatalogue
+      name: vm-networking
+    fieldref: spec.vm_infra_bridge.host_oam_interface
+  target:
+    objref:
+      kind: KubeadmControlPlane
+    fieldrefs:
+      -  "spec.kubeadmConfigSpec.files[path=/usr/bin/vm-infra-bridge.sh].content%REPLACEMENT_MGMT_INTF%"
+- source:
+    objref:
+      kind: VariableCatalogue
+      name: vm-networking
+    fieldref: spec.vm_infra_bridge.vm_interface
+  target:
+    objref:
+      kind: KubeadmControlPlane
+    fieldrefs:
+      -  "spec.kubeadmConfigSpec.files[path=/usr/bin/vm-infra-bridge.sh].content%REPLACEMENT_VM_INFRA_INTF%"
+- source:
+    objref:
+      kind: VariableCatalogue
+      name: vm-networking
+    fieldref: spec.vm_infra_bridge.vm_network
+  target:
+    objref:
+      kind: KubeadmControlPlane
+    fieldrefs:
+      -  "spec.kubeadmConfigSpec.files[path=/usr/bin/vm-infra-bridge.sh].content%REPLACEMENT_VM_NETWORK%"
+- source:
+    objref:
+      kind: VariableCatalogue
+      name: vm-networking
+    fieldref: spec.vm_infra_bridge.vm_network_subnet_mask
+  target:
+    objref:
+      kind: KubeadmControlPlane
+    fieldrefs:
+      -  "spec.kubeadmConfigSpec.files[path=/usr/bin/vm-infra-bridge.sh].content%REPLACEMENT_VMNETWORK_SUBNET_MASK%"
--- a/manifests/function/k8scontrol-vm-infra/vm-infra-bridge.yaml
+++ b/manifests/function/k8scontrol-vm-infra/vm-infra-bridge.yaml
@ -0,0 +1,66 @@
+- op: add
+  path: "/spec/kubeadmConfigSpec/preKubeadmCommands/-"
+  value:
+     systemctl enable --now vm-infra-bridge.service
+- op: add
+  path: "/spec/kubeadmConfigSpec/files/-"
+  value:
+    path: /etc/systemd/system/vm-infra-bridge.service
+    permissions: "0644"
+    owner: root:root
+    content: |
+      [Unit]
+      Description=Service to setup vm-infra-bridge and NAT using iptables
+      Wants=network-online.target
+      After=network.target network-online.target
+
+      [Service]
+      User=root
+      WorkingDirectory=/usr/bin
+      ExecStart=/usr/bin/vm-infra-bridge.sh
+
+      [Install]
+      WantedBy=multi-user.target
+- op: add
+  path: "/spec/kubeadmConfigSpec/files/-"
+  value:
+    path: /usr/bin/vm-infra-bridge.sh
+    permissions: "0744"
+    owner: root:root
+    content: |
+      #!/bin/bash
+      export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin
+      export DEBCONF_NONINTERACTIVE_SEEN=true
+      export DEBIAN_FRONTEND=noninteractive
+
+      # activate ip_forwarding
+      echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward
+
+      # retrieve the last octet as the vm-bridge ip addresses will follow
+      # the pattern of  <first three octet from the VM_NETWORK>:<last octet from the oam
+      # IP's last octet>
+      octet=$(ip addr show dev REPLACEMENT_MGMT_INTF | grep 'inet ' | awk 'NR==1{print $2}' | awk -F "/" '{print $1}' | awk -F "." '{print $4}')
+      # Given the CIDR for oam network is /26 (with 62 possible hosts), the below modulo 44 division
+      # is to ensure that the octet is within the range of the OAM last octet.
+      # TODO : Need to make the modulo value configurable for future release.
+      mgmtoctet=$(($octet % 44))
+      vm_ip_address=$(awk -F"." '{print $1"."$2"."$3"."}'<<<"REPLACEMENT_VM_NETWORK")${mgmtoctet}
+      echo "Going to use ${vm_ip_address}"
+      # convert the subnet information to CIDR format
+      vm_ip_cidr=$(awk -F. '{
+                  split($0, octets)
+                  for (i in octets) {
+                    mask += 8 - log(2**8 - octets[i])/log(2);
+                    }
+                    print "/" mask
+                   }' <<< "REPLACEMENT_VMNETWORK_SUBNET_MASK")
+      # add bridge if it doesn't exist
+      if ! brctl show | grep -q vm-infra-bridge; then
+           brctl addbr vm-infra-bridge
+           ip link set vm-infra-bridge up
+           ip addr add  ${vm_ip_address}${vm_ip_cidr} dev vm-infra-bridge
+           brctl addif vm-infra-bridge REPLACEMENT_VM_INFRA_INTF
+      fi;
+      # add iptables postrouting nat
+      iptables -t nat -A POSTROUTING -s REPLACEMENT_VM_NETWORK${vm_ip_cidr} -o REPLACEMENT_VM_INFRA_INTF -j MASQUERADE
+      exit 0
--- a/manifests/function/vm-infra-bridge/dataplane/kustomization.yaml
+++ b/manifests/function/vm-infra-bridge/dataplane/kustomization.yaml
@ -0,0 +1,13 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- ../../workers-capm3
+
+patchesJson6902:
+- target:
+    group: bootstrap.cluster.x-k8s.io
+    version: v1alpha3
+    kind: KubeadmConfigTemplate
+    name: worker-1
+  path: vm-infra-bridge.yaml
--- a/manifests/function/vm-infra-bridge/dataplane/replacements/kustomization.yaml
+++ b/manifests/function/vm-infra-bridge/dataplane/replacements/kustomization.yaml
@ -0,0 +1,2 @@
+resources:
+  - vm-infra-bridge.yaml
--- a/manifests/function/vm-infra-bridge/dataplane/replacements/vm-infra-bridge.yaml
+++ b/manifests/function/vm-infra-bridge/dataplane/replacements/vm-infra-bridge.yaml
@ -0,0 +1,49 @@
+apiVersion: airshipit.org/v1alpha1
+kind: ReplacementTransformer
+metadata:
+  name: vm-infra-bridge-replacements
+  annotations:
+    config.kubernetes.io/function: |-
+      container:
+        image: quay.io/airshipit/replacement-transformer:v2
+replacements:
+- source:
+    objref:
+      kind: VariableCatalogue
+      name: vm-networking
+    fieldref: spec.vm_infra_bridge.host_oam_interface
+  target:
+    objref:
+      kind: KubeadmConfigTemplate
+    fieldrefs:
+      -  "spec.template.spec.files[path=/usr/bin/vm-infra-bridge.sh].content%REPLACEMENT_MGMT_INTF%"
+- source:
+    objref:
+      kind: VariableCatalogue
+      name: vm-networking
+    fieldref: spec.vm_infra_bridge.vm_interface
+  target:
+    objref:
+      kind: KubeadmConfigTemplate
+    fieldrefs:
+      -  "spec.template.spec.files[path=/usr/bin/vm-infra-bridge.sh].content%REPLACEMENT_VM_INFRA_INTF%"
+- source:
+    objref:
+      kind: VariableCatalogue
+      name: vm-networking
+    fieldref: spec.vm_infra_bridge.vm_network
+  target:
+    objref:
+      kind: KubeadmConfigTemplate
+    fieldrefs:
+      -  "spec.template.spec.files[path=/usr/bin/vm-infra-bridge.sh].content%REPLACEMENT_VM_NETWORK%"
+- source:
+    objref:
+      kind: VariableCatalogue
+      name: vm-networking
+    fieldref: spec.vm_infra_bridge.vm_network_subnet_mask
+  target:
+    objref:
+      kind: KubeadmConfigTemplate
+    fieldrefs:
+      -  "spec.template.spec.files[path=/usr/bin/vm-infra-bridge.sh].content%REPLACEMENT_VMNETWORK_SUBNET_MASK%"
--- a/manifests/function/vm-infra-bridge/dataplane/vm-infra-bridge.yaml
+++ b/manifests/function/vm-infra-bridge/dataplane/vm-infra-bridge.yaml
@ -0,0 +1,66 @@
+- op: add
+  path: "/spec/template/spec/preKubeadmCommands/-"
+  value:
+     systemctl enable --now vm-infra-bridge.service
+- op: add
+  path: "/spec/template/spec/files/-"
+  value:
+    path: /etc/systemd/system/vm-infra-bridge.service
+    permissions: "0644"
+    owner: root:root
+    content: |
+      [Unit]
+      Description=Service to setup vm-infra-bridge and NAT using iptables
+      Wants=network-online.target
+      After=network.target network-online.target
+
+      [Service]
+      User=root
+      WorkingDirectory=/usr/bin
+      ExecStart=/usr/bin/vm-infra-bridge.sh
+
+      [Install]
+      WantedBy=multi-user.target
+- op: add
+  path: "/spec/template/spec/files/-"
+  value:
+    path: /usr/bin/vm-infra-bridge.sh
+    permissions: "0744"
+    owner: root:root
+    content: |
+      #!/bin/bash
+      export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin
+      export DEBCONF_NONINTERACTIVE_SEEN=true
+      export DEBIAN_FRONTEND=noninteractive
+
+      # activate ip_forwarding
+      echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward
+
+      # retrieve the last octet as the vm-bridge ip addresses will follow
+      # the pattern of  <first three octet from the VM_NETWORK>:<last octet from the oam
+      # IP's last octet>
+      octet=$(ip addr show dev REPLACEMENT_MGMT_INTF | grep 'inet ' | awk 'NR==1{print $2}' | awk -F "/" '{print $1}' | awk -F "." '{print $4}')
+      # Given the CIDR for oam network is /26 (with 62 possible hosts), the below modulo 44 division
+      # is to ensure that the octet is within the range of the OAM last octet.
+      # TODO : Need to make the modulo value configurable for future release.
+      mgmtoctet=$(($octet % 44))
+      vm_ip_address=$(awk -F"." '{print $1"."$2"."$3"."}'<<<"REPLACEMENT_VM_NETWORK")${mgmtoctet}
+      echo "Going to use ${vm_ip_address}"
+      # convert the subnet information to CIDR format
+      vm_ip_cidr=$(awk -F. '{
+                  split($0, octets)
+                  for (i in octets) {
+                    mask += 8 - log(2**8 - octets[i])/log(2);
+                    }
+                    print "/" mask
+                   }' <<< "REPLACEMENT_VMNETWORK_SUBNET_MASK")
+      # add bridge if it doesn't exist
+      if ! brctl show | grep -q vm-infra-bridge; then
+           brctl addbr vm-infra-bridge
+           ip link set vm-infra-bridge up
+           ip addr add  ${vm_ip_address}${vm_ip_cidr} dev vm-infra-bridge
+           brctl addif vm-infra-bridge REPLACEMENT_VM_INFRA_INTF
+      fi;
+      # add iptables postrouting nat
+      iptables -t nat -A POSTROUTING -s REPLACEMENT_VM_NETWORK${vm_ip_cidr} -o REPLACEMENT_VM_INFRA_INTF -j MASQUERADE
+      exit 0
--- a/manifests/site/test-site/target/workers/provision/kubeadmconfigtemplate.yaml
+++ b/manifests/site/test-site/target/workers/provision/kubeadmconfigtemplate.yaml
--- a/manifests/function/workers-capm3/kustomization.yaml
+++ b/manifests/function/workers-capm3/kustomization.yaml
@ -0,0 +1,4 @@
+resources:
+  - kubeadmconfigtemplate.yaml
+  - machinedeployment.yaml
+  - metal3machinetemplate.yaml
--- a/manifests/site/test-site/target/workers/provision/machinedeployment.yaml
+++ b/manifests/site/test-site/target/workers/provision/machinedeployment.yaml
--- a/manifests/function/workers-capm3/metal3machinetemplate.yaml
+++ b/manifests/function/workers-capm3/metal3machinetemplate.yaml
@ -0,0 +1,10 @@
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
+kind: Metal3MachineTemplate
+metadata:
+  name: worker-1
+spec:
+  template:
+    spec:
+      image:
+        url:  http://REPLACEMENT_PROVISIONING_IP:80/images/data-plane.qcow2
+        checksum: http://REPLACEMENT_PROVISIONING_IP:80/images/data-plane.qcow2.md5sum
--- a/manifests/function/workers-capm3/replacements/kustomization.yaml
+++ b/manifests/function/workers-capm3/replacements/kustomization.yaml
@ -0,0 +1,3 @@
+resources:
+  - workers-env-vars.yaml
+  - metal3machinetemplate_vars.yaml
--- a/manifests/function/workers-capm3/replacements/metal3machinetemplate_vars.yaml
+++ b/manifests/function/workers-capm3/replacements/metal3machinetemplate_vars.yaml
@ -2,7 +2,7 @@
 apiVersion: airshipit.org/v1alpha1
 kind: ReplacementTransformer
 metadata:
-  name: workers-generated-secret-replacements
+  name: metal3machientemplate-replacements
  annotations:
    config.kubernetes.io/function: |-
      container:
@ -10,11 +10,12 @@ metadata:
 replacements:
 - source:
    objref:
-      name: generated-secrets
-    fieldref: "{.sshKeys.publicKey}"
+      name: networking
+    fieldref: spec.ironic.provisioningIp
  target:
    objref:
-      kind: KubeadmConfigTemplate
+      kind: Metal3MachineTemplate
      name: worker-1
    fieldrefs:
-      - "spec.template.spec.users[name=deployer].sshAuthorizedKeys[0]%REPLACE_HOST_SSH_KEY%"
+      - "spec.template.spec.image.url%REPLACEMENT_PROVISIONING_IP%"
+      - "spec.template.spec.image.checksum%REPLACEMENT_PROVISIONING_IP%"
--- a/manifests/site/test-site/target/workers/replacements/workers-env-vars.yaml
+++ b/manifests/site/test-site/target/workers/replacements/workers-env-vars.yaml
--- a/manifests/site/test-site/target/workers/provision/kustomization.yaml
+++ b/manifests/site/test-site/target/workers/provision/kustomization.yaml
@ -2,9 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
  - ../../catalogues
-  - kubeadmconfigtemplate.yaml
-  - metal3machinetemplate.yaml
-  - machinedeployment.yaml
+  - ../../../../../function/workers-capm3

 transformers:
-  - ../replacements
+  - ../../../../../function/workers-capm3/replacements
--- a/manifests/site/test-site/target/workers/provision/metal3machinetemplate.yaml
+++ b/manifests/site/test-site/target/workers/provision/metal3machinetemplate.yaml
@ -1,10 +0,0 @@
-apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
-kind: Metal3MachineTemplate
-metadata:
-  name: worker-1
-spec:
-  template:
-    spec:
-      image:
-        url: http://10.23.24.102:80/images/data-plane.qcow2
-        checksum: http://10.23.24.102:80/images/data-plane.qcow2.md5sum
--- a/manifests/site/test-site/target/workers/replacements/kustomization.yaml
+++ b/manifests/site/test-site/target/workers/replacements/kustomization.yaml
@ -1,5 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - workers-env-vars.yaml
-  - generated-secrets.yaml
--- a/manifests/site/virtual-network-cloud/target/workers/provision/kubeadmconfigtemplate.yaml
+++ b/manifests/site/virtual-network-cloud/target/workers/provision/kubeadmconfigtemplate.yaml
@ -1,63 +0,0 @@
-apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3
-kind: KubeadmConfigTemplate
-metadata:
-  name: worker-1
-spec:
-  template:
-    spec:
-      joinConfiguration:
-        nodeRegistration:
-          name: '{{ ds.meta_data.name }}'
-          kubeletExtraArgs:
-            node-labels: 'metal3.io/uuid={{ ds.meta_data.uuid }},node-type=worker'
-            provider-id: 'metal3://{{ ds.meta_data.uuid }}'
-            feature-gates: "IPv6DualStack=true"
-      ntp:
-        enabled: true
-        servers:
-          - 135.188.34.124
-          - 135.38.244.16
-          - 135.188.34.84
-      preKubeadmCommands:
-        - echo 'root:r00tme' | chpasswd
-        - echo 'ubuntu:r00tme' | chpasswd
-        - |
-          cat <<EOF | tee /etc/sysctl.d/k8s.conf
-          net.bridge.bridge-nf-call-ip6tables = 1
-          net.bridge.bridge-nf-call-iptables = 1
-          EOF
-        - sysctl --system
-        - |
-          mkdir -p /etc/systemd/system/docker.service.d/
-          cat <<EOF | tee /etc/systemd/system/docker.service.d/http-proxy.conf
-          [Service]
-          Environment="HTTP_PROXY=REPLACEMENT_HTTP_PROXY"
-          Environment="HTTPS_PROXY=REPLACEMENT_HTTPS_PROXY"
-          Environment="NO_PROXY=REPLACEMENT_NO_PROXY"
-          EOF
-        - export HTTP_PROXY=REPLACEMENT_HTTP_PROXY
-        - export HTTPS_PROXY=REPLACEMENT_HTTPS_PROXY
-        - export http_proxy=${HTTP_PROXY}
-        - export https_proxy=${HTTPS_PROXY}
-        - export NO_PROXY=REPLACEMENT_NO_PROXY
-        - export no_proxy=${NO_PROXY}
-        - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
-        - curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
-        - add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
-        - echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" | tee -a /etc/apt/sources.list
-        - apt-get update
-        - apt-get install -y
-            docker-ce="$(apt policy docker-ce  | grep 19.03.9 | sort | head -n 1 | tr -s " " | cut -d ' ' -f 2 )"
-            docker-ce-cli="$(apt policy docker-ce-cli  | grep 19.03.9 | sort | head -n 1 | tr -s " " | cut -d ' ' -f 2 )"
-            containerd.io
-        - swapoff -a
-        - apt-get install -y kubelet=1.18.6-00 kubeadm=1.18.6-00
-        - apt-mark hold kubelet kubeadm
-        - systemctl mask ntp
-        - systemctl enable --now systemd-timesyncd
-        - unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY no_proxy NO_PROXY
-      users:
-        - name: deployer
-          sshAuthorizedKeys:
-          - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDK5jnOafJwnoN+vp77LgayzLZ7O6tu96cObzwjIEwOowi2KHTk+G4sUXzE4mt2QbPTbyOF45d/omcRZYixrTmYwwtJ9QGPbwWw/qpCRzVo5uV4qbwBd3iRUqXryOmZRCCFac678JXZS9f8AfOP9rHkh2jqhA6dJdtvqYTOpPLtmw8pYjScH/YqBXZObNSFS5PlSPl901UhZH4FNUAuYeR9JGY99wgM+R9XHRRgfBPJzwzvOQ7ZYfvxb+n4TuBr7u7jZtYC+pmG/eOYbIt2/vexO0y/rNomtC+hjDAXZO2VFwHejYW6r+ZPpkNrdr+5U8s0aENGg4BJkVa2n3LwUrZF segorov@node1
-          sudo: ALL=(ALL) NOPASSWD:ALL
--- a/manifests/site/virtual-network-cloud/target/workers/provision/kustomization.yaml
+++ b/manifests/site/virtual-network-cloud/target/workers/provision/kustomization.yaml
@ -1,10 +1,9 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - ../../../../../../../airshipctl/manifests/function/airshipctl-base-catalogues
-  - kubeadmconfigtemplate.yaml
-  - metal3machinetemplate.yaml
-  - machinedeployment.yaml
+  - ../../catalogues
+  - ../../../../../function/vm-infra-bridge/dataplane

 transformers:
-  - ../replacements
+  - ../../../../../function/vm-infra-bridge/dataplane/replacements
+  - ../../../../../function/workers-capm3/replacements
--- a/manifests/site/virtual-network-cloud/target/workers/provision/machinedeployment.yaml
+++ b/manifests/site/virtual-network-cloud/target/workers/provision/machinedeployment.yaml
@ -1,29 +0,0 @@
-apiVersion: cluster.x-k8s.io/v1alpha3
-kind: MachineDeployment
-metadata:
-  name: worker-1
-  labels:
-    cluster.x-k8s.io/cluster-name: target-cluster
-spec:
-  clusterName: target-cluster
-  replicas: 1
-  selector:
-    matchLabels:
-      cluster.x-k8s.io/cluster-name: target-cluster
-  template:
-    metadata:
-      labels:
-        cluster.x-k8s.io/cluster-name: target-cluster
-    spec:
-      clusterName: target-cluster
-      version: v1.18.3
-      bootstrap:
-        configRef:
-          name: worker-1
-          apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3
-          kind: KubeadmConfigTemplate
-      infrastructureRef:
-        name: worker-1
-        apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
-        kind: Metal3MachineTemplate
---
--- a/manifests/site/virtual-network-cloud/target/workers/provision/metal3machinetemplate.yaml
+++ b/manifests/site/virtual-network-cloud/target/workers/provision/metal3machinetemplate.yaml
@ -1,10 +0,0 @@
-apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
-kind: Metal3MachineTemplate
-metadata:
-  name: worker-1
-spec:
-  template:
-    spec:
-      image:
-        url: http://10.23.24.1:8099/target-image.qcow2
-        checksum: http://10.23.24.1:8099/target-image.qcow2.md5sum
--- a/manifests/site/virtual-network-cloud/target/workers/replacements/kustomization.yaml
+++ b/manifests/site/virtual-network-cloud/target/workers/replacements/kustomization.yaml
@ -1,4 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - workers-env-vars.yaml
--- a/manifests/site/virtual-network-cloud/target/workers/replacements/workers-env-vars.yaml
+++ b/manifests/site/virtual-network-cloud/target/workers/replacements/workers-env-vars.yaml
@ -1,38 +0,0 @@
-# These rules inject env vars into the workers.
-apiVersion: airshipit.org/v1alpha1
-kind: ReplacementTransformer
-metadata:
-  name: workers-env-vars-replacements
-  annotations:
-    config.kubernetes.io/function: |-
-      container:
-        image: quay.io/airshipit/replacement-transformer:v2
-replacements:
-# Replace the proxy vars
- source:
-    objref:
-      name: env-vars-catalogue
-    fieldref: env.HTTP_PROXY
-  target:
-    objref:
-      kind: KubeadmConfigTemplate
-      name: worker-1
-    fieldrefs: ["spec.template.spec.preKubeadmCommands%REPLACEMENT_HTTP_PROXY%"]
- source:
-    objref:
-      name: env-vars-catalogue
-    fieldref: env.HTTPS_PROXY
-  target:
-    objref:
-      kind: KubeadmConfigTemplate
-      name: worker-1
-    fieldrefs: ["spec.template.spec.preKubeadmCommands%REPLACEMENT_HTTPS_PROXY%"]
- source:
-    objref:
-      name: env-vars-catalogue
-    fieldref: env.NO_PROXY
-  target:
-    objref:
-      kind: KubeadmConfigTemplate
-      name: worker-1
-    fieldrefs: ["spec.template.spec.preKubeadmCommands%REPLACEMENT_NO_PROXY%"]
--- a/manifests/type/multi-tenant/ephemeral/controlplane/dex-apiserver/README.md
+++ b/manifests/type/multi-tenant/ephemeral/controlplane/dex-apiserver/README.md
@ -0,0 +1,67 @@
+# DEX-APIServer kustomizations
+
+The "dex-apiserver" folder provides the manifests and patches to configure the API server with
+"oidc" flags and CA certificate (Secret). Refer to the JSON patch file *oidc-apiserver-flags.json*.
+This patch file adds OIDC flags configuration to the API server in the KubeadmControlPlane CR,
+which is used to create the Target cluster's ControlPlane node and deploy the API server during
+the execution of ***airshipctl phase run controlplane-ephemeral*** command.
+
+>IMPORTANT: The JSON patch is tailored for baremetal provider. If deploying target cluster on a
+>different provider (e.g., Azure, GCP, Openstack), you will need to update this patch, accordingly.
+
+In order to ensure synchronization with the "dex-aio" service, the CA certificate (Secret)
+in the Ephemeral cluster SHALL be copied to the Target cluster. This is achieved by adding the label
+**clusterctl.cluster.x-k8s.io/move: "true"** to the CA Secret. This label idenfies this Secret as
+candidate to the CAPI move command executed by ***airshipctl phase run clusterctl-move*** command.
+
+Once this CA Secret has been moved to the Target cluster, it will be used during **dex-aio** deployment
+to sign Certificates to be used by Dex.
+
+>NOTES on **oidc-apiserver-flags.json**:
+* The (Dex) FQDN for the attribute **oidc-issuer-url** will have to be added to the list under **certSANs**
+* The patches for **"/spec/kubeadmConfigSpec/preKubeadmCommands/-"** are needed if your (Dex) FQDN cannot be resolved by the DNS used by the controlplane node.
+* The **oidc-issuer-url** FQDN and port number MUST match **dex-aio** HelmRelease values for **values.params.endpoints.hostname** and **values.params.endpoints.port.https**. Example below:
+
+Snippet of **oidc-apiserver-flags.json**
+```json
+  {
+    "op": "add",
+    "path": "/spec/kubeadmConfigSpec/clusterConfiguration/apiServer",
+    "value": {
+      "extraArgs":
+      {
+        "oidc-issuer-url": "https://dex.function.local:32556/dex",
+      },
+```
+
+Snippet of **treasuremap/manifests/function/dex-aio/dex-helmrelease.yaml**
+```yaml
+  values:
+    params:
+      endpoints:
+        hostname: dex.function.local
+        port:
+          https: 32556
+```
+
+Also, in case your **dex-aio** FQDN (e.g., **dex.function.local**) cannot be resolved by the DNS configured
+in the control plane node, your JSON patch will also have to include this FQDN to the nodes **/etc/hosts**
+so that the API server can reach **dex-aio** microservice.
+
+Snippet of **oidc-apiserver-flags.json**
+```json
+  {
+    "op": "add",
+    "path": "/spec/kubeadmConfigSpec/preKubeadmCommands/-",
+    "value": "echo '10.23.25.102 dex.function.local' | tee -a /etc/hosts"
+  }
+```
+
+>NOTES on **dex-ca-cert-secret.yaml**:
+* This Secret contains a Certificate Authority (CA) certificate manually generated.
+* The CA certificate was not signed by a known authority
+
+>TODO(s):
+* CA certificate shall be auto generated
+* The CA certificate shall be signed by a known authority
+* The generated CA certificate shall be secured, e.g., encrypted using SOPS
--- a/manifests/type/multi-tenant/ephemeral/controlplane/dex-apiserver/dex-ca-cert-secret.yaml
+++ b/manifests/type/multi-tenant/ephemeral/controlplane/dex-apiserver/dex-ca-cert-secret.yaml
@ -0,0 +1,12 @@
+# Certificate Authority to be used to generate certificates for Dex
+# By cert-manager in the Target-cluster. Also this CA will be used by API server
+apiVersion: v1
+kind: Secret
+metadata:
+  name: dex-apiserver-secret
+  labels:
+    clusterctl.cluster.x-k8s.io/move: "true"
+type: kubernetes.io/tls
+data:
+  tls.crt: "target-cluster-ca certificate will be copied by replacement transformer"
+  tls.key: "target-cluster-ca certificate key will be copied by replacement transformer"
--- a/manifests/type/multi-tenant/ephemeral/controlplane/dex-apiserver/oidc-apiserver-flags.json
+++ b/manifests/type/multi-tenant/ephemeral/controlplane/dex-apiserver/oidc-apiserver-flags.json
@ -0,0 +1,55 @@
+[
+  {
+    "op": "add",
+    "path": "/spec/kubeadmConfigSpec/clusterConfiguration/apiServer",
+    "value": {
+      "extraArgs":
+      {
+        "oidc-ca-file": "/etc/kubernetes/certs/dex-cert",
+        "oidc-client-id": "function-kubernetes",
+        "oidc-groups-claim": "group",
+        "oidc-issuer-url": "https://dex.function.local:32556/dex",
+        "oidc-username-claim": "email"
+      },
+      "extraVolumes":
+      [
+        {
+          "hostPath": "/etc/kubernetes/certs/dex-cert",
+          "mountPath": "/etc/kubernetes/certs/dex-cert",
+          "name": "dex-cert",
+          "readOnly": true
+        }
+      ],
+      "certSANs":
+      [
+        "dex.function.local",
+        "dex.baremetal.local"
+      ]
+    }
+  },
+  {
+    "op": "add",
+    "path": "/spec/kubeadmConfigSpec/files/-",
+    "value": {
+      "contentFrom": {
+        "secret": {
+          "key": "tls.crt",
+          "name": "dex-apiserver-secret"
+        }
+      },
+      "owner": "root:root",
+      "path": "/etc/kubernetes/certs/dex-cert",
+      "permissions": "0644"
+    }
+  },
+  {
+    "op": "add",
+    "path": "/spec/kubeadmConfigSpec/preKubeadmCommands/-",
+    "value": "echo '10.23.25.102 dex.baremetal.local' | tee -a /etc/hosts"
+  },
+  {
+    "op": "add",
+    "path": "/spec/kubeadmConfigSpec/preKubeadmCommands/-",
+    "value": "echo '10.23.25.102 dex.function.local' | tee -a /etc/hosts"
+  }
+]
--- a/manifests/type/multi-tenant/ephemeral/controlplane/kustomization.yaml
+++ b/manifests/type/multi-tenant/ephemeral/controlplane/kustomization.yaml
@ -1,4 +1,13 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - ../../../airship-core/ephemeral/controlplane
+  - ../../../../function/k8scontrol-vm-infra
+  - dex-apiserver/dex-ca-cert-secret.yaml
+
+patchesJson6902:
+  - target:
+      group: controlplane.cluster.x-k8s.io
+      version: v1alpha3
+      kind: KubeadmControlPlane
+      name: "cluster-controlplane"
+    path: dex-apiserver/oidc-apiserver-flags.json
--- a/manifests/type/multi-tenant/ephemeral/controlplane/replacements/kustomization.yaml
+++ b/manifests/type/multi-tenant/ephemeral/controlplane/replacements/kustomization.yaml
@ -2,3 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
  - ../../../../airship-core/ephemeral/controlplane/replacements
+  - ../../../../../function/k8scontrol-vm-infra/replacements
--- a/manifests/type/multi-tenant/shared/catalogues/kustomization.yaml
+++ b/manifests/type/multi-tenant/shared/catalogues/kustomization.yaml
@ -3,3 +3,4 @@ kind: Kustomization
 resources:
  - ../../../airship-core/shared/catalogues
  - subcluster-networking.yaml
+  - vm-networking.yaml
--- a/manifests/type/multi-tenant/shared/catalogues/vm-networking.yaml
+++ b/manifests/type/multi-tenant/shared/catalogues/vm-networking.yaml
@ -0,0 +1,12 @@
+apiVersion: airshipit.org/v1alpha1
+kind: VariableCatalogue
+metadata:
+  name: vm-networking
+  labels:
+    airshipit.org/deploy-k8s: "false"
+spec:
+  vm_infra_bridge:
+     host_oam_interface:  bond0.41
+     vm_interface: bond0.45
+     vm_network: 192.168.0.0
+     vm_network_subnet_mask: 255.255.240.0