From 393b75bc0800bd0386eb2d9ac7f93023292d0273 Mon Sep 17 00:00:00 2001
From: Stanislav Egorov <segorov@mirantis.com>
Date: Mon, 24 Aug 2020 16:27:29 -0700
Subject: [PATCH] Worker node deployment

Change-Id: I1a1c06ee9fa16e2c56374e98f8aaea5da80898cd
---
 .../config-file/inspector.ipxe                |  4 +-
 .../entrypoint/init-bootstrap                 |  2 +-
 .../function/k8scontrol/controlplane.yaml     |  2 +
 .../test-site/shared/catalogues/hosts.yaml    | 13 +++++
 .../target/initinfra/patch_bmo_config.yaml    |  5 +-
 .../hostgenerator/host-generation.yaml        | 10 ++++
 .../workers/hostgenerator/kustomization.yaml  |  9 ++++
 .../target/workers/kubeadmconfigtemplate.yaml | 48 +++++++++++++++++++
 .../target/workers/kustomization.yaml         |  7 +++
 .../target/workers/machinedeployment.yaml     | 29 +++++++++++
 .../target/workers/metal3machinetemplate.yaml | 10 ++++
 .../target/workers/nodes/kustomization.yaml   |  8 ++++
 playbooks/airship-airshipctl-build-gate.yaml  |  3 ++
 playbooks/airshipctl-gate-runner.yaml         |  1 +
 .../tasks/build-infra.yaml                    |  6 +--
 roles/airship-libvirt-gate/tests/vars.yaml    |  2 +-
 tools/deployment/34_deploy_worker_node.sh     | 48 +++++++++++++++++++
 zuul.d/jobs.yaml                              |  1 +
 18 files changed, 198 insertions(+), 10 deletions(-)
 create mode 100644 manifests/site/test-site/target/workers/hostgenerator/host-generation.yaml
 create mode 100644 manifests/site/test-site/target/workers/hostgenerator/kustomization.yaml
 create mode 100644 manifests/site/test-site/target/workers/kubeadmconfigtemplate.yaml
 create mode 100644 manifests/site/test-site/target/workers/kustomization.yaml
 create mode 100644 manifests/site/test-site/target/workers/machinedeployment.yaml
 create mode 100644 manifests/site/test-site/target/workers/metal3machinetemplate.yaml
 create mode 100644 manifests/site/test-site/target/workers/nodes/kustomization.yaml
 create mode 100755 tools/deployment/34_deploy_worker_node.sh

diff --git a/manifests/function/baremetal-operator/config-file/inspector.ipxe b/manifests/function/baremetal-operator/config-file/inspector.ipxe
index ac534a07c..16f7512be 100644
--- a/manifests/function/baremetal-operator/config-file/inspector.ipxe
+++ b/manifests/function/baremetal-operator/config-file/inspector.ipxe
@@ -4,6 +4,6 @@
 echo In inspector.ipxe
 imgfree
 # NOTE(dtantsur): keep inspection kernel params in [mdns]params in ironic-inspector-image
-kernel --timeout 60000 http://$(PROVISIONING_IP):80/images/ironic-python-agent.kernel ipa-inspection-callback-url=http://$(PROVISIONING_IP):5050/v1/continue ipa-inspection-collectors=default,extra-hardware,logs systemd.journald.forward_to_console=yes BOOTIF=${mac} ipa-debug=1 ipa-inspection-dhcp-all-interfaces=1 ipa-collect-lldp=1 initrd=ironic-python-agent.initramfs || goto retry_boot
-initrd --timeout 60000 http://$(PROVISIONING_IP):80/images/ironic-python-agent.initramfs || goto retry_boot
+kernel --timeout 60000 http://$(PROVISIONING_IP):80/images/tinyipa-stable-ussuri.vmlinuz ipa-inspection-callback-url=http://$(PROVISIONING_IP):5050/v1/continue ipa-inspection-collectors=default,extra-hardware,logs systemd.journald.forward_to_console=yes BOOTIF=${mac} ipa-debug=1 ipa-inspection-dhcp-all-interfaces=1 ipa-collect-lldp=1 initrd=tinyipa-stable-ussuri.gz || goto retry_boot
+initrd --timeout 60000 http://$(PROVISIONING_IP):80/images/tinyipa-stable-ussuri.gz || goto retry_boot
 boot
diff --git a/manifests/function/baremetal-operator/entrypoint/init-bootstrap b/manifests/function/baremetal-operator/entrypoint/init-bootstrap
index becd4af54..a2cc0fd0b 100644
--- a/manifests/function/baremetal-operator/entrypoint/init-bootstrap
+++ b/manifests/function/baremetal-operator/entrypoint/init-bootstrap
@@ -7,7 +7,7 @@ rm -f /shared/sed_commands
 for var_name in $( ls ) ; do
     echo -n s!\$\($var_name\)! >> /shared/sed_commands
     cat $var_name >> /shared/sed_commands
-    echo ! >> /shared/sed_commands
+    echo !g >> /shared/sed_commands
     export $var_name=$(cat $var_name)
 done
 popd
diff --git a/manifests/function/k8scontrol/controlplane.yaml b/manifests/function/k8scontrol/controlplane.yaml
index 49278ba86..42620d812 100644
--- a/manifests/function/k8scontrol/controlplane.yaml
+++ b/manifests/function/k8scontrol/controlplane.yaml
@@ -11,6 +11,8 @@ spec:
     name: cluster-controlplane
   kubeadmConfigSpec:
     preKubeadmCommands:
+      - echo 'root:r00tme' | chpasswd
+      - echo 'ubuntu:r00tme' | chpasswd
       - |
         cat <<EOF | tee /etc/sysctl.d/k8s.conf
         net.bridge.bridge-nf-call-ip6tables = 1
diff --git a/manifests/site/test-site/shared/catalogues/hosts.yaml b/manifests/site/test-site/shared/catalogues/hosts.yaml
index b7fb6b4cf..22d368c3b 100644
--- a/manifests/site/test-site/shared/catalogues/hosts.yaml
+++ b/manifests/site/test-site/shared/catalogues/hosts.yaml
@@ -32,3 +32,16 @@ hosts:
         oam: 52:54:00:9b:27:02
         pxe: 52:54:00:b6:ed:02
       hardwareProfile: example # defined in the hardwareprofile-example function
+    node03:
+      bootMode: UEFI
+      macAddress: 52:54:00:b6:ed:23
+      bmcAddress: redfish+http://10.23.25.1:8000/redfish/v1/Systems/air-worker-1
+      bmcUsername: username
+      bmcPassword: password
+      ipAddresses:
+        oam-ipv4: 10.23.25.103
+        pxe-ipv4: 10.23.24.103
+      macAddresses:
+        oam: 52:54:00:9b:27:07
+        pxe: 52:54:00:b6:ed:23
+      hardwareProfile: default # defined in the hardwareprofile-example function
diff --git a/manifests/site/test-site/target/initinfra/patch_bmo_config.yaml b/manifests/site/test-site/target/initinfra/patch_bmo_config.yaml
index e14f78a65..a11f3891e 100644
--- a/manifests/site/test-site/target/initinfra/patch_bmo_config.yaml
+++ b/manifests/site/test-site/target/initinfra/patch_bmo_config.yaml
@@ -5,6 +5,5 @@ metadata:
   labels:
   name: ironic-vars
 data:
-  PROVISIONING_IP: "10.23.25.102"
-  DHCP_RANGE: "10.23.25.200,10.23.25.250"
-  PROVISIONING_INTERFACE: "pxe0"
+  PROVISIONING_IP: "10.23.24.102"
+  DHCP_RANGE: "10.23.24.200,10.23.24.250"
diff --git a/manifests/site/test-site/target/workers/hostgenerator/host-generation.yaml b/manifests/site/test-site/target/workers/hostgenerator/host-generation.yaml
new file mode 100644
index 000000000..e197b455c
--- /dev/null
+++ b/manifests/site/test-site/target/workers/hostgenerator/host-generation.yaml
@@ -0,0 +1,10 @@
+# Site-level, phase-specific lists of hosts to generate
+# This is used by the hostgenerator-m3 function to narrow down the site-level
+# host-catalogue to just the hosts needed for a particular phase.
+apiVersion: airshipit.org/v1alpha1
+kind: VariableCatalogue
+metadata:
+  name: host-generation-catalogue
+hosts:
+  m3:
+    - node03
diff --git a/manifests/site/test-site/target/workers/hostgenerator/kustomization.yaml b/manifests/site/test-site/target/workers/hostgenerator/kustomization.yaml
new file mode 100644
index 000000000..8de561b8c
--- /dev/null
+++ b/manifests/site/test-site/target/workers/hostgenerator/kustomization.yaml
@@ -0,0 +1,9 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../../../../../function/hostgenerator-m3
+  - ../../../shared/catalogues/
+  - host-generation.yaml
+
+transformers:
+  - ../../../../../function/hostgenerator-m3/replacements
diff --git a/manifests/site/test-site/target/workers/kubeadmconfigtemplate.yaml b/manifests/site/test-site/target/workers/kubeadmconfigtemplate.yaml
new file mode 100644
index 000000000..4bd8d4b8d
--- /dev/null
+++ b/manifests/site/test-site/target/workers/kubeadmconfigtemplate.yaml
@@ -0,0 +1,48 @@
+apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3
+kind: KubeadmConfigTemplate
+metadata:
+  name: worker-1
+spec:
+  template:
+    spec:
+      joinConfiguration:
+        nodeRegistration:
+          name: '{{ ds.meta_data.name }}'
+          kubeletExtraArgs:
+            node-labels: 'metal3.io/uuid={{ ds.meta_data.uuid }},node-type=worker'
+            provider-id: 'metal3://{{ ds.meta_data.uuid }}'
+            feature-gates: "IPv6DualStack=true"
+      ntp:
+        enabled: true
+        servers:
+          - 135.188.34.124
+          - 135.38.244.16
+          - 135.188.34.84
+      preKubeadmCommands:
+        - echo 'root:r00tme' | chpasswd
+        - echo 'ubuntu:r00tme' | chpasswd
+        - |
+          cat <<EOF | tee /etc/sysctl.d/k8s.conf
+          net.bridge.bridge-nf-call-ip6tables = 1
+          net.bridge.bridge-nf-call-iptables = 1
+          EOF
+        - sysctl --system
+        - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+        - curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
+        - add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+        - echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" | tee -a /etc/apt/sources.list
+        - apt-get update
+        - apt-get install -y
+            docker-ce="$(apt policy docker-ce  | grep 19.03.9 | sort | head -n 1 | tr -s " " | cut -d ' ' -f 2 )"
+            docker-ce-cli="$(apt policy docker-ce-cli  | grep 19.03.9 | sort | head -n 1 | tr -s " " | cut -d ' ' -f 2 )"
+            containerd.io
+        - swapoff -a
+        - apt-get install -y kubelet=1.18.6-00 kubeadm=1.18.6-00
+        - apt-mark hold kubelet kubeadm
+        - systemctl mask ntp
+        - systemctl enable --now systemd-timesyncd
+      users:
+        - name: deployer
+          sshAuthorizedKeys:
+          - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDK5jnOafJwnoN+vp77LgayzLZ7O6tu96cObzwjIEwOowi2KHTk+G4sUXzE4mt2QbPTbyOF45d/omcRZYixrTmYwwtJ9QGPbwWw/qpCRzVo5uV4qbwBd3iRUqXryOmZRCCFac678JXZS9f8AfOP9rHkh2jqhA6dJdtvqYTOpPLtmw8pYjScH/YqBXZObNSFS5PlSPl901UhZH4FNUAuYeR9JGY99wgM+R9XHRRgfBPJzwzvOQ7ZYfvxb+n4TuBr7u7jZtYC+pmG/eOYbIt2/vexO0y/rNomtC+hjDAXZO2VFwHejYW6r+ZPpkNrdr+5U8s0aENGg4BJkVa2n3LwUrZF segorov@node1
+          sudo: ALL=(ALL) NOPASSWD:ALL
diff --git a/manifests/site/test-site/target/workers/kustomization.yaml b/manifests/site/test-site/target/workers/kustomization.yaml
new file mode 100644
index 000000000..ece1a696c
--- /dev/null
+++ b/manifests/site/test-site/target/workers/kustomization.yaml
@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - nodes
+  - kubeadmconfigtemplate.yaml
+  - metal3machinetemplate.yaml
+  - machinedeployment.yaml
diff --git a/manifests/site/test-site/target/workers/machinedeployment.yaml b/manifests/site/test-site/target/workers/machinedeployment.yaml
new file mode 100644
index 000000000..e9687d8ee
--- /dev/null
+++ b/manifests/site/test-site/target/workers/machinedeployment.yaml
@@ -0,0 +1,29 @@
+apiVersion: cluster.x-k8s.io/v1alpha3
+kind: MachineDeployment
+metadata:
+  name: worker-1
+  labels:
+    cluster.x-k8s.io/cluster-name: target-cluster
+spec:
+  clusterName: target-cluster
+  replicas: 1
+  selector:
+    matchLabels:
+      cluster.x-k8s.io/cluster-name: target-cluster
+  template:
+    metadata:
+      labels:
+        cluster.x-k8s.io/cluster-name: target-cluster
+    spec:
+      clusterName: target-cluster
+      version: v1.18.3
+      bootstrap:
+        configRef:
+          name: worker-1
+          apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3
+          kind: KubeadmConfigTemplate
+      infrastructureRef:
+        name: worker-1
+        apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
+        kind: Metal3MachineTemplate
+---
diff --git a/manifests/site/test-site/target/workers/metal3machinetemplate.yaml b/manifests/site/test-site/target/workers/metal3machinetemplate.yaml
new file mode 100644
index 000000000..68599b383
--- /dev/null
+++ b/manifests/site/test-site/target/workers/metal3machinetemplate.yaml
@@ -0,0 +1,10 @@
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
+kind: Metal3MachineTemplate
+metadata:
+  name: worker-1
+spec:
+  template:
+    spec:
+      image:
+        url: http://10.23.24.1:8099/target-image.qcow2
+        checksum: http://10.23.24.1:8099/target-image.qcow2.md5sum
diff --git a/manifests/site/test-site/target/workers/nodes/kustomization.yaml b/manifests/site/test-site/target/workers/nodes/kustomization.yaml
new file mode 100644
index 000000000..512dc07c8
--- /dev/null
+++ b/manifests/site/test-site/target/workers/nodes/kustomization.yaml
@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+generators:
+  - ../hostgenerator
+
+commonLabels:
+  airshipit.org/k8s-role: controlplane-host
diff --git a/playbooks/airship-airshipctl-build-gate.yaml b/playbooks/airship-airshipctl-build-gate.yaml
index fe2b91a02..403d4bdfc 100644
--- a/playbooks/airship-airshipctl-build-gate.yaml
+++ b/playbooks/airship-airshipctl-build-gate.yaml
@@ -60,6 +60,7 @@
         - name: clone document model
           command: git clone -q {{ airship_config_primary_repo_url }} {{ remote_work_dir }}
           when: remote_work_dir is defined
+
     - name: get VM config(MACs and bootMode) from BareMetalHost objects
       include_tasks:
         file: get-vm-config.yaml
@@ -69,6 +70,7 @@
       loop:
         - { name: 'target', path: 'ephemeral/controlplane' }
         - { name: 'ephemeral', path: 'ephemeral/bootstrap' }
+        - { name: 'worker', path: 'target/workers' }
       loop_control:
         loop_var: roleinputvar
 
@@ -96,6 +98,7 @@
             target_vm_vcpus: 2
             target_vms_count: 1
             worker_disk_size: 10G
+            worker_vm_cfg: "{{ worker_vm_cfg }}"
             worker_vm_memory_mb: 1024
             worker_vm_vcpus: 1
             worker_vms_count: 1
diff --git a/playbooks/airshipctl-gate-runner.yaml b/playbooks/airshipctl-gate-runner.yaml
index b67be2466..90888b8a9 100644
--- a/playbooks/airshipctl-gate-runner.yaml
+++ b/playbooks/airshipctl-gate-runner.yaml
@@ -27,6 +27,7 @@
           - ./tools/deployment/31_deploy_initinfra_target_node.sh
           - ./tools/deployment/32_cluster_init_target_node.sh
           - ./tools/deployment/33_cluster_move_target_node.sh
+          - ./tools/deployment/34_deploy_worker_node.sh
 
     - name: "Run gate scripts"
       include_role:
diff --git a/roles/airship-libvirt-gate/tasks/build-infra.yaml b/roles/airship-libvirt-gate/tasks/build-infra.yaml
index b635a0d40..c8337d152 100644
--- a/roles/airship-libvirt-gate/tasks/build-infra.yaml
+++ b/roles/airship-libvirt-gate/tasks/build-infra.yaml
@@ -170,7 +170,7 @@
     name: libvirt-domain
   vars:
     libvirt_domain:
-      boot_mode: UEFI
+      boot_mode: "{{ chosen_flavor.worker_vm_cfg[vm_index|int - 1].boot_mode | default('UEFI') }}"
       enable_vnc: true
       console_log_enabled: true
       state: shutdown
@@ -188,9 +188,9 @@
           pool: "{{ airship_gate_names.pool }}"
       interfaces:
         - network: "{{ airship_gate_names.nat_network }}"
-          mac: ""
+          mac: "{{ chosen_flavor.worker_vm_cfg[vm_index|int - 1].nat_mac | default('') }}"
         - network: "{{ airship_gate_names.provision_network }}"
-          mac: ""
+          mac: "{{ chosen_flavor.worker_vm_cfg[vm_index|int - 1].provision_mac | default('') }}"
   loop_control:
     loop_var: vm_index
   with_sequence: "start=1 end={{ chosen_flavor.worker_vms_count }}"
diff --git a/roles/airship-libvirt-gate/tests/vars.yaml b/roles/airship-libvirt-gate/tests/vars.yaml
index e0525a2a0..69d0dee6e 100644
--- a/roles/airship-libvirt-gate/tests/vars.yaml
+++ b/roles/airship-libvirt-gate/tests/vars.yaml
@@ -38,5 +38,5 @@ airship_gate_names:
   target_volume_prefix: "vol_target"
   ephemeral_volume: "vol_ephemeral"
   pool: airship
-  worker_vm_prefix: "air-target"
+  worker_vm_prefix: "air-worker"
   worker_volume_prefix: "vol_worker"
diff --git a/tools/deployment/34_deploy_worker_node.sh b/tools/deployment/34_deploy_worker_node.sh
new file mode 100755
index 000000000..e92269491
--- /dev/null
+++ b/tools/deployment/34_deploy_worker_node.sh
@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -xe
+
+#Default wait timeout is 3600 seconds
+export TIMEOUT=${TIMEOUT:-3600}
+export KUBECONFIG=${KUBECONFIG:-"$HOME/.airship/kubeconfig"}
+
+echo "Switch context to target cluster and set manifest"
+airshipctl config use-context target-cluster-admin@target-cluster
+airshipctl config set-context target-cluster-admin@target-cluster --manifest dummy_manifest
+
+echo "Stop ephemeral node"
+sudo virsh destroy air-ephemeral
+
+echo "Deploy worker node"
+airshipctl phase apply workers --debug
+
+#Wait till node is created
+end=$(($(date +%s) + $TIMEOUT))
+echo "Waiting $TIMEOUT seconds for node to be provisioned."
+while true; do
+    if (kubectl --request-timeout 20s --kubeconfig $KUBECONFIG get node node03 | grep -qw Ready) ; then
+        echo -e "\nGet node status"
+        kubectl --kubeconfig $KUBECONFIG get node
+        break
+    else
+        now=$(date +%s)
+        if [ $now -gt $end ]; then
+            echo -e "\nWorker node is not ready before TIMEOUT."
+            exit 1
+        fi
+        echo -n .
+        sleep 15
+    fi
+done
diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml
index 748c64981..2968d7c6e 100644
--- a/zuul.d/jobs.yaml
+++ b/zuul.d/jobs.yaml
@@ -98,6 +98,7 @@
         - ./tools/deployment/31_deploy_initinfra_target_node.sh
         - ./tools/deployment/32_cluster_init_target_node.sh
         - ./tools/deployment/33_cluster_move_target_node.sh
+        - ./tools/deployment/34_deploy_worker_node.sh
       serve_dir: /srv/iso
       serve_port: 8099
     voting: false