Enhance target node deployment on gates

* Add script to install clusterctl * Add controlplane yamls for target node * Add script to perform cluster move * Add script to zuul jobs Closes: #288 Change-Id: Ia6891df9c9b1da333396e76f11332deeb17ab807 Signed-off-by: Sreejith Punnapuzha <sreejith.punnapuzha@outlook.com>
2020-06-30 15:55:07 -05:00 · 2020-06-30 15:55:07 -05:00 · f95da5a56c
commit f95da5a56c
parent 69a6c3e406
24 changed files with 167 additions and 0 deletions
--- a/manifests/function/cabpk/v0.3.3/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml
+++ b/manifests/function/cabpk/v0.3.3/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml
@ -3,6 +3,8 @@
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
+  labels:
+    clusterctl.cluster.x-k8s.io: ""
  annotations:
    controller-gen.kubebuilder.io/version: v0.2.8
  creationTimestamp: null
--- a/manifests/function/cabpk/v0.3.3/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml
+++ b/manifests/function/cabpk/v0.3.3/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml
@ -3,6 +3,8 @@
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
+  labels:
+    clusterctl.cluster.x-k8s.io: ""
  annotations:
    controller-gen.kubebuilder.io/version: v0.2.8
  creationTimestamp: null
--- a/manifests/function/cacpk/v0.3.3/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml
+++ b/manifests/function/cacpk/v0.3.3/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml
@ -3,6 +3,8 @@
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
+  labels:
+    clusterctl.cluster.x-k8s.io: ""
  annotations:
    controller-gen.kubebuilder.io/version: v0.2.8
  creationTimestamp: null
--- a/manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_clusters.yaml
+++ b/manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_clusters.yaml
@ -3,6 +3,8 @@
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
+  labels:
+    clusterctl.cluster.x-k8s.io: ""
  annotations:
    controller-gen.kubebuilder.io/version: v0.2.8
  creationTimestamp: null
--- a/manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_machinedeployments.yaml
+++ b/manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_machinedeployments.yaml
@ -3,6 +3,8 @@
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
+  labels:
+    clusterctl.cluster.x-k8s.io: ""
  annotations:
    controller-gen.kubebuilder.io/version: v0.2.8
  creationTimestamp: null
--- a/manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_machinehealthchecks.yaml
+++ b/manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_machinehealthchecks.yaml
@ -3,6 +3,8 @@
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
+  labels:
+    clusterctl.cluster.x-k8s.io: ""
  annotations:
    controller-gen.kubebuilder.io/version: v0.2.8
  creationTimestamp: null
--- a/manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_machines.yaml
+++ b/manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_machines.yaml
@ -3,6 +3,8 @@
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
+  labels:
+    clusterctl.cluster.x-k8s.io: ""
  annotations:
    controller-gen.kubebuilder.io/version: v0.2.8
  creationTimestamp: null
--- a/manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_machinesets.yaml
+++ b/manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_machinesets.yaml
@ -3,6 +3,8 @@
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
+  labels:
+    clusterctl.cluster.x-k8s.io: ""
  annotations:
    controller-gen.kubebuilder.io/version: v0.2.8
  creationTimestamp: null
--- a/manifests/function/capi/v0.3.3/crd/bases/exp.cluster.x-k8s.io_machinepools.yaml
+++ b/manifests/function/capi/v0.3.3/crd/bases/exp.cluster.x-k8s.io_machinepools.yaml
@ -3,6 +3,8 @@
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
+  labels:
+    clusterctl.cluster.x-k8s.io: ""
  annotations:
    controller-gen.kubebuilder.io/version: v0.2.8
  creationTimestamp: null
--- a/manifests/function/capm3/v0.3.1/crd/bases/infrastructure.cluster.x-k8s.io_metal3clusters.yaml
+++ b/manifests/function/capm3/v0.3.1/crd/bases/infrastructure.cluster.x-k8s.io_metal3clusters.yaml
@ -3,6 +3,8 @@
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
+  labels:
+    clusterctl.cluster.x-k8s.io: ""
  annotations:
    controller-gen.kubebuilder.io/version: v0.2.8
  creationTimestamp: null
--- a/manifests/function/capm3/v0.3.1/crd/bases/infrastructure.cluster.x-k8s.io_metal3machines.yaml
+++ b/manifests/function/capm3/v0.3.1/crd/bases/infrastructure.cluster.x-k8s.io_metal3machines.yaml
@ -3,6 +3,8 @@
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
+  labels:
+    clusterctl.cluster.x-k8s.io: ""
  annotations:
    controller-gen.kubebuilder.io/version: v0.2.8
  creationTimestamp: null
--- a/manifests/function/capm3/v0.3.1/crd/bases/infrastructure.cluster.x-k8s.io_metal3machinetemplates.yaml
+++ b/manifests/function/capm3/v0.3.1/crd/bases/infrastructure.cluster.x-k8s.io_metal3machinetemplates.yaml
@ -3,6 +3,8 @@
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
+  labels:
+    clusterctl.cluster.x-k8s.io: ""
  annotations:
    controller-gen.kubebuilder.io/version: v0.2.8
  creationTimestamp: null
--- a/manifests/function/ephemeral/secret.yaml
+++ b/manifests/function/ephemeral/secret.yaml
@ -14,6 +14,7 @@ stringData:
      list: |
          root:deploY!K8s
          deployer:deploY!K8s
+      expire: False
    users:
      - default
      - name: deployer
--- a/manifests/global/crd/baremetal-operator/metal3.io_baremetalhosts_crd.yaml
+++ b/manifests/global/crd/baremetal-operator/metal3.io_baremetalhosts_crd.yaml
@ -1,6 +1,8 @@
 apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
+  labels:
+    clusterctl.cluster.x-k8s.io: ""
  name: baremetalhosts.metal3.io
 spec:
  additionalPrinterColumns:
--- a/manifests/site/test-site/target/controlplane/control-machine-template-patch.yaml
+++ b/manifests/site/test-site/target/controlplane/control-machine-template-patch.yaml
@ -0,0 +1,11 @@
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
+kind: Metal3MachineTemplate
+metadata:
+  name: cluster-controlplane
+spec:
+  template:
+    spec:
+      image:
+        url: http://10.23.24.1:8099/target-image.qcow2
+        checksum: http://10.23.24.1:8099/target-image.qcow2.md5sum
--- a/manifests/site/test-site/target/controlplane/hostgenerator/host-generation.yaml
+++ b/manifests/site/test-site/target/controlplane/hostgenerator/host-generation.yaml
@ -0,0 +1,11 @@
+# Site-level, phase-specific lists of hosts to generate
+# This is used by the hostgenerator-m3 function to narrow down the site-level
+# host-catalogue to just the hosts needed for a particular phase.
+apiVersion: airshipit.org/v1alpha1
+kind: VariableCatalogue
+metadata:
+  name: host-generation-catalogue
+hosts:
+  m3:
+    - node01
+    - node02
--- a/manifests/site/test-site/target/controlplane/hostgenerator/kustomization.yaml
+++ b/manifests/site/test-site/target/controlplane/hostgenerator/kustomization.yaml
@ -0,0 +1,9 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../../../../../function/hostgenerator-m3
+  - ../../../shared/catalogues/
+  - host-generation.yaml
+
+transformers:
+  - ../../../../../function/hostgenerator-m3/replacements
--- a/manifests/site/test-site/target/controlplane/kustomization.yaml
+++ b/manifests/site/test-site/target/controlplane/kustomization.yaml
@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  # TODO (dukov) It's recocommended to upload BareMetalHost objects separately
+  # otherwise nodes will hang in 'registering' state for quite a long time
+  - nodes
+  - ../../../../function/k8scontrol
+patchesStrategicMerge:
+  - control-machine-template-patch.yaml
+commonLabels:
+  airshipit.org/stage: initinfra
--- a/manifests/site/test-site/target/controlplane/nodes/kustomization.yaml
+++ b/manifests/site/test-site/target/controlplane/nodes/kustomization.yaml
@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+generators:
+  - ../hostgenerator
+
+commonLabels:
+  airshipit.org/k8s-role: controlplane-host
--- a/playbooks/airshipctl-gate-runner.yaml
+++ b/playbooks/airshipctl-gate-runner.yaml
@ -16,6 +16,7 @@
      set_fact:
        gate_scripts_default:
          - ./tools/deployment/01_install_kubectl.sh
+          - ./tools/deployment/02_install_clusterctl.sh
          - ./tools/deployment/21_systemwide_executable.sh
          - ./tools/deployment/22_test_configs.sh
          - ./tools/deployment/23_pull_documents.sh
@ -25,6 +26,7 @@
          - ./tools/deployment/30_deploy_controlplane.sh
          - ./tools/deployment/31_deploy_initinfra_target_node.sh
          - ./tools/deployment/32_cluster_init_target_node.sh
+          - ./tools/deployment/33_cluster_move_target_node.sh

    - name: "Run gate scripts"
      include_role:
--- a/roles/libvirt-install/tasks/main.yaml
+++ b/roles/libvirt-install/tasks/main.yaml
@ -26,6 +26,7 @@
          - dnsmasq
          - ebtables
          - ovmf
+          - jq
          RedHat:
          - edk2-ovmf
          - libguestfs-tools
@ -33,6 +34,7 @@
          - libvirt-devel
          - libvirt-daemon-kvm
          - qemu-kvm
+          - jq
    package:
      name: "{{ packages[ansible_os_family] }}"
      state: present
--- a/tools/deployment/02_install_clusterctl.sh
+++ b/tools/deployment/02_install_clusterctl.sh
@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -xe
+
+: ${CLUSTERCTL_VERSION:="v0.3.8"}
+
+# Install clusterctl
+sudo -E curl -sSLo /usr/local/bin/clusterctl \
+  https://github.com/kubernetes-sigs/cluster-api/releases/download/${CLUSTERCTL_VERSION}/clusterctl-linux-amd64
+
+sudo -E chmod +x /usr/local/bin/clusterctl
--- a/tools/deployment/33_cluster_move_target_node.sh
+++ b/tools/deployment/33_cluster_move_target_node.sh
@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -xe
+
+#Default wait timeout is 3600 seconds
+export TIMEOUT=${TIMEOUT:-3600}
+export KUBECONFIG=${KUBECONFIG:-"$HOME/.airship/kubeconfig"}
+
+echo "Switch context to ephemeral cluster and set manifest"
+airshipctl config use-context dummy_cluster
+airshipctl config set-context dummy_cluster --manifest dummy_manifest
+
+echo "Check Cluster Status"
+kubectl --kubeconfig $KUBECONFIG get cluster target-cluster -o json | jq '.status.controlPlaneReady'
+
+echo "Annotate BMH for target node"
+kubectl --kubeconfig $KUBECONFIG annotate bmh node01 baremetalhost.metal3.io/paused=true
+
+echo "Move Cluster Object to Target Cluster"
+clusterctl --v 20 move --kubeconfig $KUBECONFIG --kubeconfig-context dummy_cluster --to-kubeconfig $KUBECONFIG --to-kubeconfig-context  target-cluster-admin@target-cluster
+
+echo "Switch context to target cluster and set manifest"
+airshipctl config use-context target-cluster-admin@target-cluster
+airshipctl config set-context target-cluster-admin@target-cluster --manifest dummy_manifest
+
+echo "Waiting for pods to be ready"
+kubectl --kubeconfig $KUBECONFIG wait --all-namespaces --for=condition=Ready pods --all --timeout=3000s
+kubectl --kubeconfig $KUBECONFIG get pods --all-namespaces
+
+#Wait till crds are created
+end=$(($(date +%s) + $TIMEOUT))
+echo "Waiting $TIMEOUT seconds for crds to be created."
+while true; do
+    if (kubectl --request-timeout 20s --kubeconfig $KUBECONFIG get cluster target-cluster -o json | jq '.status.controlPlaneReady' | grep -q true) ; then
+        echo -e "\nGet CRD status"
+        kubectl --kubeconfig $KUBECONFIG get bmh
+        kubectl --kubeconfig $KUBECONFIG get machines
+        kubectl --kubeconfig $KUBECONFIG get clusters
+        break
+    else
+        now=$(date +%s)
+        if [ $now -gt $end ]; then
+            echo -e "\nCluster move failed and CRDs was not ready before TIMEOUT."
+            exit 1
+        fi
+        echo -n .
+        sleep 15
+    fi
+done
--- a/zuul.d/jobs.yaml
+++ b/zuul.d/jobs.yaml
@ -88,6 +88,7 @@
      site_name: test-site
      gate_scripts:
        - ./tools/deployment/01_install_kubectl.sh
+        - ./tools/deployment/02_install_clusterctl.sh
        # 21_systemwide_executable.sh is run in the build-gate pre-run above
        - ./tools/deployment/22_test_configs.sh
        - ./tools/deployment/24_build_ephemeral_iso.sh
@ -96,6 +97,7 @@
        - ./tools/deployment/30_deploy_controlplane.sh
        - ./tools/deployment/31_deploy_initinfra_target_node.sh
        - ./tools/deployment/32_cluster_init_target_node.sh
+        - ./tools/deployment/33_cluster_move_target_node.sh
      serve_dir: /srv/iso
      serve_port: 8099
    voting: false