diff --git a/manifests/function/cabpk/v0.3.3/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml b/manifests/function/cabpk/v0.3.3/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml index 28a6851a0..611a81242 100644 --- a/manifests/function/cabpk/v0.3.3/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml +++ b/manifests/function/cabpk/v0.3.3/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml @@ -3,6 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + labels: + clusterctl.cluster.x-k8s.io: "" annotations: controller-gen.kubebuilder.io/version: v0.2.8 creationTimestamp: null diff --git a/manifests/function/cabpk/v0.3.3/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml b/manifests/function/cabpk/v0.3.3/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml index 9d0448ac8..9f1b7e86e 100644 --- a/manifests/function/cabpk/v0.3.3/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml +++ b/manifests/function/cabpk/v0.3.3/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml @@ -3,6 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + labels: + clusterctl.cluster.x-k8s.io: "" annotations: controller-gen.kubebuilder.io/version: v0.2.8 creationTimestamp: null diff --git a/manifests/function/cacpk/v0.3.3/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml b/manifests/function/cacpk/v0.3.3/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml index 19f1b92ea..625e058d7 100644 --- a/manifests/function/cacpk/v0.3.3/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml +++ b/manifests/function/cacpk/v0.3.3/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml @@ -3,6 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + labels: + clusterctl.cluster.x-k8s.io: "" annotations: controller-gen.kubebuilder.io/version: v0.2.8 creationTimestamp: null diff --git a/manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_clusters.yaml b/manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_clusters.yaml index c9512218f..8608a39e4 100644 --- a/manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_clusters.yaml +++ b/manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_clusters.yaml @@ -3,6 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + labels: + clusterctl.cluster.x-k8s.io: "" annotations: controller-gen.kubebuilder.io/version: v0.2.8 creationTimestamp: null diff --git a/manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_machinedeployments.yaml b/manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_machinedeployments.yaml index 69c6d8d67..17f75d0a8 100644 --- a/manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_machinedeployments.yaml +++ b/manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_machinedeployments.yaml @@ -3,6 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + labels: + clusterctl.cluster.x-k8s.io: "" annotations: controller-gen.kubebuilder.io/version: v0.2.8 creationTimestamp: null diff --git a/manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_machinehealthchecks.yaml b/manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_machinehealthchecks.yaml index 86649ee7c..8c9421173 100644 --- a/manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_machinehealthchecks.yaml +++ b/manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_machinehealthchecks.yaml @@ -3,6 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + labels: + clusterctl.cluster.x-k8s.io: "" annotations: controller-gen.kubebuilder.io/version: v0.2.8 creationTimestamp: null diff --git a/manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_machines.yaml b/manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_machines.yaml index a5466dd3c..828bbeb45 100644 --- a/manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_machines.yaml +++ b/manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_machines.yaml @@ -3,6 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + labels: + clusterctl.cluster.x-k8s.io: "" annotations: controller-gen.kubebuilder.io/version: v0.2.8 creationTimestamp: null diff --git a/manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_machinesets.yaml b/manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_machinesets.yaml index 569eff13e..c8a18a7e0 100644 --- a/manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_machinesets.yaml +++ b/manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_machinesets.yaml @@ -3,6 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + labels: + clusterctl.cluster.x-k8s.io: "" annotations: controller-gen.kubebuilder.io/version: v0.2.8 creationTimestamp: null diff --git a/manifests/function/capi/v0.3.3/crd/bases/exp.cluster.x-k8s.io_machinepools.yaml b/manifests/function/capi/v0.3.3/crd/bases/exp.cluster.x-k8s.io_machinepools.yaml index 573d4c9c4..1c891fed8 100644 --- a/manifests/function/capi/v0.3.3/crd/bases/exp.cluster.x-k8s.io_machinepools.yaml +++ b/manifests/function/capi/v0.3.3/crd/bases/exp.cluster.x-k8s.io_machinepools.yaml @@ -3,6 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + labels: + clusterctl.cluster.x-k8s.io: "" annotations: controller-gen.kubebuilder.io/version: v0.2.8 creationTimestamp: null diff --git a/manifests/function/capm3/v0.3.1/crd/bases/infrastructure.cluster.x-k8s.io_metal3clusters.yaml b/manifests/function/capm3/v0.3.1/crd/bases/infrastructure.cluster.x-k8s.io_metal3clusters.yaml index ea8e43c3a..2ace8196a 100644 --- a/manifests/function/capm3/v0.3.1/crd/bases/infrastructure.cluster.x-k8s.io_metal3clusters.yaml +++ b/manifests/function/capm3/v0.3.1/crd/bases/infrastructure.cluster.x-k8s.io_metal3clusters.yaml @@ -3,6 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + labels: + clusterctl.cluster.x-k8s.io: "" annotations: controller-gen.kubebuilder.io/version: v0.2.8 creationTimestamp: null diff --git a/manifests/function/capm3/v0.3.1/crd/bases/infrastructure.cluster.x-k8s.io_metal3machines.yaml b/manifests/function/capm3/v0.3.1/crd/bases/infrastructure.cluster.x-k8s.io_metal3machines.yaml index 4b3d98ddc..1571dfe8d 100644 --- a/manifests/function/capm3/v0.3.1/crd/bases/infrastructure.cluster.x-k8s.io_metal3machines.yaml +++ b/manifests/function/capm3/v0.3.1/crd/bases/infrastructure.cluster.x-k8s.io_metal3machines.yaml @@ -3,6 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + labels: + clusterctl.cluster.x-k8s.io: "" annotations: controller-gen.kubebuilder.io/version: v0.2.8 creationTimestamp: null diff --git a/manifests/function/capm3/v0.3.1/crd/bases/infrastructure.cluster.x-k8s.io_metal3machinetemplates.yaml b/manifests/function/capm3/v0.3.1/crd/bases/infrastructure.cluster.x-k8s.io_metal3machinetemplates.yaml index 3d33196d6..64ae460f7 100644 --- a/manifests/function/capm3/v0.3.1/crd/bases/infrastructure.cluster.x-k8s.io_metal3machinetemplates.yaml +++ b/manifests/function/capm3/v0.3.1/crd/bases/infrastructure.cluster.x-k8s.io_metal3machinetemplates.yaml @@ -3,6 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + labels: + clusterctl.cluster.x-k8s.io: "" annotations: controller-gen.kubebuilder.io/version: v0.2.8 creationTimestamp: null diff --git a/manifests/function/ephemeral/secret.yaml b/manifests/function/ephemeral/secret.yaml index 9edc08825..a21adbd60 100644 --- a/manifests/function/ephemeral/secret.yaml +++ b/manifests/function/ephemeral/secret.yaml @@ -14,6 +14,7 @@ stringData: list: | root:deploY!K8s deployer:deploY!K8s + expire: False users: - default - name: deployer diff --git a/manifests/global/crd/baremetal-operator/metal3.io_baremetalhosts_crd.yaml b/manifests/global/crd/baremetal-operator/metal3.io_baremetalhosts_crd.yaml index 2df9551e1..bb28fe98b 100644 --- a/manifests/global/crd/baremetal-operator/metal3.io_baremetalhosts_crd.yaml +++ b/manifests/global/crd/baremetal-operator/metal3.io_baremetalhosts_crd.yaml @@ -1,6 +1,8 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: + labels: + clusterctl.cluster.x-k8s.io: "" name: baremetalhosts.metal3.io spec: additionalPrinterColumns: diff --git a/manifests/site/test-site/target/controlplane/control-machine-template-patch.yaml b/manifests/site/test-site/target/controlplane/control-machine-template-patch.yaml new file mode 100644 index 000000000..ee37cf2c5 --- /dev/null +++ b/manifests/site/test-site/target/controlplane/control-machine-template-patch.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: Metal3MachineTemplate +metadata: + name: cluster-controlplane +spec: + template: + spec: + image: + url: http://10.23.24.1:8099/target-image.qcow2 + checksum: http://10.23.24.1:8099/target-image.qcow2.md5sum diff --git a/manifests/site/test-site/target/controlplane/hostgenerator/host-generation.yaml b/manifests/site/test-site/target/controlplane/hostgenerator/host-generation.yaml new file mode 100644 index 000000000..f9ae0b403 --- /dev/null +++ b/manifests/site/test-site/target/controlplane/hostgenerator/host-generation.yaml @@ -0,0 +1,11 @@ +# Site-level, phase-specific lists of hosts to generate +# This is used by the hostgenerator-m3 function to narrow down the site-level +# host-catalogue to just the hosts needed for a particular phase. +apiVersion: airshipit.org/v1alpha1 +kind: VariableCatalogue +metadata: + name: host-generation-catalogue +hosts: + m3: + - node01 + - node02 diff --git a/manifests/site/test-site/target/controlplane/hostgenerator/kustomization.yaml b/manifests/site/test-site/target/controlplane/hostgenerator/kustomization.yaml new file mode 100644 index 000000000..8de561b8c --- /dev/null +++ b/manifests/site/test-site/target/controlplane/hostgenerator/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../../../../../function/hostgenerator-m3 + - ../../../shared/catalogues/ + - host-generation.yaml + +transformers: + - ../../../../../function/hostgenerator-m3/replacements diff --git a/manifests/site/test-site/target/controlplane/kustomization.yaml b/manifests/site/test-site/target/controlplane/kustomization.yaml new file mode 100644 index 000000000..b316d3074 --- /dev/null +++ b/manifests/site/test-site/target/controlplane/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + # TODO (dukov) It's recocommended to upload BareMetalHost objects separately + # otherwise nodes will hang in 'registering' state for quite a long time + - nodes + - ../../../../function/k8scontrol +patchesStrategicMerge: + - control-machine-template-patch.yaml +commonLabels: + airshipit.org/stage: initinfra diff --git a/manifests/site/test-site/target/controlplane/nodes/kustomization.yaml b/manifests/site/test-site/target/controlplane/nodes/kustomization.yaml new file mode 100644 index 000000000..512dc07c8 --- /dev/null +++ b/manifests/site/test-site/target/controlplane/nodes/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +generators: + - ../hostgenerator + +commonLabels: + airshipit.org/k8s-role: controlplane-host diff --git a/playbooks/airshipctl-gate-runner.yaml b/playbooks/airshipctl-gate-runner.yaml index 61c2c70c2..b67be2466 100644 --- a/playbooks/airshipctl-gate-runner.yaml +++ b/playbooks/airshipctl-gate-runner.yaml @@ -16,6 +16,7 @@ set_fact: gate_scripts_default: - ./tools/deployment/01_install_kubectl.sh + - ./tools/deployment/02_install_clusterctl.sh - ./tools/deployment/21_systemwide_executable.sh - ./tools/deployment/22_test_configs.sh - ./tools/deployment/23_pull_documents.sh @@ -25,6 +26,7 @@ - ./tools/deployment/30_deploy_controlplane.sh - ./tools/deployment/31_deploy_initinfra_target_node.sh - ./tools/deployment/32_cluster_init_target_node.sh + - ./tools/deployment/33_cluster_move_target_node.sh - name: "Run gate scripts" include_role: diff --git a/roles/libvirt-install/tasks/main.yaml b/roles/libvirt-install/tasks/main.yaml index d54a44c8c..3e6d46a47 100644 --- a/roles/libvirt-install/tasks/main.yaml +++ b/roles/libvirt-install/tasks/main.yaml @@ -26,6 +26,7 @@ - dnsmasq - ebtables - ovmf + - jq RedHat: - edk2-ovmf - libguestfs-tools @@ -33,6 +34,7 @@ - libvirt-devel - libvirt-daemon-kvm - qemu-kvm + - jq package: name: "{{ packages[ansible_os_family] }}" state: present diff --git a/tools/deployment/02_install_clusterctl.sh b/tools/deployment/02_install_clusterctl.sh new file mode 100755 index 000000000..0fa04cac9 --- /dev/null +++ b/tools/deployment/02_install_clusterctl.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xe + +: ${CLUSTERCTL_VERSION:="v0.3.8"} + +# Install clusterctl +sudo -E curl -sSLo /usr/local/bin/clusterctl \ + https://github.com/kubernetes-sigs/cluster-api/releases/download/${CLUSTERCTL_VERSION}/clusterctl-linux-amd64 + +sudo -E chmod +x /usr/local/bin/clusterctl diff --git a/tools/deployment/33_cluster_move_target_node.sh b/tools/deployment/33_cluster_move_target_node.sh new file mode 100755 index 000000000..fc816ee92 --- /dev/null +++ b/tools/deployment/33_cluster_move_target_node.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xe + +#Default wait timeout is 3600 seconds +export TIMEOUT=${TIMEOUT:-3600} +export KUBECONFIG=${KUBECONFIG:-"$HOME/.airship/kubeconfig"} + +echo "Switch context to ephemeral cluster and set manifest" +airshipctl config use-context dummy_cluster +airshipctl config set-context dummy_cluster --manifest dummy_manifest + +echo "Check Cluster Status" +kubectl --kubeconfig $KUBECONFIG get cluster target-cluster -o json | jq '.status.controlPlaneReady' + +echo "Annotate BMH for target node" +kubectl --kubeconfig $KUBECONFIG annotate bmh node01 baremetalhost.metal3.io/paused=true + +echo "Move Cluster Object to Target Cluster" +clusterctl --v 20 move --kubeconfig $KUBECONFIG --kubeconfig-context dummy_cluster --to-kubeconfig $KUBECONFIG --to-kubeconfig-context target-cluster-admin@target-cluster + +echo "Switch context to target cluster and set manifest" +airshipctl config use-context target-cluster-admin@target-cluster +airshipctl config set-context target-cluster-admin@target-cluster --manifest dummy_manifest + +echo "Waiting for pods to be ready" +kubectl --kubeconfig $KUBECONFIG wait --all-namespaces --for=condition=Ready pods --all --timeout=3000s +kubectl --kubeconfig $KUBECONFIG get pods --all-namespaces + +#Wait till crds are created +end=$(($(date +%s) + $TIMEOUT)) +echo "Waiting $TIMEOUT seconds for crds to be created." +while true; do + if (kubectl --request-timeout 20s --kubeconfig $KUBECONFIG get cluster target-cluster -o json | jq '.status.controlPlaneReady' | grep -q true) ; then + echo -e "\nGet CRD status" + kubectl --kubeconfig $KUBECONFIG get bmh + kubectl --kubeconfig $KUBECONFIG get machines + kubectl --kubeconfig $KUBECONFIG get clusters + break + else + now=$(date +%s) + if [ $now -gt $end ]; then + echo -e "\nCluster move failed and CRDs was not ready before TIMEOUT." + exit 1 + fi + echo -n . + sleep 15 + fi +done diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 78112e646..748c64981 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -88,6 +88,7 @@ site_name: test-site gate_scripts: - ./tools/deployment/01_install_kubectl.sh + - ./tools/deployment/02_install_clusterctl.sh # 21_systemwide_executable.sh is run in the build-gate pre-run above - ./tools/deployment/22_test_configs.sh - ./tools/deployment/24_build_ephemeral_iso.sh @@ -96,6 +97,7 @@ - ./tools/deployment/30_deploy_controlplane.sh - ./tools/deployment/31_deploy_initinfra_target_node.sh - ./tools/deployment/32_cluster_init_target_node.sh + - ./tools/deployment/33_cluster_move_target_node.sh serve_dir: /srv/iso serve_port: 8099 voting: false