From 13c4e82f7e8ec5c1f7946fe093f3bf81db8113ac Mon Sep 17 00:00:00 2001 From: Pallav Gupta Date: Fri, 5 Nov 2021 06:58:52 -0700 Subject: [PATCH] AIAP multinode deployment It deploys 3 controlplane and 2 worker nodes using AIAP gate. + kubectl --context target-cluster get node NAME STATUS ROLES AGE VERSION node01 Ready control-plane,master 48m v1.21.2 node03 Ready worker 43s v1.21.2 node04 Ready control-plane,master 16m v1.21.2 node05 Ready control-plane,master 26m v1.21.2 node06 Ready worker 54s v1.21.2 Tested with 32 GB node. https://zuul.opendev.org/t/openstack/build/da1c2c440d3d4026ab454de73ee518e2/logs Closes: #652 Closes: #228 Change-Id: Ie2267e15ed75b57e2e27f45b9be19ddcf7b0a0a8 --- .../function/k8scontrol/controlplane.yaml | 2 +- .../kubectl_check_ingress_ctrl.sh | 11 +++-- .../workers-capm3/machinedeployment.yaml | 2 +- manifests/phases/executors.yaml | 1 + .../ephemeral/controlplane/kustomization.yaml | 11 +++++ .../patch_json6902_controlplane.yaml | 3 ++ .../controlplane/versions-airshipctl.yaml | 12 ++++++ .../hostgenerator/host-generation.yaml | 2 + .../target/catalogues/shareable/hosts.yaml | 40 ++++++++++++++++--- .../hostgenerator/host-generation.yaml | 2 + .../target/controlplane/kustomization.yaml | 3 ++ .../controlplane/metal3machinetemplate.yaml | 16 ++++++++ .../hostgenerator/host-generation.yaml | 1 + manifests/type/gating/phases/plan.yaml | 3 ++ playbooks/airship-airshipctl-build-gate.yaml | 5 ++- playbooks/get-vm-config.yaml | 10 ++++- .../examples/base/airship-in-a-pod.yaml | 1 - .../roles/build-infra/defaults/main.yaml | 18 ++++----- 18 files changed, 117 insertions(+), 26 deletions(-) create mode 100644 manifests/site/test-site/ephemeral/controlplane/patch_json6902_controlplane.yaml create mode 100644 manifests/site/test-site/ephemeral/controlplane/versions-airshipctl.yaml create mode 100644 manifests/site/test-site/target/controlplane/metal3machinetemplate.yaml diff --git a/manifests/function/k8scontrol/controlplane.yaml b/manifests/function/k8scontrol/controlplane.yaml index 71e43c1a5..2af67121b 100644 --- a/manifests/function/k8scontrol/controlplane.yaml +++ b/manifests/function/k8scontrol/controlplane.yaml @@ -3,7 +3,7 @@ apiVersion: controlplane.cluster.x-k8s.io/v1alpha4 metadata: name: cluster-controlplane spec: - replicas: 1 + replicas: 3 version: v1.21.2 machineTemplate: infrastructureRef: diff --git a/manifests/function/phase-helpers/check_ingress_ctrl/kubectl_check_ingress_ctrl.sh b/manifests/function/phase-helpers/check_ingress_ctrl/kubectl_check_ingress_ctrl.sh index df4d646e5..d3eb70ebe 100644 --- a/manifests/function/phase-helpers/check_ingress_ctrl/kubectl_check_ingress_ctrl.sh +++ b/manifests/function/phase-helpers/check_ingress_ctrl/kubectl_check_ingress_ctrl.sh @@ -18,12 +18,11 @@ export TIMEOUT=${TIMEOUT:-60} end=$(($(date +%s) + $TIMEOUT)) while true; do - export TARGET_IP="$(kubectl --request-timeout 10s \ - --context $KCTL_CONTEXT \ - --namespace ingress \ - get pods \ - -l app.kubernetes.io/component=controller \ - -o jsonpath='{.items[*].status.hostIP}')" + export TARGET_IP="$(kubectl -n kube-system get po | \ + grep -i apiserver | \ + cut -f1 -d ' ' | \ + xargs kubectl -n kube-system \ + get po $1 -o=jsonpath='{.items[0].status.hostIP}')" if [ ! -z $TARGET_IP ]; then break else diff --git a/manifests/function/workers-capm3/machinedeployment.yaml b/manifests/function/workers-capm3/machinedeployment.yaml index 15dbfe58d..dafd94fbc 100644 --- a/manifests/function/workers-capm3/machinedeployment.yaml +++ b/manifests/function/workers-capm3/machinedeployment.yaml @@ -6,7 +6,7 @@ metadata: cluster.x-k8s.io/cluster-name: target-cluster spec: clusterName: target-cluster - replicas: 1 + replicas: 2 selector: matchLabels: cluster.x-k8s.io/cluster-name: target-cluster diff --git a/manifests/phases/executors.yaml b/manifests/phases/executors.yaml index 0a2dc3139..e2fd40b84 100644 --- a/manifests/phases/executors.yaml +++ b/manifests/phases/executors.yaml @@ -34,6 +34,7 @@ config: value: "provisioned" pruneOptions: prune: false + inventoryPolicy: force-adopt --- apiVersion: airshipit.org/v1alpha1 kind: KubernetesApply diff --git a/manifests/site/test-site/ephemeral/controlplane/kustomization.yaml b/manifests/site/test-site/ephemeral/controlplane/kustomization.yaml index f831ecd02..a81dd542f 100644 --- a/manifests/site/test-site/ephemeral/controlplane/kustomization.yaml +++ b/manifests/site/test-site/ephemeral/controlplane/kustomization.yaml @@ -11,3 +11,14 @@ namespace: target-infra transformers: - ../../../../type/gating/ephemeral/controlplane/replacements + +patchesJson6902: + - target: + group: controlplane.cluster.x-k8s.io + version: v1alpha4 + kind: KubeadmControlPlane + name: cluster-controlplane + path: patch_json6902_controlplane.yaml + +patchesStrategicMerge: + - versions-airshipctl.yaml diff --git a/manifests/site/test-site/ephemeral/controlplane/patch_json6902_controlplane.yaml b/manifests/site/test-site/ephemeral/controlplane/patch_json6902_controlplane.yaml new file mode 100644 index 000000000..a52beab36 --- /dev/null +++ b/manifests/site/test-site/ephemeral/controlplane/patch_json6902_controlplane.yaml @@ -0,0 +1,3 @@ +- op: replace + path: "/spec/replicas" + value: 1 diff --git a/manifests/site/test-site/ephemeral/controlplane/versions-airshipctl.yaml b/manifests/site/test-site/ephemeral/controlplane/versions-airshipctl.yaml new file mode 100644 index 000000000..2a85137c8 --- /dev/null +++ b/manifests/site/test-site/ephemeral/controlplane/versions-airshipctl.yaml @@ -0,0 +1,12 @@ +# Override default controlplane image location +apiVersion: airshipit.org/v1alpha1 +kind: VersionsCatalogue +metadata: + name: versions-airshipctl +spec: + files: + k8scontrol: + # Host the image in a locally served location for CI + cluster_controlplane_image: + url: http://10.23.24.137:80/images/control-plane.qcow2 + checksum: http://10.23.24.137:80/images/control-plane.qcow2.md5sum diff --git a/manifests/site/test-site/host-inventory/hostgenerator/host-generation.yaml b/manifests/site/test-site/host-inventory/hostgenerator/host-generation.yaml index 7ec14d69d..52924b924 100644 --- a/manifests/site/test-site/host-inventory/hostgenerator/host-generation.yaml +++ b/manifests/site/test-site/host-inventory/hostgenerator/host-generation.yaml @@ -13,3 +13,5 @@ hosts: - node02 - node03 - node04 + - node05 + - node06 diff --git a/manifests/site/test-site/target/catalogues/shareable/hosts.yaml b/manifests/site/test-site/target/catalogues/shareable/hosts.yaml index dea0abea0..23f72eb34 100644 --- a/manifests/site/test-site/target/catalogues/shareable/hosts.yaml +++ b/manifests/site/test-site/target/catalogues/shareable/hosts.yaml @@ -54,15 +54,45 @@ hosts: pxe: 52:54:00:b6:ed:23 hardwareProfile: default # defined in the hardwareprofile-example function node04: - bootMode: UEFI - macAddress: 52:54:00:36:5e:e3 - bmcAddress: redfish+http://10.23.25.2:8000/redfish/v1/Systems/air-target-2 + bootMode: legacy + macAddress: 52:54:00:01:71:d8 + bmcAddress: redfish+http://10.23.25.1:8000/redfish/v1/Systems/air-target-2 bmcUsername: username bmcPassword: password + disableCertificateVerification: false ipAddresses: oam-ipv4: 10.23.25.104 pxe-ipv4: 10.23.24.104 macAddresses: - oam: 52:54:00:dc:ab:04 - pxe: 52:54:00:51:0b:e4 + oam: 52:54:00:f0:57:e0 + pxe: 52:54:00:01:71:d8 + hardwareProfile: default # defined in the hostgenerator-m3 function + node05: + bootMode: legacy + macAddress: 52:54:00:a3:ee:18 + bmcAddress: redfish+http://10.23.25.1:8000/redfish/v1/Systems/air-target-3 + bmcUsername: username + bmcPassword: password + disableCertificateVerification: false + ipAddresses: + oam-ipv4: 10.23.25.105 + pxe-ipv4: 10.23.24.105 + macAddresses: + oam: 52:54:00:91:36:49 + pxe: 52:54:00:a3:ee:18 + hardwareProfile: default # defined in the hostgenerator-m3 function + node06: + bootMode: legacy + macAddress: 52:54:00:b6:ed:19 + bmcAddress: redfish+http://10.23.25.1:8000/redfish/v1/Systems/air-worker-2 + bmcUsername: username + bmcPassword: password + disableCertificateVerification: false + ipAddresses: + oam-ipv4: 10.23.25.106 + pxe-ipv4: 10.23.24.106 + macAddresses: + oam: 52:54:00:9b:27:08 + pxe: 52:54:00:b6:ed:19 hardwareProfile: default # defined in the hardwareprofile-example function + diff --git a/manifests/site/test-site/target/controlplane/hostgenerator/host-generation.yaml b/manifests/site/test-site/target/controlplane/hostgenerator/host-generation.yaml index bd953d144..3ab03b9ad 100644 --- a/manifests/site/test-site/target/controlplane/hostgenerator/host-generation.yaml +++ b/manifests/site/test-site/target/controlplane/hostgenerator/host-generation.yaml @@ -7,3 +7,5 @@ metadata: name: host-generation-catalogue hosts: m3: + - node04 + - node05 diff --git a/manifests/site/test-site/target/controlplane/kustomization.yaml b/manifests/site/test-site/target/controlplane/kustomization.yaml index 8b0037e5e..23edc037e 100644 --- a/manifests/site/test-site/target/controlplane/kustomization.yaml +++ b/manifests/site/test-site/target/controlplane/kustomization.yaml @@ -7,6 +7,9 @@ resources: - ../catalogues - nodes +patchesStrategicMerge: + - metal3machinetemplate.yaml + namespace: target-infra commonLabels: diff --git a/manifests/site/test-site/target/controlplane/metal3machinetemplate.yaml b/manifests/site/test-site/target/controlplane/metal3machinetemplate.yaml new file mode 100644 index 000000000..57b54ba3b --- /dev/null +++ b/manifests/site/test-site/target/controlplane/metal3machinetemplate.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha5 +kind: Metal3MachineTemplate +metadata: + annotations: + config.kubernetes.io/path: metal3machinetemplate_cluster-controlplane.yaml + name: cluster-controlplane +spec: + template: + spec: + hostSelector: + matchLabels: + airshipit.org/k8s-role: controlplane-host + image: + url: http://10.23.24.137:80/images/control-plane.qcow2 + checksum: http://10.23.24.137:80/images/control-plane.qcow2.md5sum diff --git a/manifests/site/test-site/target/workers/hostgenerator/host-generation.yaml b/manifests/site/test-site/target/workers/hostgenerator/host-generation.yaml index e197b455c..e9f628dac 100644 --- a/manifests/site/test-site/target/workers/hostgenerator/host-generation.yaml +++ b/manifests/site/test-site/target/workers/hostgenerator/host-generation.yaml @@ -8,3 +8,4 @@ metadata: hosts: m3: - node03 + - node06 diff --git a/manifests/type/gating/phases/plan.yaml b/manifests/type/gating/phases/plan.yaml index 95d408eb5..8a2965a17 100644 --- a/manifests/type/gating/phases/plan.yaml +++ b/manifests/type/gating/phases/plan.yaml @@ -99,6 +99,8 @@ phases: - name: eject-virtual-media-ephemeral # Power off Ephemeral baremetal host avoid DHCP conflict - name: power-off-ephemeral + # Create target k8s cluster resources + - name: controlplane-target # List all nodes in target cluster # Scripts for this phase placed in manifests/function/phase-helpers/get_node/ # To get ConfigMap for this phase, execute `airshipctl phase render --source config -k ConfigMap` @@ -124,6 +126,7 @@ validation: - VariableCatalogue crdList: - airshipctl/manifests/function/airshipctl-schemas + - airshipctl/manifests/function/baremetal-operator/v0.5.0/upstream/crd --- apiVersion: airshipit.org/v1alpha1 kind: PhasePlan diff --git a/playbooks/airship-airshipctl-build-gate.yaml b/playbooks/airship-airshipctl-build-gate.yaml index 47ce672ee..38a28370a 100644 --- a/playbooks/airship-airshipctl-build-gate.yaml +++ b/playbooks/airship-airshipctl-build-gate.yaml @@ -69,6 +69,7 @@ path: "{{ roleinputvar.path }}" loop: - { name: 'target', path: 'ephemeral/controlplane' } + - { name: 'target', path: 'target/controlplane' } - { name: 'ephemeral', path: 'ephemeral/bootstrap' } - { name: 'worker', path: 'target/workers' } loop_control: @@ -96,12 +97,12 @@ target_vm_cfg: "{{ target_vm_cfg }}" target_vm_memory_mb: 7168 target_vm_vcpus: 2 - target_vms_count: 1 + target_vms_count: 3 worker_disk_size: 30G worker_vm_cfg: "{{ worker_vm_cfg }}" worker_vm_memory_mb: 7168 worker_vm_vcpus: 2 - worker_vms_count: 1 + worker_vms_count: 2 airship_gate_file_exchanger: servername: "localhost" ip: diff --git a/playbooks/get-vm-config.yaml b/playbooks/get-vm-config.yaml index 61b0e44ad..301cb5779 100644 --- a/playbooks/get-vm-config.yaml +++ b/playbooks/get-vm-config.yaml @@ -17,7 +17,6 @@ {{ airship_config_manifest_directory }}/{{ airship_config_site_path }}/{{ path }} 2>/dev/null | kustomize cfg grep "kind=BareMetalHost" register: bmh_command - failed_when: "bmh_command.stdout == ''" environment: KUSTOMIZE_ENABLE_ALPHA_COMMANDS: "true" SOPS_IMPORT_PGP: "{{ airship_config_pgp }}" @@ -37,6 +36,7 @@ KUSTOMIZE_ENABLE_ALPHA_COMMANDS: "true" SOPS_IMPORT_PGP: "{{ airship_config_pgp }}" with_items: "{{ bmh }}" + when: bmh|length > 0 - name: get links from network data per BareMetalHost object set_fact: @@ -47,6 +47,8 @@ map(attribute='stringData.networkData') | map('from_yaml') | map(attribute='links') | list }} + when: bmh|length > 0 + - name: define list of VM mac addresses and VM boot mode set_fact: @@ -64,6 +66,12 @@ loop_control: index_var: idx register: vm_cfg_fact + when: bmh|length > 0 + +- set_fact: + "{{ name }}_vm_cfg": "{{ (lookup('vars', name + '_vm_cfg') |list) + (vm_cfg_fact.results | map(attribute='ansible_facts.vm_cfg') | list) }}" + when: "{{ name + '_vm_cfg' }} is defined and bmh|length > 0" - set_fact: "{{ name }}_vm_cfg": "{{ vm_cfg_fact.results | map(attribute='ansible_facts.vm_cfg') | list }}" + when: "{{ name + '_vm_cfg' }} is not defined and bmh|length > 0" diff --git a/tools/airship-in-a-pod/examples/base/airship-in-a-pod.yaml b/tools/airship-in-a-pod/examples/base/airship-in-a-pod.yaml index 255269613..3bddaa8e7 100644 --- a/tools/airship-in-a-pod/examples/base/airship-in-a-pod.yaml +++ b/tools/airship-in-a-pod/examples/base/airship-in-a-pod.yaml @@ -24,7 +24,6 @@ data: AIRSHIP_CONFIG_MANIFEST_REPO_AUTH_USERNAME: "" AIRSHIP_CONFIG_MANIFEST_REPO_AUTH_HTTP_PASSWORD: "" AIRSHIP_CONFIG_MANIFEST_REPO_AUTH_SSH_PASSWORD: "" - SOPS_IMPORT_PGP: "" --- apiVersion: v1 diff --git a/tools/airship-in-a-pod/infra-builder/assets/opt/ansible/playbooks/roles/build-infra/defaults/main.yaml b/tools/airship-in-a-pod/infra-builder/assets/opt/ansible/playbooks/roles/build-infra/defaults/main.yaml index 2f58ea712..68eb5f7e7 100644 --- a/tools/airship-in-a-pod/infra-builder/assets/opt/ansible/playbooks/roles/build-infra/defaults/main.yaml +++ b/tools/airship-in-a-pod/infra-builder/assets/opt/ansible/playbooks/roles/build-infra/defaults/main.yaml @@ -18,7 +18,7 @@ floorplan: ephemeral_node: name: air-ephemeral - cpu: 4 + cpu: 2 ram: 6144 nat_mac_address: [ "52:54:00:9b:27:02" ] prov_mac_address: [ "52:54:00:b6:ed:02" ] @@ -26,22 +26,22 @@ ephemeral_node: - 30G target_nodes: - count: 1 + count: 3 name: air-target cpu: 2 - ram: 7168 - nat_mac_address: [ "52:54:00:9b:27:4c" ] - prov_mac_address: [ "52:54:00:b6:ed:31" ] + ram: 6144 + nat_mac_address: [ "52:54:00:9b:27:4c", "52:54:00:f0:57:e0", "52:54:00:91:36:49" ] + prov_mac_address: [ "52:54:00:b6:ed:31", "52:54:00:01:71:d8", "52:54:00:a3:ee:18" ] block: - 30G worker_nodes: - count: 1 + count: 2 name: air-worker cpu: 2 - ram: 7168 - nat_mac_address: [ "52:54:00:9b:27:07" ] - prov_mac_address: [ "52:54:00:b6:ed:23" ] + ram: 6144 + nat_mac_address: [ "52:54:00:9b:27:07", "52:54:00:9b:27:08" ] + prov_mac_address: [ "52:54:00:b6:ed:23", "52:54:00:b6:ed:19" ] block: - 30G