diff --git a/tools/deployment/22_test_configs.sh b/tools/deployment/22_test_configs.sh index 2fbfe08f3..2ddf02919 100755 --- a/tools/deployment/22_test_configs.sh +++ b/tools/deployment/22_test_configs.sh @@ -46,11 +46,16 @@ export TARGET_IP=${TARGET_IP:-"10.23.25.102"} export TARGET_CONFIG_CA_DATA=$(cat tools/deployment/certificates/target_config_ca_data| base64 -w0) export TARGET_CONFIG_CLIENT_CERT_DATA=$(cat tools/deployment/certificates/target_config_client_cert_data| base64 -w0) export TARGET_CONFIG_CLIENT_KEY_DATA=$(cat tools/deployment/certificates/target_config_client_key_data| base64 -w0) +export SITE=${SITE:-"test-site"} +export EXTERNAL_KUBECONFIG=${EXTERNAL_KUBECONFIG:-""} # Remove the contents of the .airship folder, preserving the kustomize plugin directory -rm -rf $HOME/.airship/*config* +rm -rf $HOME/.airship/config mkdir -p $HOME/.airship echo "Generate ~/.airship/config and ~/.airship/kubeconfig" envsubst <"${AIRSHIPCTL_WS}/tools/deployment/templates/airshipconfig_template" > ~/.airship/config -envsubst <"${AIRSHIPCTL_WS}/tools/deployment/templates/kubeconfig_template" > ~/.airship/kubeconfig + +if [[ -z "$EXTERNAL_KUBECONFIG" ]]; then + envsubst <"${AIRSHIPCTL_WS}/tools/deployment/templates/kubeconfig_template" > ~/.airship/kubeconfig +fi diff --git a/tools/deployment/26_deploy_capi_ephemeral_node.sh b/tools/deployment/26_deploy_capi_ephemeral_node.sh new file mode 100755 index 000000000..4e1863bef --- /dev/null +++ b/tools/deployment/26_deploy_capi_ephemeral_node.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xe +export PROVIDER=${PROVIDER:-"metal3"} +export KUBECONFIG=${KUBECONFIG:-"$HOME/.airship/kubeconfig"} +export KUBECONFIG_EPHEMERAL_CONTEXT=${KUBECONFIG_EPHEMERAL_CONTEXT:-"ephemeral-cluster"} + +airshipctl document pull -n --debug + +if [ "$PROVIDER" = "metal3" ]; then + echo "Deploy metal3.io components to ephemeral node" + airshipctl phase run initinfra-ephemeral --debug + + echo "Getting metal3 pods as debug information" + kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_EPHEMERAL_CONTEXT --namespace ${PROVIDER} get pods + echo "Deploy cluster-api components to ephemeral node" + airshipctl phase run clusterctl-init-ephemeral --debug +else + echo "Deploy cluster-api components to ephemeral node" + airshipctl phase run clusterctl-init-ephemeral --debug --kubeconfig ${KUBECONFIG} + kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_EPHEMERAL_CONTEXT get pods -A +fi + +echo "Waiting for clusterapi pods to come up" +kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_EPHEMERAL_CONTEXT wait --for=condition=available deploy --all --timeout=1000s -A +kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_EPHEMERAL_CONTEXT get pods --all-namespaces diff --git a/tools/deployment/provider_common/01_install_kind.sh b/tools/deployment/provider_common/01_install_kind.sh new file mode 100755 index 000000000..629c16d6b --- /dev/null +++ b/tools/deployment/provider_common/01_install_kind.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +: ${KIND_VERSION:="v0.9.0"} +export KIND_URL="https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-$(uname)-amd64" +TMP=$(KIND_URL=${KIND_URL} tools/document/get_kind.sh) +export KIND=${TMP}/kind +sudo cp ${KIND} /usr/local/bin/ +${KIND} version \ No newline at end of file diff --git a/tools/deployment/provider_common/02_install_jq.sh b/tools/deployment/provider_common/02_install_jq.sh new file mode 100755 index 000000000..dc086bb17 --- /dev/null +++ b/tools/deployment/provider_common/02_install_jq.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +sudo apt-get install jq -y diff --git a/tools/deployment/provider_common/30_deploy_controlplane.sh b/tools/deployment/provider_common/30_deploy_controlplane.sh new file mode 100755 index 000000000..3633aeba4 --- /dev/null +++ b/tools/deployment/provider_common/30_deploy_controlplane.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Example Usage +# CONTROLPLANE_COUNT=1 \ +# TEST_SITE=docker-test-site \ +# ./tools/deployment/provider_common/30_deploy_controlplane.sh + +export AIRSHIP_SRC=${AIRSHIP_SRC:-"/tmp/airship"} +export KUBECONFIG=${KUBECONFIG:-"$HOME/.airship/kubeconfig"} +export CONTROLPLANE_COUNT=${CONTROLPLANE_COUNT:-"1"} +export SITE=${SITE:-"docker-test-site"} +export TARGET_CLUSTER_NAME=${TARGET_CLUSTER_NAME:-"target-cluster"} + +# Adjust Control Plane Count (default 1) +# No. of control plane can be changed using +# CONTROLPLANE_COUNT= tools/deployment/docker/30_deploy_controlplane.sh + +sed -i "/value.*/s//value\": $CONTROLPLANE_COUNT }/g" \ +${AIRSHIP_SRC}/airshipctl/manifests/site/${SITE}/ephemeral/controlplane/machine_count.json + +echo "create control plane" +airshipctl phase run controlplane-ephemeral --debug --kubeconfig ${KUBECONFIG} --wait-timeout 1000s + +TARGET_KUBECONFIG="" +TARGET_KUBECONFIG=$(kubectl --kubeconfig "${KUBECONFIG}" --namespace=default get secret/"${TARGET_CLUSTER_NAME}"-kubeconfig -o jsonpath={.data.value} || true) + +if [[ -z "$TARGET_KUBECONFIG" ]]; then + echo "Error: Could not get kubeconfig from secret." + exit 1 +fi + +echo "Generate kubeconfig" +echo ${TARGET_KUBECONFIG} | base64 -d > /tmp/${TARGET_CLUSTER_NAME}.kubeconfig +echo "Generate kubeconfig: /tmp/${TARGET_CLUSTER_NAME}.kubeconfig" + +echo "add context target-cluster" +kubectl config set-context ${TARGET_CLUSTER_NAME} --user ${TARGET_CLUSTER_NAME}-admin --cluster ${TARGET_CLUSTER_NAME} \ +--kubeconfig "/tmp/${TARGET_CLUSTER_NAME}.kubeconfig" + +if [ "$TEST_SITE" != "docker-test-site" ]; then + echo "apply cni as a part of initinfra" + airshipctl phase run initinfra-target --debug --kubeconfig "/tmp/${TARGET_CLUSTER_NAME}.kubeconfig" + kubectl --kubeconfig /tmp/"${TARGET_CLUSTER_NAME}".kubeconfig wait --for=condition=Ready nodes --all --timeout 4000s +else + kubectl --kubeconfig /tmp/"${TARGET_CLUSTER_NAME}".kubeconfig wait --for=condition=Ready nodes --all --timeout 4000s +fi + +echo "Check nodes status" + +kubectl --kubeconfig /tmp/"${TARGET_CLUSTER_NAME}".kubeconfig wait --for=condition=Ready nodes --all --timeout 4000s +kubectl get nodes --kubeconfig /tmp/"${TARGET_CLUSTER_NAME}".kubeconfig + + +echo "Waiting for pods to come up" +kubectl --kubeconfig /tmp/${TARGET_CLUSTER_NAME}.kubeconfig wait --for=condition=ready pods --all --timeout=4000s -A +kubectl --kubeconfig /tmp/${TARGET_CLUSTER_NAME}.kubeconfig get pods -A + +echo "Check machine status" +kubectl get machines --kubeconfig ${KUBECONFIG} + +echo "Get cluster state for target workload cluster " +kubectl --kubeconfig ${KUBECONFIG} get cluster + +echo "Target Cluster Kubeconfig" +echo "/tmp/${TARGET_CLUSTER_NAME}.kubeconfig" diff --git a/tools/deployment/provider_common/32_cluster_init_target_node.sh b/tools/deployment/provider_common/32_cluster_init_target_node.sh new file mode 100755 index 000000000..ab9b8d213 --- /dev/null +++ b/tools/deployment/provider_common/32_cluster_init_target_node.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xe + +# Example Usage +# KUBECONFIG=/tmp/$KUBECONFIG \ +# ./tools/deployment/provider_common/32_cluster_init_target_node.sh + +export KUBECONFIG=${KUBECONFIG:-"$HOME/.airship/kubeconfig"} +export KUBECONFIG_TARGET_CONTEXT=${KUBECONFIG_TARGET_CONTEXT:-"target-cluster"} + +# Get control plane node +CONTROL_PLANE_NODES=( $(kubectl --kubeconfig $KUBECONFIG get --no-headers=true nodes \ +| grep cluster-control-plane | awk '{print $1}') ) + +# Remove noschedule taint to prevent cluster init from timing out +for i in "${CONTROL_PLANE_NODES}"; do + echo untainting node $i + kubectl taint node $i node-role.kubernetes.io/master- --kubeconfig $KUBECONFIG --request-timeout 10s +done + +echo "Deploy CAPI components to target cluster" +airshipctl phase run clusterctl-init-target --debug --kubeconfig "$KUBECONFIG" + +echo "Waiting for pods to be ready" +kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT wait --all-namespaces --for=condition=Ready pods --all --timeout=600s +kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT get pods --all-namespaces diff --git a/tools/deployment/provider_common/33_cluster_move_target_node.sh b/tools/deployment/provider_common/33_cluster_move_target_node.sh new file mode 100755 index 000000000..eb584b262 --- /dev/null +++ b/tools/deployment/provider_common/33_cluster_move_target_node.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xe + +#Default wait timeout is 3600 seconds +export TIMEOUT=${TIMEOUT:-3600} +export KUBECONFIG=${KUBECONFIG:-"$HOME/.airship/kubeconfig"} +export KUBECONFIG_TARGET_CONTEXT=${KUBECONFIG_TARGET_CONTEXT:-"target-cluster"} +export TARGET_KUBECONFIG="/tmp/target-cluster.kubeconfig" + +echo "Waiting for machines to come up" +kubectl --kubeconfig ${KUBECONFIG} wait --for=condition=Ready machines --all --timeout 4000s + +#add wait condition +end=$(($(date +%s) + $TIMEOUT)) +echo "Waiting $TIMEOUT seconds for Machine to be Running." +while true; do + if (kubectl --request-timeout 20s --kubeconfig $KUBECONFIG get machines -o json | jq '.items[0].status.phase' | grep -q "Running") ; then + echo -e "\nMachine is Running" + kubectl --kubeconfig $KUBECONFIG get machines + break + else + now=$(date +%s) + if [ $now -gt $end ]; then + echo -e "\nMachine is not in Ruuning phase, Move can't be performed." + exit 1 + fi + echo -n . + sleep 15 + fi +done + +echo "Move Cluster Object to Target Cluster" +KUBECONFIG=$KUBECONFIG:$TARGET_KUBECONFIG kubectl config view --merge --flatten > "/tmp/merged_target_ephemeral.kubeconfig" +airshipctl phase run clusterctl-move --kubeconfig "/tmp/merged_target_ephemeral.kubeconfig" + +echo "Waiting for pods to be ready" +kubectl --kubeconfig $TARGET_KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT wait --all-namespaces --for=condition=Ready pods --all --timeout=3000s +kubectl --kubeconfig $TARGET_KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT get pods --all-namespaces + +#Wait till crds are created +end=$(($(date +%s) + $TIMEOUT)) +echo "Waiting $TIMEOUT seconds for crds to be created." +while true; do + if (kubectl --request-timeout 20s --kubeconfig $TARGET_KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT get cluster target-cluster -o json | jq '.status.controlPlaneReady' | grep -q true) ; then + echo -e "\nGet CRD status" + kubectl --kubeconfig $TARGET_KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT get machines + kubectl --kubeconfig $TARGET_KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT get clusters + break + else + now=$(date +%s) + if [ $now -gt $end ]; then + echo -e "\nCluster move failed and CRDs was not ready before TIMEOUT." + exit 1 + fi + echo -n . + sleep 15 + fi +done diff --git a/tools/deployment/provider_common/34_deploy_worker_node.sh b/tools/deployment/provider_common/34_deploy_worker_node.sh new file mode 100755 index 000000000..8bf9b569c --- /dev/null +++ b/tools/deployment/provider_common/34_deploy_worker_node.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xe + +# Example Usage +# TEST_SITE=docker-test-site \ +# WORKERS_COUNT=3 ./tools/deployment/provider_common/34_deploy_worker_node.sh + +export KUBECONFIG=${KUBECONFIG:-"$HOME/.airship/kubeconfig"} +export TARGET_KUBECONFIG=${TARGET_KUBECONFIG:-"/tmp/target-cluster.kubeconfig"} +export KUBECONFIG_TARGET_CONTEXT=${KUBECONFIG_TARGET_CONTEXT:-"target-cluster"} +export SITE=${SITE:-"docker-test-site"} +export WORKERS_COUNT=${WORKERS_COUNT:-"1"} +export AIRSHIP_SRC=${AIRSHIP_SRC:-"/tmp/airship"} + +# Adjust wokers replicas, default - 1 +sed -i "/value.*/s//value\": $WORKERS_COUNT }/g" \ +${AIRSHIP_SRC}/airshipctl/manifests/site/${SITE}/target/workers/machine_count.json + +echo "Stop/Delete ephemeral node" +kind delete cluster --name "ephemeral-cluster" + +echo "Deploy worker node" +airshipctl phase run workers-target --debug --kubeconfig "$TARGET_KUBECONFIG" + +#Wait till node is created +kubectl wait --for=condition=ready node --all --timeout=1000s --context $KUBECONFIG_TARGET_CONTEXT --kubeconfig $TARGET_KUBECONFIG -A diff --git a/tools/deployment/templates/kind-cluster-with-extramounts b/tools/deployment/templates/kind-cluster-with-extramounts new file mode 100644 index 000000000..ca335758a --- /dev/null +++ b/tools/deployment/templates/kind-cluster-with-extramounts @@ -0,0 +1,7 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + extraMounts: + - hostPath: /var/run/docker.sock + containerPath: /var/run/docker.sock diff --git a/tools/document/start_kind.sh b/tools/document/start_kind.sh index 8e6cef5bd..03219eaa9 100755 --- a/tools/document/start_kind.sh +++ b/tools/document/start_kind.sh @@ -12,16 +12,51 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -e # This starts up a kubernetes cluster which is sufficient for # assisting with tasks like `kubectl apply --dry-run` style validation -set -e +# Usage +# example 1: create a kind cluster, with name as airship +# +# ./tools/document/start_kind.sh +# +# example 2: create a kind cluster, with a custom name +# +# CLUSTER=ephemeral-cluster ./tools/document/start_kind.sh +# +# example 3: create a kind cluster, using custom name and config +# +# CLUSTER=ephemeral-cluster KIND_CONFIG=./tools/deployment/templates/kind-cluster-with-extramounts.yaml \ +# ./tools/document/start_kind.sh +# +# example 4: create a kind cluster with name as airship, using custom config +# +# KIND_CONFIG=./tools/deployment/templates/kind-cluster-with-extramounts.yaml \ +# ./tools/document/start_kind.sh : ${KIND:="/usr/local/bin/kind"} : ${CLUSTER:="airship"} # NB: kind prepends "kind-" : ${KUBECONFIG:="${HOME}/.airship/kubeconfig"} +: ${TIMEOUT:=3600} +: ${KIND_CONFIG:=""} +export KIND_EXPERIMENTAL_DOCKER_NETWORK=bridge -${KIND} create cluster --name ${CLUSTER} \ - --kubeconfig ${KUBECONFIG} +echo "cluster name: $CLUSTER"; + +if [ -z "$KIND_CONFIG" ] +then + ${KIND} create cluster --name $CLUSTER --wait 120s --kubeconfig ${KUBECONFIG} +else + echo "Using kind configuration file: $KIND_CONFIG" + ${KIND} create cluster --config $KIND_CONFIG --name $CLUSTER --wait 120s --kubeconfig ${KUBECONFIG} +fi + +# Wait till Control Plane Node is ready +kubectl wait --for=condition=ready node --all --timeout=1000s --kubeconfig $KUBECONFIG -A + +# Add context e.g ephemeral-cluster. +# By default, kind creates context kind- +kubectl config set-context ${CLUSTER} --cluster kind-${CLUSTER} --user kind-${CLUSTER} --kubeconfig $KUBECONFIG echo "This cluster can be deleted via:" echo "${KIND} delete cluster --name ${CLUSTER}" diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 0751f6598..ddbc64dfb 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -152,7 +152,42 @@ - airship-airshipctl-gather-configs - describe-kubernetes-objects - airship-gather-pod-logs - +- job: + name: airship-airshipctl-gate-script-runner-docker + attempts: 1 + timeout: 3600 + pre-run: playbooks/airship-airshipctl-deploy-docker.yaml + run: playbooks/airshipctl-gate-runner.yaml + nodeset: airship-airshipctl-single-node + irrelevant-files: *noncodefiles + dependencies: + - name: openstack-tox-docs + soft: true + - name: airship-airshipctl-lint + soft: true + - name: airship-airshipctl-unit + soft: true + - name: airship-airshipctl-golint + soft: true + - name: airship-airshipctl-build-image + soft: true + - name: airship-airshipctl-validate-site-docs + soft: true + vars: + site_name: docker-test-site + gate_scripts: + - ./tools/deployment/21_systemwide_executable.sh + - ./tools/deployment/01_install_kubectl.sh + - ./tools/deployment/provider_common/01_install_kind.sh + - ./tools/deployment/provider_common/02_install_jq.sh + - CLUSTER=ephemeral-cluster KIND_CONFIG=./tools/deployment/templates/kind-cluster-with-extramounts ./tools/document/start_kind.sh + - AIRSHIP_CONFIG_METADATA_PATH=manifests/site/docker-test-site/metadata.yaml SITE=docker-test-site EXTERNAL_KUBECONFIG="true" ./tools/deployment/22_test_configs.sh + - PROVIDER=default SITE=docker-test-site ./tools/deployment/26_deploy_capi_ephemeral_node.sh + - CONTROLPLANE_COUNT=1 SITE=docker-test-site ./tools/deployment/provider_common/30_deploy_controlplane.sh + - KUBECONFIG=/tmp/target-cluster.kubeconfig ./tools/deployment/provider_common/32_cluster_init_target_node.sh + - ./tools/deployment/provider_common/33_cluster_move_target_node.sh + - WORKERS_COUNT=2 KUBECONFIG=/tmp/target-cluster.kubeconfig SITE=docker-test-site ./tools/deployment/provider_common/34_deploy_worker_node.sh + voting: false - job: name: airship-airshipctl-publish-image parent: airship-airshipctl-build-image diff --git a/zuul.d/projects.yaml b/zuul.d/projects.yaml index ed032932a..0c07c8463 100644 --- a/zuul.d/projects.yaml +++ b/zuul.d/projects.yaml @@ -32,6 +32,7 @@ - airship-airshipctl-build-image - airship-airshipctl-validate-site-docs # - airship-airshipctl-functional-existing-k8s TODO: Enable this when functional tests exist, and a cluster is up + - airship-airshipctl-gate-script-runner-docker experimental: jobs: - airship-airshipctl-gate-script-runner