From 151eccbfbab1c90c7e9bcaeb51ed69ee78023824 Mon Sep 17 00:00:00 2001 From: Michael Beaver Date: Wed, 21 Apr 2021 15:58:22 -0500 Subject: [PATCH] Add airship-in-a-pod to gate jobs This change aims to add the airship-in-a-pod test to the gates. This is accomplished by creating new scripts to run which install a minimal k8s environment, and then apply the airship-in-a-pod.yaml. Brief description of scripts 01_dns_settings: Running k8s pods in zuul has a lot of issues with DNS resolution which this resolves. 10_install_minikube: grabs latest minikube and other dependencies. 11_build_images: builds the images under tools/airship-in-a-pod and then adds patches to the kustomize configuration to use and configure them. 12_start_minikube: starts minikube and makes sure kubectl will work 13_apply_dns.sh: This takes some of the effort from 01_dns_settings and incorporates it into the minikube cluster by modifying coredns 20_apply_aiap: Applies the airship-in-a-pod.yaml to the cluster. Has a few loops to check that the cluster is fully running before applying, and to check on the pod to see if it is ready before finishing. Closes: #478 Change-Id: I4c02d04b0eda9395642032bf56b56a0f823c1969 --- playbooks/airshipctl-gate-runner.yaml | 1 + tools/airship-in-a-pod/Makefile | 11 ++- .../scripts/01_dns_settings.sh | 49 ++++++++++++ .../scripts/10_install_minikube.sh | 35 +++++++++ .../scripts/11_build_images.sh | 77 +++++++++++++++++++ .../scripts/12_start_minikube.sh | 43 +++++++++++ .../airship-in-a-pod/scripts/13_apply_dns.sh | 32 ++++++++ .../airship-in-a-pod/scripts/20_apply_aiap.sh | 64 +++++++++++++++ zuul.d/jobs.yaml | 41 ++++++++++ zuul.d/projects.yaml | 2 + 10 files changed, 353 insertions(+), 2 deletions(-) create mode 100755 tools/airship-in-a-pod/scripts/01_dns_settings.sh create mode 100755 tools/airship-in-a-pod/scripts/10_install_minikube.sh create mode 100755 tools/airship-in-a-pod/scripts/11_build_images.sh create mode 100755 tools/airship-in-a-pod/scripts/12_start_minikube.sh create mode 100755 tools/airship-in-a-pod/scripts/13_apply_dns.sh create mode 100755 tools/airship-in-a-pod/scripts/20_apply_aiap.sh diff --git a/playbooks/airshipctl-gate-runner.yaml b/playbooks/airshipctl-gate-runner.yaml index e28b21e67..40bb7f20c 100644 --- a/playbooks/airshipctl-gate-runner.yaml +++ b/playbooks/airshipctl-gate-runner.yaml @@ -29,6 +29,7 @@ GCP_REGION: "dXMtd2VzdDE=" GCP_NETWORK_NAME: "ZGVmYXVsdA==" GCP_B64ENCODED_CREDENTIALS: "bjEtc3RhbmRhcmQtNA==" + AIRSHIPCTL_REF: "{{ zuul.ref | default('master') }}" tasks: - name: "set_default_gate_scripts" set_fact: diff --git a/tools/airship-in-a-pod/Makefile b/tools/airship-in-a-pod/Makefile index 5a30e1fb0..8bdd53496 100644 --- a/tools/airship-in-a-pod/Makefile +++ b/tools/airship-in-a-pod/Makefile @@ -1,10 +1,10 @@ DOCKER_REGISTRY ?= quay.io DOCKER_IMAGE_PREFIX ?= airshipit DOCKER_IMAGE_TAG ?= latest -IMAGES ?= infra-builder artifact-setup runner +IMAGES ?= infra-builder runner PUBLISH ?= false -.PHONY: help base libvirt $(IMAGES) images test +.PHONY: help base libvirt artifact-setup $(IMAGES) images test SHELL:=/bin/bash .ONESHELL: @@ -12,10 +12,17 @@ SHELL:=/bin/bash help: ## This help. @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z0-9_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) +images: artifact-setup images: base images: libvirt images: $(IMAGES) ## Build the containers. +artifact-setup: + docker build --tag $(DOCKER_REGISTRY)/$(DOCKER_IMAGE_PREFIX)/aiap-artifact-setup:$(DOCKER_IMAGE_TAG) --build-arg AIRSHIPCTL_REF=$(AIRSHIPCTL_REF) ./artifact-setup +ifeq (true, $(PUBLISH)) + docker push $(DOCKER_REGISTRY)/$(DOCKER_IMAGE_PREFIX)/aiap-artifact-setup:$(DOCKER_IMAGE_TAG) +endif + base: docker build --tag $(DOCKER_REGISTRY)/$(DOCKER_IMAGE_PREFIX)/aiap-base:$(DOCKER_IMAGE_TAG) --build-arg BASE_IMAGE=ubuntu:20.04 ./base ifeq (true, $(PUBLISH)) diff --git a/tools/airship-in-a-pod/scripts/01_dns_settings.sh b/tools/airship-in-a-pod/scripts/01_dns_settings.sh new file mode 100755 index 000000000..e63858c5b --- /dev/null +++ b/tools/airship-in-a-pod/scripts/01_dns_settings.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# Running K8s pods in zuul can cause a lot of issues with resolving domains +# inside of the running pods and catching coredns in a loop. This aims to +# tackle a few issues that have been hit to resolve this. + + +NAMESERVER="1.0.0.1" + +# Grab the real nameservers instead of the local one listed in the original +sudo ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf + +# Add the known good DNS server +sudo sed -i "1i\nameserver $NAMESERVER\n" /etc/resolv.conf +# Remove DNS servers pointing to localhost so coredns doesn't get caught in a loop +sudo sed -i '/127\.0/d' /etc/resolv.conf +# Spit out the nameservers for the logs +cat /etc/resolv.conf + +# Running unbound server can cause issues with coredns, disabling +if [[ -f "/etc/unbound/unbound.pid" ]]; then + sudo kill "$(cat /etc/unbound/unbound.pid)" +fi + +# flush iptables so coredns doesn't get caught up +# be sure to stop docker if it is installed +dpkg -l | grep -i docker | head -1 | if [[ "$(cut -d ' ' -f 1)" == "ii" ]]; then + sudo systemctl stop docker +fi +sudo iptables --flush +sudo iptables -tnat --flush + +dpkg -l | grep -i docker | head -1 | if [[ "$(cut -d ' ' -f 1)" == "ii" ]]; then + sudo systemctl start docker +fi diff --git a/tools/airship-in-a-pod/scripts/10_install_minikube.sh b/tools/airship-in-a-pod/scripts/10_install_minikube.sh new file mode 100755 index 000000000..746b24039 --- /dev/null +++ b/tools/airship-in-a-pod/scripts/10_install_minikube.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# Installs minikube and other dependencies required for the scripts to run + + +MINIKUBE_VERSION="latest" + +install_pkg(){ + for i in "$@"; do + dpkg -l "$i" 2> /dev/null | grep ^ii > /dev/null || sudo DEBIAN_FRONTEND=noninteractive -E apt -y install "$i" + done +} + +# Grab usefull packages needed for minikube and other scripts +install_pkg curl conntrack make docker.io jq + +curl -Lo minikube "https://storage.googleapis.com/minikube/releases/$MINIKUBE_VERSION/minikube-linux-amd64" \ + && chmod +x minikube + +sudo mkdir -p /usr/local/bin/ +sudo install minikube /usr/local/bin/ \ No newline at end of file diff --git a/tools/airship-in-a-pod/scripts/11_build_images.sh b/tools/airship-in-a-pod/scripts/11_build_images.sh new file mode 100755 index 000000000..354092a25 --- /dev/null +++ b/tools/airship-in-a-pod/scripts/11_build_images.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# Builds all of the images under tools/airship-in-a-pod and then configures +# the AIAP pod to never pull down images so it is sure to use the built +# images instead. This also requires a few other images to be pulled. + + +AIRSHIPCTL_REF=${AIRSHIPCTL_REF:-"master"} +export AIRSHIPCTL_REF +# Images that are required by airship-in-a-pod but not built +PULL_LIST="docker:stable-dind nginx quay.io/metal3-io/sushy-tools quay.io/airshipit/libvirt:aiap-v1" + + +pushd tools/airship-in-a-pod/ || exit + +make -e images artifact-setup base infra-builder runner libvirt + +for IMAGE in $PULL_LIST; do + docker pull "$IMAGE" +done + +# Now that we have built/pulled the images, lets change the imagePullPolicy to +# Never to be 100% confident they are used +echo "- op: add + path: \"/spec/containers/0/imagePullPolicy\" + value: Never + +- op: add + path: \"/spec/containers/1/imagePullPolicy\" + value: Never + +- op: add + path: \"/spec/containers/2/imagePullPolicy\" + value: Never + +- op: add + path: \"/spec/containers/3/imagePullPolicy\" + value: Never + +- op: add + path: \"/spec/containers/4/imagePullPolicy\" + value: Never + +- op: add + path: \"/spec/containers/5/imagePullPolicy\" + value: Never + +- op: add + path: \"/spec/containers/6/imagePullPolicy\" + value: Never + +" >> examples/airshipctl/patchset.yaml + +# Also add the patchset to the environment variables +# while being sure to escape the slashes from the ref +echo "- op: replace + path: \"/spec/containers/4/env/6/value\" + value: $AIRSHIPCTL_REF + +" >> examples/airshipctl/patchset.yaml + + +popd || exit diff --git a/tools/airship-in-a-pod/scripts/12_start_minikube.sh b/tools/airship-in-a-pod/scripts/12_start_minikube.sh new file mode 100755 index 000000000..f9a706ade --- /dev/null +++ b/tools/airship-in-a-pod/scripts/12_start_minikube.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# This script starts up minikube, and accounts for an issue that sometimes +# comes up when running minikube for the first time in some environments + + +set +e +sudo -E minikube start --driver=none +status=$? +sudo chown -R "$USER" "$HOME"/.minikube; chmod -R u+wrx "$HOME"/.minikube +if [[ $status -gt 0 ]]; then + # Sometimes minikube fails to start if the directory permissions are not correct + sudo -E minikube delete + set -e + sudo -E minikube start --driver=none +fi + +set -e +sudo -E minikube status + +# Ensure .kube and .minikube have proper ownership +sudo chown -R "$USER" "$HOME"/.kube "$HOME"/.minikube + +# Make a copy of the kubeconfig for the log playbooks +mkdir -p "$HOME"/.airship +cp "$HOME"/.kube/config "$HOME"/.airship/kubeconfig + +# Give cluster a chance to start up +sleep 10 \ No newline at end of file diff --git a/tools/airship-in-a-pod/scripts/13_apply_dns.sh b/tools/airship-in-a-pod/scripts/13_apply_dns.sh new file mode 100755 index 000000000..9fccdbdf4 --- /dev/null +++ b/tools/airship-in-a-pod/scripts/13_apply_dns.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# This utilizes some of the work that was done to the nameservers in +# 01_dns_settings.sh to take those DNS servers and force the coredns pod +# of the minikube cluster to use those instead of the default. + + +# Grab a list of the nameservers IPs in /etc/resolv.conf +NAMESERVERS=$(grep nameserver /etc/resolv.conf | awk '{print $2}' | tr '\n' ' ') + + +kubectl -n kube-system get pods -o wide +# Configure coredns with an upstream DNS to ensure the pod can resolve +# domains outside of the cluster +kubectl -n kube-system get cm -o yaml coredns | sed "s/\/etc\/resolv\.conf/$NAMESERVERS/" > tools/airship-in-a-pod/coredns-upstream-dns.yaml +cat tools/airship-in-a-pod/coredns-upstream-dns.yaml +kubectl apply -f tools/airship-in-a-pod/coredns-upstream-dns.yaml +kubectl rollout restart -n kube-system deployment/coredns diff --git a/tools/airship-in-a-pod/scripts/20_apply_aiap.sh b/tools/airship-in-a-pod/scripts/20_apply_aiap.sh new file mode 100755 index 000000000..309bb2006 --- /dev/null +++ b/tools/airship-in-a-pod/scripts/20_apply_aiap.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# This script first loops to check if the K8s cluster is ready before applying +# airship-in-a-pod to it. Once all of the kube-system pods are ready, it applies +# the yaml and then checks every so often to determine if all of the containers +# are ready. + + +INTERVAL=15 +READY=false +KUBE_READY=false + + +# Wait for the Kubernetes environment to become completely ready +while [[ $KUBE_READY == "false" ]]; +do + # Grab the readiness from the kubectl output + kube_pods=$(kubectl get pods -n kube-system | tail -n +2 | awk '{print $2}') + for POD in $kube_pods; do + # Compare the two values to determine if each pod is completely ready + kube_ready_pod=$(echo "$POD" | cut -f1 -d/) + kube_ready_total=$(echo "$POD" | cut -f2 -d/) + if [[ "$kube_ready_pod" != "$kube_ready_total" ]]; then + # If a pod is not ready yet, break out and try again next time + KUBE_READY=false + break + fi + # This will only stay "true" as long as the previous 'if' is never reached + KUBE_READY=true + done + sleep "$INTERVAL" +done + +kustomize build tools/airship-in-a-pod/examples/airshipctl | kubectl apply -f - + +while [[ $READY == "false" ]]; +do + # Grab the number of ready containers from the kubectl output + kubectl get pod airship-in-a-pod -o wide + readiness=$(kubectl get pods | grep airship-in-a-pod | awk '{print $2}') + ready_pod=$(echo "$readiness" | cut -f1 -d/) + ready_total=$(echo "$readiness" | cut -f2 -d/) + # if it is 7/7 ready (example with 7 containers), then mark ready + if [[ "$ready_pod" == "$ready_total" ]]; then + READY=true + fi + + sleep "$INTERVAL" +done + diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 53cb41dfb..4789c618f 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -140,6 +140,47 @@ - airship-airshipctl-gather-configs - describe-kubernetes-objects - airship-gather-pod-logs +- job: + name: airship-airship-in-a-pod-script-runner + attempts: 1 + timeout: 9000 + pre-run: + - playbooks/airship-airshipctl-deploy-docker.yaml + post-run: playbooks/airship-collect-logs.yaml + run: playbooks/airshipctl-gate-runner.yaml + nodeset: airship-airshipctl-single-16GB-bionic-node + irrelevant-files: *noncodefiles + dependencies: + - name: openstack-tox-docs + soft: true + - name: airship-airshipctl-lint + soft: true + - name: airship-airshipctl-golint + soft: true + - name: airship-airshipctl-unit + soft: true + - name: airship-airshipctl-build-image + soft: true + vars: + site_name: test-site + gate_scripts: + - ./tools/deployment/01_install_kubectl.sh + - ./tools/install_kustomize.sh + - ./tools/airship-in-a-pod/scripts/01_dns_settings.sh + - ./tools/airship-in-a-pod/scripts/10_install_minikube.sh + - ./tools/airship-in-a-pod/scripts/11_build_images.sh + - ./tools/airship-in-a-pod/scripts/12_start_minikube.sh + - ./tools/airship-in-a-pod/scripts/13_apply_dns.sh + - ./tools/airship-in-a-pod/scripts/20_apply_aiap.sh + serve_dir: /srv/images + serve_port: 8099 + log_roles: + - gather-system-logs + - airship-gather-runtime-logs + - airship-airshipctl-gather-configs + - describe-kubernetes-objects + - airship-gather-pod-logs + voting: false - job: name: airship-airshipctl-gate-script-runner-docker attempts: 1 diff --git a/zuul.d/projects.yaml b/zuul.d/projects.yaml index 93a936a7c..702b8b20b 100644 --- a/zuul.d/projects.yaml +++ b/zuul.d/projects.yaml @@ -41,6 +41,7 @@ # - airship-airshipctl-functional-existing-k8s TODO: Enable this when functional tests exist, and a cluster is up - airship-airshipctl-gate-script-runner-docker - airship-aiap-build-image + - airship-airship-in-a-pod-script-runner experimental: jobs: - airship-airshipctl-docker-kubebench-conformance @@ -55,6 +56,7 @@ - airship-airshipctl-validate-site-docs - airship-airshipctl-gate-script-runner - airship-aiap-build-image + - airship-airship-in-a-pod-script-runner # - airship-airshipctl-functional-existing-k8s TODO: Enable this when functional tests exist, and a cluster is up post: