Add airship-in-a-pod to gate jobs

This change aims to add the airship-in-a-pod test to the gates.
This is accomplished by creating new scripts to run which
install a minimal k8s environment, and then apply the
airship-in-a-pod.yaml.

Brief description of scripts

01_dns_settings: Running k8s pods in zuul has a lot of issues with
DNS resolution which this resolves.

10_install_minikube: grabs latest minikube and other dependencies.

11_build_images: builds the images under tools/airship-in-a-pod
and then adds patches to the kustomize configuration to use and
configure them.

12_start_minikube: starts minikube and makes sure kubectl will work

13_apply_dns.sh: This takes some of the effort from 01_dns_settings
and incorporates it into the minikube cluster by modifying coredns

20_apply_aiap: Applies the airship-in-a-pod.yaml to the cluster.
Has a few loops to check that the cluster is fully running
before applying, and to check on the pod to see if it is ready
before finishing.

Closes: #478
Change-Id: I4c02d04b0eda9395642032bf56b56a0f823c1969

playbooks/airshipctl-gate-runner.yaml

@@ -29,6 +29,7 @@
     GCP_REGION: "dXMtd2VzdDE="
     GCP_NETWORK_NAME: "ZGVmYXVsdA=="
     GCP_B64ENCODED_CREDENTIALS: "bjEtc3RhbmRhcmQtNA=="
+    AIRSHIPCTL_REF: "{{ zuul.ref | default('master') }}"
   tasks:
     - name: "set_default_gate_scripts"
       set_fact:

tools/airship-in-a-pod/Makefile

@@ -1,10 +1,10 @@
 DOCKER_REGISTRY     ?= quay.io
 DOCKER_IMAGE_PREFIX ?= airshipit
 DOCKER_IMAGE_TAG    ?= latest
-IMAGES              ?= infra-builder artifact-setup runner
+IMAGES              ?= infra-builder runner
 PUBLISH             ?= false
 
-.PHONY: help base libvirt $(IMAGES) images test
+.PHONY: help base libvirt artifact-setup $(IMAGES) images test
 
 SHELL:=/bin/bash
 .ONESHELL:
@@ -12,10 +12,17 @@ SHELL:=/bin/bash
 help: ## This help.
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z0-9_-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 
+images: artifact-setup
 images: base
 images: libvirt
 images: $(IMAGES) ## Build the containers.
 
+artifact-setup:
+	docker build --tag $(DOCKER_REGISTRY)/$(DOCKER_IMAGE_PREFIX)/aiap-artifact-setup:$(DOCKER_IMAGE_TAG) --build-arg AIRSHIPCTL_REF=$(AIRSHIPCTL_REF) ./artifact-setup
+ifeq (true, $(PUBLISH))
+	docker push $(DOCKER_REGISTRY)/$(DOCKER_IMAGE_PREFIX)/aiap-artifact-setup:$(DOCKER_IMAGE_TAG)
+endif
+
 base:
 	docker build --tag $(DOCKER_REGISTRY)/$(DOCKER_IMAGE_PREFIX)/aiap-base:$(DOCKER_IMAGE_TAG) --build-arg BASE_IMAGE=ubuntu:20.04 ./base
 ifeq (true, $(PUBLISH))

tools/airship-in-a-pod/scripts/01_dns_settings.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# Running K8s pods in zuul can cause a lot of issues with resolving domains
+# inside of the running pods and catching coredns in a loop. This aims to
+# tackle a few issues that have been hit to resolve this.
+
+
+NAMESERVER="1.0.0.1"
+
+# Grab the real nameservers instead of the local one listed in the original
+sudo ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf
+
+# Add the known good DNS server
+sudo sed -i "1i\nameserver $NAMESERVER\n" /etc/resolv.conf
+# Remove DNS servers pointing to localhost so coredns doesn't get caught in a loop
+sudo sed -i '/127\.0/d' /etc/resolv.conf
+# Spit out the nameservers for the logs
+cat /etc/resolv.conf
+
+# Running unbound server can cause issues with coredns, disabling
+if [[ -f "/etc/unbound/unbound.pid" ]]; then
+    sudo kill "$(cat /etc/unbound/unbound.pid)"
+fi
+
+# flush iptables so coredns doesn't get caught up
+# be sure to stop docker if it is installed
+dpkg -l | grep -i docker | head -1 | if [[ "$(cut -d ' ' -f 1)" == "ii" ]]; then
+    sudo systemctl stop docker
+fi
+sudo iptables --flush
+sudo iptables -tnat --flush
+
+dpkg -l | grep -i docker | head -1 | if [[ "$(cut -d ' ' -f 1)" == "ii" ]]; then
+    sudo systemctl start docker
+fi

tools/airship-in-a-pod/scripts/10_install_minikube.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# Installs minikube and other dependencies required for the scripts to run
+
+
+MINIKUBE_VERSION="latest"
+
+install_pkg(){
+  for i in "$@"; do
+    dpkg -l "$i" 2> /dev/null | grep ^ii > /dev/null || sudo DEBIAN_FRONTEND=noninteractive -E apt -y install "$i"
+  done
+}
+
+# Grab usefull packages needed for minikube and other scripts
+install_pkg curl conntrack make docker.io jq
+
+curl -Lo minikube "https://storage.googleapis.com/minikube/releases/$MINIKUBE_VERSION/minikube-linux-amd64" \
+  && chmod +x minikube
+
+sudo mkdir -p /usr/local/bin/
+sudo install minikube /usr/local/bin/

tools/airship-in-a-pod/scripts/11_build_images.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# Builds all of the images under tools/airship-in-a-pod and then configures
+# the AIAP pod to never pull down images so it is sure to use the built
+# images instead. This also requires a few other images to be pulled.
+
+
+AIRSHIPCTL_REF=${AIRSHIPCTL_REF:-"master"}
+export AIRSHIPCTL_REF
+# Images that are required by airship-in-a-pod but not built
+PULL_LIST="docker:stable-dind nginx quay.io/metal3-io/sushy-tools quay.io/airshipit/libvirt:aiap-v1"
+
+
+pushd tools/airship-in-a-pod/ || exit
+
+make -e images artifact-setup base infra-builder runner libvirt
+
+for IMAGE in $PULL_LIST; do
+    docker pull "$IMAGE"
+done
+
+# Now that we have built/pulled the images, lets change the imagePullPolicy to
+# Never to be 100% confident they are used
+echo "- op: add
+  path: \"/spec/containers/0/imagePullPolicy\"
+  value: Never
+
+- op: add
+  path: \"/spec/containers/1/imagePullPolicy\"
+  value: Never
+
+- op: add
+  path: \"/spec/containers/2/imagePullPolicy\"
+  value: Never
+
+- op: add
+  path: \"/spec/containers/3/imagePullPolicy\"
+  value: Never
+
+- op: add
+  path: \"/spec/containers/4/imagePullPolicy\"
+  value: Never
+
+- op: add
+  path: \"/spec/containers/5/imagePullPolicy\"
+  value: Never
+
+- op: add
+  path: \"/spec/containers/6/imagePullPolicy\"
+  value: Never
+
+" >> examples/airshipctl/patchset.yaml
+
+# Also add the patchset to the environment variables
+# while being sure to escape the slashes from the ref
+echo "- op: replace
+  path: \"/spec/containers/4/env/6/value\"
+  value: $AIRSHIPCTL_REF
+
+" >> examples/airshipctl/patchset.yaml
+
+
+popd || exit

tools/airship-in-a-pod/scripts/12_start_minikube.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# This script starts up minikube, and accounts for an issue that sometimes
+# comes up when running minikube for the first time in some environments
+
+
+set +e
+sudo -E minikube start --driver=none
+status=$?
+sudo chown -R "$USER" "$HOME"/.minikube; chmod -R u+wrx "$HOME"/.minikube
+if [[ $status -gt 0 ]]; then
+    # Sometimes minikube fails to start if the directory permissions are not correct
+    sudo -E minikube delete
+    set -e
+    sudo -E minikube start --driver=none
+fi
+
+set -e
+sudo -E minikube status
+
+# Ensure .kube and .minikube have proper ownership
+sudo chown -R "$USER" "$HOME"/.kube "$HOME"/.minikube
+
+# Make a copy of the kubeconfig for the log playbooks
+mkdir -p "$HOME"/.airship
+cp "$HOME"/.kube/config "$HOME"/.airship/kubeconfig
+
+# Give cluster a chance to start up
+sleep 10

tools/airship-in-a-pod/scripts/13_apply_dns.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# This utilizes some of the work that was done to the nameservers in
+# 01_dns_settings.sh to take those DNS servers and force the coredns pod
+# of the minikube cluster to use those instead of the default.
+
+
+# Grab a list of the nameservers IPs in /etc/resolv.conf
+NAMESERVERS=$(grep nameserver /etc/resolv.conf | awk '{print $2}' | tr '\n' ' ')
+
+
+kubectl -n kube-system get pods -o wide
+# Configure coredns with an upstream DNS to ensure the pod can resolve
+# domains outside of the cluster
+kubectl -n kube-system get cm -o yaml coredns | sed "s/\/etc\/resolv\.conf/$NAMESERVERS/" > tools/airship-in-a-pod/coredns-upstream-dns.yaml
+cat tools/airship-in-a-pod/coredns-upstream-dns.yaml
+kubectl apply -f tools/airship-in-a-pod/coredns-upstream-dns.yaml
+kubectl rollout restart -n kube-system deployment/coredns

tools/airship-in-a-pod/scripts/20_apply_aiap.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# This script first loops to check if the K8s cluster is ready before applying
+# airship-in-a-pod to it. Once all of the kube-system pods are ready, it applies
+# the yaml and then checks every so often to determine if all of the containers
+# are ready.
+
+
+INTERVAL=15
+READY=false
+KUBE_READY=false
+
+
+# Wait for the Kubernetes environment to become completely ready
+while [[ $KUBE_READY == "false" ]];
+do
+    # Grab the readiness from the kubectl output
+    kube_pods=$(kubectl get pods -n kube-system | tail -n +2 | awk '{print $2}')
+    for POD in $kube_pods; do
+        # Compare the two values to determine if each pod is completely ready
+        kube_ready_pod=$(echo "$POD" | cut -f1 -d/)
+        kube_ready_total=$(echo "$POD" | cut -f2 -d/)
+        if [[ "$kube_ready_pod" != "$kube_ready_total" ]]; then
+            # If a pod is not ready yet, break out and try again next time
+            KUBE_READY=false
+            break
+        fi
+        # This will only stay "true" as long as the previous 'if' is never reached
+        KUBE_READY=true
+    done
+    sleep "$INTERVAL"
+done
+
+kustomize build tools/airship-in-a-pod/examples/airshipctl | kubectl apply -f -
+
+while [[ $READY == "false" ]];
+do
+    # Grab the number of ready containers from the kubectl output
+    kubectl get pod airship-in-a-pod -o wide
+    readiness=$(kubectl get pods | grep airship-in-a-pod | awk '{print $2}')
+    ready_pod=$(echo "$readiness" | cut -f1 -d/)
+    ready_total=$(echo "$readiness" | cut -f2 -d/)
+    # if it is 7/7 ready (example with 7 containers), then mark ready
+    if [[ "$ready_pod" == "$ready_total" ]]; then
+        READY=true
+    fi
+
+    sleep "$INTERVAL"
+done
+

zuul.d/jobs.yaml

@@ -140,6 +140,47 @@
         - airship-airshipctl-gather-configs
         - describe-kubernetes-objects
         - airship-gather-pod-logs
+- job:
+    name: airship-airship-in-a-pod-script-runner
+    attempts: 1
+    timeout: 9000
+    pre-run:
+      - playbooks/airship-airshipctl-deploy-docker.yaml
+    post-run: playbooks/airship-collect-logs.yaml
+    run: playbooks/airshipctl-gate-runner.yaml
+    nodeset: airship-airshipctl-single-16GB-bionic-node
+    irrelevant-files: *noncodefiles
+    dependencies:
+      - name: openstack-tox-docs
+        soft: true
+      - name: airship-airshipctl-lint
+        soft: true
+      - name: airship-airshipctl-golint
+        soft: true
+      - name: airship-airshipctl-unit
+        soft: true
+      - name: airship-airshipctl-build-image
+        soft: true
+    vars:
+      site_name: test-site
+      gate_scripts:
+        - ./tools/deployment/01_install_kubectl.sh
+        - ./tools/install_kustomize.sh
+        - ./tools/airship-in-a-pod/scripts/01_dns_settings.sh
+        - ./tools/airship-in-a-pod/scripts/10_install_minikube.sh
+        - ./tools/airship-in-a-pod/scripts/11_build_images.sh
+        - ./tools/airship-in-a-pod/scripts/12_start_minikube.sh
+        - ./tools/airship-in-a-pod/scripts/13_apply_dns.sh
+        - ./tools/airship-in-a-pod/scripts/20_apply_aiap.sh
+      serve_dir: /srv/images
+      serve_port: 8099
+      log_roles:
+        - gather-system-logs
+        - airship-gather-runtime-logs
+        - airship-airshipctl-gather-configs
+        - describe-kubernetes-objects
+        - airship-gather-pod-logs
+    voting: false
 - job:
     name: airship-airshipctl-gate-script-runner-docker
     attempts: 1

zuul.d/projects.yaml

@@ -41,6 +41,7 @@
 #        - airship-airshipctl-functional-existing-k8s TODO: Enable this when functional tests exist, and a cluster is up
         - airship-airshipctl-gate-script-runner-docker
         - airship-aiap-build-image
+        - airship-airship-in-a-pod-script-runner
     experimental:
       jobs:
         - airship-airshipctl-docker-kubebench-conformance
@@ -55,6 +56,7 @@
         - airship-airshipctl-validate-site-docs
         - airship-airshipctl-gate-script-runner
         - airship-aiap-build-image
+        - airship-airship-in-a-pod-script-runner
 
 #        - airship-airshipctl-functional-existing-k8s TODO: Enable this when functional tests exist, and a cluster is up
     post: