Browse Source

Add airship-in-a-pod to gate jobs

This change aims to add the airship-in-a-pod test to the gates.
This is accomplished by creating new scripts to run which
install a minimal k8s environment, and then apply the
airship-in-a-pod.yaml.

Brief description of scripts

01_dns_settings: Running k8s pods in zuul has a lot of issues with
DNS resolution which this resolves.

10_install_minikube: grabs latest minikube and other dependencies.

11_build_images: builds the images under tools/airship-in-a-pod
and then adds patches to the kustomize configuration to use and
configure them.

12_start_minikube: starts minikube and makes sure kubectl will work

13_apply_dns.sh: This takes some of the effort from 01_dns_settings
and incorporates it into the minikube cluster by modifying coredns

20_apply_aiap: Applies the airship-in-a-pod.yaml to the cluster.
Has a few loops to check that the cluster is fully running
before applying, and to check on the pod to see if it is ready
before finishing.

Closes: #478
Change-Id: I4c02d04b0eda9395642032bf56b56a0f823c1969
changes/42/787442/76
Michael Beaver 1 year ago
parent
commit
151eccbfba
  1. 1
      playbooks/airshipctl-gate-runner.yaml
  2. 11
      tools/airship-in-a-pod/Makefile
  3. 49
      tools/airship-in-a-pod/scripts/01_dns_settings.sh
  4. 35
      tools/airship-in-a-pod/scripts/10_install_minikube.sh
  5. 77
      tools/airship-in-a-pod/scripts/11_build_images.sh
  6. 43
      tools/airship-in-a-pod/scripts/12_start_minikube.sh
  7. 32
      tools/airship-in-a-pod/scripts/13_apply_dns.sh
  8. 64
      tools/airship-in-a-pod/scripts/20_apply_aiap.sh
  9. 41
      zuul.d/jobs.yaml
  10. 2
      zuul.d/projects.yaml

1
playbooks/airshipctl-gate-runner.yaml

@ -29,6 +29,7 @@
GCP_REGION: "dXMtd2VzdDE="
GCP_NETWORK_NAME: "ZGVmYXVsdA=="
GCP_B64ENCODED_CREDENTIALS: "bjEtc3RhbmRhcmQtNA=="
AIRSHIPCTL_REF: "{{ zuul.ref | default('master') }}"
tasks:
- name: "set_default_gate_scripts"
set_fact:

11
tools/airship-in-a-pod/Makefile

@ -1,10 +1,10 @@
DOCKER_REGISTRY ?= quay.io
DOCKER_IMAGE_PREFIX ?= airshipit
DOCKER_IMAGE_TAG ?= latest
IMAGES ?= infra-builder artifact-setup runner
IMAGES ?= infra-builder runner
PUBLISH ?= false
.PHONY: help base libvirt $(IMAGES) images test
.PHONY: help base libvirt artifact-setup $(IMAGES) images test
SHELL:=/bin/bash
.ONESHELL:
@ -12,10 +12,17 @@ SHELL:=/bin/bash
help: ## This help.
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z0-9_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
images: artifact-setup
images: base
images: libvirt
images: $(IMAGES) ## Build the containers.
artifact-setup:
docker build --tag $(DOCKER_REGISTRY)/$(DOCKER_IMAGE_PREFIX)/aiap-artifact-setup:$(DOCKER_IMAGE_TAG) --build-arg AIRSHIPCTL_REF=$(AIRSHIPCTL_REF) ./artifact-setup
ifeq (true, $(PUBLISH))
docker push $(DOCKER_REGISTRY)/$(DOCKER_IMAGE_PREFIX)/aiap-artifact-setup:$(DOCKER_IMAGE_TAG)
endif
base:
docker build --tag $(DOCKER_REGISTRY)/$(DOCKER_IMAGE_PREFIX)/aiap-base:$(DOCKER_IMAGE_TAG) --build-arg BASE_IMAGE=ubuntu:20.04 ./base
ifeq (true, $(PUBLISH))

49
tools/airship-in-a-pod/scripts/01_dns_settings.sh

@ -0,0 +1,49 @@
#!/usr/bin/env bash
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -ex
# Running K8s pods in zuul can cause a lot of issues with resolving domains
# inside of the running pods and catching coredns in a loop. This aims to
# tackle a few issues that have been hit to resolve this.
NAMESERVER="1.0.0.1"
# Grab the real nameservers instead of the local one listed in the original
sudo ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf
# Add the known good DNS server
sudo sed -i "1i\nameserver $NAMESERVER\n" /etc/resolv.conf
# Remove DNS servers pointing to localhost so coredns doesn't get caught in a loop
sudo sed -i '/127\.0/d' /etc/resolv.conf
# Spit out the nameservers for the logs
cat /etc/resolv.conf
# Running unbound server can cause issues with coredns, disabling
if [[ -f "/etc/unbound/unbound.pid" ]]; then
sudo kill "$(cat /etc/unbound/unbound.pid)"
fi
# flush iptables so coredns doesn't get caught up
# be sure to stop docker if it is installed
dpkg -l | grep -i docker | head -1 | if [[ "$(cut -d ' ' -f 1)" == "ii" ]]; then
sudo systemctl stop docker
fi
sudo iptables --flush
sudo iptables -tnat --flush
dpkg -l | grep -i docker | head -1 | if [[ "$(cut -d ' ' -f 1)" == "ii" ]]; then
sudo systemctl start docker
fi

35
tools/airship-in-a-pod/scripts/10_install_minikube.sh

@ -0,0 +1,35 @@
#!/usr/bin/env bash
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -ex
# Installs minikube and other dependencies required for the scripts to run
MINIKUBE_VERSION="latest"
install_pkg(){
for i in "$@"; do
dpkg -l "$i" 2> /dev/null | grep ^ii > /dev/null || sudo DEBIAN_FRONTEND=noninteractive -E apt -y install "$i"
done
}
# Grab usefull packages needed for minikube and other scripts
install_pkg curl conntrack make docker.io jq
curl -Lo minikube "https://storage.googleapis.com/minikube/releases/$MINIKUBE_VERSION/minikube-linux-amd64" \
&& chmod +x minikube
sudo mkdir -p /usr/local/bin/
sudo install minikube /usr/local/bin/

77
tools/airship-in-a-pod/scripts/11_build_images.sh

@ -0,0 +1,77 @@
#!/usr/bin/env bash
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -ex
# Builds all of the images under tools/airship-in-a-pod and then configures
# the AIAP pod to never pull down images so it is sure to use the built
# images instead. This also requires a few other images to be pulled.
AIRSHIPCTL_REF=${AIRSHIPCTL_REF:-"master"}
export AIRSHIPCTL_REF
# Images that are required by airship-in-a-pod but not built
PULL_LIST="docker:stable-dind nginx quay.io/metal3-io/sushy-tools quay.io/airshipit/libvirt:aiap-v1"
pushd tools/airship-in-a-pod/ || exit
make -e images artifact-setup base infra-builder runner libvirt
for IMAGE in $PULL_LIST; do
docker pull "$IMAGE"
done
# Now that we have built/pulled the images, lets change the imagePullPolicy to
# Never to be 100% confident they are used
echo "- op: add
path: \"/spec/containers/0/imagePullPolicy\"
value: Never
- op: add
path: \"/spec/containers/1/imagePullPolicy\"
value: Never
- op: add
path: \"/spec/containers/2/imagePullPolicy\"
value: Never
- op: add
path: \"/spec/containers/3/imagePullPolicy\"
value: Never
- op: add
path: \"/spec/containers/4/imagePullPolicy\"
value: Never
- op: add
path: \"/spec/containers/5/imagePullPolicy\"
value: Never
- op: add
path: \"/spec/containers/6/imagePullPolicy\"
value: Never
" >> examples/airshipctl/patchset.yaml
# Also add the patchset to the environment variables
# while being sure to escape the slashes from the ref
echo "- op: replace
path: \"/spec/containers/4/env/6/value\"
value: $AIRSHIPCTL_REF
" >> examples/airshipctl/patchset.yaml
popd || exit

43
tools/airship-in-a-pod/scripts/12_start_minikube.sh

@ -0,0 +1,43 @@
#!/usr/bin/env bash
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -ex
# This script starts up minikube, and accounts for an issue that sometimes
# comes up when running minikube for the first time in some environments
set +e
sudo -E minikube start --driver=none
status=$?
sudo chown -R "$USER" "$HOME"/.minikube; chmod -R u+wrx "$HOME"/.minikube
if [[ $status -gt 0 ]]; then
# Sometimes minikube fails to start if the directory permissions are not correct
sudo -E minikube delete
set -e
sudo -E minikube start --driver=none
fi
set -e
sudo -E minikube status
# Ensure .kube and .minikube have proper ownership
sudo chown -R "$USER" "$HOME"/.kube "$HOME"/.minikube
# Make a copy of the kubeconfig for the log playbooks
mkdir -p "$HOME"/.airship
cp "$HOME"/.kube/config "$HOME"/.airship/kubeconfig
# Give cluster a chance to start up
sleep 10

32
tools/airship-in-a-pod/scripts/13_apply_dns.sh

@ -0,0 +1,32 @@
#!/usr/bin/env bash
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -ex
# This utilizes some of the work that was done to the nameservers in
# 01_dns_settings.sh to take those DNS servers and force the coredns pod
# of the minikube cluster to use those instead of the default.
# Grab a list of the nameservers IPs in /etc/resolv.conf
NAMESERVERS=$(grep nameserver /etc/resolv.conf | awk '{print $2}' | tr '\n' ' ')
kubectl -n kube-system get pods -o wide
# Configure coredns with an upstream DNS to ensure the pod can resolve
# domains outside of the cluster
kubectl -n kube-system get cm -o yaml coredns | sed "s/\/etc\/resolv\.conf/$NAMESERVERS/" > tools/airship-in-a-pod/coredns-upstream-dns.yaml
cat tools/airship-in-a-pod/coredns-upstream-dns.yaml
kubectl apply -f tools/airship-in-a-pod/coredns-upstream-dns.yaml
kubectl rollout restart -n kube-system deployment/coredns

64
tools/airship-in-a-pod/scripts/20_apply_aiap.sh

@ -0,0 +1,64 @@
#!/usr/bin/env bash
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -ex
# This script first loops to check if the K8s cluster is ready before applying
# airship-in-a-pod to it. Once all of the kube-system pods are ready, it applies
# the yaml and then checks every so often to determine if all of the containers
# are ready.
INTERVAL=15
READY=false
KUBE_READY=false
# Wait for the Kubernetes environment to become completely ready
while [[ $KUBE_READY == "false" ]];
do
# Grab the readiness from the kubectl output
kube_pods=$(kubectl get pods -n kube-system | tail -n +2 | awk '{print $2}')
for POD in $kube_pods; do
# Compare the two values to determine if each pod is completely ready
kube_ready_pod=$(echo "$POD" | cut -f1 -d/)
kube_ready_total=$(echo "$POD" | cut -f2 -d/)
if [[ "$kube_ready_pod" != "$kube_ready_total" ]]; then
# If a pod is not ready yet, break out and try again next time
KUBE_READY=false
break
fi
# This will only stay "true" as long as the previous 'if' is never reached
KUBE_READY=true
done
sleep "$INTERVAL"
done
kustomize build tools/airship-in-a-pod/examples/airshipctl | kubectl apply -f -
while [[ $READY == "false" ]];
do
# Grab the number of ready containers from the kubectl output
kubectl get pod airship-in-a-pod -o wide
readiness=$(kubectl get pods | grep airship-in-a-pod | awk '{print $2}')
ready_pod=$(echo "$readiness" | cut -f1 -d/)
ready_total=$(echo "$readiness" | cut -f2 -d/)
# if it is 7/7 ready (example with 7 containers), then mark ready
if [[ "$ready_pod" == "$ready_total" ]]; then
READY=true
fi
sleep "$INTERVAL"
done

41
zuul.d/jobs.yaml

@ -140,6 +140,47 @@
- airship-airshipctl-gather-configs
- describe-kubernetes-objects
- airship-gather-pod-logs
- job:
name: airship-airship-in-a-pod-script-runner
attempts: 1
timeout: 9000
pre-run:
- playbooks/airship-airshipctl-deploy-docker.yaml
post-run: playbooks/airship-collect-logs.yaml
run: playbooks/airshipctl-gate-runner.yaml
nodeset: airship-airshipctl-single-16GB-bionic-node
irrelevant-files: *noncodefiles
dependencies:
- name: openstack-tox-docs
soft: true
- name: airship-airshipctl-lint
soft: true
- name: airship-airshipctl-golint
soft: true
- name: airship-airshipctl-unit
soft: true
- name: airship-airshipctl-build-image
soft: true
vars:
site_name: test-site
gate_scripts:
- ./tools/deployment/01_install_kubectl.sh
- ./tools/install_kustomize.sh
- ./tools/airship-in-a-pod/scripts/01_dns_settings.sh
- ./tools/airship-in-a-pod/scripts/10_install_minikube.sh
- ./tools/airship-in-a-pod/scripts/11_build_images.sh
- ./tools/airship-in-a-pod/scripts/12_start_minikube.sh
- ./tools/airship-in-a-pod/scripts/13_apply_dns.sh
- ./tools/airship-in-a-pod/scripts/20_apply_aiap.sh
serve_dir: /srv/images
serve_port: 8099
log_roles:
- gather-system-logs
- airship-gather-runtime-logs
- airship-airshipctl-gather-configs
- describe-kubernetes-objects
- airship-gather-pod-logs
voting: false
- job:
name: airship-airshipctl-gate-script-runner-docker
attempts: 1

2
zuul.d/projects.yaml

@ -41,6 +41,7 @@
# - airship-airshipctl-functional-existing-k8s TODO: Enable this when functional tests exist, and a cluster is up
- airship-airshipctl-gate-script-runner-docker
- airship-aiap-build-image
- airship-airship-in-a-pod-script-runner
experimental:
jobs:
- airship-airshipctl-docker-kubebench-conformance
@ -55,6 +56,7 @@
- airship-airshipctl-validate-site-docs
- airship-airshipctl-gate-script-runner
- airship-aiap-build-image
- airship-airship-in-a-pod-script-runner
# - airship-airshipctl-functional-existing-k8s TODO: Enable this when functional tests exist, and a cluster is up
post:

Loading…
Cancel
Save