Browse Source
This change aims to add the airship-in-a-pod test to the gates. This is accomplished by creating new scripts to run which install a minimal k8s environment, and then apply the airship-in-a-pod.yaml. Brief description of scripts 01_dns_settings: Running k8s pods in zuul has a lot of issues with DNS resolution which this resolves. 10_install_minikube: grabs latest minikube and other dependencies. 11_build_images: builds the images under tools/airship-in-a-pod and then adds patches to the kustomize configuration to use and configure them. 12_start_minikube: starts minikube and makes sure kubectl will work 13_apply_dns.sh: This takes some of the effort from 01_dns_settings and incorporates it into the minikube cluster by modifying coredns 20_apply_aiap: Applies the airship-in-a-pod.yaml to the cluster. Has a few loops to check that the cluster is fully running before applying, and to check on the pod to see if it is ready before finishing. Closes: #478 Change-Id: I4c02d04b0eda9395642032bf56b56a0f823c1969changes/42/787442/76
Michael Beaver
1 year ago
10 changed files with 353 additions and 2 deletions
@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env bash |
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
set -ex |
||||
|
||||
# Running K8s pods in zuul can cause a lot of issues with resolving domains |
||||
# inside of the running pods and catching coredns in a loop. This aims to |
||||
# tackle a few issues that have been hit to resolve this. |
||||
|
||||
|
||||
NAMESERVER="1.0.0.1" |
||||
|
||||
# Grab the real nameservers instead of the local one listed in the original |
||||
sudo ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf |
||||
|
||||
# Add the known good DNS server |
||||
sudo sed -i "1i\nameserver $NAMESERVER\n" /etc/resolv.conf |
||||
# Remove DNS servers pointing to localhost so coredns doesn't get caught in a loop |
||||
sudo sed -i '/127\.0/d' /etc/resolv.conf |
||||
# Spit out the nameservers for the logs |
||||
cat /etc/resolv.conf |
||||
|
||||
# Running unbound server can cause issues with coredns, disabling |
||||
if [[ -f "/etc/unbound/unbound.pid" ]]; then |
||||
sudo kill "$(cat /etc/unbound/unbound.pid)" |
||||
fi |
||||
|
||||
# flush iptables so coredns doesn't get caught up |
||||
# be sure to stop docker if it is installed |
||||
dpkg -l | grep -i docker | head -1 | if [[ "$(cut -d ' ' -f 1)" == "ii" ]]; then |
||||
sudo systemctl stop docker |
||||
fi |
||||
sudo iptables --flush |
||||
sudo iptables -tnat --flush |
||||
|
||||
dpkg -l | grep -i docker | head -1 | if [[ "$(cut -d ' ' -f 1)" == "ii" ]]; then |
||||
sudo systemctl start docker |
||||
fi |
||||
@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env bash |
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
set -ex |
||||
|
||||
# Installs minikube and other dependencies required for the scripts to run |
||||
|
||||
|
||||
MINIKUBE_VERSION="latest" |
||||
|
||||
install_pkg(){ |
||||
for i in "$@"; do |
||||
dpkg -l "$i" 2> /dev/null | grep ^ii > /dev/null || sudo DEBIAN_FRONTEND=noninteractive -E apt -y install "$i" |
||||
done |
||||
} |
||||
|
||||
# Grab usefull packages needed for minikube and other scripts |
||||
install_pkg curl conntrack make docker.io jq |
||||
|
||||
curl -Lo minikube "https://storage.googleapis.com/minikube/releases/$MINIKUBE_VERSION/minikube-linux-amd64" \ |
||||
&& chmod +x minikube |
||||
|
||||
sudo mkdir -p /usr/local/bin/ |
||||
sudo install minikube /usr/local/bin/ |
||||
@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env bash |
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
set -ex |
||||
|
||||
# Builds all of the images under tools/airship-in-a-pod and then configures |
||||
# the AIAP pod to never pull down images so it is sure to use the built |
||||
# images instead. This also requires a few other images to be pulled. |
||||
|
||||
|
||||
AIRSHIPCTL_REF=${AIRSHIPCTL_REF:-"master"} |
||||
export AIRSHIPCTL_REF |
||||
# Images that are required by airship-in-a-pod but not built |
||||
PULL_LIST="docker:stable-dind nginx quay.io/metal3-io/sushy-tools quay.io/airshipit/libvirt:aiap-v1" |
||||
|
||||
|
||||
pushd tools/airship-in-a-pod/ || exit |
||||
|
||||
make -e images artifact-setup base infra-builder runner libvirt |
||||
|
||||
for IMAGE in $PULL_LIST; do |
||||
docker pull "$IMAGE" |
||||
done |
||||
|
||||
# Now that we have built/pulled the images, lets change the imagePullPolicy to |
||||
# Never to be 100% confident they are used |
||||
echo "- op: add |
||||
path: \"/spec/containers/0/imagePullPolicy\" |
||||
value: Never |
||||
|
||||
- op: add |
||||
path: \"/spec/containers/1/imagePullPolicy\" |
||||
value: Never |
||||
|
||||
- op: add |
||||
path: \"/spec/containers/2/imagePullPolicy\" |
||||
value: Never |
||||
|
||||
- op: add |
||||
path: \"/spec/containers/3/imagePullPolicy\" |
||||
value: Never |
||||
|
||||
- op: add |
||||
path: \"/spec/containers/4/imagePullPolicy\" |
||||
value: Never |
||||
|
||||
- op: add |
||||
path: \"/spec/containers/5/imagePullPolicy\" |
||||
value: Never |
||||
|
||||
- op: add |
||||
path: \"/spec/containers/6/imagePullPolicy\" |
||||
value: Never |
||||
|
||||
" >> examples/airshipctl/patchset.yaml |
||||
|
||||
# Also add the patchset to the environment variables |
||||
# while being sure to escape the slashes from the ref |
||||
echo "- op: replace |
||||
path: \"/spec/containers/4/env/6/value\" |
||||
value: $AIRSHIPCTL_REF |
||||
|
||||
" >> examples/airshipctl/patchset.yaml |
||||
|
||||
|
||||
popd || exit |
||||
@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env bash |
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
set -ex |
||||
|
||||
# This script starts up minikube, and accounts for an issue that sometimes |
||||
# comes up when running minikube for the first time in some environments |
||||
|
||||
|
||||
set +e |
||||
sudo -E minikube start --driver=none |
||||
status=$? |
||||
sudo chown -R "$USER" "$HOME"/.minikube; chmod -R u+wrx "$HOME"/.minikube |
||||
if [[ $status -gt 0 ]]; then |
||||
# Sometimes minikube fails to start if the directory permissions are not correct |
||||
sudo -E minikube delete |
||||
set -e |
||||
sudo -E minikube start --driver=none |
||||
fi |
||||
|
||||
set -e |
||||
sudo -E minikube status |
||||
|
||||
# Ensure .kube and .minikube have proper ownership |
||||
sudo chown -R "$USER" "$HOME"/.kube "$HOME"/.minikube |
||||
|
||||
# Make a copy of the kubeconfig for the log playbooks |
||||
mkdir -p "$HOME"/.airship |
||||
cp "$HOME"/.kube/config "$HOME"/.airship/kubeconfig |
||||
|
||||
# Give cluster a chance to start up |
||||
sleep 10 |
||||
@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env bash |
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
set -ex |
||||
|
||||
# This utilizes some of the work that was done to the nameservers in |
||||
# 01_dns_settings.sh to take those DNS servers and force the coredns pod |
||||
# of the minikube cluster to use those instead of the default. |
||||
|
||||
|
||||
# Grab a list of the nameservers IPs in /etc/resolv.conf |
||||
NAMESERVERS=$(grep nameserver /etc/resolv.conf | awk '{print $2}' | tr '\n' ' ') |
||||
|
||||
|
||||
kubectl -n kube-system get pods -o wide |
||||
# Configure coredns with an upstream DNS to ensure the pod can resolve |
||||
# domains outside of the cluster |
||||
kubectl -n kube-system get cm -o yaml coredns | sed "s/\/etc\/resolv\.conf/$NAMESERVERS/" > tools/airship-in-a-pod/coredns-upstream-dns.yaml |
||||
cat tools/airship-in-a-pod/coredns-upstream-dns.yaml |
||||
kubectl apply -f tools/airship-in-a-pod/coredns-upstream-dns.yaml |
||||
kubectl rollout restart -n kube-system deployment/coredns |
||||
@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env bash |
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
set -ex |
||||
|
||||
# This script first loops to check if the K8s cluster is ready before applying |
||||
# airship-in-a-pod to it. Once all of the kube-system pods are ready, it applies |
||||
# the yaml and then checks every so often to determine if all of the containers |
||||
# are ready. |
||||
|
||||
|
||||
INTERVAL=15 |
||||
READY=false |
||||
KUBE_READY=false |
||||
|
||||
|
||||
# Wait for the Kubernetes environment to become completely ready |
||||
while [[ $KUBE_READY == "false" ]]; |
||||
do |
||||
# Grab the readiness from the kubectl output |
||||
kube_pods=$(kubectl get pods -n kube-system | tail -n +2 | awk '{print $2}') |
||||
for POD in $kube_pods; do |
||||
# Compare the two values to determine if each pod is completely ready |
||||
kube_ready_pod=$(echo "$POD" | cut -f1 -d/) |
||||
kube_ready_total=$(echo "$POD" | cut -f2 -d/) |
||||
if [[ "$kube_ready_pod" != "$kube_ready_total" ]]; then |
||||
# If a pod is not ready yet, break out and try again next time |
||||
KUBE_READY=false |
||||
break |
||||
fi |
||||
# This will only stay "true" as long as the previous 'if' is never reached |
||||
KUBE_READY=true |
||||
done |
||||
sleep "$INTERVAL" |
||||
done |
||||
|
||||
kustomize build tools/airship-in-a-pod/examples/airshipctl | kubectl apply -f - |
||||
|
||||
while [[ $READY == "false" ]]; |
||||
do |
||||
# Grab the number of ready containers from the kubectl output |
||||
kubectl get pod airship-in-a-pod -o wide |
||||
readiness=$(kubectl get pods | grep airship-in-a-pod | awk '{print $2}') |
||||
ready_pod=$(echo "$readiness" | cut -f1 -d/) |
||||
ready_total=$(echo "$readiness" | cut -f2 -d/) |
||||
# if it is 7/7 ready (example with 7 containers), then mark ready |
||||
if [[ "$ready_pod" == "$ready_total" ]]; then |
||||
READY=true |
||||
fi |
||||
|
||||
sleep "$INTERVAL" |
||||
done |
||||
|
||||
Loading…
Reference in new issue