AIAP: Cause AIAP to fail faster

Currently, when a container in AIAP fails, the container continues to
run in the backgroud. This causes the container to be marked as failed
so that the deployment will fail faster.

Closes: #626

Change-Id: I34de366390b6ddc0ef434f7e642e73797a790edf

tools/airship-in-a-pod/artifact-setup/assets/entrypoint.sh

@@ -14,7 +14,19 @@
 
 set -ex
 
-function cloneRepo(){
+# Create the "canary" file, indicating that the container is healthy
+mkdir -p /tmp/healthy
+touch /tmp/healthy/artifact-setup
+
+success=false
+function cleanup() {
+  if [[ "$success" == "false" ]]; then
+    rm /tmp/healthy/artifact-setup
+  fi
+}
+trap cleanup EXIT
+
+function cloneRepo() {
   repo_dir=$1
   repo_url=$2
   repo_ref=$3
@@ -44,4 +56,5 @@ else
   cp "$(command -v airshipctl)" bin
 fi
 
+success=true
 /signal_complete artifact-setup

tools/airship-in-a-pod/examples/base/airship-in-a-pod.yaml

@@ -121,9 +121,7 @@ spec:
           --interface 10.23.25.1 \
           --port 8443 \
           --ssl-key /airship_gate_redfish_auth.key \
-          --ssl-certificate /airship_gate_redfish_auth.pem || true
+          --ssl-certificate /airship_gate_redfish_auth.pem
-
-        tail -f /dev/null
     volumeMounts:
       - name: var-run-libvirt
         mountPath: /var/run/libvirt
@@ -200,15 +198,21 @@ spec:
     command:
       - bash
       - -cex
-      - |
+      - /entrypoint.sh
-        /entrypoint.sh || true
-        tail -f /dev/null
     readinessProbe:
       exec:
         command:
         - test
         - -e
         - /tmp/completed/artifact-setup
+    livenessProbe:
+      exec:
+        command:
+        - test
+        - -e
+        - /tmp/healthy/artifact-setup
+      initialDelaySeconds: 5
+      periodSeconds: 5
     env:
       - name: CACHE_DIR
         value: /opt/aiap-cache
@@ -249,15 +253,21 @@ spec:
     command:
       - bash
       - -cex
-      - |
+      - /entrypoint.sh
-        /entrypoint.sh || true
-        tail -f /dev/null
     readinessProbe:
       exec:
         command:
         - test
         - -e
         - /tmp/completed/infra-builder
+    livenessProbe:
+      exec:
+        command:
+        - test
+        - -e
+        - /tmp/healthy/infra-builder
+      initialDelaySeconds: 5
+      periodSeconds: 5
     env:
       - name: CACHE_DIR
         value: /opt/aiap-cache
@@ -298,9 +308,7 @@ spec:
     command:
       - bash
       - -cex
-      - |
+      - /entrypoint.sh
-        /entrypoint.sh || true
-        tail -f /dev/null
     readinessProbe:
       exec:
         command:
@@ -309,6 +317,14 @@ spec:
         - /tmp/completed/runner
       initialDelaySeconds: 600
       periodSeconds: 30
+    livenessProbe:
+      exec:
+        command:
+        - test
+        - -e
+        - /tmp/healthy/runner
+      initialDelaySeconds: 5
+      periodSeconds: 5
     env:
       - name: CACHE_DIR
         value: /opt/aiap-cache

tools/airship-in-a-pod/infra-builder/assets/entrypoint.sh

@@ -14,10 +14,23 @@
 
 set -ex
 
+# Create the "canary" file, indicating that the container is healthy
+mkdir -p /tmp/healthy
+touch /tmp/healthy/infra-builder
+
+success=false
+function cleanup() {
+  if [[ "$success" == "false" ]]; then
+    rm /tmp/healthy/infra-builder
+  fi
+}
+trap cleanup EXIT
+
 printf "Waiting 30 seconds for the libvirt and docker services to be ready\n"
 sleep 30
 
 ansible-playbook -v /opt/ansible/playbooks/build-infra.yaml \
   -e local_src_dir="$(pwd)"
 
+success=true
 /signal_complete infra-builder

tools/airship-in-a-pod/runner/assets/entrypoint.sh

@@ -14,6 +14,18 @@
 
 set -ex
 
+# Create the "canary" file, indicating that the container is healthy
+mkdir -p /tmp/healthy
+touch /tmp/healthy/runner
+
+success=false
+function cleanup() {
+  if [[ "$success" == "false" ]]; then
+    rm /tmp/healthy/runner
+  fi
+}
+trap cleanup EXIT
+
 # Wait until artifact-setup and libvirt infrastructure has been built
 /wait_for artifact-setup
 /wait_for infra-builder
@@ -67,4 +79,5 @@ fi
 
 ./tools/deployment/25_deploy_gating.sh
 
+success=true
 /signal_complete runner