From 3b7cf3dbaa8b95904cfb8e7eca91cf3325e178e0 Mon Sep 17 00:00:00 2001
From: Clark Boylan <clark.boylan@gmail.com>
Date: Tue, 5 Oct 2021 10:25:38 -0700
Subject: [PATCH] Quickstart wait for certs in quickstart docker-compose.yaml

Zookeeper, Zuul, and Nodepool services need the zk certs to be present
before starting. Without this they cannot communicate with each other
with TLS which is required. Docker-compose doesn't do a strict startup
ordering waiting for each service to be ready. It just ensures processes
begin in the right order.

Fix this with a new wait script and override container commands to run
the wait script as necessary to ensure certs are present before we
begin.

Change-Id: I8179159ae7d6a15155066549dfe245607646d433
---
 doc/source/examples/docker-compose.yaml       | 23 +++++++++++--------
 .../examples/playbooks/wait-to-start-certs.sh | 19 +++++++++++++++
 2 files changed, 32 insertions(+), 10 deletions(-)
 create mode 100755 doc/source/examples/playbooks/wait-to-start-certs.sh

diff --git a/doc/source/examples/docker-compose.yaml b/doc/source/examples/docker-compose.yaml
index 1123a577a0..ba628f1a2e 100644
--- a/doc/source/examples/docker-compose.yaml
+++ b/doc/source/examples/docker-compose.yaml
@@ -31,16 +31,10 @@ services:
     image: zookeeper
     hostname: examples_zk_1.examples_default
     volumes:
+      - "./playbooks/:/var/playbooks/:z"
       - "certs:/var/certs:z"
       - "./zoo.cfg:/conf/zoo.cfg:z"
-    # introduced for 3.7.0: zookeeper shall wait for certificates to be available
-    # examples_zk_1.examples_default.pem is the last file created by ./tools/zk-ca.sh
-    command: |
-      /bin/sh -c '\
-        while [ ! -f /var/certs/keystores/examples_zk_1.examples_default.pem ] ; do \
-          sleep 1; \
-        done; \
-      zkServer.sh start-foreground'
+    command: "sh -c '/var/playbooks/wait-to-start-certs.sh && zkServer.sh start-foreground'"
   mysql:
     image: mariadb
     environment:
@@ -60,7 +54,10 @@ services:
       - https_proxy
       - no_proxy=${no_proxy},gerrit
       - ZUUL_MYSQL_PASSWORD=secret
-    command: "sh -c '/var/playbooks/wait-to-start.sh && zuul-scheduler -f'"
+    command: |
+      sh -c '/var/playbooks/wait-to-start-certs.sh && \
+             /var/playbooks/wait-to-start.sh && \
+             zuul-scheduler -f'
     # FIXME: The scheduler has no ansible anymore so use the executor image.
     #        This needs to be changes such that ansible is not required for startup.
     image: zuul/zuul-scheduler
@@ -70,7 +67,10 @@ services:
       - "sshkey:/var/ssh:z"
       - "certs:/var/certs:z"
   web:
-    command: "sh -c '/var/playbooks/wait-to-start-gearman.sh && zuul-web -f'"
+    command: |
+      sh -c '/var/playbooks/wait-to-start-certs.sh && \
+             /var/playbooks/wait-to-start-gearman.sh && \
+             zuul-web -f'
     depends_on:
       - scheduler
       - mysql
@@ -99,6 +99,7 @@ services:
       - "sshkey:/var/ssh:z"
       - "logs:/srv/static/logs:z"
       - "certs:/var/certs:z"
+    command: "sh -c '/var/playbooks/wait-to-start-certs.sh && zuul-executor -f'"
   node:
     build:
       dockerfile: node-Dockerfile
@@ -114,10 +115,12 @@ services:
       - zk
     image: zuul/nodepool-launcher
     volumes:
+      - "./playbooks/:/var/playbooks/:z"
       - "./etc_nodepool/:/etc/nodepool/:z"
       - "certs:/var/certs:z"
     ports:
       - "8022:8022"
+    command: "sh -c '/var/playbooks/wait-to-start-certs.sh && nodepool-launcher -f'"
   logs:
     build:
       dockerfile: logs-Dockerfile
diff --git a/doc/source/examples/playbooks/wait-to-start-certs.sh b/doc/source/examples/playbooks/wait-to-start-certs.sh
new file mode 100755
index 0000000000..e761b8766c
--- /dev/null
+++ b/doc/source/examples/playbooks/wait-to-start-certs.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# Zuul needs ssl certs to be present to talk to zookeeper before it
+# starts.
+
+wait_for_certs() {
+    echo `date -Iseconds` "Wait for certs to be present"
+    for i in $(seq 1 120); do
+        # Introduced for 3.7.0: zookeeper shall wait for certificates to be available
+        # examples_zk_1.examples_default.pem is the last file created by ./tools/zk-ca.sh
+        [ -f /var/certs/keystores/examples_zk_1.examples_default.pem ] && return
+        sleep 1
+    done;
+
+    echo `date -Iseconds` "Timeout waiting for certs"
+    exit 1
+}
+
+wait_for_certs