From a8884314292a99827da08f92d31a9014559cf3db Mon Sep 17 00:00:00 2001
From: Monty Taylor <mordred@inaugust.com>
Date: Sat, 6 Jun 2020 11:14:56 -0500
Subject: [PATCH] Run zuul-executor using docker

It looks like we don't actually have an issue with executor and AFS.

Change-Id: I9df29b328daad907c631c4a102cf2168dfef815a
---
 .../roles/zuul-executor/defaults/main.yaml    |   1 +
 .../zuul-executor/files/docker-compose.yaml   |   5 +-
 .../zuul-executor/files/zuul-executor.init    | 122 ------------------
 playbooks/roles/zuul-executor/tasks/main.yaml | 121 +++++------------
 .../roles/zuul-executor/tasks/start.yaml      |  10 +-
 playbooks/roles/zuul-executor/tasks/stop.yaml |  12 +-
 .../zuul-executor/vars/Ubuntu.focal.yaml      |   2 -
 .../roles/zuul-executor/vars/default.yaml     |  15 ---
 playbooks/roles/zuul-executor/vars/main.yaml  |  31 -----
 .../group_vars/zuul-executor.yaml.j2          |   1 +
 10 files changed, 47 insertions(+), 273 deletions(-)
 create mode 100644 playbooks/roles/zuul-executor/defaults/main.yaml
 delete mode 100644 playbooks/roles/zuul-executor/files/zuul-executor.init

diff --git a/playbooks/roles/zuul-executor/defaults/main.yaml b/playbooks/roles/zuul-executor/defaults/main.yaml
new file mode 100644
index 0000000000..1186ea715a
--- /dev/null
+++ b/playbooks/roles/zuul-executor/defaults/main.yaml
@@ -0,0 +1 @@
+zuul_executor_start: false
diff --git a/playbooks/roles/zuul-executor/files/docker-compose.yaml b/playbooks/roles/zuul-executor/files/docker-compose.yaml
index 15df22da8c..4e3d9e830d 100644
--- a/playbooks/roles/zuul-executor/files/docker-compose.yaml
+++ b/playbooks/roles/zuul-executor/files/docker-compose.yaml
@@ -8,6 +8,9 @@ services:
     image: docker.io/zuul/zuul-executor:latest
     network_mode: host
     user: zuul
+    privileged: true
+    stop_signal: SIGHUP
+    stop_grace_period: 3h10m
     volumes:
       - /etc/zuul:/etc/zuul
       - /opt/project-config:/opt/project-config
@@ -15,5 +18,5 @@ services:
       - /home/zuuld:/home/zuul
       - /var/lib/zuul:/var/lib/zuul
       - /var/log/zuul:/var/log/zuul
-      - /etc/openafs:/etc/openafs
+      - /etc/openafs:/etc/openafs:ro
       - /etc/ssl/certs:/etc/ssl/certs
diff --git a/playbooks/roles/zuul-executor/files/zuul-executor.init b/playbooks/roles/zuul-executor/files/zuul-executor.init
deleted file mode 100644
index d0cbe95c38..0000000000
--- a/playbooks/roles/zuul-executor/files/zuul-executor.init
+++ /dev/null
@@ -1,122 +0,0 @@
-#! /bin/sh
-### BEGIN INIT INFO
-# Provides:          zuul-executor
-# Required-Start:    $remote_fs $syslog
-# Required-Stop:     $remote_fs $syslog
-# Default-Start:     2 3 4 5
-# Default-Stop:      0 1 6
-# Short-Description: Zuul
-# Description:       Zuul Executor
-### END INIT INFO
-
-# Do NOT "set -e"
-
-# PATH should only include /usr/* if it runs after the mountnfs.sh script
-PATH=/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/bin
-DESC="Zuul Executor"
-NAME=zuul-executor
-DAEMON=/usr/local/bin/zuul-executor
-PIDFILE=/var/run/$NAME/$NAME.pid
-SCRIPTNAME=/etc/init.d/$NAME
-USER=zuuld
-
-# Exit if the package is not installed
-[ -x "$DAEMON" ] || exit 0
-
-# Read configuration variable file if it is present
-[ -r /etc/default/$NAME ] && . /etc/default/$NAME
-
-# Load the VERBOSE setting and other rcS variables
-. /lib/init/vars.sh
-
-# Define LSB log_* functions.
-# Depend on lsb-base (>= 3.0-6) to ensure that this file is present.
-. /lib/lsb/init-functions
-
-PIDFILE_DIR=$(dirname $PIDFILE)
-
-#
-# Function that starts the daemon/service
-#
-do_start()
-{
-    # Return
-    #   0 if daemon has been started
-    #   1 if daemon was already running
-    #   2 if daemon could not be started
-    #   3 if pid file already exist
-
-    if [ ! -d "$PIDFILE_DIR" ] ; then
-        mkdir -p $PIDFILE_DIR
-        chown $USER $PIDFILE_DIR
-    fi
-    ulimit -n 8192
-    ulimit -c unlimited
-    if [ -f $PIDFILE ]; then
-        return 3
-    fi
-    start-stop-daemon \
-        --start --quiet --pidfile $PIDFILE -c $USER \
-        --exec $DAEMON --test > /dev/null || return 1
-    start-stop-daemon \
-        --start --quiet --pidfile $PIDFILE -c $USER \
-        --exec $DAEMON -- $DAEMON_ARGS || return 2
-    # Add code here, if necessary, that waits for the process to be ready
-    # to handle requests from services started subsequently which depend
-    # on this one.  As a last resort, sleep for some time.
-}
-
-#
-# Function that stops the daemon/service
-#
-do_stop()
-{
-    $DAEMON stop
-    return 0
-}
-
-#
-# Function that sends a SIGHUP to the daemon/service
-#
-do_reload() {
-    $DAEMON reconfigure
-    return 0
-}
-
-case "$1" in
-    start)
-        [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME"
-        do_start
-        case "$?" in
-            0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;;
-            2) [ "$VERBOSE" != no ] && log_end_msg 1 ;;
-            3) echo "Pidfile at $PIDFILE already exists, run service zuul-executor stop to clean up."
-        esac
-        ;;
-    stop)
-        [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME"
-        do_stop
-        case "$?" in
-            0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;;
-            2) [ "$VERBOSE" != no ] && log_end_msg 1 ;;
-        esac
-        ;;
-    status)
-       status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $?
-       ;;
-    reload|force-reload)
-        #
-        # If do_reload() is not implemented then leave this commented out
-        # and leave 'force-reload' as an alias for 'restart'.
-        #
-        log_daemon_msg "Reloading $DESC" "$NAME"
-        do_reload
-        log_end_msg $?
-        ;;
-    *)
-        echo "Usage: $SCRIPTNAME {start|stop|status|force-reload}" >&2
-        exit 3
-        ;;
-esac
-
-:
diff --git a/playbooks/roles/zuul-executor/tasks/main.yaml b/playbooks/roles/zuul-executor/tasks/main.yaml
index dcc294009b..395d5f8bd7 100644
--- a/playbooks/roles/zuul-executor/tasks/main.yaml
+++ b/playbooks/roles/zuul-executor/tasks/main.yaml
@@ -15,81 +15,6 @@
     repo_content: "{{ item.content }}"
   loop: '{{ zuul_executor_ppas }}'
 
-- name: Atomic for focal
-  when: ansible_distribution_version is version('20.04', '>=')
-  include_role:
-    name: install-apt-repo
-  vars:
-    repo_name: projectatomic
-    repo_key: "{{ libcontainers_apt_key }}"
-    repo_content: "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_{{ ansible_lsb.release }}/ /"
-
-- name: Install bindep
-  pip:
-    name: bindep
-    state: present
-    executable: pip3
-  become: yes
-
-- name: Install extra packages
-  package:
-    name: '{{ item }}'
-    state: present
-  loop:
-    - bubblewrap
-    - skopeo
-    - socat
-
-- name: Install distro-specific executor extra packages
-  package:
-    name: '{{ item }}'
-    state: present
-  loop: '{{ zuul_executor_extra_packages }}'
-
-- name: Clone zuul repo
-  git:
-    repo: https://opendev.org/zuul/zuul
-    dest: /opt/zuul
-    force: yes
-  register: zuul_repo
-
-- name: Install zuul bindep packages
-  shell:
-    cmd: apt-get install -y $(bindep -b compile)
-    chdir: /opt/zuul
-  when: zuul_repo is changed
-
-- name: Install zuul
-  shell:
-    cmd: pip3 install .
-    chdir: /opt/zuul
-  when: zuul_repo is changed
-
-- name: Run zuul-manage-ansible
-  shell:
-    cmd: zuul-manage-ansible
-  environment:
-    ANSIBLE_EXTRA_PACKAGES: gear
-  when: zuul_repo is changed
-
-- name: Install kubectl
-  include_role:
-    name: install-kubectl
-
-# This checks the current installed ara version with pip list and the
-# latest version of ara on pypi with pip search and if they are different
-# then we know we need to upgrade to reconcile the local version with
-# the upstream version.
-#
-# We do this using this check here rather than a pip package resource so
-# that ara's deps don't inadverdently update zuuls deps (specifically
-# ansible).
-- name: Install ARA safely
-  shell: |
-    if test $(pip3 list --format columns | sed -ne 's/^ara\s\+\([.0-9]\+\)\s\+$/\1/p') != $(pip3 search 'ara$' | sed -ne 's/^ara (\(.*\)).*$/\1/p') ; then
-      pip3 install --upgrade --upgrade-strategy=only-if-needed "ara<1.0.0"
-    fi
-
 - name: Write key for talking to nodepool nodes
   copy:
     content: '{{ nodepool_test_node_ssh_private_key_contents }}'
@@ -133,19 +58,41 @@
   vars:
     logrotate_file_name: /var/log/zuul/executor-debug.log
 
-- name: Install defaults file
-  template:
-    src: zuul-executor.defaults.j2
-    dest: /etc/default/zuul-executor
+- name: Make docker-compose directory
+  file:
+    state: directory
+    path: /etc/zuul-executor
 
-- name: Install init script
+- name: Install docker-compose file
   copy:
-    src: zuul-executor.init
-    dest: /etc/init.d/zuul-executor
-    mode: 0555
-  register: install_init_script
+    src: docker-compose.yaml
+    dest: /etc/zuul-executor/docker-compose.yaml
 
-- name: Register script with systemd
+- name: Run docker-compose pull
   shell:
-    cmd: /bin/systemctl daemon-reload
-  when: install_init_script is changed
+    cmd: docker-compose pull
+    chdir: /etc/zuul-executor
+
+- name: Start containers
+  include_tasks: start.yaml
+  when: zuul_executor_start | bool
+
+# We can prune here as it should leave the "latest" tagged images
+# as well as the currently running images.
+- name: Run docker prune to cleanup unneeded images
+  shell:
+    cmd: docker image prune -f
+
+- name: Disable old service
+  service:
+    name: zuul-executor
+    enabled: no
+  failed_when: false
+
+- name: Remove old init script files
+  file:
+    state: absent
+    path: '{{ item }}'
+  loop:
+    - /etc/default/zuul-executor
+    - /etc/init.d/zuul-executor
diff --git a/playbooks/roles/zuul-executor/tasks/start.yaml b/playbooks/roles/zuul-executor/tasks/start.yaml
index a6519f0d51..791e15dd4c 100644
--- a/playbooks/roles/zuul-executor/tasks/start.yaml
+++ b/playbooks/roles/zuul-executor/tasks/start.yaml
@@ -1,6 +1,4 @@
-- name: Start Zuul Executor
-  service:
-    name: zuul-executor
-    state: started
-  become: true
-  become_user: root
+- name: Run docker-compose up
+  shell:
+    cmd: docker-compose up -d
+    chdir: /etc/zuul-executor
diff --git a/playbooks/roles/zuul-executor/tasks/stop.yaml b/playbooks/roles/zuul-executor/tasks/stop.yaml
index 7916800895..a5acbf04e6 100644
--- a/playbooks/roles/zuul-executor/tasks/stop.yaml
+++ b/playbooks/roles/zuul-executor/tasks/stop.yaml
@@ -1,12 +1,6 @@
 - name: Stop Zuul Executor
-  service:
-    name: zuul-executor
-    state: stopped
+  shell:
+    cmd: docker-compose down
+    chdir: /etc/zuul-executor
   become: true
   become_user: root
-
-- name: Wait for (3h10m) to stop Zuul Executor
-  wait_for:
-    path: /var/run/zuul/executor.pid
-    state: absent
-    timeout: 11400
diff --git a/playbooks/roles/zuul-executor/vars/Ubuntu.focal.yaml b/playbooks/roles/zuul-executor/vars/Ubuntu.focal.yaml
index e29c34859f..ba50cdc59a 100644
--- a/playbooks/roles/zuul-executor/vars/Ubuntu.focal.yaml
+++ b/playbooks/roles/zuul-executor/vars/Ubuntu.focal.yaml
@@ -4,5 +4,3 @@ zuul_executor_ppas:
   - name: openafs
     content: "deb http://ppa.launchpad.net/openstack-ci-core/openafs/ubuntu focal main"
     key: "{{ openstack_ci_core_ppa_key }}"
-
-zuul_executor_extra_packages: []
diff --git a/playbooks/roles/zuul-executor/vars/default.yaml b/playbooks/roles/zuul-executor/vars/default.yaml
index ac6be2a677..37ff5c66b4 100644
--- a/playbooks/roles/zuul-executor/vars/default.yaml
+++ b/playbooks/roles/zuul-executor/vars/default.yaml
@@ -1,21 +1,6 @@
 zuul_executor_ppas:
-  # For bubblewrap
-  - name: bubblewrap
-    content: "deb http://ppa.launchpad.net/openstack-ci-core/bubblewrap/ubuntu xenial main"
-    key: "{{ openstack_ci_core_ppa_key }}"
-  # Temporary PPA needed for bpo-27945 while waiting for SRU to be published
-  - name: python-bpo-27945-backport
-    content: "deb http://ppa.launchpad.net/openstack-ci-core/python-bpo-27945-backport/ubuntu xenial main"
-    key: "{{ openstack_ci_core_ppa_key }}"
   # We use later HWE kernels for better memory managment, requiring an
   # updated AFS version which we install from our custom ppa.
   - name: openafs
     content: "deb http://ppa.launchpad.net/openstack-ci-core/openafs-amd64-hwe/ubuntu xenial main"
     key: "{{ openstack_ci_core_ppa_key }}"
-  # For skopeo
-  - name: projectatomic
-    content: "deb http://ppa.launchpad.net/projectatomic/ppa/ubuntu xenial main"
-    key: "{{ projectatomic_ppa_key }}"
-
-zuul_executor_extra_packages:
-  - libjemalloc1
diff --git a/playbooks/roles/zuul-executor/vars/main.yaml b/playbooks/roles/zuul-executor/vars/main.yaml
index f0023b35be..4beafbd218 100644
--- a/playbooks/roles/zuul-executor/vars/main.yaml
+++ b/playbooks/roles/zuul-executor/vars/main.yaml
@@ -28,34 +28,3 @@ openstack_ci_core_ppa_key: |
   5pX70jigBqjtJ0PZGm7jELb8bB70SVSGsvwHmEz0pSs=
   =cc1L
   -----END PGP PUBLIC KEY BLOCK-----
-
-projectatomic_ppa_key: |
-  -----BEGIN PGP PUBLIC KEY BLOCK-----
-  Version: GnuPG v1
-
-  mQINBFlRJjABEADuE3ZLY/2W++bPsxtcaoi7VaNnkvsXuVYbbHalEh/YwKFVsDTo
-  PQpuw1UlPpmVTwT3ufWfv2v42eZiiWMZaKG9/aWF/TeIdH5+3anfVi+X+tuIW9sv
-  GKTHZdtDqd7fIhtY6AuNQ/D629TJxLvafZ5MoGeyxjsebt5dOvOrl0SHpwR75uPP
-  aCXTWrokhH7W2BbJQUB+47k62BMd03EKe8stz9FzUxptROFJJ2bITijJlDXNfSbV
-  bwCiyREIkzXS6ZdWliJAqencOIZ4UbUax+5BT8SRbSLtr/c4YxvARilpSVCkxo8/
-  EkPHBGygmgfw0kRPSGtLL7IqfWip9mFObji2geoU3A8gV/i3s9Ccc9GPKApX8r7b
-  QFs1tIlgUJKPqVwB2FAh+Xrqlsy/+8r95jL2gfRptSw7u8OP4AySj5WVm7cCEQ69
-  aLyemCsf+v72bFOUXuYQ22Kr3yqz2O/1IsG/0Usr4riTdG65Aq6gnq4KRHMNgXu8
-  7fC9omoy3sKHvzeAJsw/eC9chYNwO8pv8KRIvpDSGL5L7Ems8mq2C5xMyzSVegTr
-  AvXu7nJoZWVBFRluh42bZa9QesX9MzzfOQ+G3085aW8BE++lhtX5QOkfRd74E49H
-  1I2piAq/aE8P9jUHr60Po1C1Tw9iXeEaULLKut8eTMLkQ/02DXhBfq0I5QARAQAB
-  tCBMYXVuY2hwYWQgUFBBIGZvciBQcm9qZWN0IEF0b21pY4kCOAQTAQIAIgUCWVEm
-  MAIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQi+zxY3rYx50HLw/5Ad6k
-  EHf2uT4owvzu393S/bUR6VVwCWYMbg14XgphxnoOfrHZWUjbrETTURyd1UexoHt7
-  ZDtMCVmzeY0jpvMb1W3WDebFVo+wR4CI15sPjyycsOxWTviD743wxaPCL1s009co
-  CzWg5AgP88B0D353Y39meC07BBgOJgIfk1OkFdeRjqHfAtucT99NrCuKr/bbBwDn
-  0E+wWaJoIbQvBzsPIFzMWWQ6RcnrZtyQv35epo+VBmW3VEIkorv1VoStF0RjvJM+
-  cMW/ogZsIEZk0IUREOtrtTKUXVrMw1hZ9IGYZRpbJ2g670UGuNjW/vo3rRCRSDaF
-  6Txp5Pn6ZLTgQWsWMw/6M6ooFIEpz3rhYmQSJLNmUN6SgKeWGVmOrQlg4f7YM75o
-  UEw56GKQWl9FAthO0qH0qF1OMfUKp/Tv2OSV/FNZsokf6alWXOB6Bzj6gYmmGXIv
-  MfFW5fZ1cuu5/0ULDckxWhVQ1ywLHREEoBQ6oKYONwUjSdWcM+VsKCEFeCqsNwak
-  qweP8C0fooycfiEZuncc/9ZujgkQ2p7xXTlv3t2SPF9h43xHs3515VS/OTJPGW59
-  98AqllpfqGxggYs5cwi2LO3xwvHyPoTqj3hcl1dRMspZINRsIo4VC8bSrCOqbjDc
-  CD2WFOo2c4mwTDmJpz0PLK87ev/WZ8K0OEflTfc=
-  =jPWv
-  -----END PGP PUBLIC KEY BLOCK-----
diff --git a/playbooks/zuul/templates/group_vars/zuul-executor.yaml.j2 b/playbooks/zuul/templates/group_vars/zuul-executor.yaml.j2
index 0bfb9160b5..0d5a5b4ce4 100644
--- a/playbooks/zuul/templates/group_vars/zuul-executor.yaml.j2
+++ b/playbooks/zuul/templates/group_vars/zuul-executor.yaml.j2
@@ -1,3 +1,4 @@
+zuul_executor_start: true
 nodepool_test_node_ssh_private_key_contents: |
   -----BEGIN OPENSSH PRIVATE KEY-----
   b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABFwAAAAdzc2gtcn