Run zuul-launcher

This doesn't do anything substantial yet, but we'd like to start running
the server soon in order to test it out.

Change-Id: I9eb2bccd6e5e9a064cbaff10676aeb1af6653f98

doc/source/zuul.rst

@@ -16,6 +16,7 @@ At a Glance
   * zuul*.opendev.org
   * ze*.opendev.org
   * zm*.opendev.org
+  * zl*.opendev.org
 :Configuration:
   * :config:`zuul/main.yaml`
   * :config:`zuul.d`

hiera/common.yaml

@@ -76,6 +76,7 @@ cacti_hosts:
 - zk04.opendev.org
 - zk05.opendev.org
 - zk06.opendev.org
+- zl01.opendev.org
 - zm01.opendev.org
 - zm02.opendev.org
 - zm03.opendev.org

inventory/service/groups.yaml

@@ -183,10 +183,13 @@ groups:
     - zuul-lb[0-9]*.opendev.org
   zuul:
     - ze[0-9]*.opendev.org
+    - zl[0-9]*.opendev.org
     - zm[0-9]*.opendev.org
     - zuul[0-9]*.opendev.org
   zuul-executor:
     - ze[0-9]*.opendev.org
+  zuul-launcher:
+    - zl[0-9]*.opendev.org
   zuul-merger:
     - zm[0-9]*.opendev.org
   zuul-preview:

playbooks/roles/zuul-launcher/README.rst

@@ -0,0 +1 @@
+Run zuul launcher

playbooks/roles/zuul-launcher/defaults/main.yaml

@@ -0,0 +1 @@
+zuul_launcher_start: false

playbooks/roles/zuul-launcher/files/docker-compose.yaml

@@ -0,0 +1,15 @@
+# Version 2 is the latest that is supported by docker-compose in
+# Ubuntu Xenial.
+version: '2'
+
+services:
+  launcher:
+    restart: on-failure
+    image: quay.io/zuul-ci/zuul-launcher:latest
+    network_mode: host
+    user: zuul
+    volumes:
+      - /etc/zuul:/etc/zuul
+      - /home/zuuld:/home/zuul
+      - /var/lib/zuul:/var/lib/zuul
+      - /var/log/zuul:/var/log/zuul

playbooks/roles/zuul-launcher/files/logging.conf

@@ -0,0 +1,50 @@
+[loggers]
+keys=root,zuul,gerrit,gear
+
+[handlers]
+keys=console,debug,normal
+
+[formatters]
+keys=simple
+
+[logger_root]
+level=WARNING
+handlers=console
+
+[logger_zuul]
+level=DEBUG
+handlers=debug,normal
+qualname=zuul
+
+[logger_gerrit]
+level=INFO
+handlers=debug,normal
+qualname=gerrit
+
+[logger_gear]
+level=WARNING
+handlers=debug,normal
+qualname=gear
+
+[handler_console]
+level=WARNING
+class=StreamHandler
+formatter=simple
+args=(sys.stdout,)
+
+[handler_debug]
+level=DEBUG
+class=logging.handlers.WatchedFileHandler
+formatter=simple
+args=('/var/log/zuul/launcher-debug.log',)
+
+[handler_normal]
+level=INFO
+class=logging.handlers.WatchedFileHandler
+formatter=simple
+args=('/var/log/zuul/launcher.log',)
+
+[formatter_simple]
+format=%(asctime)s %(levelname)s %(name)s: %(message)s
+datefmt=
+class=zuul.lib.logutil.MultiLineFormatter

playbooks/roles/zuul-launcher/tasks/graceful.yaml

@@ -0,0 +1,42 @@
+- name: Check if Zuul launcher containers are running
+  # It is possible they are stopped due to some external circumstance.
+  # NOTE: docker-compose ps -q reports exited containers unlike docker ps -q
+  command:
+    cmd: docker-compose ps -q
+    chdir: /etc/zuul-launcher
+  become: true
+  become_user: root
+  register: launcher_container_list
+- name: Gracefully stop Zuul Launcher
+  shell:
+    cmd: docker-compose exec -T launcher zuul-launcher stop
+    chdir: /etc/zuul-launcher
+  become: true
+  become_user: root
+  # Only run the docker exec command if a container is running
+  when: launcher_container_list.stdout_lines | length > 0
+  register: zl_graceful
+  failed_when:
+    - zl_graceful.rc != 0
+    # There is a fun race with docker exec and shutting down the
+    # container running the exec. If the container is stopped while
+    # the exec is running then the command in the exec effectively gets
+    # kill -9'd and the docker exec command rc is 137. Since this
+    # should only happen when the container is stopped we proceed with
+    # the overall graceful shutdown.
+    - zl_graceful.rc != 137
+    # If the exec fails because the container is not running we continue.
+    - "'No container found' not in zl_graceful.stderr"
+- name: Wait for Zuul Launcher to stop
+  shell:
+    cmd: docker-compose ps -q | xargs docker wait
+    chdir: /etc/zuul-launcher
+  become: true
+  become_user: root
+  when: launcher_container_list.stdout_lines | length > 0
+- name: Down Zuul Launcher containers
+  shell:
+    cmd: docker-compose down
+    chdir: /etc/zuul-launcher
+  become: true
+  become_user: root

playbooks/roles/zuul-launcher/tasks/main.yaml

@@ -0,0 +1,39 @@
+- name: Install logging config
+  copy:
+    src: logging.conf
+    dest: /etc/zuul/launcher-logging.conf
+
+- name: Rotate launcher logs
+  include_role:
+    name: logrotate
+  vars:
+    logrotate_file_name: /var/log/zuul/launcher.log
+
+- name: Rotate launcher debug logs
+  include_role:
+    name: logrotate
+  vars:
+    logrotate_file_name: /var/log/zuul/launcher-debug.log
+
+- name: Make docker-compose directory
+  file:
+    state: directory
+    path: /etc/zuul-launcher
+
+- name: Install docker-compose file
+  copy:
+    src: docker-compose.yaml
+    dest: /etc/zuul-launcher/docker-compose.yaml
+
+- name: Update container images
+  include_tasks: pull.yaml
+
+- name: Start containers
+  include_tasks: start.yaml
+  when: zuul_launcher_start | bool
+
+# We can prune here as it should leave the "latest" tagged images
+# as well as the currently running images.
+- name: Run docker prune to cleanup unneeded images
+  shell:
+    cmd: docker image prune -f

playbooks/roles/zuul-launcher/tasks/pull.yaml

@@ -0,0 +1,4 @@
+- name: Run docker-compose pull
+  shell:
+    cmd: docker-compose pull
+    chdir: /etc/zuul-launcher

playbooks/roles/zuul-launcher/tasks/start.yaml

@@ -0,0 +1,4 @@
+- name: Run docker-compose up
+  shell:
+    cmd: docker-compose up -d
+    chdir: /etc/zuul-launcher

playbooks/roles/zuul-launcher/tasks/stop.yaml

@@ -0,0 +1,6 @@
+- name: Stop Zuul Launcher
+  shell:
+    cmd: docker-compose down
+    chdir: /etc/zuul-launcher
+  become: true
+  become_user: root

playbooks/service-zuul.yaml

@@ -15,6 +15,11 @@
     - install-docker
     - zuul
 
+- hosts: "zuul-launcher:!disabled"
+  name: "Configure zuul launcher"
+  roles:
+    - zuul-launcher
+
 - hosts: "zuul-merger:!disabled"
   name: "Configure zuul merger"
   roles:

playbooks/zuul/run-base.yaml

@@ -136,6 +136,7 @@
         - group_vars/zuul-lb.yaml
         - group_vars/zuul.yaml
         - group_vars/zuul-executor.yaml
+        - group_vars/zuul-launcher.yaml
         - group_vars/zuul-merger.yaml
         - group_vars/zuul-scheduler.yaml
         - group_vars/zuul-web.yaml

playbooks/zuul/templates/group_vars/zuul-launcher.yaml.j2

@@ -0,0 +1 @@
+zuul_launcher_start: true

playbooks/zuul_pull.yaml

@@ -16,6 +16,12 @@
         name: zuul-merger
         tasks_from: pull
 
+- hosts: 'zuul-launcher:!disabled'
+  tasks:
+    - include_role:
+        name: zuul-launcher
+        tasks_from: pull
+
 - hosts: 'zuul-executor:!disabled'
   tasks:
     - include_role:

playbooks/zuul_reboot.yaml

@@ -54,6 +54,30 @@
         name: zuul-merger
         tasks_from: start
 
+- hosts: "zuul-launcher:!disabled"
+  name: "Reboot zuul-launchers gracefully one at a time"
+  serial: 1
+  tasks:
+    - name: Gracefully stop the launcher
+      include_role:
+        name: zuul-launcher
+        tasks_from: graceful
+    - name: Upgrade launcher server packages
+      apt:
+        update_cache: yes
+        upgrade: yes
+      register: apt_action
+      # 20 minute wait for unattended-upgrades to complete
+      delay: 30
+      retries: 40
+      until: apt_action is success or 'Failed to lock apt for exclusive operation' not in apt_action.msg
+    - name: Reboot the launcher server
+      reboot:
+    - name: Start the launcher
+      include_role:
+        name: zuul-launcher
+        tasks_from: start
+
 # TODO should we do both schedulers with reboots then do the webs without
 # reboots?
 - hosts: "zuul-scheduler:!disabled"

playbooks/zuul_start.yaml

@@ -18,6 +18,12 @@
         name: zuul-merger
         tasks_from: start
 
+- hosts: 'zuul-launcher:!disabled'
+  tasks:
+    - include_role:
+        name: zuul-launcher
+        tasks_from: start
+
 - hosts: 'zuul-executor:!disabled'
   tasks:
     - include_role:

playbooks/zuul_stop.yaml

@@ -19,6 +19,12 @@
         name: zuul-merger
         tasks_from: stop
 
+- hosts: 'zuul-launcher:!disabled'
+  tasks:
+    - include_role:
+        name: zuul-launcher
+        tasks_from: stop
+
 - hosts: 'zuul-executor:!disabled'
   tasks:
     - include_role:

testinfra/test_zuul_launcher.py

@@ -0,0 +1,25 @@
+# Copyright 2018 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import util
+
+testinfra_hosts = ['zl01.opendev.org']
+
+
+def test_iptables(host):
+    rules = util.verify_iptables(host)
+
+    for rule in rules:
+        assert '--dport 79' not in rule
+        assert '--dport 7900' not in rule

zuul.d/system-config-run.yaml

@@ -962,6 +962,8 @@
           label: ubuntu-focal
         - name: zm01.opendev.org
           label: ubuntu-jammy
+        - name: zl01.opendev.org
+          label: ubuntu-jammy
         - name: ze01.opendev.org
           label: ubuntu-jammy
         - name: zuul02.opendev.org
@@ -990,6 +992,11 @@
           '/etc/hosts': logs
           '/etc/zuul/zuul.conf': logs
           '/var/log/zuul/merger-debug.log': logs
+      zl01.opendev.org:
+        host_copy_output:
+          '/etc/hosts': logs
+          '/etc/zuul/zuul.conf': logs
+          '/var/log/zuul/launcher-debug.log': logs
       ze01.opendev.org:
         host_copy_output:
           '/etc/hosts': logs
@@ -1032,6 +1039,7 @@
       - testinfra/test_zuul_executor.py
       - testinfra/test_zuul_scheduler.py
       - testinfra/test_zuul_merger.py
+      - testinfra/test_zuul_launcher.py
       - testinfra/test_zuul_db.py
       - testinfra/util.py