diff --git a/doc/source/zuul.rst b/doc/source/zuul.rst index 4c684220e9..b16eedfb27 100644 --- a/doc/source/zuul.rst +++ b/doc/source/zuul.rst @@ -16,6 +16,7 @@ At a Glance * zuul*.opendev.org * ze*.opendev.org * zm*.opendev.org + * zl*.opendev.org :Configuration: * :config:`zuul/main.yaml` * :config:`zuul.d` diff --git a/hiera/common.yaml b/hiera/common.yaml index d9faaa7b4c..dcf032a9dc 100644 --- a/hiera/common.yaml +++ b/hiera/common.yaml @@ -76,6 +76,7 @@ cacti_hosts: - zk04.opendev.org - zk05.opendev.org - zk06.opendev.org +- zl01.opendev.org - zm01.opendev.org - zm02.opendev.org - zm03.opendev.org diff --git a/inventory/service/groups.yaml b/inventory/service/groups.yaml index f6dc3419fb..f0d09a1364 100644 --- a/inventory/service/groups.yaml +++ b/inventory/service/groups.yaml @@ -183,10 +183,13 @@ groups: - zuul-lb[0-9]*.opendev.org zuul: - ze[0-9]*.opendev.org + - zl[0-9]*.opendev.org - zm[0-9]*.opendev.org - zuul[0-9]*.opendev.org zuul-executor: - ze[0-9]*.opendev.org + zuul-launcher: + - zl[0-9]*.opendev.org zuul-merger: - zm[0-9]*.opendev.org zuul-preview: diff --git a/playbooks/roles/zuul-launcher/README.rst b/playbooks/roles/zuul-launcher/README.rst new file mode 100644 index 0000000000..c32b2b397b --- /dev/null +++ b/playbooks/roles/zuul-launcher/README.rst @@ -0,0 +1 @@ +Run zuul launcher diff --git a/playbooks/roles/zuul-launcher/defaults/main.yaml b/playbooks/roles/zuul-launcher/defaults/main.yaml new file mode 100644 index 0000000000..0b7d4ec732 --- /dev/null +++ b/playbooks/roles/zuul-launcher/defaults/main.yaml @@ -0,0 +1 @@ +zuul_launcher_start: false diff --git a/playbooks/roles/zuul-launcher/files/docker-compose.yaml b/playbooks/roles/zuul-launcher/files/docker-compose.yaml new file mode 100644 index 0000000000..d5e9006ddf --- /dev/null +++ b/playbooks/roles/zuul-launcher/files/docker-compose.yaml @@ -0,0 +1,15 @@ +# Version 2 is the latest that is supported by docker-compose in +# Ubuntu Xenial. +version: '2' + +services: + launcher: + restart: on-failure + image: quay.io/zuul-ci/zuul-launcher:latest + network_mode: host + user: zuul + volumes: + - /etc/zuul:/etc/zuul + - /home/zuuld:/home/zuul + - /var/lib/zuul:/var/lib/zuul + - /var/log/zuul:/var/log/zuul diff --git a/playbooks/roles/zuul-launcher/files/logging.conf b/playbooks/roles/zuul-launcher/files/logging.conf new file mode 100644 index 0000000000..28031a4371 --- /dev/null +++ b/playbooks/roles/zuul-launcher/files/logging.conf @@ -0,0 +1,50 @@ +[loggers] +keys=root,zuul,gerrit,gear + +[handlers] +keys=console,debug,normal + +[formatters] +keys=simple + +[logger_root] +level=WARNING +handlers=console + +[logger_zuul] +level=DEBUG +handlers=debug,normal +qualname=zuul + +[logger_gerrit] +level=INFO +handlers=debug,normal +qualname=gerrit + +[logger_gear] +level=WARNING +handlers=debug,normal +qualname=gear + +[handler_console] +level=WARNING +class=StreamHandler +formatter=simple +args=(sys.stdout,) + +[handler_debug] +level=DEBUG +class=logging.handlers.WatchedFileHandler +formatter=simple +args=('/var/log/zuul/launcher-debug.log',) + +[handler_normal] +level=INFO +class=logging.handlers.WatchedFileHandler +formatter=simple +args=('/var/log/zuul/launcher.log',) + +[formatter_simple] +format=%(asctime)s %(levelname)s %(name)s: %(message)s +datefmt= +class=zuul.lib.logutil.MultiLineFormatter diff --git a/playbooks/roles/zuul-launcher/tasks/graceful.yaml b/playbooks/roles/zuul-launcher/tasks/graceful.yaml new file mode 100644 index 0000000000..db591c03a4 --- /dev/null +++ b/playbooks/roles/zuul-launcher/tasks/graceful.yaml @@ -0,0 +1,42 @@ +- name: Check if Zuul launcher containers are running + # It is possible they are stopped due to some external circumstance. + # NOTE: docker-compose ps -q reports exited containers unlike docker ps -q + command: + cmd: docker-compose ps -q + chdir: /etc/zuul-launcher + become: true + become_user: root + register: launcher_container_list +- name: Gracefully stop Zuul Launcher + shell: + cmd: docker-compose exec -T launcher zuul-launcher stop + chdir: /etc/zuul-launcher + become: true + become_user: root + # Only run the docker exec command if a container is running + when: launcher_container_list.stdout_lines | length > 0 + register: zl_graceful + failed_when: + - zl_graceful.rc != 0 + # There is a fun race with docker exec and shutting down the + # container running the exec. If the container is stopped while + # the exec is running then the command in the exec effectively gets + # kill -9'd and the docker exec command rc is 137. Since this + # should only happen when the container is stopped we proceed with + # the overall graceful shutdown. + - zl_graceful.rc != 137 + # If the exec fails because the container is not running we continue. + - "'No container found' not in zl_graceful.stderr" +- name: Wait for Zuul Launcher to stop + shell: + cmd: docker-compose ps -q | xargs docker wait + chdir: /etc/zuul-launcher + become: true + become_user: root + when: launcher_container_list.stdout_lines | length > 0 +- name: Down Zuul Launcher containers + shell: + cmd: docker-compose down + chdir: /etc/zuul-launcher + become: true + become_user: root diff --git a/playbooks/roles/zuul-launcher/tasks/main.yaml b/playbooks/roles/zuul-launcher/tasks/main.yaml new file mode 100644 index 0000000000..bf20699ca2 --- /dev/null +++ b/playbooks/roles/zuul-launcher/tasks/main.yaml @@ -0,0 +1,39 @@ +- name: Install logging config + copy: + src: logging.conf + dest: /etc/zuul/launcher-logging.conf + +- name: Rotate launcher logs + include_role: + name: logrotate + vars: + logrotate_file_name: /var/log/zuul/launcher.log + +- name: Rotate launcher debug logs + include_role: + name: logrotate + vars: + logrotate_file_name: /var/log/zuul/launcher-debug.log + +- name: Make docker-compose directory + file: + state: directory + path: /etc/zuul-launcher + +- name: Install docker-compose file + copy: + src: docker-compose.yaml + dest: /etc/zuul-launcher/docker-compose.yaml + +- name: Update container images + include_tasks: pull.yaml + +- name: Start containers + include_tasks: start.yaml + when: zuul_launcher_start | bool + +# We can prune here as it should leave the "latest" tagged images +# as well as the currently running images. +- name: Run docker prune to cleanup unneeded images + shell: + cmd: docker image prune -f diff --git a/playbooks/roles/zuul-launcher/tasks/pull.yaml b/playbooks/roles/zuul-launcher/tasks/pull.yaml new file mode 100644 index 0000000000..630a02e88b --- /dev/null +++ b/playbooks/roles/zuul-launcher/tasks/pull.yaml @@ -0,0 +1,4 @@ +- name: Run docker-compose pull + shell: + cmd: docker-compose pull + chdir: /etc/zuul-launcher diff --git a/playbooks/roles/zuul-launcher/tasks/start.yaml b/playbooks/roles/zuul-launcher/tasks/start.yaml new file mode 100644 index 0000000000..593d17c042 --- /dev/null +++ b/playbooks/roles/zuul-launcher/tasks/start.yaml @@ -0,0 +1,4 @@ +- name: Run docker-compose up + shell: + cmd: docker-compose up -d + chdir: /etc/zuul-launcher diff --git a/playbooks/roles/zuul-launcher/tasks/stop.yaml b/playbooks/roles/zuul-launcher/tasks/stop.yaml new file mode 100644 index 0000000000..9e54b666d6 --- /dev/null +++ b/playbooks/roles/zuul-launcher/tasks/stop.yaml @@ -0,0 +1,6 @@ +- name: Stop Zuul Launcher + shell: + cmd: docker-compose down + chdir: /etc/zuul-launcher + become: true + become_user: root diff --git a/playbooks/service-zuul.yaml b/playbooks/service-zuul.yaml index 17fe95fb69..02014d7a20 100644 --- a/playbooks/service-zuul.yaml +++ b/playbooks/service-zuul.yaml @@ -15,6 +15,11 @@ - install-docker - zuul +- hosts: "zuul-launcher:!disabled" + name: "Configure zuul launcher" + roles: + - zuul-launcher + - hosts: "zuul-merger:!disabled" name: "Configure zuul merger" roles: diff --git a/playbooks/zuul/run-base.yaml b/playbooks/zuul/run-base.yaml index cde4585646..554365399a 100644 --- a/playbooks/zuul/run-base.yaml +++ b/playbooks/zuul/run-base.yaml @@ -136,6 +136,7 @@ - group_vars/zuul-lb.yaml - group_vars/zuul.yaml - group_vars/zuul-executor.yaml + - group_vars/zuul-launcher.yaml - group_vars/zuul-merger.yaml - group_vars/zuul-scheduler.yaml - group_vars/zuul-web.yaml diff --git a/playbooks/zuul/templates/group_vars/zuul-launcher.yaml.j2 b/playbooks/zuul/templates/group_vars/zuul-launcher.yaml.j2 new file mode 100644 index 0000000000..004442b760 --- /dev/null +++ b/playbooks/zuul/templates/group_vars/zuul-launcher.yaml.j2 @@ -0,0 +1 @@ +zuul_launcher_start: true diff --git a/playbooks/zuul_pull.yaml b/playbooks/zuul_pull.yaml index a6dc7fc126..381d6e2b06 100644 --- a/playbooks/zuul_pull.yaml +++ b/playbooks/zuul_pull.yaml @@ -16,6 +16,12 @@ name: zuul-merger tasks_from: pull +- hosts: 'zuul-launcher:!disabled' + tasks: + - include_role: + name: zuul-launcher + tasks_from: pull + - hosts: 'zuul-executor:!disabled' tasks: - include_role: diff --git a/playbooks/zuul_reboot.yaml b/playbooks/zuul_reboot.yaml index 198cc3fee5..66a524469e 100644 --- a/playbooks/zuul_reboot.yaml +++ b/playbooks/zuul_reboot.yaml @@ -54,6 +54,30 @@ name: zuul-merger tasks_from: start +- hosts: "zuul-launcher:!disabled" + name: "Reboot zuul-launchers gracefully one at a time" + serial: 1 + tasks: + - name: Gracefully stop the launcher + include_role: + name: zuul-launcher + tasks_from: graceful + - name: Upgrade launcher server packages + apt: + update_cache: yes + upgrade: yes + register: apt_action + # 20 minute wait for unattended-upgrades to complete + delay: 30 + retries: 40 + until: apt_action is success or 'Failed to lock apt for exclusive operation' not in apt_action.msg + - name: Reboot the launcher server + reboot: + - name: Start the launcher + include_role: + name: zuul-launcher + tasks_from: start + # TODO should we do both schedulers with reboots then do the webs without # reboots? - hosts: "zuul-scheduler:!disabled" diff --git a/playbooks/zuul_start.yaml b/playbooks/zuul_start.yaml index a295778b7c..e46cc5b103 100644 --- a/playbooks/zuul_start.yaml +++ b/playbooks/zuul_start.yaml @@ -18,6 +18,12 @@ name: zuul-merger tasks_from: start +- hosts: 'zuul-launcher:!disabled' + tasks: + - include_role: + name: zuul-launcher + tasks_from: start + - hosts: 'zuul-executor:!disabled' tasks: - include_role: diff --git a/playbooks/zuul_stop.yaml b/playbooks/zuul_stop.yaml index 6f53fdaf79..3944344b01 100644 --- a/playbooks/zuul_stop.yaml +++ b/playbooks/zuul_stop.yaml @@ -19,6 +19,12 @@ name: zuul-merger tasks_from: stop +- hosts: 'zuul-launcher:!disabled' + tasks: + - include_role: + name: zuul-launcher + tasks_from: stop + - hosts: 'zuul-executor:!disabled' tasks: - include_role: diff --git a/testinfra/test_zuul_launcher.py b/testinfra/test_zuul_launcher.py new file mode 100644 index 0000000000..491ed22141 --- /dev/null +++ b/testinfra/test_zuul_launcher.py @@ -0,0 +1,25 @@ +# Copyright 2018 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import util + +testinfra_hosts = ['zl01.opendev.org'] + + +def test_iptables(host): + rules = util.verify_iptables(host) + + for rule in rules: + assert '--dport 79' not in rule + assert '--dport 7900' not in rule diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml index e1c8b212aa..3066c49aa7 100644 --- a/zuul.d/system-config-run.yaml +++ b/zuul.d/system-config-run.yaml @@ -962,6 +962,8 @@ label: ubuntu-focal - name: zm01.opendev.org label: ubuntu-jammy + - name: zl01.opendev.org + label: ubuntu-jammy - name: ze01.opendev.org label: ubuntu-jammy - name: zuul02.opendev.org @@ -990,6 +992,11 @@ '/etc/hosts': logs '/etc/zuul/zuul.conf': logs '/var/log/zuul/merger-debug.log': logs + zl01.opendev.org: + host_copy_output: + '/etc/hosts': logs + '/etc/zuul/zuul.conf': logs + '/var/log/zuul/launcher-debug.log': logs ze01.opendev.org: host_copy_output: '/etc/hosts': logs @@ -1032,6 +1039,7 @@ - testinfra/test_zuul_executor.py - testinfra/test_zuul_scheduler.py - testinfra/test_zuul_merger.py + - testinfra/test_zuul_launcher.py - testinfra/test_zuul_db.py - testinfra/util.py