Add zuul02 to inventory

This zuul02 instance will replace zuul01. There are a few items to
coordinate when doing an actual switch so we haven't removed zuul01 from
inventory here. In particular we need to update gearman server config
values in the zuul cluster and we need to save queues, shutdown zuul01,
then start zuul02's scheduler and restore queues there.

I believe landing this change is safe as we don't appear to start zuul
on new instances by default. Reviewers should double check this.

Depends-On: https://review.opendev.org/c/opendev/zone-opendev.org/+/791039
Change-Id: I524b456e494124d8293fbe8e1468de40f3800772

hiera/common.yaml

@@ -323,6 +323,7 @@ cacti_hosts:
 - zm07.opendev.org
 - zm08.opendev.org
 - zuul01.openstack.org
+- zuul02.opendev.org
 statusbot_auth_nicks:
 - jeblair
 - corvus

inventory/base/hosts.yaml

@@ -892,6 +892,13 @@ all:
         region_name: DFW
       public_v4: 104.130.246.196
       public_v6: 2001:4800:7819:103:be76:4eff:fe04:5870
+    zuul02.opendev.org:
+      ansible_host: 104.130.246.31
+      location:
+        cloud: openstackci-rax
+        region_name: DFW
+      public_v4: 104.130.246.31
+      public_v6: 2001:4800:7819:103:be76:4eff:fe04:42c2
     zp01.opendev.org:
       ansible_host: 174.143.130.226
       location:

inventory/service/group_vars/zuul.yaml

@@ -6,6 +6,8 @@ zuul_known_hosts: |
   [review.opendev.org]:29418,[review.openstack.org]:29418,[104.130.246.32]:29418,[2001:4800:7819:103:be76:4eff:fe04:9229]:29418 {{ gerrit_ssh_rsa_pubkey_contents }}
   [git.opendaylight.org]:29418,[52.35.122.251]:29418,[2600:1f14:421:f500:7b21:2a58:ab0a:2d17]:29418 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAyRXyHEw/P1iZr/fFFzbodT5orVV/ftnNRW59Zh9rnSY5Rmbc9aygsZHdtiWBERVVv8atrJSdZool75AglPDDYtPICUGWLR91YBSDcZwReh5S9es1dlQ6fyWTnv9QggSZ98KTQEuE3t/b5SfH0T6tXWmrNydv4J2/mejKRRLU2+oumbeVN1yB+8Uau/3w9/K5F5LgsDDzLkW35djLhPV8r0OfmxV/cAnLl7AaZlaqcJMA+2rGKqM3m3Yu+pQw4pxOfCSpejlAwL6c8tA9naOvBkuJk+hYpg5tDEq2QFGRX5y1F9xQpwpdzZROc5hdGYntM79VMMXTj+95dwVv/8yTsw==
 gearman_server: zuul01.openstack.org
+# TODO this swap needs to happen as part of the server swap
+#gearman_server: zuul02.opendev.org
 zuul_connections:
   - name: 'smtp'
     driver: 'smtp'

inventory/service/host_vars/zuul02.opendev.org.yaml

@@ -0,0 +1,10 @@
+gearman_server: 127.0.0.1
+letsencrypt_certs:
+  zuul02-opendev-main:
+    - zuul.opendev.org
+    - zuul.openstack.org
+    - zuul02.opendev.org
+borg_backup_excludes_extra:
+  - /var/lib/zuul/backup/*
+  - /var/lib/zuul/times/*
+  - /var/log/zuul/*debug*

playbooks/roles/letsencrypt-create-certs/handlers/main.yaml

@@ -26,6 +26,9 @@
 - name: letsencrypt updated zuul-opendev-main
   include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
 
+- name: letsencrypt updated zuul02-opendev-main
+  include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
+
 - name: letsencrypt updated logs-main
   include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
 

testinfra/test_zuul_scheduler.py

@@ -15,7 +15,7 @@
 import util
 import socket
 
-testinfra_hosts = ['zuul01.openstack.org']
+testinfra_hosts = ['zuul02.opendev.org']
 
 
 def test_iptables(host):

zuul.d/infra-prod.yaml

@@ -357,6 +357,7 @@
       - inventory/service/group_vars/zookeeper.yaml
       - inventory/service/host_vars/zk\d+
       - inventory/service/host_vars/zuul01.openstack.org
+      - inventory/service/host_vars/zuul02.opendev.org
       - playbooks/roles/install-docker/
       - playbooks/roles/iptables/
       - playbooks/roles/zookeeper/

zuul.d/system-config-run.yaml

@@ -746,8 +746,8 @@
           label: ubuntu-focal
         - name: ze01.opendev.org
           label: ubuntu-focal
-        - name: zuul01.openstack.org
-          label: ubuntu-xenial
+        - name: zuul02.opendev.org
+          label: ubuntu-focal
     required-projects:
       - openstack/project-config
       - opendev/system-config
@@ -769,7 +769,7 @@
           '/etc/hosts': logs
           '/etc/zuul/zuul.conf': logs
           '/var/log/zuul/executor-debug.log': logs
-      zuul01.openstack.org:
+      zuul02.opendev.org:
         host_copy_output:
           '/etc/hosts': logs
           '/etc/zuul/zuul.conf': logs
@@ -784,14 +784,14 @@
       - inventory/service/group_vars/zuul
       - inventory/service/group_vars/zookeeper.yaml
       - inventory/service/host_vars/zk\d+
-      - inventory/service/host_vars/zuul01.openstack.org
+      - inventory/service/host_vars/zuul02.opendev.org
       - playbooks/roles/zookeeper/
       - playbooks/roles/install-apt-repo
       - playbooks/roles/zuul
       - playbooks/zuul/templates/group_vars/zuul
       - playbooks/zuul/templates/group_vars/zookeeper.yaml
       - playbooks/zuul/templates/host_vars/zk\d+
-      - playbooks/zuul/templates/host_vars/zuul01.openstack.org
+      - playbooks/zuul/templates/host_vars/zuul02.opendev.org
       - playbooks/zuul_restart.yaml
       - testinfra/test_zuul_executor.py
       - testinfra/test_zuul_scheduler.py