Switch Gerrit replication to a larger RSA key

This change is related to a similar change [0] in gitea that
adds/rotates public keys for the gerrit user in gitea. We should be
happy with the approach on both sides of the gitea and gerrit
replication interaction before proceeding.

This is motivated by changes in gitea that make it more picky about the
keys it will accept by default. Rather than disable those checks we're
switching keys to be more acceptable.

The end result is the use of 4096 bit RSA keys. We did consider ed25519
keys but there is concern that the Gerrit replication plugin may not be
able to handle them as they only come in the new openssh key file
format. The replication plugin docs indicate PEM format should be used
instead. It is possible that new MINA in gerrit handles this fine but we
stick with what we know works to avoid problems.

[0] https://review.opendev.org/c/opendev/system-config/+/901082

Change-Id: I36704b7f8c0710fb5142153f99418eb200860bee

playbooks/roles/gerrit/tasks/main.yaml

@@ -158,9 +158,9 @@
     group: "{{ gerrit_user_name }}"
     mode: 0700
 
-# Private key for gerrit user to connect to other systems,
+# Private RSA A key for gerrit user to connect to other systems,
 # such as for replication.
-- name: Write Gerrit SSH private key
+- name: Write Gerrit SSH private RSA A key
   copy:
     content: "{{ gerrit_replication_ssh_rsa_key_contents }}"
     dest: "{{ gerrit_home_dir }}/.ssh/id_rsa"
@@ -168,7 +168,7 @@
     group: "{{ gerrit_user_name }}"
     mode: 0600
 
-- name: Write Gerrit SSH public key
+- name: Write Gerrit SSH public RSA A key
   copy:
     content: "{{ gerrit_replication_ssh_rsa_pubkey_contents }}"
     dest: "{{ gerrit_home_dir }}/.ssh/id_rsa.pub"
@@ -176,6 +176,32 @@
     group: "{{ gerrit_user_name }}"
     mode: 0644
 
+# Private RSA B key for gerrit user to connect to other systems,
+# such as for replication.
+- name: Write Gerrit SSH private RSA B key
+  copy:
+    content: "{{ gerrit_replication_ssh_rsa_B_key_contents }}"
+    dest: "{{ gerrit_home_dir }}/.ssh/replication_id_rsa_B"
+    owner: "{{ gerrit_user_name }}"
+    group: "{{ gerrit_user_name }}"
+    mode: 0600
+
+- name: Write Gerrit SSH public RSA B key
+  copy:
+    content: "{{ gerrit_replication_ssh_rsa_B_pubkey_contents }}"
+    dest: "{{ gerrit_home_dir }}/.ssh/replication_id_rsa_B.pub"
+    owner: "{{ gerrit_user_name }}"
+    group: "{{ gerrit_user_name }}"
+    mode: 0644
+
+- name: SSH config to select the appropriate key above for replication
+  template:
+    src: gerrit_ssh_config.j2
+    dest: "{{ gerrit_home_dir }}/.ssh/config"
+    owner: "{{ gerrit_user_name }}"
+    group: "{{ gerrit_user_name }}"
+    mode: 0644
+
 # Make the directory even if we don't have creds to make
 # bind mounting in the docker-compose file simple.
 - name: Ensure launchpadlib directory exists

playbooks/roles/gerrit/templates/gerrit_ssh_config.j2

@@ -0,0 +1,4 @@
+Host gitea*.opendev.org:
+  IdentityFile {{ gerrit_home_dir }}/.ssh/replication_id_rsa_B
+  Port 222
+  PreferredAuthentications publickey

playbooks/zuul/templates/host_vars/review99.opendev.org.yaml.j2

@@ -90,6 +90,59 @@ gerrit_replication_ssh_rsa_key_contents: |
   edHQJDKx5PktPWsAAAAgbW9yZHJlZEBNb250eXMtTWFjQm9vay1BaXIubG9jYWwBAgM=
   -----END OPENSSH PRIVATE KEY-----
 gerrit_replication_ssh_rsa_pubkey_contents: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDQhZQ0z+RVPmOzY2f56N9/PrqDeHftvnagPJyOOXnCd/9N0j+stFWNmavvb8y4dRZ+y6lOJpzPYEahwUUXZHAanz5l5as+VihWq7ldcMxSPnmkC9zr65Z8eNDcM2Bzk8gx5e4DE6OgpWkc6ke9MpwI5dmfW7o53gQZkdSc94TuLr+ZCYUKo7fScsVeE+F9dT0PLyW0zU7c23PzYnkKcrB9ihpQfSfbJj9EAtsA3aA8ZdHt78i5r7+0u0JZxaWoKjkCfYqC8ofbTU61YuUO8TTgNgMC6ZzBmTRdRRRKdGun+m1fqtgIqPSi+iZpKnERgg/hPwY+gqcKh+svW6pgCDhJ gerrit-code-review-replication
+gerrit_replication_ssh_rsa_B_key_contents: |
+  -----BEGIN RSA PRIVATE KEY-----
+  MIIJKAIBAAKCAgEA09s+O5KsDuhspPzW9bDMqSI/x4Txe5vcFyYQGBKqin0WXu1K
+  64y9FMMCg/QKfNxKOe3Pt74UepCXo0LSo/LcZQLGbazvspl5Eo0+48YoE73HHw3P
+  L3xZZD5E4ympKcMLkDWocRWvxdQgQ/EmBKkpv8HM1JAtEpB+yuL8cTv8Yj8S3oBm
+  MaNoXN5ODTWRbDYR0CPaSXXmY4+BMf9mwK6K1ZEGpcE6x7dzXf6u+46sdeoJdpW0
+  w24FOGzIgkI+BSb3Vecnv0cd5og9BUBatLicTUHgQzYrz2BS6dtZC/Sn1MPDkTWv
+  kJhP51OYZ6wQDH6CvP3qDn2XLiNZymy8oemfi8XYe/xobE6TA0etcmKdGVAJvhne
+  A498h5jY7yWXfIyyFfsOsPFcJvWHNBPDlLNkRT9y2VQK8xAaDCv1jegq4WyXy4VO
+  hfqGOjeeoNAw+1gpJcZ33dPwJDZHxCMS7HnEuHMIIjZWCfD7WXSbFYc8MHJaT81I
+  L5utfvZPp8lqLqe71JFKwHdca88kZXSYPaapXwAQ1xHLscswH+VYsvqqEmgZYZpQ
+  H37h84e3Qzb8BxDnlj2Xs3NGxLzzpjcm7rvlazDD1wmC1s0n9FWYyv0VEXOCclIp
+  YDqaWZAA9xVMnd+jud2oeEhpAhWcM9HCN71tcO8j6cM2kk1YiR6lTyfw1gcCAwEA
+  AQKCAgBDhyMfhwFb4R7cOhFkj920XYvZ01jLjyMIp+PCYJTGfteWG2nhieMtDnmr
+  SKrdILRyIYivpyFM7fC/o8mTY5J3ifpotBJVKdErJiVxIdTdcgTZs6OiHa86ohSA
+  GePnQVnathfCL+julE5SibeWDbuWeTYKXQhY3gDkN5TCnR21zSf9Dw1D7jOSQnO7
+  hyMazGNCJmNqPe/ZNUE3iBKfASOUrlzhkaVkSme2AruQyGnVTeuFRnOvRU7ZrOb+
+  ihHNv51f3sXPFOKFfFCC73/aEewUPha3JbmyKKBVFUsdYfbq/RlFnEihPMNfV0iB
+  ZxlYeiy/A+pKgyKgnLj+qkk4DMkDBktdZZlNkIaNvoUju8FLPpRWtC0foJcNdgJS
+  Aq5BK72kHGj87kvryrbAyCtIaeQ1srzeoaSZ7qqNoUuxeCYE8gpnr+VrRc/5b+j+
+  R9+hEwhf3m14ZNMAdULeWfcpEKnK16onplkM6IoIksLt5ulPoYVv5sIPrTURDSS0
+  J+LLZA5+lsqMNTZXt37RJHCjMJd3O6w+I+2iMrWWrUzYPZzX3Df0oeVs7/K/9czb
+  dvZkq6Y9adMyHRu8yu/Wjv5ElGrCr7xnOJTT0WqT8WoqviHSBc3Y5J3CRCFxSyEi
+  YnruZuMU7Bue9NXp9o19uV84eiiP/VpHeNTi43mojqKO+YND4QKCAQEA8zFAu2S8
+  FWkwLpfCHlwjvIiwEeZaqGy0NWMcHGNngU1Z19elAFrPH2ik8CUBwJ3m+Fu/ZYqg
+  I0ZbD8o5c08xC9wJlNxz6bRvC1ke5lxVAcbk6RJ3gN4skAuSwouJj6MM0q6Z5c2l
+  d5rYL+RVeZAmbhOxPbbnaZIxZn93A3fy1LCNeqOYmxmRFnTKEehu/Mrrw7FgKsW9
+  wcO+IHAMkfgoSoAr0T0irN0U5VwTLNZ9bQQ+hWNn1kcYMWmhVHQsryRL2coZzFlz
+  /GbtpKd0oDLPUFnzw8JLf0x/NlptYTzF6tPad83qBHLvYvjDKiZJIqXitsDScKeE
+  0GUMHguTFAIo4QKCAQEA3wOD3XPharPeB0xOSIrrAG/8fny9IgY8UJJoqCDvhqf8
+  Xw4Gbejc3MLRjLq8IpebvjttNceGOisMNYoIcnAdIK23e2jPVBcPzuoA44CIR7ir
+  oemYnYCA8D61u5CPELMbKMcywayb3x/e9DeVqMldXvF/U59xhCNswqTJMXWom3zT
+  AYk18bzC78DS0VIzyebJIRAiXyrjXzqlhBX+LfS3dX/bPdIB+BGBcmYN94h4Zy8o
+  PjeRdOohiPCB42Frwqge/AGA1ZtNn6ZP4k978fPPynh65grKUiXaig1peK7HlGu6
+  OetOtjc/VK4in3j1Tz7eNy7Lkr7y0R4cU1ODLV1T5wKCAQBtoX50++xuGoVF+9Pe
+  q9rQWy5EY3vrAVYb2xoJEibO+3fM/cG8bzOADUSNnaE0m/pLa9DUjbGzNTxH2foc
+  KU8K8Z7AJMF8UYLdssdjQaxwqKD5EQIebgnYxd7bJNxWjEJzl5J5LkOxr3RV4rFF
+  o94vMWFtWM7poKX0dvHH9oLZrt2Ys7dP9C6b2PpfKFEgVLoD9ipMHeh1OTC0ns6L
+  3zsKms0l/lFrbB7HZsKeK/NO+eLVbwKYbmRRojTARb7/FXW8MIeAv7KxzhTDbVn9
+  /enHZ0WksiomsO2IKyuz8hmmyuppp8IfT1DrZQlWLvw5Sl7x0+sKLfqJl4Pm54De
+  PDsBAoIBAQCgGR3pNO92cnnKM3Vfjpr2TW6uP05nxqI2FWUcjchmmuIKOz9SWAF2
+  WkWlCclV7BDamD7mhL5Ps+en59f4j5PZidxWs/9jFss6d7L7n6I2GtTb/56YM1Bd
+  KCe+5yBNlMbCl35Qm2Gq5G5iVCUUbrqhFi2aErSjb+r8MOBeqWDJfurcB2y6hhBL
+  ndm6e5DCOPPa0IJcX6WrD6cTE9bNlwi9SXRTBRh0xdxwC+Oq+EW3jZsOT0YU8J/y
+  dvZIDgAWVisoLswWjM9E9VgT14vbPnTFnYhc7RIhtxsUUFyPTqnoWw3t1odDOJY2
+  bGxen687nJ5abzWlu38FsOAU0bcyMfWxAoIBAGHBqhAZlhJvQPLCpf44NYnirbxH
+  fpHjIdZo2OgHG8zppYPZLUBTlwc3z+tw5gjq99mbmjmtKwCmaftbMRdnvbgosfPq
+  Hk9DJeb4PEgzXWxemV91ShXVe/2N3L+xHMLjw9LyUm5pV78ew2Wp0gBuxUm0eYAu
+  oIRAQez/Att/bjV1hZBJa/xQddla61ZH5BSRh5VBgnLr8rLPzEk51HJSKggNXVXo
+  Qr0sgoks9cGQE5fj2a8v+iGAPeyKqiRAMg4ufcieeFl0OxhX8gmt03ltET2+LBA2
+  kZradknMgpElfrDIKEp/3ekxTnhSCaerQ1avmBZMSawhDkDGG3udmui2AnI=
+  -----END RSA PRIVATE KEY-----
+gerrit_replication_ssh_rsa_B_pubkey_contents: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDT2z47kqwO6Gyk/Nb1sMypIj/HhPF7m9wXJhAYEqqKfRZe7UrrjL0UwwKD9Ap83Eo57c+3vhR6kJejQtKj8txlAsZtrO+ymXkSjT7jxigTvccfDc8vfFlkPkTjKakpwwuQNahxFa/F1CBD8SYEqSm/wczUkC0SkH7K4vxxO/xiPxLegGYxo2hc3k4NNZFsNhHQI9pJdeZjj4Ex/2bArorVkQalwTrHt3Nd/q77jqx16gl2lbTDbgU4bMiCQj4FJvdV5ye/Rx3miD0FQFq0uJxNQeBDNivPYFLp21kL9KfUw8ORNa+QmE/nU5hnrBAMfoK8/eoOfZcuI1nKbLyh6Z+Lxdh7/GhsTpMDR61yYp0ZUAm+Gd4Dj3yHmNjvJZd8jLIV+w6w8Vwm9Yc0E8OUs2RFP3LZVArzEBoMK/WN6CrhbJfLhU6F+oY6N56g0DD7WCklxnfd0/AkNkfEIxLsecS4cwgiNlYJ8PtZdJsVhzwwclpPzUgvm61+9k+nyWoup7vUkUrAd1xrzyRldJg9pqlfABDXEcuxyzAf5Viy+qoSaBlhmlAffuHzh7dDNvwHEOeWPZezc0bEvPOmNybuu+VrMMPXCYLWzSf0VZjK/RURc4JyUilgOppZkAD3FUyd36O53ah4SGkCFZwz0cI3vW1w7yPpwzaSTViJHqVPJ/DWBw== testgerrit@review99-20231130"
 gerrit_reviewdb_mariadb_password: password
 gerrit_run_compose_up: true
 gerrit_run_init: true