Improve gerrit known_hosts management

Previously we were only managing root's known_hosts via ansible but even
then this wasn't happening because the gerrit_self_hostkey var wasn't
set anywhere. On top of that we need to manage multiple known_hosts
because gerrit must recognize itself and all of the gitea servers.
Update the code to take a dict of host key values and add each entry to
known_hosts for both the root and gerrit2 user.

We remove keyscans from tests to ensure that this update is actually
working.

Change-Id: If64c34322f64c1fb63bf2ebdcc04355fff6ebba2

inventory/service/group_vars/all.yaml

@@ -1 +1,6 @@
+# This value is currently set to the value we use in testing.
+# We then override this pubkey everywhere we need to in prod
+# host/group vars.
+# TODO replace this value with the prod pubkey value and force
+# testing to override rather than the other way around.
 gerrit_ssh_rsa_pubkey_contents: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC+pCQlTAQYmCrOY6aPbvbyKQDcOCXibPNGIjnPPMuEItCS0vtRnqEBz7znWZS5Drq9yKpROh6uFF01ao2VnNjw6f+NdRNV19RWVe6mYN+qa2VrH2caLwBrKPiH0Xc/eK41D55dZU7IWwKYAw/NpiBaBfHavFwipI+rmEb68MH2hcimDdr/bji+0hkh3X+42dkNvmMdtkuCW6nKdAEhnXaHZc5SJR/EvzgRCfB8vbML13p46O9xhoJgn7ZWvMb3vaR5jxIkQwstUR36raEVhttBDEuWasWnHYbrM1zd3ooudbTEQf5vXISZKFygHyJFFqb4iQ76i+hDlb0VQKZCdaol gerrit-code-review@829f141b0fa5

playbooks/roles/gerrit/tasks/main.yaml

@@ -237,12 +237,22 @@
     group: gerrit2
     mode: 0600
 
-- name: Accept own own hostkey
+- name: Accept own own hostkey for root
   known_hosts:
     state: present
-    key: '{{ gerrit_self_hostkey }}'
+    key: '{{ item.value }}'
-    name: '[{{ gerrit_vhost_name }}]:29418'
+    name: '{{ item.key }}'
-  when: gerrit_self_hostkey is defined
+  loop: '{{ gerrit_known_hosts_keys | dict2items }}'
+  when: gerrit_known_hosts_keys is defined
+
+- name: Accept own own hostkey for gerrit2
+  known_hosts:
+    state: present
+    key: '{{ item.value }}'
+    name: '{{ item.key }}'
+    path: '/home/gerrit2/.ssh/known_hosts'
+  loop: '{{ gerrit_known_hosts_keys | dict2items }}'
+  when: gerrit_known_hosts_keys is defined
 
 - name: Install apache2
   apt:

playbooks/test-review.yaml

@@ -92,7 +92,6 @@
 
           ssh-keygen -t ed25519 -f /root/.ssh/id_25519 -P ""
           curl -X POST --user "admin:secret" -H "Content-Type: text/plain" -d@/root/.ssh/id_25519.pub http://localhost:8081/a/accounts/admin/sshkeys
-          ssh-keyscan -p 29418 localhost >> /root/.ssh/known_hosts
 
           git config --global user.name "Admin"
           git config --global user.email "admin@example.com"
@@ -123,7 +122,6 @@
 
           # openstack-project-creator bootstrapping
           curl -X POST --user "openstack-project-creator:secret" -H "Content-Type: text/plain" -d@/home/gerrit2/review_site/etc/ssh_project_rsa_key.pub http://localhost:8081/a/accounts/openstack-project-creator/sshkeys
-          ssh-keyscan -p 29418 localhost >> /home/gerrit2/.ssh/known_hosts
 
     # This is helpful on a held node when you're trying to fix/enhance
     # the Zuul summary plugin.  You can build it locally, scp the new

playbooks/zuul/templates/group_vars/review.yaml.j2

@@ -26,6 +26,11 @@ gerrit_ssh_rsa_key_contents: |
   pHMmNylg7j2NyL/9aLKs1NzdGBxpxVa5A4vgcr1DjoS1cuRVEiQoSkI6D6DCmENA
   Pb95AevPUxqqAKNZYsj4yDsXnmbFSHARijPWcpfkCDJmVhMFPObr4OE=
   -----END RSA PRIVATE KEY-----
+gerrit_ssh_rsa_pubkey_contents: |
+  ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC+pCQlTAQYmCrOY6aPbvbyKQDcOCXibPNGIjnPPMuEItCS0vtRnqEBz7znWZS5Drq9yKpROh6uFF01ao2VnNjw6f+NdRNV19RWVe6mYN+qa2VrH2caLwBrKPiH0Xc/eK41D55dZU7IWwKYAw/NpiBaBfHavFwipI+rmEb68MH2hcimDdr/bji+0hkh3X+42dkNvmMdtkuCW6nKdAEhnXaHZc5SJR/EvzgRCfB8vbML13p46O9xhoJgn7ZWvMb3vaR5jxIkQwstUR36raEVhttBDEuWasWnHYbrM1zd3ooudbTEQf5vXISZKFygHyJFFqb4iQ76i+hDlb0VQKZCdaol test-gerrit-hostkey
+gerrit_known_hosts_keys:
+  '[{% raw %}{{ gerrit_vhost_name }}{% endraw %}]:29418': |
+    [{% raw %}{{ gerrit_vhost_name }}{% endraw %}]:29418,[localhost]:29418,[127.0.0.1]:29418,[::1]:29418 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC+pCQlTAQYmCrOY6aPbvbyKQDcOCXibPNGIjnPPMuEItCS0vtRnqEBz7znWZS5Drq9yKpROh6uFF01ao2VnNjw6f+NdRNV19RWVe6mYN+qa2VrH2caLwBrKPiH0Xc/eK41D55dZU7IWwKYAw/NpiBaBfHavFwipI+rmEb68MH2hcimDdr/bji+0hkh3X+42dkNvmMdtkuCW6nKdAEhnXaHZc5SJR/EvzgRCfB8vbML13p46O9xhoJgn7ZWvMb3vaR5jxIkQwstUR36raEVhttBDEuWasWnHYbrM1zd3ooudbTEQf5vXISZKFygHyJFFqb4iQ76i+hDlb0VQKZCdaol
 gerrit_project_ssh_rsa_key_contents: |
   -----BEGIN OPENSSH PRIVATE KEY-----
   b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABFwAAAAdzc2gtcn