Fix concurrency issues while adding SSH keys to known_hosts

Co-Authored-By: Mark Goddard <mark@stackhpc.com>

Change-Id: Ie0b98e7364c1c00dc017129e86843962f924247b
Story: 2007628
Task: 39672

ansible/roles/ssh-known-host/tasks/main.yml

@@ -21,11 +21,15 @@
   register: keyscan_result
   changed_when: False
 
+# NOTE(priteau): Run this task serially as known_hosts is not safe to execute
+# concurrently, and some keys can end up being dropped. For more details see
+# https://github.com/ansible/proposals/issues/113
 - name: Ensure SSH keys are in known hosts
   local_action:
     module: known_hosts
     host: "{{ item[0].item }}"
     key: "{{ item[1] }}"
-  with_subelements:
-    - "{{ keyscan_result.results }}"
-    - stdout_lines
+  # TODO(priteau): Replace the following with "throttle: 1" once we require
+  # Ansible >= 2.9
+  loop: "{{ query('subelements', ansible_play_batch | map('extract', hostvars, ['keyscan_result', 'results']) | map('first') | list, 'stdout_lines') }}"
+  run_once: True

releasenotes/notes/concurrent-known-hosts-update-8dc94557e9a48021.yaml

@@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    Fixes concurrency issues while adding SSH keys to the known hosts file by
+    performing the action serially. See `story 2007628
+    <https://storyboard.openstack.org/#!/story/2007628>`__ for details.