diff --git a/ansible/gather-facts.yml b/ansible/gather-facts.yml
index 48f17ef3c2..cf5f3b042f 100644
--- a/ansible/gather-facts.yml
+++ b/ansible/gather-facts.yml
@@ -22,6 +22,7 @@
         filter: "{{ kolla_ansible_setup_filter }}"
         gather_subset: "{{ kolla_ansible_setup_gather_subset }}"
       when:
+        # Don't gather if fact caching is in use
         - not ansible_facts
   tags: always
 
@@ -54,7 +55,10 @@
       delegate_facts: True
       delegate_to: "{{ item }}"
       with_items: "{{ delegate_hosts }}"
-      # We gathered facts for all hosts in the batch during the first play.
       when:
+        # We gathered facts for all hosts in the batch during the first play.
+        # Ensure that we don't try again if they failed.
+        - item not in groups["all_using_limit_True"]
+        # Don't gather if fact caching is in use
         - not hostvars[item].ansible_facts
   tags: always
diff --git a/releasenotes/notes/fact-gathering-double-failure-4662ef2f54d58291.yaml b/releasenotes/notes/fact-gathering-double-failure-4662ef2f54d58291.yaml
new file mode 100644
index 0000000000..fe0595cf4e
--- /dev/null
+++ b/releasenotes/notes/fact-gathering-double-failure-4662ef2f54d58291.yaml
@@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    Fixes an issue during fact gathering when using the ``--limit`` argument
+    where a host that fails to gather facts could cause another host to fail
+    during delegated fact gathering.