diff --git a/playbooks/test-lists.yaml b/playbooks/test-lists.yaml
new file mode 100644
index 0000000000..c50488af00
--- /dev/null
+++ b/playbooks/test-lists.yaml
@@ -0,0 +1,37 @@
+- hosts: "lists.openstack.org"
+  tasks:
+
+    # Make sure Mailman services for each site are running so that they
+    # will attempt to deliver any pending list admin notifications and
+    # we can capture that activity in the Exim logs.
+    - name: Restart Mailman services
+      service:
+        name: "mailman-{{ zuul_mailman_site.name }}"
+        state: restarted
+      loop: "{{ mailman_sites }}"
+      loop_control:
+        loop_var: zuul_mailman_site
+
+- hosts: "lists.katacontainers.io"
+  tasks:
+
+    # Make sure Mailman services are running so that they will attempt to
+    # deliver any pending list admin notifications and we can capture that
+    # activity in the Exim logs.
+    - name: Restart Mailman services
+      service:
+        name: "mailman"
+        state: restarted
+
+- hosts: "localhost"
+  tasks:
+
+    # Pause briefly so that mail deliveries are attempted, otherwise the
+    # job may finish too quickly and we won't have a complete picture in
+    # the MTA logs by the time we start collecting them.
+    #
+    # TODO(fungi): We could improve this to wait for specific log entries
+    # instead, I suppose.
+    - name: Wait for delivery attempts
+      wait_for:
+        timeout: 10
diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml
index 92a8a78c05..d81954887f 100644
--- a/zuul.d/system-config-run.yaml
+++ b/zuul.d/system-config-run.yaml
@@ -252,6 +252,7 @@
       - playbooks/roles/base/exim
       - playbooks/roles/mailman
       - playbooks/service-lists.yaml
+      - playbooks/test-lists.yaml
       - playbooks/zuul/templates/host_vars/lists.openstack.org.yaml.j2
       - playbooks/zuul/templates/host_vars/lists.katacontainers.io.yaml.j2
       - testinfra/test_lists_k_i.py
@@ -262,6 +263,7 @@
         - playbooks/service-lists.yaml
         # Run this twice to check idempotency
         - playbooks/service-lists.yaml
+      run_test_playbook: playbooks/test-lists.yaml
     host-vars:
       lists.katacontainers.io:
         host_copy_output: