diff --git a/playbooks/check-main-yaml/run.yaml b/playbooks/check-main-yaml/run.yaml
deleted file mode 100644
index 1e59a7c374..0000000000
--- a/playbooks/check-main-yaml/run.yaml
+++ /dev/null
@@ -1,9 +0,0 @@
-- hosts: all
-  roles:
-    - role: tox
-      tox_envlist: add-projects-to-main
-
-- hosts: all
-  tasks:
-    - name: Check for changes to zuul/main.yaml
-      command: git diff-files --quiet
diff --git a/playbooks/docs-infra-index/run.yaml b/playbooks/docs-infra-index/run.yaml
deleted file mode 100644
index 11f4e3d4ff..0000000000
--- a/playbooks/docs-infra-index/run.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-- hosts: all
-  roles:
-    - role: tox
-      tox_envlist: infra-docs
diff --git a/playbooks/yaml2ical/post.yaml b/playbooks/yaml2ical/post.yaml
index d5708223d2..c21d1454e9 100644
--- a/playbooks/yaml2ical/post.yaml
+++ b/playbooks/yaml2ical/post.yaml
@@ -1,3 +1,12 @@
+- hosts: all
+  tasks:
+    - name: Collect yaml2ical output
+      synchronize:
+        dest: "{{ zuul.executor.work_root }}/yaml2ical"
+        mode: pull
+        src: "src/{{ zuul.project.canonical_name }}/output"
+        verify_host: true
+
 - hosts: localhost
   roles:
     - add-fileserver
diff --git a/playbooks/yaml2ical/run.yaml b/playbooks/yaml2ical/run.yaml
deleted file mode 100644
index 0ff5a7f019..0000000000
--- a/playbooks/yaml2ical/run.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-- hosts: all
-  roles:
-    - role: tox
-      tox_envlist: ical
-
-  post_tasks:
-
-    - name: Collect yaml2ical output
-      synchronize:
-        dest: "{{ zuul.executor.work_root }}/yaml2ical"
-        mode: pull
-        src: "src/{{ zuul.project.canonical_name }}/output"
-        verify_host: true
diff --git a/tox.ini b/tox.ini
index 8cb46170ea..e7d682d42f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -118,6 +118,10 @@ commands = nodepool -c {toxinidir}/nodepool/nodepool.yaml config-validate
 
 [testenv:add-projects-to-main]
 basepython = python3
+whitelist_externals =
+  git
 deps =
-    ruamel.yaml
-commands = python tools/add-projects-to-main.py
+  ruamel.yaml
+commands =
+  python tools/add-projects-to-main.py
+  git diff-files --quiet
diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml
index 0dce04912c..215528916f 100644
--- a/zuul.d/jobs.yaml
+++ b/zuul.d/jobs.yaml
@@ -263,8 +263,10 @@
       Publish the IRC meeting calendars to eavesdrop.openstack.org.
     final: True
     pre-run: playbooks/yaml2ical/pre
-    run: playbooks/yaml2ical/run
+    run: playbooks/project-config/run-tox
     post-run: playbooks/yaml2ical/post
+    vars:
+      tox_envlist: ical
     secrets:
       - name: fileserver
         secret: eavesdrop_ssh_key
@@ -431,10 +433,13 @@
 - job:
     name: project-config-check-main-yaml
     pre-run: playbooks/check-main-yaml/pre
-    run: playbooks/check-main-yaml/run
+    run: playbooks/project-config/run-tox
     post-run: playbooks/check-main-yaml/post
     allowed-projects:
       - openstack-infra/project-config
+    final: true
+    vars:
+      tox_envlist: add-projects-to-main
     files:
       - zuul/main.yaml
       - gerrit/projects.yaml