diff --git a/tests/base.py b/tests/base.py
index 52c073f94a..ad72f12ebf 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -1326,14 +1326,14 @@ class ZuulTestCase(BaseTestCase):
         self.rpc = zuul.rpclistener.RPCListener(self.config, self.sched)
 
         self.sched.start()
-        self.sched.reconfigure(self.config)
-        self.sched.resume()
         self.webapp.start()
         self.rpc.start()
         self.launch_client.gearman.waitForServer()
-
         self.addCleanup(self.shutdown)
 
+        self.sched.reconfigure(self.config)
+        self.sched.resume()
+
     def tearDown(self):
         super(ZuulTestCase, self).tearDown()
         self.assertFinalState()
diff --git a/tests/fixtures/config/broken/git/common-config/zuul.yaml b/tests/fixtures/config/broken/git/common-config/zuul.yaml
new file mode 100644
index 0000000000..6abb87fac8
--- /dev/null
+++ b/tests/fixtures/config/broken/git/common-config/zuul.yaml
@@ -0,0 +1,17 @@
+- pipeline:
+    name: check
+    manager: independent
+    source:
+      gerrit
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        verified: 1
+    failure:
+      gerrit:
+        verified: -1
+
+- project:
+error: true
diff --git a/tests/fixtures/config/broken/main.yaml b/tests/fixtures/config/broken/main.yaml
new file mode 100644
index 0000000000..a22ed5c60c
--- /dev/null
+++ b/tests/fixtures/config/broken/main.yaml
@@ -0,0 +1,6 @@
+- tenant:
+    name: tenant-one
+    source:
+      gerrit:
+        config-repos:
+          - common-config
diff --git a/tests/make_playbooks.py b/tests/make_playbooks.py
index 33d45ca21d..17acba8b25 100755
--- a/tests/make_playbooks.py
+++ b/tests/make_playbooks.py
@@ -39,7 +39,11 @@ def handle_repo(path):
         if os.path.exists(os.path.join(path, fn)):
             config_path = os.path.join(path, fn)
             break
-    config = yaml.safe_load(open(config_path))
+    try:
+        config = yaml.safe_load(open(config_path))
+    except Exception:
+        print("  Has yaml errors")
+        return
     for block in config:
         if 'job' not in block:
             continue
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index e566479e5a..6ed9de4f30 100644
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -17,6 +17,9 @@
 import os
 import textwrap
 
+import testtools
+
+import zuul.configloader
 from tests.base import AnsibleZuulTestCase, ZuulTestCase
 
 
@@ -280,3 +283,19 @@ class TestAnsible(AnsibleZuulTestCase):
         bare_role_flag_path = os.path.join(self.test_root,
                                            build.uuid + '.bare-role.flag')
         self.assertTrue(os.path.exists(bare_role_flag_path))
+
+
+class TestBrokenConfig(ZuulTestCase):
+    # Test that we get an appropriate syntax error if we start with a
+    # broken config.
+
+    tenant_config_file = 'config/broken/main.yaml'
+
+    def setUp(self):
+        with testtools.ExpectedException(
+                zuul.configloader.ConfigurationSyntaxError,
+                "\nZuul encountered a syntax error"):
+            super(TestBrokenConfig, self).setUp()
+
+    def test_broken_config_on_startup(self):
+        pass