diff --git a/tests/base.py b/tests/base.py
index 97adc275bb..a9632ad714 100644
--- a/tests/base.py
+++ b/tests/base.py
@@ -2009,8 +2009,8 @@ class SchedulerTestApp:
         ]
 
     def start(self, validate_tenants=None):
-        self.sched.start()
         if validate_tenants is None:
+            self.sched.start()
             self.sched.prime(self.config)
         else:
             self.sched.validateTenants(self.config, validate_tenants)
@@ -2815,8 +2815,11 @@ class ZuulTestCase(BaseTestCase):
         self.executor_server.join()
         self.launcher.stop()
         self.launcher.join()
-        self.scheds.execute(lambda app: app.sched.stop())
-        self.scheds.execute(lambda app: app.sched.join())
+        if self.validate_tenants is None:
+            self.scheds.execute(lambda app: app.sched.stop())
+            self.scheds.execute(lambda app: app.sched.join())
+        else:
+            self.scheds.execute(lambda app: app.sched.stopConnections())
         self.statsd.stop()
         self.statsd.join()
         self.fake_nodepool.stop()
diff --git a/tests/upgrade/test_upgrade_old.py b/tests/upgrade/test_upgrade_old.py
index 0e3d1f19b4..678ec578c7 100644
--- a/tests/upgrade/test_upgrade_old.py
+++ b/tests/upgrade/test_upgrade_old.py
@@ -32,8 +32,11 @@ class TestUpgradeOld(ZuulTestCase):
 
     def shutdown(self):
         # Shutdown the scheduler now before it gets any aborted events
-        self.scheds.execute(lambda app: app.sched.stop())
-        self.scheds.execute(lambda app: app.sched.join())
+        if self.validate_tenants is None:
+            self.scheds.execute(lambda app: app.sched.stop())
+            self.scheds.execute(lambda app: app.sched.join())
+        else:
+            self.scheds.execute(lambda app: app.sched.stopConnections())
         # Then release the executor jobs and stop the executors
         self.executor_server.hold_jobs_in_build = False
         self.executor_server.release()
diff --git a/zuul/cmd/scheduler.py b/zuul/cmd/scheduler.py
index 0309ff86ac..d81880a199 100755
--- a/zuul/cmd/scheduler.py
+++ b/zuul/cmd/scheduler.py
@@ -118,6 +118,7 @@ class Scheduler(zuul.cmd.ZuulDaemonApp):
 
         if self.args.validate_tenants is not None:
             self.sched.stop()
+            self.sched.stopConnections()
             sys.exit(0)
 
         if self.args.nodaemon:
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index 6fc6cad676..4dbbea2ba0 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -332,6 +332,10 @@ class Scheduler(threading.Thread):
         self.connection_cleanup_lock = ConnectionCleanupLock(self.zk_client)
         self.node_request_cleanup_lock = NodeRequestCleanupLock(self.zk_client)
 
+        self.keystore = KeyStorage(
+            self.zk_client,
+            password=self._get_key_store_password())
+
         self.abide = Abide()
         self.unparsed_abide = UnparsedAbideConfig()
         self.tenant_layout_state = LayoutStateStore(
@@ -365,9 +369,6 @@ class Scheduler(threading.Thread):
 
     def start(self):
         super(Scheduler, self).start()
-        self.keystore = KeyStorage(
-            self.zk_client,
-            password=self._get_key_store_password())
 
         self._command_running = True
         self.log.debug("Starting command processor")