Fix races in timer trigger tests.

Timer trigger tests are vulnerable to a race where the timer trigger can
enqueue new builds while we are checking the number of builds queued.
This can happen during intermediate build queue checks or in
assertFinalState.

To fix this race swap in a layout that does not use the timer trigger
which will prevent further build queueing once the timer triggered
builds have done their thing.

Change-Id: Ic9b64e34c6a39275b57b1299b47d9b1d0e0e5c79

tests/fixtures/layout-idle.yaml

@@ -8,5 +8,5 @@ pipelines:
 projects:
   - name: org/project
     periodic:
-      - project-test1
-      - project-test2
+      - project-bitrot-stable-old
+      - project-bitrot-stable-older

tests/fixtures/layout-no-timer.yaml

@@ -0,0 +1,14 @@
+pipelines:
+  - name: periodic
+    manager: IndependentPipelineManager
+    # Trigger is required, set it to one that is a noop
+    # during tests that check the timer trigger.
+    trigger:
+      gerrit:
+        - event: patchset-created
+
+projects:
+  - name: org/project
+    periodic:
+      - project-bitrot-stable-old
+      - project-bitrot-stable-older

tests/fixtures/layout-timer-smtp.yaml

@@ -3,7 +3,7 @@ pipelines:
     manager: IndependentPipelineManager
     trigger:
       timer:
-        - time: '* * * * * */10'
+        - time: '* * * * * */1'
     success:
       smtp:
         to: alternative_me@example.com

tests/fixtures/layout-timer.yaml

@@ -15,7 +15,7 @@ pipelines:
     manager: IndependentPipelineManager
     trigger:
       timer:
-        - time: '* * * * * */10'
+        - time: '* * * * * */1'
 
 projects:
   - name: org/project

tests/test_scheduler.py

@@ -3175,25 +3175,25 @@ class TestScheduler(testtools.TestCase):
         self.sched.reconfigure(self.config)
         self.registerJobs()
 
-        start = time.time()
-        failed = True
-        while ((time.time() - start) < 30):
-            if len(self.builds) == 2:
-                failed = False
-                break
-            else:
-                time.sleep(1)
-
-        if failed:
-            raise Exception("Expected jobs never ran")
-
+        # The pipeline triggers every second, so we should have seen
+        # several by now.
+        time.sleep(5)
         self.waitUntilSettled()
+
+        self.assertEqual(len(self.builds), 2)
+
         port = self.webapp.server.socket.getsockname()[1]
 
         f = urllib.urlopen("http://localhost:%s/status.json" % port)
         data = f.read()
 
         self.worker.hold_jobs_in_build = False
+        # Stop queuing timer triggered jobs so that the assertions
+        # below don't race against more jobs being queued.
+        self.config.set('zuul', 'layout_config',
+                        'tests/fixtures/layout-no-timer.yaml')
+        self.sched.reconfigure(self.config)
+        self.registerJobs()
         self.worker.release()
         self.waitUntilSettled()
 
@@ -3217,29 +3217,33 @@ class TestScheduler(testtools.TestCase):
     def test_idle(self):
         "Test that frequent periodic jobs work"
         self.worker.hold_jobs_in_build = True
-        self.config.set('zuul', 'layout_config',
-                        'tests/fixtures/layout-idle.yaml')
-        self.sched.reconfigure(self.config)
-        self.registerJobs()
 
-        # The pipeline triggers every second, so we should have seen
-        # several by now.
-        time.sleep(5)
-        self.waitUntilSettled()
-        self.assertEqual(len(self.builds), 2)
-        self.worker.release('.*')
-        self.waitUntilSettled()
-        self.assertEqual(len(self.builds), 0)
-        self.assertEqual(len(self.history), 2)
+        for x in range(1, 3):
+            # Test that timer triggers periodic jobs even across
+            # layout config reloads.
+            # Start timer trigger
+            self.config.set('zuul', 'layout_config',
+                            'tests/fixtures/layout-idle.yaml')
+            self.sched.reconfigure(self.config)
+            self.registerJobs()
 
-        time.sleep(5)
-        self.waitUntilSettled()
-        self.assertEqual(len(self.builds), 2)
-        self.assertEqual(len(self.history), 2)
-        self.worker.release('.*')
-        self.waitUntilSettled()
-        self.assertEqual(len(self.builds), 0)
-        self.assertEqual(len(self.history), 4)
+            # The pipeline triggers every second, so we should have seen
+            # several by now.
+            time.sleep(5)
+            self.waitUntilSettled()
+
+            # Stop queuing timer triggered jobs so that the assertions
+            # below don't race against more jobs being queued.
+            self.config.set('zuul', 'layout_config',
+                            'tests/fixtures/layout-no-timer.yaml')
+            self.sched.reconfigure(self.config)
+            self.registerJobs()
+
+            self.assertEqual(len(self.builds), 2)
+            self.worker.release('.*')
+            self.waitUntilSettled()
+            self.assertEqual(len(self.builds), 0)
+            self.assertEqual(len(self.history), x * 2)
 
     def test_check_smtp_pool(self):
         self.config.set('zuul', 'layout_config',
@@ -3274,25 +3278,22 @@ class TestScheduler(testtools.TestCase):
 
     def test_timer_smtp(self):
         "Test that a periodic job is triggered"
+        self.worker.hold_jobs_in_build = True
         self.config.set('zuul', 'layout_config',
                         'tests/fixtures/layout-timer-smtp.yaml')
         self.sched.reconfigure(self.config)
         self.registerJobs()
 
-        start = time.time()
-        failed = True
-        while ((time.time() - start) < 30):
-            if len(self.history) == 2:
-                failed = False
-                break
-            else:
-                time.sleep(1)
-
-        if failed:
-            raise Exception("Expected jobs never ran")
-
+        # The pipeline triggers every second, so we should have seen
+        # several by now.
+        time.sleep(5)
         self.waitUntilSettled()
 
+        self.assertEqual(len(self.builds), 2)
+        self.worker.release('.*')
+        self.waitUntilSettled()
+        self.assertEqual(len(self.history), 2)
+
         self.assertEqual(self.getJobFromHistory(
             'project-bitrot-stable-old').result, 'SUCCESS')
         self.assertEqual(self.getJobFromHistory(
@@ -3311,6 +3312,15 @@ class TestScheduler(testtools.TestCase):
         self.assertIn('Subject: Periodic check for org/project succeeded',
                       self.smtp_messages[0]['headers'])
 
+        # Stop queuing timer triggered jobs and let any that may have
+        # queued through so that end of test assertions pass.
+        self.config.set('zuul', 'layout_config',
+                        'tests/fixtures/layout-no-timer.yaml')
+        self.sched.reconfigure(self.config)
+        self.registerJobs()
+        self.worker.release('.*')
+        self.waitUntilSettled()
+
     def test_client_enqueue(self):
         "Test that the RPC client can enqueue a change"
         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')