Move re-enqueue to pipeline processing

Moving the re-enqueue step after a reconfiguration to the pipeline
processing phase allows us to re-enqueue multiple pipelines in
parallel in a multi-scheduler environment.

In case of a failure the re-enqueue can also be retried the next time
the pipeline is processed.

Change-Id: Iad483c37610b51d94ead72573c3540b1dea9ab84

tests/unit/test_cross_crd.py

@@ -347,6 +347,7 @@ class TestGerritToGithubCRD(ZuulTestCase):
         self.waitUntilSettled()
 
         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
+        self.waitUntilSettled()
 
         # Make sure the items still share a change queue, and the
         # first one is not live.
@@ -798,6 +799,7 @@ class TestGithubToGerritCRD(ZuulTestCase):
         self.waitUntilSettled()
 
         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
+        self.waitUntilSettled()
 
         # Make sure the items still share a change queue, and the
         # first one is not live.

tests/unit/test_gerrit_crd.py

@@ -517,6 +517,7 @@ class TestGerritCRD(ZuulTestCase):
         self.waitUntilSettled()
 
         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
+        self.waitUntilSettled()
 
         # Make sure the items still share a change queue, and the
         # first one is not live.

tests/unit/test_gerrit_legacy_crd.py

@@ -446,6 +446,7 @@ class TestGerritLegacyCRD(ZuulTestCase):
         self.waitUntilSettled()
 
         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
+        self.waitUntilSettled()
 
         # Make sure the items still share a change queue, and the
         # first one is not live.

tests/unit/test_scheduler.py

@@ -5481,6 +5481,7 @@ class TestScheduler(ZuulTestCase):
         self.commitConfigUpdate('org/common-config',
                                 'layouts/reconfigure-window2.yaml')
         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
+        self.waitUntilSettled()
         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
         queue = tenant.layout.pipelines['gate'].queues[0]
         # Even though we have configured a smaller window, the value
@@ -5489,6 +5490,7 @@ class TestScheduler(ZuulTestCase):
         self.assertTrue(len(self.builds), 4)
 
         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
+        self.waitUntilSettled()
         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
         queue = tenant.layout.pipelines['gate'].queues[0]
         self.assertEqual(queue.window, 20)
@@ -5539,6 +5541,7 @@ class TestScheduler(ZuulTestCase):
         self.assertEqual(len(self.builds), 4)
 
         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
+        self.waitUntilSettled()
         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
         queue = tenant.layout.pipelines['gate'].queues[0]
         self.assertEqual(queue.window, 1)