Optimize layout re-calculation after re-enqueue
Since we no longer re-freeze the job graph during re-enqueue as of I2e4bd2fb9222b49cb10661d28d4c52a3c994ba62 and the queue items no longer have a layout attribute, we can save some time by not re-calculating the layout for items that already have a job graph and for non-live items. Instead we only re-generate the 'nearest' fallback layout on demand in case a following item needs it. Previous change for optimizing the layout re-calculation was I2b626f944cd09cf7e1dee73a0e121c4705ce850a. Change-Id: I9b9657a18664e515ecc11a47f8624b5f248084c4
This commit is contained in:
parent
55c1945314
commit
07188e2ee7
|
@ -3453,6 +3453,142 @@ class TestScheduler(ZuulTestCase):
|
|||
self.assertEqual(A.data['status'], 'MERGED')
|
||||
self.assertEqual(A.reported, 2)
|
||||
|
||||
def test_live_reconfiguration_layout_cache_fallback(self):
|
||||
# Test that re-calculating a dynamic fallback layout works after it
|
||||
# was removed during a reconfiguration.
|
||||
self.executor_server.hold_jobs_in_build = True
|
||||
|
||||
in_repo_conf = textwrap.dedent(
|
||||
"""
|
||||
- job:
|
||||
name: project-test3
|
||||
parent: project-test1
|
||||
|
||||
# add a job by the canonical project name
|
||||
- project:
|
||||
gate:
|
||||
jobs:
|
||||
- project-test3:
|
||||
dependencies:
|
||||
- project-merge
|
||||
""")
|
||||
|
||||
file_dict = {'zuul.d/a.yaml': in_repo_conf}
|
||||
|
||||
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
|
||||
files=file_dict)
|
||||
A.addApproval('Code-Review', 2)
|
||||
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
|
||||
self.waitUntilSettled()
|
||||
|
||||
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
|
||||
self.waitUntilSettled()
|
||||
|
||||
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
|
||||
pipeline = tenant.layout.pipelines['gate']
|
||||
items = pipeline.getAllItems()
|
||||
self.assertEqual(len(items), 1)
|
||||
self.assertIsNone(items[0].layout_uuid)
|
||||
|
||||
# Assert that the layout cache is empty after a reconfiguration.
|
||||
self.assertEqual(pipeline.manager._layout_cache, {})
|
||||
|
||||
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B',
|
||||
parent='refs/changes/01/1/1')
|
||||
B.addApproval('Code-Review', 2)
|
||||
|
||||
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
|
||||
self.waitUntilSettled()
|
||||
|
||||
items = pipeline.getAllItems()
|
||||
self.assertEqual(len(items), 2)
|
||||
for item in items:
|
||||
# Layout UUID should be set again for all live items. It had to
|
||||
# be re-calculated for the first item in the queue as it was reset
|
||||
# during re-enqueue.
|
||||
self.assertIsNotNone(item.layout_uuid)
|
||||
|
||||
self.executor_server.hold_jobs_in_build = False
|
||||
self.executor_server.release()
|
||||
self.waitUntilSettled()
|
||||
|
||||
self.assertEqual(A.data['status'], 'MERGED')
|
||||
self.assertEqual(A.reported, 2)
|
||||
self.assertEqual(B.data['status'], 'MERGED')
|
||||
self.assertEqual(B.reported, 2)
|
||||
|
||||
self.assertHistory([
|
||||
dict(name='project-merge', result='SUCCESS', changes='1,1'),
|
||||
dict(name='project-merge', result='SUCCESS', changes='1,1 2,1'),
|
||||
dict(name='project-test1', result='SUCCESS', changes='1,1'),
|
||||
dict(name='project-test1', result='SUCCESS', changes='1,1 2,1'),
|
||||
dict(name='project-test2', result='SUCCESS', changes='1,1'),
|
||||
dict(name='project-test2', result='SUCCESS', changes='1,1 2,1'),
|
||||
dict(name='project-test3', result='SUCCESS', changes='1,1'),
|
||||
dict(name='project-test3', result='SUCCESS', changes='1,1 2,1'),
|
||||
], ordered=False)
|
||||
|
||||
def test_live_reconfiguration_layout_cache_non_live(self):
|
||||
# Test that the layout UUID is only reset for live items.
|
||||
self.executor_server.hold_jobs_in_build = True
|
||||
|
||||
in_repo_conf = textwrap.dedent(
|
||||
"""
|
||||
- job:
|
||||
name: project-test3
|
||||
parent: project-test1
|
||||
|
||||
# add a job by the canonical project name
|
||||
- project:
|
||||
check:
|
||||
jobs:
|
||||
- project-test3:
|
||||
dependencies:
|
||||
- project-merge
|
||||
""")
|
||||
|
||||
file_dict = {'zuul.d/a.yaml': in_repo_conf}
|
||||
|
||||
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
|
||||
files=file_dict)
|
||||
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B',
|
||||
parent='refs/changes/01/1/1')
|
||||
B.setDependsOn(A, 1)
|
||||
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
|
||||
self.waitUntilSettled()
|
||||
|
||||
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
|
||||
self.waitUntilSettled()
|
||||
|
||||
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
|
||||
pipeline = tenant.layout.pipelines['check']
|
||||
items = pipeline.getAllItems()
|
||||
self.assertEqual(len(items), 2)
|
||||
|
||||
# Assert that the layout UUID of the live item is reset during a
|
||||
# reconfiguration, but non-live items keep their UUID.
|
||||
self.assertIsNotNone(items[0].layout_uuid)
|
||||
self.assertIsNone(items[1].layout_uuid)
|
||||
|
||||
# Cache should be empty after a reconfiguration
|
||||
self.assertEqual(pipeline.manager._layout_cache, {})
|
||||
|
||||
self.executor_server.hold_jobs_in_build = False
|
||||
self.executor_server.release()
|
||||
self.waitUntilSettled()
|
||||
|
||||
self.assertEqual(A.data['status'], 'NEW')
|
||||
self.assertEqual(A.reported, 0)
|
||||
self.assertEqual(B.data['status'], 'NEW')
|
||||
self.assertEqual(B.reported, 1)
|
||||
|
||||
self.assertHistory([
|
||||
dict(name='project-merge', result='SUCCESS', changes='1,1 2,1'),
|
||||
dict(name='project-test1', result='SUCCESS', changes='1,1 2,1'),
|
||||
dict(name='project-test2', result='SUCCESS', changes='1,1 2,1'),
|
||||
dict(name='project-test3', result='SUCCESS', changes='1,1 2,1'),
|
||||
], ordered=False)
|
||||
|
||||
def test_live_reconfiguration_command_socket(self):
|
||||
"Test that live reconfiguration via command socket works"
|
||||
|
||||
|
|
|
@ -301,7 +301,10 @@ class PipelineManager(metaclass=ABCMeta):
|
|||
# necessary, or it will do nothing if we're waiting on
|
||||
# a merge job.
|
||||
has_job_graph = bool(item.job_graph)
|
||||
item.layout_uuid = None
|
||||
if item.live:
|
||||
# Only reset the layout for live items as we don't need to
|
||||
# re-create the layout in independent pipelines.
|
||||
item.layout_uuid = None
|
||||
|
||||
# If the item is no longer active, but has a job graph we
|
||||
# will make sure to update it.
|
||||
|
@ -660,7 +663,7 @@ class PipelineManager(metaclass=ABCMeta):
|
|||
def executeJobs(self, item):
|
||||
# TODO(jeblair): This should return a value indicating a job
|
||||
# was executed. Appears to be a longstanding bug.
|
||||
if not item.layout_uuid:
|
||||
if not item.job_graph:
|
||||
return False
|
||||
|
||||
jobs = item.findJobsToRun(
|
||||
|
@ -1071,18 +1074,6 @@ class PipelineManager(metaclass=ABCMeta):
|
|||
if not ready:
|
||||
return False
|
||||
|
||||
# With the merges done, we have the info needed to get a
|
||||
# layout. This may return the pipeline layout, a layout from
|
||||
# a change ahead, a newly generated layout for this change, or
|
||||
# None if there was an error that makes the layout unusable.
|
||||
# In the last case, it will have set the config_errors on this
|
||||
# item, which may be picked up by the next itme.
|
||||
if not item.layout_uuid:
|
||||
self.getLayout(item)
|
||||
|
||||
if not item.layout_uuid:
|
||||
return False
|
||||
|
||||
# If the change can not be merged or has config errors, don't
|
||||
# run jobs.
|
||||
if build_set.unable_to_merge:
|
||||
|
@ -1090,6 +1081,17 @@ class PipelineManager(metaclass=ABCMeta):
|
|||
if build_set.config_errors:
|
||||
return False
|
||||
|
||||
# With the merges done, we have the info needed to get a
|
||||
# layout. This may return the pipeline layout, a layout from
|
||||
# a change ahead, a newly generated layout for this change, or
|
||||
# None if there was an error that makes the layout unusable.
|
||||
# In the last case, it will have set the config_errors on this
|
||||
# item, which may be picked up by the next item.
|
||||
if not (item.layout_uuid or item.job_graph):
|
||||
layout = self.getLayout(item)
|
||||
if not layout:
|
||||
return False
|
||||
|
||||
# We don't need to build a job graph for a non-live item, we
|
||||
# just need the layout.
|
||||
if not item.live:
|
||||
|
|
Loading…
Reference in New Issue