zuul/zuul
Code Issues Proposed changes

Deduplicate jobs in dependency cycles

Browse Source 
This adds support for deduplicating jobs within dependency cycles.

By default, this will happen automatically if we can determine that the
results of two builds would be expected to be identical.  This uses a
heuristic which should almost always be correct; the behavior can be
overidden otherwise.

Change-Id: I890407df822035d52ead3516942fd95e3633094b
This commit is contained in:
James E. Blair 2022-04-30 14:40:15 -07:00
parent 282182f7c2
commit 959a0b9834
24 changed files with 1424 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
doc/source/config/job.rst
View File

@ -906,6 +906,44 @@ Here is an example of two job definitions:
self-testing without requiring that the file matchers include
the Zuul configuration file defining the job.
.. attr:: deduplicate
:default: auto
In the case of a dependency cycle where multiple changes within
the cycle run the same job, this setting indicates whether Zuul
should attempt to deduplicate the job. If it is deduplicated,
then the job will only run for one queue item within the cycle
and other items which run the same job will use the results of
that build.
This setting determins whether Zuul will consider deduplication.
If it is set to ``false``, Zuul will never attempt to
deduplicate the job. If it is set to ``auto`` (the default),
then Zuul will compare the job with other jobs of other queue
items in the dependency cycle, and if they are equivalent and
meet certain project criteria, it will deduplicate them.
The project criteria that Zuul considers under the ``auto``
setting are either:
* The job must specify :attr:`job.required-projects`.
* Or the queue items must be for the same project.
This is because of the following heuristic: if a job specifies
:attr:`job.required-projects`, it is most likely to be one which
operates in the same way regardless of which project the change
under test belongs to, therefore the result of the same job
running on two queue items in the same dependency cycle should
be the same. If a job does not specify
:attr:`job.required-projects` and runs with two different
projects under test, the outcome is likely different for those
two items.
If this is not true for a job (e.g., the job ignores the project
under test and interacts only with external resources)
:attr:`job.deduplicate` may be set to ``true`` to ignore the
heuristic and deduplicate anyway.
.. attr:: workspace-scheme
:default: golang

7
doc/source/developer/model-changelog.rst
View File

@ -79,3 +79,10 @@ Version 7
Playbook secret references are now either an integer
index into the job secret list, or a dict with a blob
store key. This affects schedulers and executors.
Version 8
---------
:Prior Zuul version: 6.0.0
:Description: Deduplicates jobs in dependency cycles. Affects
schedulers only.

6
releasenotes/notes/deduplicate-ac171d3206eb43b3.yaml Normal file
View File

@ -0,0 +1,6 @@
---
features:
- |
If identical jobs are run for multiple changes in a dependency
cycle, Zuul may now deduplicate them under certain circumstances.
See :attr:`job.deduplicate` for details.

29
tests/base.py
View File

@ -3119,6 +3119,8 @@ class FakeBuild(object):
result = (RecordingAnsibleJob.RESULT_NORMAL, 0) # Success
if self.shouldFail():
result = (RecordingAnsibleJob.RESULT_NORMAL, 1) # Failure
if self.shouldRetry():
result = (RecordingAnsibleJob.RESULT_NORMAL, None)
if self.aborted:
result = (RecordingAnsibleJob.RESULT_ABORTED, None)
if self.requeue:
@ -3133,6 +3135,17 @@ class FakeBuild(object):
return True
return False
def shouldRetry(self):
entries = self.executor_server.retry_tests.get(self.name, [])
for entry in entries:
if self.hasChanges(entry['change']):
if entry['retries'] is None:
return True
if entry['retries']:
entry['retries'] = entry['retries'] - 1
return True
return False
def writeReturnData(self):
changes = self.executor_server.return_data.get(self.name, {})
data = changes.get(self.change)
@ -3478,6 +3491,7 @@ class RecordingExecutorServer(zuul.executor.server.ExecutorServer):
self.running_builds = []
self.build_history = []
self.fail_tests = {}
self.retry_tests = {}
self.return_data = {}
self.job_builds = {}
@ -3494,6 +3508,19 @@ class RecordingExecutorServer(zuul.executor.server.ExecutorServer):
l.append(change)
self.fail_tests[name] = l
def retryJob(self, name, change, retries=None):
"""Instruct the executor to report matching builds as retries.
:arg str name: The name of the job to fail.
:arg Change change: The :py:class:`~tests.base.FakeChange`
instance which should cause the job to fail. This job
will also fail for changes depending on this change.
"""
self.retry_tests.setdefault(name, []).append(
dict(change=change,
retries=retries))
def returnData(self, name, change, data):
"""Instruct the executor to return data for this build.
@ -3634,6 +3661,7 @@ class FakeNodepool(object):
self.python_path = 'auto'
self.shell_type = None
self.connection_port = None
self.history = []
def stop(self):
self._running = False
@ -3792,6 +3820,7 @@ class FakeNodepool(object):
if request['state'] != 'requested':
return
request = request.copy()
self.history.append(request)
oid = request['_oid']
del request['_oid']

1
tests/fixtures/config/circular-dependencies/git/common-config/zuul.yaml vendored
View File

@ -67,6 +67,7 @@
name: base
parent: null
run: playbooks/run.yaml
deduplicate: false
required-projects:
- common-config
- org/project

65
tests/fixtures/layouts/job-dedup-auto-shared.yaml vendored Normal file
View File

@ -0,0 +1,65 @@
- queue:
name: integrated
allow-circular-dependencies: true
- pipeline:
name: gate
manager: dependent
success-message: Build succeeded (gate).
require:
gerrit:
approval:
- Approved: 1
trigger:
gerrit:
- event: comment-added
approval:
- Approved: 1
success:
gerrit:
Verified: 2
submit: true
failure:
gerrit:
Verified: -2
start:
gerrit:
Verified: 0
precedence: high
- job:
name: base
parent: null
run: playbooks/run.yaml
nodeset:
nodes:
- label: debian
name: controller
- job:
name: common-job
required-projects:
- org/project1
- org/project2
- job:
name: project1-job
- job:
name: project2-job
- project:
name: org/project1
queue: integrated
gate:
jobs:
- common-job
- project1-job
- project:
name: org/project2
queue: integrated
gate:
jobs:
- common-job
- project2-job

62
tests/fixtures/layouts/job-dedup-auto-unshared.yaml vendored Normal file
View File

@ -0,0 +1,62 @@
- queue:
name: integrated
allow-circular-dependencies: true
- pipeline:
name: gate
manager: dependent
success-message: Build succeeded (gate).
require:
gerrit:
approval:
- Approved: 1
trigger:
gerrit:
- event: comment-added
approval:
- Approved: 1
success:
gerrit:
Verified: 2
submit: true
failure:
gerrit:
Verified: -2
start:
gerrit:
Verified: 0
precedence: high
- job:
name: base
parent: null
run: playbooks/run.yaml
nodeset:
nodes:
- label: debian
name: controller
- job:
name: common-job
- job:
name: project1-job
- job:
name: project2-job
- project:
name: org/project1
queue: integrated
gate:
jobs:
- common-job
- project1-job
- project:
name: org/project2
queue: integrated
gate:
jobs:
- common-job
- project2-job

61
tests/fixtures/layouts/job-dedup-auto.yaml vendored Normal file
View File

@ -0,0 +1,61 @@
- queue:
name: integrated
allow-circular-dependencies: true
- pipeline:
name: gate
manager: dependent
success-message: Build succeeded (gate).
require:
gerrit:
approval:
- Approved: 1
trigger:
gerrit:
- event: comment-added
approval:
- Approved: 1
success:
gerrit:
Verified: 2
submit: true
failure:
gerrit:
Verified: -2
start:
gerrit:
Verified: 0
precedence: high
- job:
name: base
parent: null
run: playbooks/run.yaml
- job:
name: common-job
required-projects:
- org/project1
- org/project2
- job:
name: project1-job
- job:
name: project2-job
- project:
name: org/project1
queue: integrated
gate:
jobs:
- common-job
- project1-job
- project:
name: org/project2
queue: integrated
gate:
jobs:
- common-job
- project2-job

61
tests/fixtures/layouts/job-dedup-empty-nodeset.yaml vendored Normal file
View File

@ -0,0 +1,61 @@
- queue:
name: integrated
allow-circular-dependencies: true
- pipeline:
name: gate
manager: dependent
success-message: Build succeeded (gate).
require:
gerrit:
approval:
- Approved: 1
trigger:
gerrit:
- event: comment-added
approval:
- Approved: 1
success:
gerrit:
Verified: 2
submit: true
failure:
gerrit:
Verified: -2
start:
gerrit:
Verified: 0
precedence: high
- job:
name: base
parent: null
run: playbooks/run.yaml
- job:
name: common-job
required-projects:
- org/project1
- org/project2
- job:
name: project1-job
- job:
name: project2-job
- project:
name: org/project1
queue: integrated
gate:
jobs:
- common-job
- project1-job
- project:
name: org/project2
queue: integrated
gate:
jobs:
- common-job
- project2-job

66
tests/fixtures/layouts/job-dedup-false.yaml vendored Normal file
View File

@ -0,0 +1,66 @@
- queue:
name: integrated
allow-circular-dependencies: true
- pipeline:
name: gate
manager: dependent
success-message: Build succeeded (gate).
require:
gerrit:
approval:
- Approved: 1
trigger:
gerrit:
- event: comment-added
approval:
- Approved: 1
success:
gerrit:
Verified: 2
submit: true
failure:
gerrit:
Verified: -2
start:
gerrit:
Verified: 0
precedence: high
- job:
name: base
parent: null
run: playbooks/run.yaml
nodeset:
nodes:
- label: debian
name: controller
- job:
name: common-job
deduplicate: false
required-projects:
- org/project1
- org/project2
- job:
name: project1-job
- job:
name: project2-job
- project:
name: org/project1
queue: integrated
gate:
jobs:
- common-job
- project1-job
- project:
name: org/project2
queue: integrated
gate:
jobs:
- common-job
- project2-job

81
tests/fixtures/layouts/job-dedup-parent-data.yaml vendored Normal file
View File

@ -0,0 +1,81 @@
- queue:
name: integrated
allow-circular-dependencies: true
- pipeline:
name: gate
manager: dependent
success-message: Build succeeded (gate).
require:
gerrit:
approval:
- Approved: 1
trigger:
gerrit:
- event: comment-added
approval:
- Approved: 1
success:
gerrit:
Verified: 2
submit: true
failure:
gerrit:
Verified: -2
start:
gerrit:
Verified: 0
precedence: high
- job:
name: base
parent: null
run: playbooks/run.yaml
nodeset:
nodes:
- label: debian
name: controller
- job:
name: parent-job
deduplicate: true
- job:
name: forked-child-job
deduplicate: true
- job:
name: common-child-job
deduplicate: true
- job:
name: project1-job
- job:
name: project2-job
- project:
name: org/project1
queue: integrated
gate:
jobs:
- parent-job
- common-child-job:
dependencies: parent-job
- project1-job:
dependencies: parent-job
- forked-child-job:
dependencies: project1-job
- project:
name: org/project2
queue: integrated
gate:
jobs:
- parent-job
- common-child-job:
dependencies: parent-job
- project2-job:
dependencies: parent-job
- forked-child-job:
dependencies: project2-job

65
tests/fixtures/layouts/job-dedup-retry-child.yaml vendored Normal file
View File

@ -0,0 +1,65 @@
- queue:
name: integrated
allow-circular-dependencies: true
- pipeline:
name: gate
manager: dependent
success-message: Build succeeded (gate).
require:
gerrit:
approval:
- Approved: 1
trigger:
gerrit:
- event: comment-added
approval:
- Approved: 1
success:
gerrit:
Verified: 2
submit: true
failure:
gerrit:
Verified: -2
start:
gerrit:
Verified: 0
precedence: high
- job:
name: base
parent: null
run: playbooks/run.yaml
nodeset:
nodes:
- label: debian
name: controller
- job:
name: parent-job
deduplicate: true
- job:
name: project1-job
- job:
name: project2-job
- project:
name: org/project1
queue: integrated
gate:
jobs:
- parent-job
- project1-job:
dependencies: parent-job
- project:
name: org/project2
queue: integrated
gate:
jobs:
- parent-job
- project2-job:
dependencies: parent-job

66
tests/fixtures/layouts/job-dedup-retry.yaml vendored Normal file
View File

@ -0,0 +1,66 @@
- queue:
name: integrated
allow-circular-dependencies: true
- pipeline:
name: gate
manager: dependent
success-message: Build succeeded (gate).
require:
gerrit:
approval:
- Approved: 1
trigger:
gerrit:
- event: comment-added
approval:
- Approved: 1
success:
gerrit:
Verified: 2
submit: true
failure:
gerrit:
Verified: -2
start:
gerrit:
Verified: 0
precedence: high
- job:
name: base
parent: null
pre-run: playbooks/pre.yaml
run: playbooks/run.yaml
nodeset:
nodes:
- label: debian
name: controller
- job:
name: common-job
required-projects:
- org/project1
- org/project2
- job:
name: project1-job
- job:
name: project2-job
- project:
name: org/project1
queue: integrated
gate:
jobs:
- common-job
- project1-job
- project:
name: org/project2
queue: integrated
gate:
jobs:
- common-job
- project2-job

71
tests/fixtures/layouts/job-dedup-semaphore-first.yaml vendored Normal file
View File

@ -0,0 +1,71 @@
- queue:
name: integrated
allow-circular-dependencies: true
- semaphore:
name: test-semaphore
- pipeline:
name: gate
manager: dependent
success-message: Build succeeded (gate).
require:
gerrit:
approval:
- Approved: 1
trigger:
gerrit:
- event: comment-added
approval:
- Approved: 1
success:
gerrit:
Verified: 2
submit: true
failure:
gerrit:
Verified: -2
start:
gerrit:
Verified: 0
precedence: high
- job:
name: base
parent: null
run: playbooks/run.yaml
nodeset:
nodes:
- label: debian
name: controller
- job:
name: common-job
semaphore:
name: test-semaphore
resources-first: true
required-projects:
- org/project1
- org/project2
- job:
name: project1-job
- job:
name: project2-job
- project:
name: org/project1
queue: integrated
gate:
jobs:
- common-job
- project1-job
- project:
name: org/project2
queue: integrated
gate:
jobs:
- common-job
- project2-job

70
tests/fixtures/layouts/job-dedup-semaphore.yaml vendored Normal file
View File

@ -0,0 +1,70 @@
- queue:
name: integrated
allow-circular-dependencies: true
- semaphore:
name: test-semaphore
- pipeline:
name: gate
manager: dependent
success-message: Build succeeded (gate).
require:
gerrit:
approval:
- Approved: 1
trigger:
gerrit:
- event: comment-added
approval:
- Approved: 1
success:
gerrit:
Verified: 2
submit: true
failure:
gerrit:
Verified: -2
start:
gerrit:
Verified: 0
precedence: high
- job:
name: base
parent: null
run: playbooks/run.yaml
nodeset:
nodes:
- label: debian
name: controller
- job:
name: common-job
semaphore:
name: test-semaphore
required-projects:
- org/project1
- org/project2
- job:
name: project1-job
- job:
name: project2-job
- project:
name: org/project1
queue: integrated
gate:
jobs:
- common-job
- project1-job
- project:
name: org/project2
queue: integrated
gate:
jobs:
- common-job
- project2-job

63
tests/fixtures/layouts/job-dedup-true.yaml vendored Normal file
View File

@ -0,0 +1,63 @@
- queue:
name: integrated
allow-circular-dependencies: true
- pipeline:
name: gate
manager: dependent
success-message: Build succeeded (gate).
require:
gerrit:
approval:
- Approved: 1
trigger:
gerrit:
- event: comment-added
approval:
- Approved: 1
success:
gerrit:
Verified: 2
submit: true
failure:
gerrit:
Verified: -2
start:
gerrit:
Verified: 0
precedence: high
- job:
name: base
parent: null
run: playbooks/run.yaml
nodeset:
nodes:
- label: debian
name: controller
- job:
name: common-job
deduplicate: true
- job:
name: project1-job
- job:
name: project2-job
- project:
name: org/project1
queue: integrated
gate:
jobs:
- common-job
- project1-job
- project:
name: org/project2
queue: integrated
gate:
jobs:
- common-job
- project2-job

1
tests/fixtures/layouts/sos-circular.yaml vendored
View File

@ -45,6 +45,7 @@
name: base
parent: null
run: playbooks/base.yaml
deduplicate: false
nodeset:
nodes:
- label: ubuntu-xenial

359
tests/unit/test_circular_dependencies.py
View File

@ -17,7 +17,7 @@ import textwrap
from zuul.model import PromoteEvent
from tests.base import ZuulTestCase, simple_layout
from tests.base import ZuulTestCase, simple_layout, iterate_timeout
class TestGerritCircularDependencies(ZuulTestCase):
@ -1113,6 +1113,7 @@ class TestGerritCircularDependencies(ZuulTestCase):
"""
- job:
name: project-vars-job
deduplicate: false
vars:
test_var: pass
@ -1536,6 +1537,362 @@ class TestGerritCircularDependencies(ZuulTestCase):
self.assertEqual(B.data['status'], 'NEW')
self.assertEqual(C.data['status'], 'MERGED')
def _test_job_deduplication(self):
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
A.addApproval('Code-Review', 2)
B.addApproval('Code-Review', 2)
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(B.data['status'], 'MERGED')
@simple_layout('layouts/job-dedup-auto-shared.yaml')
def test_job_deduplication_auto_shared(self):
self._test_job_deduplication()
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
# This is deduplicated
# dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self.assertEqual(len(self.fake_nodepool.history), 3)
@simple_layout('layouts/job-dedup-auto-unshared.yaml')
def test_job_deduplication_auto_unshared(self):
self._test_job_deduplication()
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
# This is not deduplicated
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self.assertEqual(len(self.fake_nodepool.history), 4)
@simple_layout('layouts/job-dedup-true.yaml')
def test_job_deduplication_true(self):
self._test_job_deduplication()
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
# This is deduplicated
# dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self.assertEqual(len(self.fake_nodepool.history), 3)
@simple_layout('layouts/job-dedup-false.yaml')
def test_job_deduplication_false(self):
self._test_job_deduplication()
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
# This is not deduplicated, though it would be under auto
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self.assertEqual(len(self.fake_nodepool.history), 4)
@simple_layout('layouts/job-dedup-empty-nodeset.yaml')
def test_job_deduplication_empty_nodeset(self):
# Make sure that jobs with empty nodesets can still be
# deduplicated
self._test_job_deduplication()
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
# This is deduplicated
# dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self.assertEqual(len(self.fake_nodepool.history), 0)
@simple_layout('layouts/job-dedup-auto-shared.yaml')
def test_job_deduplication_failed_node_request(self):
# Pause nodepool so we can fail the node request later
self.fake_nodepool.pause()
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
A.addApproval('Code-Review', 2)
B.addApproval('Code-Review', 2)
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
self.waitUntilSettled()
# Fail the node request and unpause
for req in self.fake_nodepool.getNodeRequests():
if req['requestor_data']['job_name'] == 'common-job':
self.fake_nodepool.addFailRequest(req)
self.fake_nodepool.unpause()
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
self.assertHistory([])
self.assertEqual(len(self.fake_nodepool.history), 3)
@simple_layout('layouts/job-dedup-auto-shared.yaml')
def test_job_deduplication_failed_job(self):
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
A.addApproval('Code-Review', 2)
B.addApproval('Code-Review', 2)
self.executor_server.failJob("common-job", A)
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result="FAILURE", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
# This is deduplicated
# dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self.assertEqual(len(self.fake_nodepool.history), 3)
@simple_layout('layouts/job-dedup-retry.yaml')
def test_job_deduplication_retry(self):
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.executor_server.retryJob('common-job', A)
A.addApproval('Code-Review', 2)
B.addApproval('Code-Review', 2)
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
# There should be exactly 3 runs of the job (not 6)
dict(name="common-job", result=None, changes="2,1 1,1"),
dict(name="common-job", result=None, changes="2,1 1,1"),
dict(name="common-job", result=None, changes="2,1 1,1"),
], ordered=False)
self.assertEqual(len(self.fake_nodepool.history), 5)
@simple_layout('layouts/job-dedup-retry-child.yaml')
def test_job_deduplication_retry_child(self):
# This tests retrying a paused build (simulating an executor restart)
# See test_data_return_child_from_retried_paused_job
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.executor_server.returnData(
'parent-job', A,
{'zuul': {'pause': True}}
)
A.addApproval('Code-Review', 2)
B.addApproval('Code-Review', 2)
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
self.waitUntilSettled()
self.executor_server.release('parent-job')
self.waitUntilSettled("till job is paused")
paused_job = self.builds[0]
self.assertTrue(paused_job.paused)
# Stop the job worker to simulate an executor restart
for job_worker in self.executor_server.job_workers.values():
if job_worker.build_request.uuid == paused_job.uuid:
job_worker.stop()
self.waitUntilSettled("stop job worker")
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled("all jobs are done")
# The "pause" job might be paused during the waitUntilSettled
# call and appear settled; it should automatically resume
# though, so just wait for it.
for x in iterate_timeout(60, 'paused job'):
if not self.builds:
break
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(B.data['status'], 'MERGED')
self.assertHistory([
dict(name="parent-job", result="ABORTED", changes="2,1 1,1"),
dict(name="project1-job", result="ABORTED", changes="2,1 1,1"),
dict(name="project2-job", result="ABORTED", changes="2,1 1,1"),
dict(name="parent-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self.assertEqual(len(self.fake_nodepool.history), 6)
@simple_layout('layouts/job-dedup-parent-data.yaml')
def test_job_deduplication_parent_data(self):
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
# The parent job returns data
self.executor_server.returnData(
'parent-job', A,
{'zuul':
{'artifacts': [
{'name': 'image',
'url': 'http://example.com/image',
'metadata': {
'type': 'container_image'
}},
]}}
)
A.addApproval('Code-Review', 2)
B.addApproval('Code-Review', 2)
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(B.data['status'], 'MERGED')
self.assertHistory([
dict(name="parent-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
# Only one run of the common job since it's the same
dict(name="common-child-job", result="SUCCESS", changes="2,1 1,1"),
# The forked job depends on different parents
# so it should run twice
dict(name="forked-child-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="forked-child-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self.assertEqual(len(self.fake_nodepool.history), 6)
def _test_job_deduplication_semaphore(self):
"Test semaphores with max=1 (mutex) and get resources first"
self.executor_server.hold_jobs_in_build = True
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
self.assertEqual(
len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
0)
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
A.addApproval('Code-Review', 2)
B.addApproval('Code-Review', 2)
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
self.waitUntilSettled()
self.assertEqual(
len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
1)
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
# This is deduplicated
# dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self.assertEqual(len(self.fake_nodepool.history), 3)
self.assertEqual(
len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
0)
@simple_layout('layouts/job-dedup-semaphore.yaml')
def test_job_deduplication_semaphore(self):
self._test_job_deduplication_semaphore()
@simple_layout('layouts/job-dedup-semaphore-first.yaml')
def test_job_deduplication_semaphore_resources_first(self):
self._test_job_deduplication_semaphore()
def test_submitted_together(self):
self.fake_gerrit._fake_submit_whole_topic = True
A = self.fake_gerrit.addFakeChange('org/project1', "master", "A",

42
tests/unit/test_model_upgrade.py
View File

@ -360,3 +360,45 @@ class TestGithubModelUpgrade(ZuulTestCase):
dict(name='project-test2', result='SUCCESS'),
], ordered=False)
self.assertTrue(A.is_merged)
class TestDeduplication(ZuulTestCase):
config_file = "zuul-gerrit-github.conf"
tenant_config_file = "config/circular-dependencies/main.yaml"
scheduler_count = 1
def _test_job_deduplication(self):
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
A.addApproval('Code-Review', 2)
B.addApproval('Code-Review', 2)
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(B.data['status'], 'MERGED')
@simple_layout('layouts/job-dedup-auto-shared.yaml')
@model_version(7)
def test_job_deduplication_auto_shared(self):
self._test_job_deduplication()
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
# This would be deduplicated
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self.assertEqual(len(self.fake_nodepool.history), 4)

8
tests/unit/test_web.py
View File

@ -371,6 +371,7 @@ class TestWeb(BaseTestWeb):
'ansible_version': None,
'attempts': 4,
'branches': [],
'deduplicate': 'auto',
'dependencies': [],
'description': None,
'files': [],
@ -421,6 +422,7 @@ class TestWeb(BaseTestWeb):
'ansible_version': None,
'attempts': 3,
'branches': ['stable'],
'deduplicate': 'auto',
'dependencies': [],
'description': None,
'files': [],
@ -475,6 +477,7 @@ class TestWeb(BaseTestWeb):
'ansible_version': None,
'attempts': 3,
'branches': [],
'deduplicate': 'auto',
'dependencies': [],
'description': None,
'files': [],
@ -598,6 +601,7 @@ class TestWeb(BaseTestWeb):
'ansible_version': None,
'attempts': 3,
'branches': [],
'deduplicate': 'auto',
'dependencies': [],
'description': None,
'files': [],
@ -636,6 +640,7 @@ class TestWeb(BaseTestWeb):
'ansible_version': None,
'attempts': 3,
'branches': [],
'deduplicate': 'auto',
'dependencies': [{'name': 'project-merge',
'soft': False}],
'description': None,
@ -675,6 +680,7 @@ class TestWeb(BaseTestWeb):
'ansible_version': None,
'attempts': 3,
'branches': [],
'deduplicate': 'auto',
'dependencies': [{'name': 'project-merge',
'soft': False}],
'description': None,
@ -714,6 +720,7 @@ class TestWeb(BaseTestWeb):
'ansible_version': None,
'attempts': 3,
'branches': [],
'deduplicate': 'auto',
'dependencies': [{'name': 'project-merge',
'soft': False}],
'description': None,
@ -778,6 +785,7 @@ class TestWeb(BaseTestWeb):
'ansible_version': None,
'attempts': 3,
'branches': [],
'deduplicate': 'auto',
'dependencies': [],
'description': None,
'files': [],

2
zuul/configloader.py
View File

@ -633,6 +633,7 @@ class JobParser(object):
'post-review': bool,
'match-on-config-updates': bool,
'workspace-scheme': vs.Any('golang', 'flat', 'unique'),
'deduplicate': vs.Any(bool, 'auto'),
}
job_name = {vs.Required('name'): str}
@ -658,6 +659,7 @@ class JobParser(object):
'override-checkout',
'match-on-config-updates',
'workspace-scheme',
'deduplicate',
]
def __init__(self, pcontext):

51
zuul/manager/__init__.py
View File

@ -1448,6 +1448,7 @@ class PipelineManager(metaclass=ABCMeta):
if build_set.repo_state_state == build_set.PENDING:
return False
item.deduplicateJobs(log)
return True
def _processOneItem(self, item, nnfi):
@ -1746,25 +1747,45 @@ class PipelineManager(metaclass=ABCMeta):
build, item)
return
item.setResult(build)
log.debug("Item %s status is now:\n %s", item, item.formatStatus())
# If the build was for deduplicated jobs, apply the results to
# all the items that use this build.
build_in_items = [item]
if item.bundle:
for other_item in item.bundle.items:
if other_item not in build_in_items:
build_in_items.append(other_item)
for item in build_in_items:
# We don't care about some actions below if this build
# isn't in the current buildset, so determine that before
# it is potentially removed with setResult.
if item.current_build_set.getBuild(build.job.name) is not build:
current = False
else:
current = True
item.setResult(build)
log.debug("Item %s status is now:\n %s", item, item.formatStatus())
if build.retry:
if build.build_set.getJobNodeSetInfo(build.job.name):
build.build_set.removeJobNodeSetInfo(build.job.name)
if not current:
continue
build_set = item.current_build_set
# in case this was a paused build we need to retry all child jobs
self._resetDependentBuilds(build.build_set, build)
if build.retry:
if build_set.getJobNodeSetInfo(build.job.name):
build_set.removeJobNodeSetInfo(build.job.name)
self._resumeBuilds(build.build_set)
# in case this was a paused build we need to retry all
# child jobs
self._resetDependentBuilds(build_set, build)
if (item.current_build_set.fail_fast and
build.failed and build.job.voting and not build.retry):
# If fail-fast is set and the build is not successful
# cancel all remaining jobs.
log.debug("Build %s failed and fail-fast enabled, canceling "
"running builds", build)
self._cancelRunningBuilds(build.build_set)
self._resumeBuilds(build_set)
if (build_set.fail_fast and
build.failed and build.job.voting and not build.retry):
# If fail-fast is set and the build is not successful
# cancel all remaining jobs.
log.debug("Build %s failed and fail-fast enabled, canceling "
"running builds", build)
self._cancelRunningBuilds(build_set)
return True

180
zuul/model.py
View File

@ -2005,6 +2005,7 @@ class FrozenJob(zkobject.ZKObject):
'requires',
'workspace_scheme',
'config_hash',
'deduplicate',
)
job_data_attributes = ('artifact_data',
@ -2018,9 +2019,33 @@ class FrozenJob(zkobject.ZKObject):
'affected_projects',
)
def __init__(self):
super().__init__()
self._set(_ready_to_run=False)
def __repr__(self):
return '<FrozenJob %s>' % (self.name)
def isEqual(self, other):
# Compare two frozen jobs to determine whether they are
# effectively equal. The inheritance path will always be
# different, so it is ignored. But if otherwise they have the
# same attributes, they will probably produce the same
# results.
if not isinstance(other, FrozenJob):
return False
if self.name != other.name:
return False
for k in self.attributes:
if k in ['inheritance_path', 'waiting_status', 'queued']:
continue
if getattr(self, k) != getattr(other, k):
return False
for k in self.job_data_attributes:
if getattr(self, k) != getattr(other, k):
return False
return True
@classmethod
def new(klass, context, **kw):
obj = klass()
@ -2367,6 +2392,7 @@ class Job(ConfigObject):
self.cleanup_run))
d['post_review'] = self.post_review
d['match_on_config_updates'] = self.match_on_config_updates
d['deduplicate'] = self.deduplicate
if self.isBase():
d['parent'] = None
elif self.parent:
@ -2404,6 +2430,7 @@ class Job(ConfigObject):
irrelevant_file_matcher=None, # skip-if
_irrelevant_files=(),
match_on_config_updates=True,
deduplicate='auto',
tags=frozenset(),
provides=frozenset(),
requires=frozenset(),
@ -3041,6 +3068,16 @@ class JobDependency(ConfigObject):
self.name = name
self.soft = soft
def __ne__(self, other):
return not self.__eq__(other)
def __eq__(self, other):
if not isinstance(other, JobDependency):
return False
return self.toDict() == other.toDict()
__hash__ = object.__hash__
def toDict(self):
return {'name': self.name,
'soft': self.soft}
@ -3698,6 +3735,7 @@ class BuildSet(zkobject.ZKObject):
fail_fast=False,
job_graph=None,
jobs={},
deduplicated_jobs=[],
# Cached job graph of previous layout; not serialized
_old_job_graph=None,
_old_jobs={},
@ -4028,7 +4066,22 @@ class BuildSet(zkobject.ZKObject):
def removeJobNodeRequestID(self, job_name):
if job_name in self.node_requests:
del self.node_requests[job_name]
with self.activeContext(
self.item.pipeline.manager.current_context):
del self.node_requests[job_name]
def setJobNodeRequestDuplicate(self, job_name, other_item):
with self.activeContext(
self.item.pipeline.manager.current_context):
self.node_requests[job_name] = {
'deduplicated_item': other_item.uuid}
def setJobNodeSetInfoDuplicate(self, job_name, other_item):
# Nothing uses this value yet; we just need an entry in the
# nodset_info dict.
with self.activeContext(self.item.pipeline.manager.current_context):
self.nodeset_info[job_name] = {
'deduplicated_item': other_item.uuid}
def jobNodeRequestComplete(self, job_name, nodeset):
if job_name in self.nodeset_info:
@ -4415,7 +4468,7 @@ class QueueItem(zkobject.ZKObject):
continue
build = self.current_build_set.getBuild(job.name)
if (build and build.result and
build.result not in ['SUCCESS', 'SKIPPED']):
build.result not in ['SUCCESS', 'SKIPPED', 'RETRY']):
return True
return False
@ -4626,7 +4679,8 @@ class QueueItem(zkobject.ZKObject):
data = []
ret = self.item_ahead.providesRequirements(job, data)
data.reverse()
job.setArtifactData(data)
if data:
job.setArtifactData(data)
except RequirementsError as e:
self.warning(str(e))
fakebuild = Build.new(self.pipeline.manager.current_context,
@ -4637,24 +4691,36 @@ class QueueItem(zkobject.ZKObject):
ret = False
return ret
def findJobsToRun(self, semaphore_handler):
torun = []
if not self.live:
return []
if not self.current_build_set.job_graph:
return []
if self.item_ahead:
# Only run jobs if any 'hold' jobs on the change ahead
# have completed successfully.
if self.item_ahead.isHoldingFollowingChanges():
return []
def findDuplicateJob(self, job):
"""
If another item in the bundle has a duplicate job,
return the other item
"""
if not self.bundle:
return None
if job.deduplicate is False:
return None
for other_item in self.bundle.items:
if other_item is self:
continue
for other_job in other_item.getJobs():
if other_job.isEqual(job):
if job.deduplicate == 'auto':
# Deduplicate if there are required projects
# or the item project is the same.
if (not job.required_projects and
self.change.project != other_item.change.project):
continue
return other_item
def updateJobParentData(self):
job_graph = self.current_build_set.job_graph
failed_job_names = set() # Jobs that run and failed
ignored_job_names = set() # Jobs that were skipped or canceled
unexecuted_job_names = set() # Jobs that were not started yet
jobs_not_started = set()
for job in job_graph.getJobs():
job._set(_ready_to_run=False)
build = self.current_build_set.getBuild(job.name)
if build:
if build.result == 'SUCCESS' or build.paused:
@ -4667,8 +4733,6 @@ class QueueItem(zkobject.ZKObject):
unexecuted_job_names.add(job.name)
jobs_not_started.add(job)
# Attempt to run jobs in the order they appear in
# configuration.
for job in job_graph.getJobs():
if job not in jobs_not_started:
continue
@ -4719,7 +4783,91 @@ class QueueItem(zkobject.ZKObject):
job.setParentData(new_parent_data,
new_secret_parent_data,
new_artifact_data)
job._set(_ready_to_run=True)
def deduplicateJobs(self, log):
"""Sync node request and build info with deduplicated jobs"""
if not self.live:
return
if not self.current_build_set.job_graph:
return
if self.item_ahead:
# Only run jobs if any 'hold' jobs on the change ahead
# have completed successfully.
if self.item_ahead.isHoldingFollowingChanges():
return
self.updateJobParentData()
if COMPONENT_REGISTRY.model_api < 8:
return
if not self.bundle:
return
build_set = self.current_build_set
job_graph = build_set.job_graph
for job in job_graph.getJobs():
this_request = build_set.getJobNodeRequestID(job.name)
this_nodeset = build_set.getJobNodeSetInfo(job.name)
this_build = build_set.getBuild(job.name)
if this_build:
# Nothing more possible for this job
continue
other_item = self.findDuplicateJob(job)
if not other_item:
continue
other_build_set = other_item.current_build_set
# Handle node requests
other_request = other_build_set.getJobNodeRequestID(job.name)
if (isinstance(other_request, dict) and
other_request.get('deduplicated_item') == self.uuid):
# We're the original, but we're probably in the middle
# of a retry
return
if other_request is not None and this_request is None:
log.info("Deduplicating request of bundle job %s for item %s "
"with item %s", job, self, other_item)
build_set.setJobNodeRequestDuplicate(job.name, other_item)
# Handle provisioned nodes
other_nodeset = other_build_set.getJobNodeSetInfo(job.name)
if (isinstance(other_nodeset, dict) and
other_nodeset.get('deduplicated_item') == self.uuid):
# We're the original, but we're probably in the middle
# of a retry
return
if other_nodeset is not None and this_nodeset is None:
log.info("Deduplicating nodeset of bundle job %s for item %s "
"with item %s", job, self, other_item)
build_set.setJobNodeSetInfoDuplicate(job.name, other_item)
# Handle builds
other_build = other_build_set.getBuild(job.name)
if other_build and not this_build:
log.info("Deduplicating build of bundle job %s for item %s "
"with item %s", job, self, other_item)
self.addBuild(other_build)
job._set(_ready_to_run=False)
def findJobsToRun(self, semaphore_handler):
torun = []
if not self.live:
return []
if not self.current_build_set.job_graph:
return []
if self.item_ahead:
# Only run jobs if any 'hold' jobs on the change ahead
# have completed successfully.
if self.item_ahead.isHoldingFollowingChanges():
return []
job_graph = self.current_build_set.job_graph
for job in job_graph.getJobs():
if job._ready_to_run:
nodeset = self.current_build_set.getJobNodeSetInfo(job.name)
if nodeset is None:
# The nodes for this job are not ready, skip

2
zuul/model_api.py
View File

@ -14,4 +14,4 @@
# When making ZK schema changes, increment this and add a record to
# docs/developer/model-changelog.rst
MODEL_API = 7
MODEL_API = 8