Merge "Deduplicate jobs in dependency cycles"
This commit is contained in:
commit
c44a86edde
|
@ -946,6 +946,44 @@ Here is an example of two job definitions:
|
|||
self-testing without requiring that the file matchers include
|
||||
the Zuul configuration file defining the job.
|
||||
|
||||
.. attr:: deduplicate
|
||||
:default: auto
|
||||
|
||||
In the case of a dependency cycle where multiple changes within
|
||||
the cycle run the same job, this setting indicates whether Zuul
|
||||
should attempt to deduplicate the job. If it is deduplicated,
|
||||
then the job will only run for one queue item within the cycle
|
||||
and other items which run the same job will use the results of
|
||||
that build.
|
||||
|
||||
This setting determins whether Zuul will consider deduplication.
|
||||
If it is set to ``false``, Zuul will never attempt to
|
||||
deduplicate the job. If it is set to ``auto`` (the default),
|
||||
then Zuul will compare the job with other jobs of other queue
|
||||
items in the dependency cycle, and if they are equivalent and
|
||||
meet certain project criteria, it will deduplicate them.
|
||||
|
||||
The project criteria that Zuul considers under the ``auto``
|
||||
setting are either:
|
||||
|
||||
* The job must specify :attr:`job.required-projects`.
|
||||
* Or the queue items must be for the same project.
|
||||
|
||||
This is because of the following heuristic: if a job specifies
|
||||
:attr:`job.required-projects`, it is most likely to be one which
|
||||
operates in the same way regardless of which project the change
|
||||
under test belongs to, therefore the result of the same job
|
||||
running on two queue items in the same dependency cycle should
|
||||
be the same. If a job does not specify
|
||||
:attr:`job.required-projects` and runs with two different
|
||||
projects under test, the outcome is likely different for those
|
||||
two items.
|
||||
|
||||
If this is not true for a job (e.g., the job ignores the project
|
||||
under test and interacts only with external resources)
|
||||
:attr:`job.deduplicate` may be set to ``true`` to ignore the
|
||||
heuristic and deduplicate anyway.
|
||||
|
||||
.. attr:: workspace-scheme
|
||||
:default: golang
|
||||
|
||||
|
|
|
@ -79,3 +79,10 @@ Version 7
|
|||
Playbook secret references are now either an integer
|
||||
index into the job secret list, or a dict with a blob
|
||||
store key. This affects schedulers and executors.
|
||||
|
||||
Version 8
|
||||
---------
|
||||
|
||||
:Prior Zuul version: 6.0.0
|
||||
:Description: Deduplicates jobs in dependency cycles. Affects
|
||||
schedulers only.
|
||||
|
|
|
@ -0,0 +1,6 @@
|
|||
---
|
||||
features:
|
||||
- |
|
||||
If identical jobs are run for multiple changes in a dependency
|
||||
cycle, Zuul may now deduplicate them under certain circumstances.
|
||||
See :attr:`job.deduplicate` for details.
|
|
@ -3120,6 +3120,8 @@ class FakeBuild(object):
|
|||
result = (RecordingAnsibleJob.RESULT_NORMAL, 0) # Success
|
||||
if self.shouldFail():
|
||||
result = (RecordingAnsibleJob.RESULT_NORMAL, 1) # Failure
|
||||
if self.shouldRetry():
|
||||
result = (RecordingAnsibleJob.RESULT_NORMAL, None)
|
||||
if self.aborted:
|
||||
result = (RecordingAnsibleJob.RESULT_ABORTED, None)
|
||||
if self.requeue:
|
||||
|
@ -3134,6 +3136,17 @@ class FakeBuild(object):
|
|||
return True
|
||||
return False
|
||||
|
||||
def shouldRetry(self):
|
||||
entries = self.executor_server.retry_tests.get(self.name, [])
|
||||
for entry in entries:
|
||||
if self.hasChanges(entry['change']):
|
||||
if entry['retries'] is None:
|
||||
return True
|
||||
if entry['retries']:
|
||||
entry['retries'] = entry['retries'] - 1
|
||||
return True
|
||||
return False
|
||||
|
||||
def writeReturnData(self):
|
||||
changes = self.executor_server.return_data.get(self.name, {})
|
||||
data = changes.get(self.change)
|
||||
|
@ -3479,6 +3492,7 @@ class RecordingExecutorServer(zuul.executor.server.ExecutorServer):
|
|||
self.running_builds = []
|
||||
self.build_history = []
|
||||
self.fail_tests = {}
|
||||
self.retry_tests = {}
|
||||
self.return_data = {}
|
||||
self.job_builds = {}
|
||||
|
||||
|
@ -3495,6 +3509,19 @@ class RecordingExecutorServer(zuul.executor.server.ExecutorServer):
|
|||
l.append(change)
|
||||
self.fail_tests[name] = l
|
||||
|
||||
def retryJob(self, name, change, retries=None):
|
||||
"""Instruct the executor to report matching builds as retries.
|
||||
|
||||
:arg str name: The name of the job to fail.
|
||||
:arg Change change: The :py:class:`~tests.base.FakeChange`
|
||||
instance which should cause the job to fail. This job
|
||||
will also fail for changes depending on this change.
|
||||
|
||||
"""
|
||||
self.retry_tests.setdefault(name, []).append(
|
||||
dict(change=change,
|
||||
retries=retries))
|
||||
|
||||
def returnData(self, name, change, data):
|
||||
"""Instruct the executor to return data for this build.
|
||||
|
||||
|
@ -3635,6 +3662,7 @@ class FakeNodepool(object):
|
|||
self.python_path = 'auto'
|
||||
self.shell_type = None
|
||||
self.connection_port = None
|
||||
self.history = []
|
||||
|
||||
def stop(self):
|
||||
self._running = False
|
||||
|
@ -3793,6 +3821,7 @@ class FakeNodepool(object):
|
|||
if request['state'] != 'requested':
|
||||
return
|
||||
request = request.copy()
|
||||
self.history.append(request)
|
||||
oid = request['_oid']
|
||||
del request['_oid']
|
||||
|
||||
|
|
|
@ -67,6 +67,7 @@
|
|||
name: base
|
||||
parent: null
|
||||
run: playbooks/run.yaml
|
||||
deduplicate: false
|
||||
required-projects:
|
||||
- common-config
|
||||
- org/project
|
||||
|
|
|
@ -0,0 +1,65 @@
|
|||
- queue:
|
||||
name: integrated
|
||||
allow-circular-dependencies: true
|
||||
|
||||
- pipeline:
|
||||
name: gate
|
||||
manager: dependent
|
||||
success-message: Build succeeded (gate).
|
||||
require:
|
||||
gerrit:
|
||||
approval:
|
||||
- Approved: 1
|
||||
trigger:
|
||||
gerrit:
|
||||
- event: comment-added
|
||||
approval:
|
||||
- Approved: 1
|
||||
success:
|
||||
gerrit:
|
||||
Verified: 2
|
||||
submit: true
|
||||
failure:
|
||||
gerrit:
|
||||
Verified: -2
|
||||
start:
|
||||
gerrit:
|
||||
Verified: 0
|
||||
precedence: high
|
||||
|
||||
- job:
|
||||
name: base
|
||||
parent: null
|
||||
run: playbooks/run.yaml
|
||||
nodeset:
|
||||
nodes:
|
||||
- label: debian
|
||||
name: controller
|
||||
|
||||
- job:
|
||||
name: common-job
|
||||
required-projects:
|
||||
- org/project1
|
||||
- org/project2
|
||||
|
||||
- job:
|
||||
name: project1-job
|
||||
|
||||
- job:
|
||||
name: project2-job
|
||||
|
||||
- project:
|
||||
name: org/project1
|
||||
queue: integrated
|
||||
gate:
|
||||
jobs:
|
||||
- common-job
|
||||
- project1-job
|
||||
|
||||
- project:
|
||||
name: org/project2
|
||||
queue: integrated
|
||||
gate:
|
||||
jobs:
|
||||
- common-job
|
||||
- project2-job
|
|
@ -0,0 +1,62 @@
|
|||
- queue:
|
||||
name: integrated
|
||||
allow-circular-dependencies: true
|
||||
|
||||
- pipeline:
|
||||
name: gate
|
||||
manager: dependent
|
||||
success-message: Build succeeded (gate).
|
||||
require:
|
||||
gerrit:
|
||||
approval:
|
||||
- Approved: 1
|
||||
trigger:
|
||||
gerrit:
|
||||
- event: comment-added
|
||||
approval:
|
||||
- Approved: 1
|
||||
success:
|
||||
gerrit:
|
||||
Verified: 2
|
||||
submit: true
|
||||
failure:
|
||||
gerrit:
|
||||
Verified: -2
|
||||
start:
|
||||
gerrit:
|
||||
Verified: 0
|
||||
precedence: high
|
||||
|
||||
- job:
|
||||
name: base
|
||||
parent: null
|
||||
run: playbooks/run.yaml
|
||||
nodeset:
|
||||
nodes:
|
||||
- label: debian
|
||||
name: controller
|
||||
|
||||
- job:
|
||||
name: common-job
|
||||
|
||||
- job:
|
||||
name: project1-job
|
||||
|
||||
- job:
|
||||
name: project2-job
|
||||
|
||||
- project:
|
||||
name: org/project1
|
||||
queue: integrated
|
||||
gate:
|
||||
jobs:
|
||||
- common-job
|
||||
- project1-job
|
||||
|
||||
- project:
|
||||
name: org/project2
|
||||
queue: integrated
|
||||
gate:
|
||||
jobs:
|
||||
- common-job
|
||||
- project2-job
|
|
@ -0,0 +1,61 @@
|
|||
- queue:
|
||||
name: integrated
|
||||
allow-circular-dependencies: true
|
||||
|
||||
- pipeline:
|
||||
name: gate
|
||||
manager: dependent
|
||||
success-message: Build succeeded (gate).
|
||||
require:
|
||||
gerrit:
|
||||
approval:
|
||||
- Approved: 1
|
||||
trigger:
|
||||
gerrit:
|
||||
- event: comment-added
|
||||
approval:
|
||||
- Approved: 1
|
||||
success:
|
||||
gerrit:
|
||||
Verified: 2
|
||||
submit: true
|
||||
failure:
|
||||
gerrit:
|
||||
Verified: -2
|
||||
start:
|
||||
gerrit:
|
||||
Verified: 0
|
||||
precedence: high
|
||||
|
||||
- job:
|
||||
name: base
|
||||
parent: null
|
||||
run: playbooks/run.yaml
|
||||
|
||||
- job:
|
||||
name: common-job
|
||||
required-projects:
|
||||
- org/project1
|
||||
- org/project2
|
||||
|
||||
- job:
|
||||
name: project1-job
|
||||
|
||||
- job:
|
||||
name: project2-job
|
||||
|
||||
- project:
|
||||
name: org/project1
|
||||
queue: integrated
|
||||
gate:
|
||||
jobs:
|
||||
- common-job
|
||||
- project1-job
|
||||
|
||||
- project:
|
||||
name: org/project2
|
||||
queue: integrated
|
||||
gate:
|
||||
jobs:
|
||||
- common-job
|
||||
- project2-job
|
|
@ -0,0 +1,61 @@
|
|||
- queue:
|
||||
name: integrated
|
||||
allow-circular-dependencies: true
|
||||
|
||||
- pipeline:
|
||||
name: gate
|
||||
manager: dependent
|
||||
success-message: Build succeeded (gate).
|
||||
require:
|
||||
gerrit:
|
||||
approval:
|
||||
- Approved: 1
|
||||
trigger:
|
||||
gerrit:
|
||||
- event: comment-added
|
||||
approval:
|
||||
- Approved: 1
|
||||
success:
|
||||
gerrit:
|
||||
Verified: 2
|
||||
submit: true
|
||||
failure:
|
||||
gerrit:
|
||||
Verified: -2
|
||||
start:
|
||||
gerrit:
|
||||
Verified: 0
|
||||
precedence: high
|
||||
|
||||
- job:
|
||||
name: base
|
||||
parent: null
|
||||
run: playbooks/run.yaml
|
||||
|
||||
- job:
|
||||
name: common-job
|
||||
required-projects:
|
||||
- org/project1
|
||||
- org/project2
|
||||
|
||||
- job:
|
||||
name: project1-job
|
||||
|
||||
- job:
|
||||
name: project2-job
|
||||
|
||||
- project:
|
||||
name: org/project1
|
||||
queue: integrated
|
||||
gate:
|
||||
jobs:
|
||||
- common-job
|
||||
- project1-job
|
||||
|
||||
- project:
|
||||
name: org/project2
|
||||
queue: integrated
|
||||
gate:
|
||||
jobs:
|
||||
- common-job
|
||||
- project2-job
|
|
@ -0,0 +1,66 @@
|
|||
- queue:
|
||||
name: integrated
|
||||
allow-circular-dependencies: true
|
||||
|
||||
- pipeline:
|
||||
name: gate
|
||||
manager: dependent
|
||||
success-message: Build succeeded (gate).
|
||||
require:
|
||||
gerrit:
|
||||
approval:
|
||||
- Approved: 1
|
||||
trigger:
|
||||
gerrit:
|
||||
- event: comment-added
|
||||
approval:
|
||||
- Approved: 1
|
||||
success:
|
||||
gerrit:
|
||||
Verified: 2
|
||||
submit: true
|
||||
failure:
|
||||
gerrit:
|
||||
Verified: -2
|
||||
start:
|
||||
gerrit:
|
||||
Verified: 0
|
||||
precedence: high
|
||||
|
||||
- job:
|
||||
name: base
|
||||
parent: null
|
||||
run: playbooks/run.yaml
|
||||
nodeset:
|
||||
nodes:
|
||||
- label: debian
|
||||
name: controller
|
||||
|
||||
- job:
|
||||
name: common-job
|
||||
deduplicate: false
|
||||
required-projects:
|
||||
- org/project1
|
||||
- org/project2
|
||||
|
||||
- job:
|
||||
name: project1-job
|
||||
|
||||
- job:
|
||||
name: project2-job
|
||||
|
||||
- project:
|
||||
name: org/project1
|
||||
queue: integrated
|
||||
gate:
|
||||
jobs:
|
||||
- common-job
|
||||
- project1-job
|
||||
|
||||
- project:
|
||||
name: org/project2
|
||||
queue: integrated
|
||||
gate:
|
||||
jobs:
|
||||
- common-job
|
||||
- project2-job
|
|
@ -0,0 +1,81 @@
|
|||
- queue:
|
||||
name: integrated
|
||||
allow-circular-dependencies: true
|
||||
|
||||
- pipeline:
|
||||
name: gate
|
||||
manager: dependent
|
||||
success-message: Build succeeded (gate).
|
||||
require:
|
||||
gerrit:
|
||||
approval:
|
||||
- Approved: 1
|
||||
trigger:
|
||||
gerrit:
|
||||
- event: comment-added
|
||||
approval:
|
||||
- Approved: 1
|
||||
success:
|
||||
gerrit:
|
||||
Verified: 2
|
||||
submit: true
|
||||
failure:
|
||||
gerrit:
|
||||
Verified: -2
|
||||
start:
|
||||
gerrit:
|
||||
Verified: 0
|
||||
precedence: high
|
||||
|
||||
- job:
|
||||
name: base
|
||||
parent: null
|
||||
run: playbooks/run.yaml
|
||||
nodeset:
|
||||
nodes:
|
||||
- label: debian
|
||||
name: controller
|
||||
|
||||
- job:
|
||||
name: parent-job
|
||||
deduplicate: true
|
||||
|
||||
- job:
|
||||
name: forked-child-job
|
||||
deduplicate: true
|
||||
|
||||
- job:
|
||||
name: common-child-job
|
||||
deduplicate: true
|
||||
|
||||
- job:
|
||||
name: project1-job
|
||||
|
||||
- job:
|
||||
name: project2-job
|
||||
|
||||
- project:
|
||||
name: org/project1
|
||||
queue: integrated
|
||||
gate:
|
||||
jobs:
|
||||
- parent-job
|
||||
- common-child-job:
|
||||
dependencies: parent-job
|
||||
- project1-job:
|
||||
dependencies: parent-job
|
||||
- forked-child-job:
|
||||
dependencies: project1-job
|
||||
|
||||
- project:
|
||||
name: org/project2
|
||||
queue: integrated
|
||||
gate:
|
||||
jobs:
|
||||
- parent-job
|
||||
- common-child-job:
|
||||
dependencies: parent-job
|
||||
- project2-job:
|
||||
dependencies: parent-job
|
||||
- forked-child-job:
|
||||
dependencies: project2-job
|
|
@ -0,0 +1,65 @@
|
|||
- queue:
|
||||
name: integrated
|
||||
allow-circular-dependencies: true
|
||||
|
||||
- pipeline:
|
||||
name: gate
|
||||
manager: dependent
|
||||
success-message: Build succeeded (gate).
|
||||
require:
|
||||
gerrit:
|
||||
approval:
|
||||
- Approved: 1
|
||||
trigger:
|
||||
gerrit:
|
||||
- event: comment-added
|
||||
approval:
|
||||
- Approved: 1
|
||||
success:
|
||||
gerrit:
|
||||
Verified: 2
|
||||
submit: true
|
||||
failure:
|
||||
gerrit:
|
||||
Verified: -2
|
||||
start:
|
||||
gerrit:
|
||||
Verified: 0
|
||||
precedence: high
|
||||
|
||||
- job:
|
||||
name: base
|
||||
parent: null
|
||||
run: playbooks/run.yaml
|
||||
nodeset:
|
||||
nodes:
|
||||
- label: debian
|
||||
name: controller
|
||||
|
||||
- job:
|
||||
name: parent-job
|
||||
deduplicate: true
|
||||
|
||||
- job:
|
||||
name: project1-job
|
||||
|
||||
- job:
|
||||
name: project2-job
|
||||
|
||||
- project:
|
||||
name: org/project1
|
||||
queue: integrated
|
||||
gate:
|
||||
jobs:
|
||||
- parent-job
|
||||
- project1-job:
|
||||
dependencies: parent-job
|
||||
|
||||
- project:
|
||||
name: org/project2
|
||||
queue: integrated
|
||||
gate:
|
||||
jobs:
|
||||
- parent-job
|
||||
- project2-job:
|
||||
dependencies: parent-job
|
|
@ -0,0 +1,66 @@
|
|||
- queue:
|
||||
name: integrated
|
||||
allow-circular-dependencies: true
|
||||
|
||||
- pipeline:
|
||||
name: gate
|
||||
manager: dependent
|
||||
success-message: Build succeeded (gate).
|
||||
require:
|
||||
gerrit:
|
||||
approval:
|
||||
- Approved: 1
|
||||
trigger:
|
||||
gerrit:
|
||||
- event: comment-added
|
||||
approval:
|
||||
- Approved: 1
|
||||
success:
|
||||
gerrit:
|
||||
Verified: 2
|
||||
submit: true
|
||||
failure:
|
||||
gerrit:
|
||||
Verified: -2
|
||||
start:
|
||||
gerrit:
|
||||
Verified: 0
|
||||
precedence: high
|
||||
|
||||
- job:
|
||||
name: base
|
||||
parent: null
|
||||
pre-run: playbooks/pre.yaml
|
||||
run: playbooks/run.yaml
|
||||
nodeset:
|
||||
nodes:
|
||||
- label: debian
|
||||
name: controller
|
||||
|
||||
- job:
|
||||
name: common-job
|
||||
required-projects:
|
||||
- org/project1
|
||||
- org/project2
|
||||
|
||||
- job:
|
||||
name: project1-job
|
||||
|
||||
- job:
|
||||
name: project2-job
|
||||
|
||||
- project:
|
||||
name: org/project1
|
||||
queue: integrated
|
||||
gate:
|
||||
jobs:
|
||||
- common-job
|
||||
- project1-job
|
||||
|
||||
- project:
|
||||
name: org/project2
|
||||
queue: integrated
|
||||
gate:
|
||||
jobs:
|
||||
- common-job
|
||||
- project2-job
|
|
@ -0,0 +1,71 @@
|
|||
- queue:
|
||||
name: integrated
|
||||
allow-circular-dependencies: true
|
||||
|
||||
- semaphore:
|
||||
name: test-semaphore
|
||||
|
||||
- pipeline:
|
||||
name: gate
|
||||
manager: dependent
|
||||
success-message: Build succeeded (gate).
|
||||
require:
|
||||
gerrit:
|
||||
approval:
|
||||
- Approved: 1
|
||||
trigger:
|
||||
gerrit:
|
||||
- event: comment-added
|
||||
approval:
|
||||
- Approved: 1
|
||||
success:
|
||||
gerrit:
|
||||
Verified: 2
|
||||
submit: true
|
||||
failure:
|
||||
gerrit:
|
||||
Verified: -2
|
||||
start:
|
||||
gerrit:
|
||||
Verified: 0
|
||||
precedence: high
|
||||
|
||||
- job:
|
||||
name: base
|
||||
parent: null
|
||||
run: playbooks/run.yaml
|
||||
nodeset:
|
||||
nodes:
|
||||
- label: debian
|
||||
name: controller
|
||||
|
||||
- job:
|
||||
name: common-job
|
||||
semaphore:
|
||||
name: test-semaphore
|
||||
resources-first: true
|
||||
required-projects:
|
||||
- org/project1
|
||||
- org/project2
|
||||
|
||||
- job:
|
||||
name: project1-job
|
||||
|
||||
- job:
|
||||
name: project2-job
|
||||
|
||||
- project:
|
||||
name: org/project1
|
||||
queue: integrated
|
||||
gate:
|
||||
jobs:
|
||||
- common-job
|
||||
- project1-job
|
||||
|
||||
- project:
|
||||
name: org/project2
|
||||
queue: integrated
|
||||
gate:
|
||||
jobs:
|
||||
- common-job
|
||||
- project2-job
|
|
@ -0,0 +1,70 @@
|
|||
- queue:
|
||||
name: integrated
|
||||
allow-circular-dependencies: true
|
||||
|
||||
- semaphore:
|
||||
name: test-semaphore
|
||||
|
||||
- pipeline:
|
||||
name: gate
|
||||
manager: dependent
|
||||
success-message: Build succeeded (gate).
|
||||
require:
|
||||
gerrit:
|
||||
approval:
|
||||
- Approved: 1
|
||||
trigger:
|
||||
gerrit:
|
||||
- event: comment-added
|
||||
approval:
|
||||
- Approved: 1
|
||||
success:
|
||||
gerrit:
|
||||
Verified: 2
|
||||
submit: true
|
||||
failure:
|
||||
gerrit:
|
||||
Verified: -2
|
||||
start:
|
||||
gerrit:
|
||||
Verified: 0
|
||||
precedence: high
|
||||
|
||||
- job:
|
||||
name: base
|
||||
parent: null
|
||||
run: playbooks/run.yaml
|
||||
nodeset:
|
||||
nodes:
|
||||
- label: debian
|
||||
name: controller
|
||||
|
||||
- job:
|
||||
name: common-job
|
||||
semaphore:
|
||||
name: test-semaphore
|
||||
required-projects:
|
||||
- org/project1
|
||||
- org/project2
|
||||
|
||||
- job:
|
||||
name: project1-job
|
||||
|
||||
- job:
|
||||
name: project2-job
|
||||
|
||||
- project:
|
||||
name: org/project1
|
||||
queue: integrated
|
||||
gate:
|
||||
jobs:
|
||||
- common-job
|
||||
- project1-job
|
||||
|
||||
- project:
|
||||
name: org/project2
|
||||
queue: integrated
|
||||
gate:
|
||||
jobs:
|
||||
- common-job
|
||||
- project2-job
|
|
@ -0,0 +1,63 @@
|
|||
- queue:
|
||||
name: integrated
|
||||
allow-circular-dependencies: true
|
||||
|
||||
- pipeline:
|
||||
name: gate
|
||||
manager: dependent
|
||||
success-message: Build succeeded (gate).
|
||||
require:
|
||||
gerrit:
|
||||
approval:
|
||||
- Approved: 1
|
||||
trigger:
|
||||
gerrit:
|
||||
- event: comment-added
|
||||
approval:
|
||||
- Approved: 1
|
||||
success:
|
||||
gerrit:
|
||||
Verified: 2
|
||||
submit: true
|
||||
failure:
|
||||
gerrit:
|
||||
Verified: -2
|
||||
start:
|
||||
gerrit:
|
||||
Verified: 0
|
||||
precedence: high
|
||||
|
||||
- job:
|
||||
name: base
|
||||
parent: null
|
||||
run: playbooks/run.yaml
|
||||
nodeset:
|
||||
nodes:
|
||||
- label: debian
|
||||
name: controller
|
||||
|
||||
- job:
|
||||
name: common-job
|
||||
deduplicate: true
|
||||
|
||||
- job:
|
||||
name: project1-job
|
||||
|
||||
- job:
|
||||
name: project2-job
|
||||
|
||||
- project:
|
||||
name: org/project1
|
||||
queue: integrated
|
||||
gate:
|
||||
jobs:
|
||||
- common-job
|
||||
- project1-job
|
||||
|
||||
- project:
|
||||
name: org/project2
|
||||
queue: integrated
|
||||
gate:
|
||||
jobs:
|
||||
- common-job
|
||||
- project2-job
|
|
@ -45,6 +45,7 @@
|
|||
name: base
|
||||
parent: null
|
||||
run: playbooks/base.yaml
|
||||
deduplicate: false
|
||||
nodeset:
|
||||
nodes:
|
||||
- label: ubuntu-xenial
|
||||
|
|
|
@ -17,7 +17,7 @@ import textwrap
|
|||
|
||||
from zuul.model import PromoteEvent
|
||||
|
||||
from tests.base import ZuulTestCase, simple_layout
|
||||
from tests.base import ZuulTestCase, simple_layout, iterate_timeout
|
||||
|
||||
|
||||
class TestGerritCircularDependencies(ZuulTestCase):
|
||||
|
@ -1113,6 +1113,7 @@ class TestGerritCircularDependencies(ZuulTestCase):
|
|||
"""
|
||||
- job:
|
||||
name: project-vars-job
|
||||
deduplicate: false
|
||||
vars:
|
||||
test_var: pass
|
||||
|
||||
|
@ -1536,6 +1537,362 @@ class TestGerritCircularDependencies(ZuulTestCase):
|
|||
self.assertEqual(B.data['status'], 'NEW')
|
||||
self.assertEqual(C.data['status'], 'MERGED')
|
||||
|
||||
def _test_job_deduplication(self):
|
||||
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
|
||||
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
|
||||
|
||||
# A <-> B
|
||||
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
|
||||
A.subject, B.data["url"]
|
||||
)
|
||||
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
|
||||
B.subject, A.data["url"]
|
||||
)
|
||||
|
||||
A.addApproval('Code-Review', 2)
|
||||
B.addApproval('Code-Review', 2)
|
||||
|
||||
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
|
||||
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
|
||||
|
||||
self.waitUntilSettled()
|
||||
|
||||
self.assertEqual(A.data['status'], 'MERGED')
|
||||
self.assertEqual(B.data['status'], 'MERGED')
|
||||
|
||||
@simple_layout('layouts/job-dedup-auto-shared.yaml')
|
||||
def test_job_deduplication_auto_shared(self):
|
||||
self._test_job_deduplication()
|
||||
self.assertHistory([
|
||||
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
# This is deduplicated
|
||||
# dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
], ordered=False)
|
||||
self.assertEqual(len(self.fake_nodepool.history), 3)
|
||||
|
||||
@simple_layout('layouts/job-dedup-auto-unshared.yaml')
|
||||
def test_job_deduplication_auto_unshared(self):
|
||||
self._test_job_deduplication()
|
||||
self.assertHistory([
|
||||
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
# This is not deduplicated
|
||||
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
], ordered=False)
|
||||
self.assertEqual(len(self.fake_nodepool.history), 4)
|
||||
|
||||
@simple_layout('layouts/job-dedup-true.yaml')
|
||||
def test_job_deduplication_true(self):
|
||||
self._test_job_deduplication()
|
||||
self.assertHistory([
|
||||
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
# This is deduplicated
|
||||
# dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
], ordered=False)
|
||||
self.assertEqual(len(self.fake_nodepool.history), 3)
|
||||
|
||||
@simple_layout('layouts/job-dedup-false.yaml')
|
||||
def test_job_deduplication_false(self):
|
||||
self._test_job_deduplication()
|
||||
self.assertHistory([
|
||||
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
# This is not deduplicated, though it would be under auto
|
||||
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
], ordered=False)
|
||||
self.assertEqual(len(self.fake_nodepool.history), 4)
|
||||
|
||||
@simple_layout('layouts/job-dedup-empty-nodeset.yaml')
|
||||
def test_job_deduplication_empty_nodeset(self):
|
||||
# Make sure that jobs with empty nodesets can still be
|
||||
# deduplicated
|
||||
self._test_job_deduplication()
|
||||
self.assertHistory([
|
||||
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
# This is deduplicated
|
||||
# dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
], ordered=False)
|
||||
self.assertEqual(len(self.fake_nodepool.history), 0)
|
||||
|
||||
@simple_layout('layouts/job-dedup-auto-shared.yaml')
|
||||
def test_job_deduplication_failed_node_request(self):
|
||||
# Pause nodepool so we can fail the node request later
|
||||
self.fake_nodepool.pause()
|
||||
|
||||
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
|
||||
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
|
||||
|
||||
# A <-> B
|
||||
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
|
||||
A.subject, B.data["url"]
|
||||
)
|
||||
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
|
||||
B.subject, A.data["url"]
|
||||
)
|
||||
|
||||
A.addApproval('Code-Review', 2)
|
||||
B.addApproval('Code-Review', 2)
|
||||
|
||||
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
|
||||
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
|
||||
|
||||
self.waitUntilSettled()
|
||||
|
||||
# Fail the node request and unpause
|
||||
for req in self.fake_nodepool.getNodeRequests():
|
||||
if req['requestor_data']['job_name'] == 'common-job':
|
||||
self.fake_nodepool.addFailRequest(req)
|
||||
|
||||
self.fake_nodepool.unpause()
|
||||
self.waitUntilSettled()
|
||||
|
||||
self.assertEqual(A.data['status'], 'NEW')
|
||||
self.assertEqual(B.data['status'], 'NEW')
|
||||
self.assertHistory([])
|
||||
self.assertEqual(len(self.fake_nodepool.history), 3)
|
||||
|
||||
@simple_layout('layouts/job-dedup-auto-shared.yaml')
|
||||
def test_job_deduplication_failed_job(self):
|
||||
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
|
||||
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
|
||||
|
||||
# A <-> B
|
||||
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
|
||||
A.subject, B.data["url"]
|
||||
)
|
||||
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
|
||||
B.subject, A.data["url"]
|
||||
)
|
||||
|
||||
A.addApproval('Code-Review', 2)
|
||||
B.addApproval('Code-Review', 2)
|
||||
self.executor_server.failJob("common-job", A)
|
||||
|
||||
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
|
||||
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
|
||||
|
||||
self.waitUntilSettled()
|
||||
|
||||
self.assertEqual(A.data['status'], 'NEW')
|
||||
self.assertEqual(B.data['status'], 'NEW')
|
||||
self.assertHistory([
|
||||
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
dict(name="common-job", result="FAILURE", changes="2,1 1,1"),
|
||||
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
# This is deduplicated
|
||||
# dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
], ordered=False)
|
||||
self.assertEqual(len(self.fake_nodepool.history), 3)
|
||||
|
||||
@simple_layout('layouts/job-dedup-retry.yaml')
|
||||
def test_job_deduplication_retry(self):
|
||||
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
|
||||
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
|
||||
|
||||
# A <-> B
|
||||
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
|
||||
A.subject, B.data["url"]
|
||||
)
|
||||
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
|
||||
B.subject, A.data["url"]
|
||||
)
|
||||
|
||||
self.executor_server.retryJob('common-job', A)
|
||||
|
||||
A.addApproval('Code-Review', 2)
|
||||
B.addApproval('Code-Review', 2)
|
||||
|
||||
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
|
||||
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
|
||||
|
||||
self.waitUntilSettled()
|
||||
|
||||
self.assertEqual(A.data['status'], 'NEW')
|
||||
self.assertEqual(B.data['status'], 'NEW')
|
||||
self.assertHistory([
|
||||
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
# There should be exactly 3 runs of the job (not 6)
|
||||
dict(name="common-job", result=None, changes="2,1 1,1"),
|
||||
dict(name="common-job", result=None, changes="2,1 1,1"),
|
||||
dict(name="common-job", result=None, changes="2,1 1,1"),
|
||||
], ordered=False)
|
||||
self.assertEqual(len(self.fake_nodepool.history), 5)
|
||||
|
||||
@simple_layout('layouts/job-dedup-retry-child.yaml')
|
||||
def test_job_deduplication_retry_child(self):
|
||||
# This tests retrying a paused build (simulating an executor restart)
|
||||
# See test_data_return_child_from_retried_paused_job
|
||||
self.executor_server.hold_jobs_in_build = True
|
||||
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
|
||||
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
|
||||
|
||||
# A <-> B
|
||||
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
|
||||
A.subject, B.data["url"]
|
||||
)
|
||||
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
|
||||
B.subject, A.data["url"]
|
||||
)
|
||||
|
||||
self.executor_server.returnData(
|
||||
'parent-job', A,
|
||||
{'zuul': {'pause': True}}
|
||||
)
|
||||
|
||||
A.addApproval('Code-Review', 2)
|
||||
B.addApproval('Code-Review', 2)
|
||||
|
||||
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
|
||||
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
|
||||
|
||||
self.waitUntilSettled()
|
||||
|
||||
self.executor_server.release('parent-job')
|
||||
self.waitUntilSettled("till job is paused")
|
||||
|
||||
paused_job = self.builds[0]
|
||||
self.assertTrue(paused_job.paused)
|
||||
|
||||
# Stop the job worker to simulate an executor restart
|
||||
for job_worker in self.executor_server.job_workers.values():
|
||||
if job_worker.build_request.uuid == paused_job.uuid:
|
||||
job_worker.stop()
|
||||
self.waitUntilSettled("stop job worker")
|
||||
|
||||
self.executor_server.hold_jobs_in_build = False
|
||||
self.executor_server.release()
|
||||
self.waitUntilSettled("all jobs are done")
|
||||
# The "pause" job might be paused during the waitUntilSettled
|
||||
# call and appear settled; it should automatically resume
|
||||
# though, so just wait for it.
|
||||
for x in iterate_timeout(60, 'paused job'):
|
||||
if not self.builds:
|
||||
break
|
||||
self.waitUntilSettled()
|
||||
|
||||
self.assertEqual(A.data['status'], 'MERGED')
|
||||
self.assertEqual(B.data['status'], 'MERGED')
|
||||
self.assertHistory([
|
||||
dict(name="parent-job", result="ABORTED", changes="2,1 1,1"),
|
||||
dict(name="project1-job", result="ABORTED", changes="2,1 1,1"),
|
||||
dict(name="project2-job", result="ABORTED", changes="2,1 1,1"),
|
||||
dict(name="parent-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
], ordered=False)
|
||||
self.assertEqual(len(self.fake_nodepool.history), 6)
|
||||
|
||||
@simple_layout('layouts/job-dedup-parent-data.yaml')
|
||||
def test_job_deduplication_parent_data(self):
|
||||
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
|
||||
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
|
||||
|
||||
# A <-> B
|
||||
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
|
||||
A.subject, B.data["url"]
|
||||
)
|
||||
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
|
||||
B.subject, A.data["url"]
|
||||
)
|
||||
|
||||
# The parent job returns data
|
||||
self.executor_server.returnData(
|
||||
'parent-job', A,
|
||||
{'zuul':
|
||||
{'artifacts': [
|
||||
{'name': 'image',
|
||||
'url': 'http://example.com/image',
|
||||
'metadata': {
|
||||
'type': 'container_image'
|
||||
}},
|
||||
]}}
|
||||
)
|
||||
|
||||
A.addApproval('Code-Review', 2)
|
||||
B.addApproval('Code-Review', 2)
|
||||
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
|
||||
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
|
||||
|
||||
self.waitUntilSettled()
|
||||
|
||||
self.assertEqual(A.data['status'], 'MERGED')
|
||||
self.assertEqual(B.data['status'], 'MERGED')
|
||||
self.assertHistory([
|
||||
dict(name="parent-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
# Only one run of the common job since it's the same
|
||||
dict(name="common-child-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
# The forked job depends on different parents
|
||||
# so it should run twice
|
||||
dict(name="forked-child-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
dict(name="forked-child-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
], ordered=False)
|
||||
self.assertEqual(len(self.fake_nodepool.history), 6)
|
||||
|
||||
def _test_job_deduplication_semaphore(self):
|
||||
"Test semaphores with max=1 (mutex) and get resources first"
|
||||
self.executor_server.hold_jobs_in_build = True
|
||||
|
||||
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
|
||||
self.assertEqual(
|
||||
len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
|
||||
0)
|
||||
|
||||
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
|
||||
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
|
||||
|
||||
# A <-> B
|
||||
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
|
||||
A.subject, B.data["url"]
|
||||
)
|
||||
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
|
||||
B.subject, A.data["url"]
|
||||
)
|
||||
|
||||
A.addApproval('Code-Review', 2)
|
||||
B.addApproval('Code-Review', 2)
|
||||
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
|
||||
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
|
||||
|
||||
self.waitUntilSettled()
|
||||
self.assertEqual(
|
||||
len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
|
||||
1)
|
||||
|
||||
self.executor_server.hold_jobs_in_build = False
|
||||
self.executor_server.release()
|
||||
self.waitUntilSettled()
|
||||
|
||||
self.assertHistory([
|
||||
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
# This is deduplicated
|
||||
# dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
], ordered=False)
|
||||
self.assertEqual(len(self.fake_nodepool.history), 3)
|
||||
self.assertEqual(
|
||||
len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
|
||||
0)
|
||||
|
||||
@simple_layout('layouts/job-dedup-semaphore.yaml')
|
||||
def test_job_deduplication_semaphore(self):
|
||||
self._test_job_deduplication_semaphore()
|
||||
|
||||
@simple_layout('layouts/job-dedup-semaphore-first.yaml')
|
||||
def test_job_deduplication_semaphore_resources_first(self):
|
||||
self._test_job_deduplication_semaphore()
|
||||
|
||||
def test_submitted_together(self):
|
||||
self.fake_gerrit._fake_submit_whole_topic = True
|
||||
A = self.fake_gerrit.addFakeChange('org/project1', "master", "A",
|
||||
|
|
|
@ -360,3 +360,45 @@ class TestGithubModelUpgrade(ZuulTestCase):
|
|||
dict(name='project-test2', result='SUCCESS'),
|
||||
], ordered=False)
|
||||
self.assertTrue(A.is_merged)
|
||||
|
||||
|
||||
class TestDeduplication(ZuulTestCase):
|
||||
config_file = "zuul-gerrit-github.conf"
|
||||
tenant_config_file = "config/circular-dependencies/main.yaml"
|
||||
scheduler_count = 1
|
||||
|
||||
def _test_job_deduplication(self):
|
||||
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
|
||||
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
|
||||
|
||||
# A <-> B
|
||||
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
|
||||
A.subject, B.data["url"]
|
||||
)
|
||||
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
|
||||
B.subject, A.data["url"]
|
||||
)
|
||||
|
||||
A.addApproval('Code-Review', 2)
|
||||
B.addApproval('Code-Review', 2)
|
||||
|
||||
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
|
||||
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
|
||||
|
||||
self.waitUntilSettled()
|
||||
|
||||
self.assertEqual(A.data['status'], 'MERGED')
|
||||
self.assertEqual(B.data['status'], 'MERGED')
|
||||
|
||||
@simple_layout('layouts/job-dedup-auto-shared.yaml')
|
||||
@model_version(7)
|
||||
def test_job_deduplication_auto_shared(self):
|
||||
self._test_job_deduplication()
|
||||
self.assertHistory([
|
||||
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
# This would be deduplicated
|
||||
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
|
||||
], ordered=False)
|
||||
self.assertEqual(len(self.fake_nodepool.history), 4)
|
||||
|
|
|
@ -371,6 +371,7 @@ class TestWeb(BaseTestWeb):
|
|||
'ansible_version': None,
|
||||
'attempts': 4,
|
||||
'branches': [],
|
||||
'deduplicate': 'auto',
|
||||
'dependencies': [],
|
||||
'description': None,
|
||||
'files': [],
|
||||
|
@ -421,6 +422,7 @@ class TestWeb(BaseTestWeb):
|
|||
'ansible_version': None,
|
||||
'attempts': 3,
|
||||
'branches': ['stable'],
|
||||
'deduplicate': 'auto',
|
||||
'dependencies': [],
|
||||
'description': None,
|
||||
'files': [],
|
||||
|
@ -475,6 +477,7 @@ class TestWeb(BaseTestWeb):
|
|||
'ansible_version': None,
|
||||
'attempts': 3,
|
||||
'branches': [],
|
||||
'deduplicate': 'auto',
|
||||
'dependencies': [],
|
||||
'description': None,
|
||||
'files': [],
|
||||
|
@ -598,6 +601,7 @@ class TestWeb(BaseTestWeb):
|
|||
'ansible_version': None,
|
||||
'attempts': 3,
|
||||
'branches': [],
|
||||
'deduplicate': 'auto',
|
||||
'dependencies': [],
|
||||
'description': None,
|
||||
'files': [],
|
||||
|
@ -636,6 +640,7 @@ class TestWeb(BaseTestWeb):
|
|||
'ansible_version': None,
|
||||
'attempts': 3,
|
||||
'branches': [],
|
||||
'deduplicate': 'auto',
|
||||
'dependencies': [{'name': 'project-merge',
|
||||
'soft': False}],
|
||||
'description': None,
|
||||
|
@ -675,6 +680,7 @@ class TestWeb(BaseTestWeb):
|
|||
'ansible_version': None,
|
||||
'attempts': 3,
|
||||
'branches': [],
|
||||
'deduplicate': 'auto',
|
||||
'dependencies': [{'name': 'project-merge',
|
||||
'soft': False}],
|
||||
'description': None,
|
||||
|
@ -714,6 +720,7 @@ class TestWeb(BaseTestWeb):
|
|||
'ansible_version': None,
|
||||
'attempts': 3,
|
||||
'branches': [],
|
||||
'deduplicate': 'auto',
|
||||
'dependencies': [{'name': 'project-merge',
|
||||
'soft': False}],
|
||||
'description': None,
|
||||
|
@ -778,6 +785,7 @@ class TestWeb(BaseTestWeb):
|
|||
'ansible_version': None,
|
||||
'attempts': 3,
|
||||
'branches': [],
|
||||
'deduplicate': 'auto',
|
||||
'dependencies': [],
|
||||
'description': None,
|
||||
'files': [],
|
||||
|
|
|
@ -633,6 +633,7 @@ class JobParser(object):
|
|||
'post-review': bool,
|
||||
'match-on-config-updates': bool,
|
||||
'workspace-scheme': vs.Any('golang', 'flat', 'unique'),
|
||||
'deduplicate': vs.Any(bool, 'auto'),
|
||||
}
|
||||
|
||||
job_name = {vs.Required('name'): str}
|
||||
|
@ -658,6 +659,7 @@ class JobParser(object):
|
|||
'override-checkout',
|
||||
'match-on-config-updates',
|
||||
'workspace-scheme',
|
||||
'deduplicate',
|
||||
]
|
||||
|
||||
def __init__(self, pcontext):
|
||||
|
|
|
@ -1447,6 +1447,7 @@ class PipelineManager(metaclass=ABCMeta):
|
|||
if build_set.repo_state_state == build_set.PENDING:
|
||||
return False
|
||||
|
||||
item.deduplicateJobs(log)
|
||||
return True
|
||||
|
||||
def _processOneItem(self, item, nnfi):
|
||||
|
@ -1745,25 +1746,45 @@ class PipelineManager(metaclass=ABCMeta):
|
|||
build, item)
|
||||
return
|
||||
|
||||
item.setResult(build)
|
||||
log.debug("Item %s status is now:\n %s", item, item.formatStatus())
|
||||
# If the build was for deduplicated jobs, apply the results to
|
||||
# all the items that use this build.
|
||||
build_in_items = [item]
|
||||
if item.bundle:
|
||||
for other_item in item.bundle.items:
|
||||
if other_item not in build_in_items:
|
||||
build_in_items.append(other_item)
|
||||
for item in build_in_items:
|
||||
# We don't care about some actions below if this build
|
||||
# isn't in the current buildset, so determine that before
|
||||
# it is potentially removed with setResult.
|
||||
if item.current_build_set.getBuild(build.job.name) is not build:
|
||||
current = False
|
||||
else:
|
||||
current = True
|
||||
item.setResult(build)
|
||||
log.debug("Item %s status is now:\n %s", item, item.formatStatus())
|
||||
|
||||
if build.retry:
|
||||
if build.build_set.getJobNodeSetInfo(build.job.name):
|
||||
build.build_set.removeJobNodeSetInfo(build.job.name)
|
||||
if not current:
|
||||
continue
|
||||
build_set = item.current_build_set
|
||||
|
||||
# in case this was a paused build we need to retry all child jobs
|
||||
self._resetDependentBuilds(build.build_set, build)
|
||||
if build.retry:
|
||||
if build_set.getJobNodeSetInfo(build.job.name):
|
||||
build_set.removeJobNodeSetInfo(build.job.name)
|
||||
|
||||
self._resumeBuilds(build.build_set)
|
||||
# in case this was a paused build we need to retry all
|
||||
# child jobs
|
||||
self._resetDependentBuilds(build_set, build)
|
||||
|
||||
if (item.current_build_set.fail_fast and
|
||||
build.failed and build.job.voting and not build.retry):
|
||||
# If fail-fast is set and the build is not successful
|
||||
# cancel all remaining jobs.
|
||||
log.debug("Build %s failed and fail-fast enabled, canceling "
|
||||
"running builds", build)
|
||||
self._cancelRunningBuilds(build.build_set)
|
||||
self._resumeBuilds(build_set)
|
||||
|
||||
if (build_set.fail_fast and
|
||||
build.failed and build.job.voting and not build.retry):
|
||||
# If fail-fast is set and the build is not successful
|
||||
# cancel all remaining jobs.
|
||||
log.debug("Build %s failed and fail-fast enabled, canceling "
|
||||
"running builds", build)
|
||||
self._cancelRunningBuilds(build_set)
|
||||
|
||||
return True
|
||||
|
||||
|
|
180
zuul/model.py
180
zuul/model.py
|
@ -2005,6 +2005,7 @@ class FrozenJob(zkobject.ZKObject):
|
|||
'requires',
|
||||
'workspace_scheme',
|
||||
'config_hash',
|
||||
'deduplicate',
|
||||
)
|
||||
|
||||
job_data_attributes = ('artifact_data',
|
||||
|
@ -2018,9 +2019,33 @@ class FrozenJob(zkobject.ZKObject):
|
|||
'affected_projects',
|
||||
)
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._set(_ready_to_run=False)
|
||||
|
||||
def __repr__(self):
|
||||
return '<FrozenJob %s>' % (self.name)
|
||||
|
||||
def isEqual(self, other):
|
||||
# Compare two frozen jobs to determine whether they are
|
||||
# effectively equal. The inheritance path will always be
|
||||
# different, so it is ignored. But if otherwise they have the
|
||||
# same attributes, they will probably produce the same
|
||||
# results.
|
||||
if not isinstance(other, FrozenJob):
|
||||
return False
|
||||
if self.name != other.name:
|
||||
return False
|
||||
for k in self.attributes:
|
||||
if k in ['inheritance_path', 'waiting_status', 'queued']:
|
||||
continue
|
||||
if getattr(self, k) != getattr(other, k):
|
||||
return False
|
||||
for k in self.job_data_attributes:
|
||||
if getattr(self, k) != getattr(other, k):
|
||||
return False
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def new(klass, context, **kw):
|
||||
obj = klass()
|
||||
|
@ -2367,6 +2392,7 @@ class Job(ConfigObject):
|
|||
self.cleanup_run))
|
||||
d['post_review'] = self.post_review
|
||||
d['match_on_config_updates'] = self.match_on_config_updates
|
||||
d['deduplicate'] = self.deduplicate
|
||||
if self.isBase():
|
||||
d['parent'] = None
|
||||
elif self.parent:
|
||||
|
@ -2404,6 +2430,7 @@ class Job(ConfigObject):
|
|||
irrelevant_file_matcher=None, # skip-if
|
||||
_irrelevant_files=(),
|
||||
match_on_config_updates=True,
|
||||
deduplicate='auto',
|
||||
tags=frozenset(),
|
||||
provides=frozenset(),
|
||||
requires=frozenset(),
|
||||
|
@ -3041,6 +3068,16 @@ class JobDependency(ConfigObject):
|
|||
self.name = name
|
||||
self.soft = soft
|
||||
|
||||
def __ne__(self, other):
|
||||
return not self.__eq__(other)
|
||||
|
||||
def __eq__(self, other):
|
||||
if not isinstance(other, JobDependency):
|
||||
return False
|
||||
return self.toDict() == other.toDict()
|
||||
|
||||
__hash__ = object.__hash__
|
||||
|
||||
def toDict(self):
|
||||
return {'name': self.name,
|
||||
'soft': self.soft}
|
||||
|
@ -3698,6 +3735,7 @@ class BuildSet(zkobject.ZKObject):
|
|||
fail_fast=False,
|
||||
job_graph=None,
|
||||
jobs={},
|
||||
deduplicated_jobs=[],
|
||||
# Cached job graph of previous layout; not serialized
|
||||
_old_job_graph=None,
|
||||
_old_jobs={},
|
||||
|
@ -4028,7 +4066,22 @@ class BuildSet(zkobject.ZKObject):
|
|||
|
||||
def removeJobNodeRequestID(self, job_name):
|
||||
if job_name in self.node_requests:
|
||||
del self.node_requests[job_name]
|
||||
with self.activeContext(
|
||||
self.item.pipeline.manager.current_context):
|
||||
del self.node_requests[job_name]
|
||||
|
||||
def setJobNodeRequestDuplicate(self, job_name, other_item):
|
||||
with self.activeContext(
|
||||
self.item.pipeline.manager.current_context):
|
||||
self.node_requests[job_name] = {
|
||||
'deduplicated_item': other_item.uuid}
|
||||
|
||||
def setJobNodeSetInfoDuplicate(self, job_name, other_item):
|
||||
# Nothing uses this value yet; we just need an entry in the
|
||||
# nodset_info dict.
|
||||
with self.activeContext(self.item.pipeline.manager.current_context):
|
||||
self.nodeset_info[job_name] = {
|
||||
'deduplicated_item': other_item.uuid}
|
||||
|
||||
def jobNodeRequestComplete(self, job_name, nodeset):
|
||||
if job_name in self.nodeset_info:
|
||||
|
@ -4415,7 +4468,7 @@ class QueueItem(zkobject.ZKObject):
|
|||
continue
|
||||
build = self.current_build_set.getBuild(job.name)
|
||||
if (build and build.result and
|
||||
build.result not in ['SUCCESS', 'SKIPPED']):
|
||||
build.result not in ['SUCCESS', 'SKIPPED', 'RETRY']):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
@ -4626,7 +4679,8 @@ class QueueItem(zkobject.ZKObject):
|
|||
data = []
|
||||
ret = self.item_ahead.providesRequirements(job, data)
|
||||
data.reverse()
|
||||
job.setArtifactData(data)
|
||||
if data:
|
||||
job.setArtifactData(data)
|
||||
except RequirementsError as e:
|
||||
self.warning(str(e))
|
||||
fakebuild = Build.new(self.pipeline.manager.current_context,
|
||||
|
@ -4637,24 +4691,36 @@ class QueueItem(zkobject.ZKObject):
|
|||
ret = False
|
||||
return ret
|
||||
|
||||
def findJobsToRun(self, semaphore_handler):
|
||||
torun = []
|
||||
if not self.live:
|
||||
return []
|
||||
if not self.current_build_set.job_graph:
|
||||
return []
|
||||
if self.item_ahead:
|
||||
# Only run jobs if any 'hold' jobs on the change ahead
|
||||
# have completed successfully.
|
||||
if self.item_ahead.isHoldingFollowingChanges():
|
||||
return []
|
||||
def findDuplicateJob(self, job):
|
||||
"""
|
||||
If another item in the bundle has a duplicate job,
|
||||
return the other item
|
||||
"""
|
||||
if not self.bundle:
|
||||
return None
|
||||
if job.deduplicate is False:
|
||||
return None
|
||||
for other_item in self.bundle.items:
|
||||
if other_item is self:
|
||||
continue
|
||||
for other_job in other_item.getJobs():
|
||||
if other_job.isEqual(job):
|
||||
if job.deduplicate == 'auto':
|
||||
# Deduplicate if there are required projects
|
||||
# or the item project is the same.
|
||||
if (not job.required_projects and
|
||||
self.change.project != other_item.change.project):
|
||||
continue
|
||||
return other_item
|
||||
|
||||
def updateJobParentData(self):
|
||||
job_graph = self.current_build_set.job_graph
|
||||
failed_job_names = set() # Jobs that run and failed
|
||||
ignored_job_names = set() # Jobs that were skipped or canceled
|
||||
unexecuted_job_names = set() # Jobs that were not started yet
|
||||
jobs_not_started = set()
|
||||
for job in job_graph.getJobs():
|
||||
job._set(_ready_to_run=False)
|
||||
build = self.current_build_set.getBuild(job.name)
|
||||
if build:
|
||||
if build.result == 'SUCCESS' or build.paused:
|
||||
|
@ -4667,8 +4733,6 @@ class QueueItem(zkobject.ZKObject):
|
|||
unexecuted_job_names.add(job.name)
|
||||
jobs_not_started.add(job)
|
||||
|
||||
# Attempt to run jobs in the order they appear in
|
||||
# configuration.
|
||||
for job in job_graph.getJobs():
|
||||
if job not in jobs_not_started:
|
||||
continue
|
||||
|
@ -4719,7 +4783,91 @@ class QueueItem(zkobject.ZKObject):
|
|||
job.setParentData(new_parent_data,
|
||||
new_secret_parent_data,
|
||||
new_artifact_data)
|
||||
job._set(_ready_to_run=True)
|
||||
|
||||
def deduplicateJobs(self, log):
|
||||
"""Sync node request and build info with deduplicated jobs"""
|
||||
if not self.live:
|
||||
return
|
||||
if not self.current_build_set.job_graph:
|
||||
return
|
||||
if self.item_ahead:
|
||||
# Only run jobs if any 'hold' jobs on the change ahead
|
||||
# have completed successfully.
|
||||
if self.item_ahead.isHoldingFollowingChanges():
|
||||
return
|
||||
|
||||
self.updateJobParentData()
|
||||
|
||||
if COMPONENT_REGISTRY.model_api < 8:
|
||||
return
|
||||
|
||||
if not self.bundle:
|
||||
return
|
||||
|
||||
build_set = self.current_build_set
|
||||
job_graph = build_set.job_graph
|
||||
for job in job_graph.getJobs():
|
||||
this_request = build_set.getJobNodeRequestID(job.name)
|
||||
this_nodeset = build_set.getJobNodeSetInfo(job.name)
|
||||
this_build = build_set.getBuild(job.name)
|
||||
|
||||
if this_build:
|
||||
# Nothing more possible for this job
|
||||
continue
|
||||
|
||||
other_item = self.findDuplicateJob(job)
|
||||
if not other_item:
|
||||
continue
|
||||
other_build_set = other_item.current_build_set
|
||||
|
||||
# Handle node requests
|
||||
other_request = other_build_set.getJobNodeRequestID(job.name)
|
||||
if (isinstance(other_request, dict) and
|
||||
other_request.get('deduplicated_item') == self.uuid):
|
||||
# We're the original, but we're probably in the middle
|
||||
# of a retry
|
||||
return
|
||||
if other_request is not None and this_request is None:
|
||||
log.info("Deduplicating request of bundle job %s for item %s "
|
||||
"with item %s", job, self, other_item)
|
||||
build_set.setJobNodeRequestDuplicate(job.name, other_item)
|
||||
|
||||
# Handle provisioned nodes
|
||||
other_nodeset = other_build_set.getJobNodeSetInfo(job.name)
|
||||
if (isinstance(other_nodeset, dict) and
|
||||
other_nodeset.get('deduplicated_item') == self.uuid):
|
||||
# We're the original, but we're probably in the middle
|
||||
# of a retry
|
||||
return
|
||||
if other_nodeset is not None and this_nodeset is None:
|
||||
log.info("Deduplicating nodeset of bundle job %s for item %s "
|
||||
"with item %s", job, self, other_item)
|
||||
build_set.setJobNodeSetInfoDuplicate(job.name, other_item)
|
||||
|
||||
# Handle builds
|
||||
other_build = other_build_set.getBuild(job.name)
|
||||
if other_build and not this_build:
|
||||
log.info("Deduplicating build of bundle job %s for item %s "
|
||||
"with item %s", job, self, other_item)
|
||||
self.addBuild(other_build)
|
||||
job._set(_ready_to_run=False)
|
||||
|
||||
def findJobsToRun(self, semaphore_handler):
|
||||
torun = []
|
||||
if not self.live:
|
||||
return []
|
||||
if not self.current_build_set.job_graph:
|
||||
return []
|
||||
if self.item_ahead:
|
||||
# Only run jobs if any 'hold' jobs on the change ahead
|
||||
# have completed successfully.
|
||||
if self.item_ahead.isHoldingFollowingChanges():
|
||||
return []
|
||||
|
||||
job_graph = self.current_build_set.job_graph
|
||||
for job in job_graph.getJobs():
|
||||
if job._ready_to_run:
|
||||
nodeset = self.current_build_set.getJobNodeSetInfo(job.name)
|
||||
if nodeset is None:
|
||||
# The nodes for this job are not ready, skip
|
||||
|
|
|
@ -14,4 +14,4 @@
|
|||
|
||||
# When making ZK schema changes, increment this and add a record to
|
||||
# docs/developer/model-changelog.rst
|
||||
MODEL_API = 7
|
||||
MODEL_API = 8
|
||||
|
|
Loading…
Reference in New Issue