zuul
/
zuul
Code Issues Proposed changes

Improve job dependencies using graph instead of tree 

This replaces the job dependency tree with a graph so that we can
indicate that a job should wait until one or more jobs are complete
before starting.

Project pipeline job definitions are now a flat list, with each job
specifying its dependencies as the job attribute 'dependencies'.

Fixes bug #1166937.

Signed-off-by: Fredrik Medley <fredrik.medley@autoliv.com>
Signed-off-by: Fredrik Medley <fredrik.medley@gmail.com>
Signed-off-by: James E. Blair <jeblair@redhat.com>
Co-Authored-By: James E. Blair <jeblair@redhat.com>
Change-Id: I921940cafeea0738c39deb99357cfd7c91592359
This commit is contained in:
Fredrik Medley 2015-09-28 13:40:20 +02:00 committed by James E. Blair
parent e06a03bb45
commit f8aec83b3b
54 changed files with 726 additions and 444 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
doc/source/zuul.rst
View File

@ -771,8 +771,11 @@ given pipeline. Within the pipeline section, the jobs that should be
executed are listed. If a job is entered as a dictionary key, then
jobs contained within that key are only executed if the key job
succeeds. In the above example, project-unittest, project-pep8, and
project-pyflakes are only executed if project-merge succeeds. This
can help avoid running unnecessary jobs.
project-pyflakes are only executed if project-merge succeeds.
Furthermore, project-finaltest is executed only if project-unittest,
project-pep8 and project-pyflakes all succeed. This can help avoid
running unnecessary jobs while maximizing parallelism. It is also
useful when distributing results between jobs.
The special job named ``noop`` is internal to Zuul and will always
return ``SUCCESS`` immediately. This can be useful if you require

8
tests/fixtures/config/ansible/git/common-config/zuul.yaml vendored
View File

@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: patchset-created
@ -17,8 +16,7 @@
name: gate
manager: dependent
success-message: Build succeeded (gate).
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: comment-added
@ -41,7 +39,7 @@
pre-run: pre
post-run: post
vars:
flagpath: "{{zuul._test.test_root}}/{{zuul.uuid}}.flag"
flagpath: '{{zuul._test.test_root}}/{{zuul.uuid}}.flag'
roles:
- zuul: bare-role

1
tests/fixtures/config/ansible/git/org_project/.zuul.yaml vendored
View File

@ -4,7 +4,6 @@
- project:
name: org/project
check:
jobs:
- python27

2
tests/fixtures/config/dependency-graph/git/common-config/playbooks/A.yaml vendored Normal file
View File

@ -0,0 +1,2 @@
- hosts: all
tasks: []

2
tests/fixtures/config/dependency-graph/git/common-config/playbooks/B.yaml vendored Normal file
View File

@ -0,0 +1,2 @@
- hosts: all
tasks: []

2
tests/fixtures/config/dependency-graph/git/common-config/playbooks/C.yaml vendored Normal file
View File

@ -0,0 +1,2 @@
- hosts: all
tasks: []

2
tests/fixtures/config/dependency-graph/git/common-config/playbooks/D.yaml vendored Normal file
View File

@ -0,0 +1,2 @@
- hosts: all
tasks: []

2
tests/fixtures/config/dependency-graph/git/common-config/playbooks/E.yaml vendored Normal file
View File

@ -0,0 +1,2 @@
- hosts: all
tasks: []

2
tests/fixtures/config/dependency-graph/git/common-config/playbooks/F.yaml vendored Normal file
View File

@ -0,0 +1,2 @@
- hosts: all
tasks: []

2
tests/fixtures/config/dependency-graph/git/common-config/playbooks/G.yaml vendored Normal file
View File

@ -0,0 +1,2 @@
- hosts: all
tasks: []

2
tests/fixtures/config/dependency-graph/git/common-config/playbooks/project-test1.yaml vendored Normal file
View File

@ -0,0 +1,2 @@
- hosts: all
tasks: []

73
tests/fixtures/config/dependency-graph/git/common-config/zuul.yaml vendored Normal file
View File

@ -0,0 +1,73 @@
- pipeline:
name: gate
manager: dependent
success-message: Build succeeded (gate).
source: gerrit
trigger:
gerrit:
- event: comment-added
approval:
- approved: 1
success:
gerrit:
verified: 2
submit: true
failure:
gerrit:
verified: -2
start:
gerrit:
verified: 0
precedence: high
- job:
name: A
- job:
name: B
- job:
name: C
- job:
name: D
- job:
name: E
- job:
name: F
- job:
name: G
- project:
name: org/project
gate:
jobs:
# Job dependencies, starting with A
# A
# / \
# B C
# / \ / \
# D F E
# |
# G
# This is intentionally not listed in the natural order to
# ensure that we can reference dependencies before they are
# defined.
- E:
dependencies: C
- A
- B:
dependencies: A
- C:
dependencies: A
- F:
dependencies:
- B
- C
- D:
dependencies: B
- G:
dependencies: F

1
tests/fixtures/config/dependency-graph/git/org_project/README vendored Normal file
View File

@ -0,0 +1 @@
test

8
tests/fixtures/config/dependency-graph/main.yaml vendored Normal file
View File

@ -0,0 +1,8 @@
- tenant:
name: tenant-one
source:
gerrit:
config-repos:
- common-config
project-repos:
- org/project

7
tests/fixtures/config/duplicate-pipeline/git/common-config/zuul.yaml vendored
View File

@ -2,8 +2,7 @@
name: dup1
manager: independent
success-message: Build succeeded (dup1).
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: change-restored
@ -18,8 +17,7 @@
name: dup2
manager: independent
success-message: Build succeeded (dup2).
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: change-restored
@ -39,7 +37,6 @@
queue: integrated
jobs:
- project-test1
dup2:
queue: integrated
jobs:

6
tests/fixtures/config/in-repo/git/common-config/zuul.yaml vendored
View File

@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: patchset-created
@ -17,8 +16,7 @@
name: tenant-one-gate
manager: dependent
success-message: Build succeeded (tenant-one-gate).
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: comment-added

21
tests/fixtures/config/merges/git/common-config/zuul.yaml vendored
View File

@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: patchset-created
@ -17,8 +16,7 @@
name: gate
manager: dependent
success-message: Build succeeded (gate).
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: comment-added
@ -37,16 +35,13 @@
precedence: high
- job:
name:
project-test1
name: project-test1
- job:
name:
project-test2
name: project-test2
- job:
name:
project-merge
name: project-merge
hold-following-changes: true
- project:
@ -75,6 +70,6 @@
merge-mode: cherry-pick
gate:
jobs:
- project-merge:
jobs:
- project-test1
- project-merge
- project-test1:
dependencies: project-merge

6
tests/fixtures/config/multi-tenant/git/common-config/zuul.yaml vendored
View File

@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: patchset-created
@ -14,8 +13,7 @@
verified: -1
- job:
name:
python27
name: python27
nodes:
- name: controller
image: ubuntu-trusty

6
tests/fixtures/config/multi-tenant/git/tenant-one-config/zuul.yaml vendored
View File

@ -2,8 +2,7 @@
name: tenant-one-gate
manager: dependent
success-message: Build succeeded (tenant-one-gate).
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: comment-added
@ -28,8 +27,7 @@
image: controller-image
- job:
name:
project1-test1
name: project1-test1
- project:
name: org/project1

6
tests/fixtures/config/multi-tenant/git/tenant-two-config/zuul.yaml vendored
View File

@ -2,8 +2,7 @@
name: tenant-two-gate
manager: dependent
success-message: Build succeeded (tenant-two-gate).
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: comment-added
@ -28,8 +27,7 @@
image: controller-image
- job:
name:
project2-test1
name: project2-test1
- project:
name: org/project2

9
tests/fixtures/config/one-job-project/git/common-config/zuul.yaml vendored
View File

@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: patchset-created
@ -17,8 +16,7 @@
name: gate
manager: dependent
success-message: Build succeeded (gate).
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: comment-added
@ -39,8 +37,7 @@
- pipeline:
name: post
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: ref-updated

12
tests/fixtures/config/openstack/git/project-config/zuul.yaml vendored
View File

@ -1,11 +1,8 @@
# Pipeline definitions
- pipeline:
name: check
manager: independent
success-message: Build succeeded (check).
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: patchset-created
@ -20,8 +17,7 @@
name: gate
manager: dependent
success-message: Build succeeded (gate).
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: comment-added
@ -39,8 +35,6 @@
verified: 0
precedence: high
# Job definitions
- job:
name: base
timeout: 30
@ -78,8 +72,6 @@
- openstack/keystone
- openstack/nova
# Project definitions
- project:
name: openstack/nova
templates:

6
tests/fixtures/config/requirements/email/git/common-config/zuul.yaml vendored
View File

@ -1,8 +1,7 @@
- pipeline:
name: pipeline
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: comment-added
@ -19,8 +18,7 @@
- pipeline:
name: trigger
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: comment-added

6
tests/fixtures/config/requirements/newer-than/git/common-config/zuul.yaml vendored
View File

@ -1,8 +1,7 @@
- pipeline:
name: pipeline
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: comment-added
@ -20,8 +19,7 @@
- pipeline:
name: trigger
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: comment-added

6
tests/fixtures/config/requirements/older-than/git/common-config/zuul.yaml vendored
View File

@ -1,8 +1,7 @@
- pipeline:
name: pipeline
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: comment-added
@ -20,8 +19,7 @@
- pipeline:
name: trigger
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: comment-added

10
tests/fixtures/config/requirements/reject-username/git/common-config/zuul.yaml vendored
View File

@ -1,11 +1,10 @@
- pipeline:
name: pipeline
manager: independent
source:
gerrit
source: gerrit
reject:
approval:
- username: 'jenkins'
- username: jenkins
trigger:
gerrit:
- event: comment-added
@ -19,13 +18,12 @@
- pipeline:
name: trigger
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: comment-added
reject-approval:
- username: 'jenkins'
- username: jenkins
success:
gerrit:
verified: 1

22
tests/fixtures/config/requirements/reject/git/common-config/zuul.yaml vendored
View File

@ -1,15 +1,18 @@
- pipeline:
name: pipeline
manager: independent
source:
gerrit
source: gerrit
require:
approval:
- username: jenkins
verified: [1, 2]
verified:
- 1
- 2
reject:
approval:
- verified: [-1, -2]
- verified:
- -1
- -2
trigger:
gerrit:
- event: comment-added
@ -23,16 +26,19 @@
- pipeline:
name: trigger
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: comment-added
require-approval:
- username: jenkins
verified: [1, 2]
verified:
- 1
- 2
reject-approval:
- verified: [-1, -2]
- verified:
- -1
- -2
success:
gerrit:
verified: 1

13
tests/fixtures/config/requirements/state/git/common-config/zuul.yaml vendored
View File

@ -1,10 +1,9 @@
- pipeline:
name: current-check
manager: independent
source:
gerrit
source: gerrit
require:
current-patchset: True
current-patchset: true
trigger:
gerrit:
- event: patchset-created
@ -19,10 +18,9 @@
- pipeline:
name: open-check
manager: independent
source:
gerrit
source: gerrit
require:
open: True
open: true
trigger:
gerrit:
- event: patchset-created
@ -37,8 +35,7 @@
- pipeline:
name: status-check
manager: independent
source:
gerrit
source: gerrit
require:
status: NEW
trigger:

6
tests/fixtures/config/requirements/username/git/common-config/zuul.yaml vendored
View File

@ -1,8 +1,7 @@
- pipeline:
name: pipeline
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: comment-added
@ -19,8 +18,7 @@
- pipeline:
name: trigger
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: comment-added

7
tests/fixtures/config/requirements/vote1/git/common-config/zuul.yaml vendored
View File

@ -1,4 +1,3 @@
- pipeline:
name: pipeline
manager: independent
@ -6,8 +5,7 @@
approval:
- username: jenkins
verified: 1
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: comment-added
@ -21,8 +19,7 @@
- pipeline:
name: trigger
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: comment-added

14
tests/fixtures/config/requirements/vote2/git/common-config/zuul.yaml vendored
View File

@ -4,9 +4,10 @@
require:
approval:
- username: jenkins
verified: [1, 2]
source:
gerrit
verified:
- 1
- 2
source: gerrit
trigger:
gerrit:
- event: comment-added
@ -20,14 +21,15 @@
- pipeline:
name: trigger
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: comment-added
require-approval:
- username: jenkins
verified: [1, 2]
verified:
- 1
- 2
success:
gerrit:
verified: 1

119
tests/fixtures/config/single-tenant/git/common-config/zuul.yaml vendored
View File

@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: patchset-created
@ -17,8 +16,7 @@
name: gate
manager: dependent
success-message: Build succeeded (gate).
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: comment-added
@ -39,8 +37,7 @@
- pipeline:
name: post
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: ref-updated
@ -49,8 +46,7 @@
- pipeline:
name: experimental
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: patchset-created
@ -107,23 +103,26 @@
- job:
name: project-testfile
files:
- '.*-requires'
- .*-requires
- project:
name: org/project
check:
jobs:
- project-merge:
jobs:
- project-test1
- project-test2
- project-merge
- project-test1:
dependencies: project-merge
- project-test2:
dependencies: project-merge
gate:
jobs:
- project-merge:
jobs:
- project-test1
- project-test2
- project-testfile
- project-merge
- project-test1:
dependencies: project-merge
- project-test2:
dependencies: project-merge
- project-testfile:
dependencies: project-merge
post:
jobs:
- project-post
@ -132,48 +131,58 @@
name: org/project1
check:
jobs:
- project-merge:
jobs:
- project-test1
- project-test2
- project1-project2-integration
- project-merge
- project-test1:
dependencies: project-merge
- project-test2:
dependencies: project-merge
- project1-project2-integration:
dependencies: project-merge
gate:
queue: integrated
jobs:
- project-merge:
jobs:
- project-test1
- project-test2
- project1-project2-integration
- project-merge
- project-test1:
dependencies: project-merge
- project-test2:
dependencies: project-merge
- project1-project2-integration:
dependencies: project-merge
- project:
name: org/project2
gate:
queue: integrated
jobs:
- project-merge:
jobs:
- project-test1
- project-test2
- project1-project2-integration
- project-merge
- project-test1:
dependencies: project-merge
- project-test2:
dependencies: project-merge
- project1-project2-integration:
dependencies: project-merge
- project:
name: org/project3
check:
jobs:
- project-merge:
jobs:
- project-test1
- project-test2
- project1-project2-integration
- project-merge
- project-test1:
dependencies: project-merge
- project-test2:
dependencies: project-merge
- project1-project2-integration:
dependencies: project-merge
gate:
queue: integrated
jobs:
- project-merge:
jobs:
- project-test1
- project-test2
- project1-project2-integration
- project-merge
- project-test1:
dependencies: project-merge
- project-test2:
dependencies: project-merge
- project1-project2-integration:
dependencies: project-merge
post:
jobs:
- project-post
@ -182,9 +191,9 @@
name: org/experimental-project
experimental:
jobs:
- project-merge:
jobs:
- experimental-project-test
- project-merge
- experimental-project-test:
dependencies: project-merge
- project:
name: org/noop-project
@ -199,16 +208,18 @@
name: org/nonvoting-project
check:
jobs:
- nonvoting-project-merge:
jobs:
- nonvoting-project-test1
- nonvoting-project-test2
- nonvoting-project-merge
- nonvoting-project-test1:
dependencies: nonvoting-project-merge
- nonvoting-project-test2:
dependencies: nonvoting-project-merge
gate:
jobs:
- nonvoting-project-merge:
jobs:
- nonvoting-project-test1
- nonvoting-project-test2
- nonvoting-project-merge
- nonvoting-project-test1:
dependencies: nonvoting-project-merge
- nonvoting-project-test2:
dependencies: nonvoting-project-merge
- project:
name: org/no-jobs-project

3
tests/fixtures/config/single-tenant/git/layout-disabled-at/zuul.yaml vendored
View File

@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: patchset-created

6
tests/fixtures/config/single-tenant/git/layout-dont-ignore-ref-deletes/zuul.yaml vendored
View File

@ -1,13 +1,12 @@
- pipeline:
name: post
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: ref-updated
ref: ^(?!refs/).*$
ignore-deletes: False
ignore-deletes: false
- job:
name: project-post
@ -20,4 +19,3 @@
post:
jobs:
- project-post

4
tests/fixtures/config/single-tenant/git/layout-footer-message/zuul.yaml vendored
View File

@ -2,8 +2,7 @@
name: gate
manager: dependent
success-message: Build succeeded (gate).
source:
gerrit
source: gerrit
failure-message: Build failed. For information on how to proceed, see http://wiki.example.org/Test_Failures
footer-message: For CI problems and help debugging, contact ci@example.org
trigger:
@ -35,4 +34,3 @@
gate:
jobs:
- project-test1

3
tests/fixtures/config/single-tenant/git/layout-idle/zuul.yaml vendored
View File

@ -1,8 +1,7 @@
- pipeline:
name: periodic
manager: independent
source:
gerrit
source: gerrit
trigger:
timer:
- time: '* * * * * */1'

4
tests/fixtures/config/single-tenant/git/layout-inheritance/zuul.yaml vendored
View File

@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: patchset-created
@ -13,7 +12,6 @@
gerrit:
verified: -1
- job:
name: project-test-irrelevant-starts-empty

4
tests/fixtures/config/single-tenant/git/layout-irrelevant-files/zuul.yaml vendored
View File

@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: patchset-created
@ -13,7 +12,6 @@
gerrit:
verified: -1
- job:
name: project-test-irrelevant-files

3
tests/fixtures/config/single-tenant/git/layout-mutex-reconfiguration/zuul.yaml vendored
View File

@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: patchset-created

3
tests/fixtures/config/single-tenant/git/layout-mutex/zuul.yaml vendored
View File

@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: patchset-created

6
tests/fixtures/config/single-tenant/git/layout-no-timer/zuul.yaml vendored
View File

@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: patchset-created
@ -18,8 +17,7 @@
manager: independent
# Trigger is required, set it to one that is a noop
# during tests that check the timer trigger.
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: ref-updated

24
tests/fixtures/config/single-tenant/git/layout-repo-deleted/zuul.yaml vendored
View File

@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: patchset-created
@ -17,8 +16,7 @@
name: gate
manager: dependent
success-message: Build succeeded (gate).
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: comment-added
@ -60,13 +58,15 @@
name: org/delete-project
check:
jobs:
- project-merge:
jobs:
- project-test1
- project-test2
- project-merge
- project-test1:
dependencies: project-merge
- project-test2:
dependencies: project-merge
gate:
jobs:
- project-merge:
jobs:
- project-test1
- project-test2
- project-merge
- project-test1:
dependencies: project-merge
- project-test2:
dependencies: project-merge

24
tests/fixtures/config/single-tenant/git/layout-smtp/zuul.yaml vendored
View File

@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: patchset-created
@ -23,8 +22,7 @@
name: gate
manager: dependent
success-message: Build succeeded (gate).
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: comment-added
@ -69,13 +67,15 @@
name: org/project
check:
jobs:
- project-merge:
jobs:
- project-test1
- project-test2
- project-merge
- project-test1:
dependencies: project-merge
- project-test2:
dependencies: project-merge
gate:
jobs:
- project-merge:
jobs:
- project-test1
- project-test2
- project-merge
- project-test1:
dependencies: project-merge
- project-test2:
dependencies: project-merge

25
tests/fixtures/config/single-tenant/git/layout-tags/zuul.yaml vendored
View File

@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: patchset-created
@ -34,19 +33,23 @@
check:
jobs:
- merge:
jobs:
- test1
- test2
- integration
tags:
- extratag
- test1:
dependencies: merge
- test2:
dependencies: merge
- integration:
dependencies: merge
- project:
name: org/project2
check:
jobs:
- merge:
jobs:
- test1
- test2
- integration
- merge
- test1:
dependencies: merge
- test2:
dependencies: merge
- integration:
dependencies: merge

5
tests/fixtures/config/single-tenant/git/layout-timer-smtp/zuul.yaml vendored
View File

@ -1,8 +1,7 @@
- pipeline:
name: periodic
manager: independent
source:
gerrit
source: gerrit
trigger:
timer:
- time: '* * * * * */1'
@ -10,7 +9,7 @@
smtp:
to: alternative_me@example.com
from: zuul_from@example.com
subject: 'Periodic check for {change.project} succeeded'
subject: Periodic check for {change.project} succeeded
- job:
name: project-bitrot-stable-old

6
tests/fixtures/config/single-tenant/git/layout-timer/zuul.yaml vendored
View File

@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: patchset-created
@ -16,8 +15,7 @@
- pipeline:
name: periodic
manager: independent
source:
gerrit
source: gerrit
trigger:
timer:
- time: '* * * * * */1'

4
tests/fixtures/config/success-url/git/common-config/zuul.yaml vendored
View File

@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: patchset-created
@ -18,7 +17,6 @@
gerrit:
verified: -1
- job:
name: docs-draft-test
success-url: http://docs-draft.example.org/{build.parameters[LOG_PATH]}/publish-docs/

17
tests/fixtures/config/templated-project/git/common-config/zuul.yaml vendored
View File

@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: patchset-created
@ -17,8 +16,7 @@
name: gate
manager: dependent
success-message: Build succeeded (gate).
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: comment-added
@ -39,8 +37,7 @@
- pipeline:
name: post
manager: independent
source:
gerrit
source: gerrit
trigger:
gerrit:
- event: ref-updated
@ -56,15 +53,15 @@
- project-template:
name: test-three-and-four
check:
jobs:
- layered-project-test3
- layered-project-test4
jobs:
- layered-project-test3
- layered-project-test4
- project-template:
name: test-five
check:
jobs:
- layered-project-foo-test5
- layered-project-foo-test5
- job:
name: project-test1

65
tests/unit/test_model.py
View File

@ -225,7 +225,7 @@ class TestJob(BaseTestCase):
self.assertFalse(python27diablo.changeMatches(change))
self.assertFalse(python27essex.changeMatches(change))
item.freezeJobTree()
item.freezeJobGraph()
self.assertEqual(len(item.getJobs()), 1)
job = item.getJobs()[0]
self.assertEqual(job.name, 'python27')
@ -253,7 +253,7 @@ class TestJob(BaseTestCase):
self.assertTrue(python27diablo.changeMatches(change))
self.assertFalse(python27essex.changeMatches(change))
item.freezeJobTree()
item.freezeJobGraph()
self.assertEqual(len(item.getJobs()), 1)
job = item.getJobs()[0]
self.assertEqual(job.name, 'python27')
@ -282,7 +282,7 @@ class TestJob(BaseTestCase):
self.assertFalse(python27diablo.changeMatches(change))
self.assertTrue(python27essex.changeMatches(change))
item.freezeJobTree()
item.freezeJobGraph()
self.assertEqual(len(item.getJobs()), 1)
job = item.getJobs()[0]
self.assertEqual(job.name, 'python27')
@ -439,7 +439,7 @@ class TestJob(BaseTestCase):
self.assertTrue(python27.changeMatches(change))
self.assertFalse(python27diablo.changeMatches(change))
item.freezeJobTree()
item.freezeJobGraph()
self.assertEqual(len(item.getJobs()), 1)
job = item.getJobs()[0]
self.assertEqual(job.name, 'python27')
@ -453,7 +453,7 @@ class TestJob(BaseTestCase):
self.assertTrue(python27.changeMatches(change))
self.assertTrue(python27diablo.changeMatches(change))
item.freezeJobTree()
item.freezeJobGraph()
self.assertEqual(len(item.getJobs()), 1)
job = item.getJobs()[0]
self.assertEqual(job.name, 'python27')
@ -506,7 +506,7 @@ class TestJob(BaseTestCase):
self.assertTrue(base.changeMatches(change))
self.assertFalse(python27.changeMatches(change))
item.freezeJobTree()
item.freezeJobGraph()
self.assertEqual([], item.getJobs())
def test_job_source_project(self):
@ -609,3 +609,56 @@ class TestTimeDataBase(BaseTestCase):
for x in range(10):
self.db.update('job-name', 100, 'SUCCESS')
self.assertEqual(self.db.getEstimatedTime('job-name'), 100)
class TestGraph(BaseTestCase):
def test_job_graph_disallows_multiple_jobs_with_same_name(self):
graph = model.JobGraph()
job1 = model.Job('job')
job2 = model.Job('job')
graph.addJob(job1)
with testtools.ExpectedException(Exception,
"Job job already added"):
graph.addJob(job2)
def test_job_graph_disallows_circular_dependencies(self):
graph = model.JobGraph()
jobs = [model.Job('job%d' % i) for i in range(0, 10)]
prevjob = None
for j in jobs[:3]:
if prevjob:
j.dependencies = frozenset([prevjob.name])
graph.addJob(j)
prevjob = j
# 0 triggers 1 triggers 2 triggers 3...
# Cannot depend on itself
with testtools.ExpectedException(
Exception,
"Dependency cycle detected in job jobX"):
j = model.Job('jobX')
j.dependencies = frozenset([j.name])
graph.addJob(j)
# Disallow circular dependencies
with testtools.ExpectedException(
Exception,
"Dependency cycle detected in job job3"):
jobs[4].dependencies = frozenset([jobs[3].name])
graph.addJob(jobs[4])
jobs[3].dependencies = frozenset([jobs[4].name])
graph.addJob(jobs[3])
jobs[5].dependencies = frozenset([jobs[4].name])
graph.addJob(jobs[5])
with testtools.ExpectedException(
Exception,
"Dependency cycle detected in job job3"):
jobs[3].dependencies = frozenset([jobs[5].name])
graph.addJob(jobs[3])
jobs[3].dependencies = frozenset([jobs[2].name])
graph.addJob(jobs[3])
jobs[6].dependencies = frozenset([jobs[2].name])
graph.addJob(jobs[6])

111
tests/unit/test_scheduler.py
View File

@ -4478,6 +4478,117 @@ For CI problems and help debugging, contact ci@example.org"""
self.assertIn('project-test2 : SKIPPED', A.messages[1])
class TestDependencyGraph(ZuulTestCase):
tenant_config_file = 'config/dependency-graph/main.yaml'
def test_dependeny_graph_dispatch_jobs_once(self):
"Test a job in a dependency graph is queued only once"
# Job dependencies, starting with A
# A
# / \
# B C
# / \ / \
# D F E
# |
# G
self.executor_server.hold_jobs_in_build = True
change = self.fake_gerrit.addFakeChange(
'org/project', 'master', 'change')
change.addApproval('code-review', 2)
self.fake_gerrit.addEvent(change.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual([b.name for b in self.builds], ['A'])
self.executor_server.release('A')
self.waitUntilSettled()
self.assertEqual(sorted(b.name for b in self.builds), ['B', 'C'])
self.executor_server.release('B')
self.waitUntilSettled()
self.assertEqual(sorted(b.name for b in self.builds), ['C', 'D'])
self.executor_server.release('D')
self.waitUntilSettled()
self.assertEqual([b.name for b in self.builds], ['C'])
self.executor_server.release('C')
self.waitUntilSettled()
self.assertEqual(sorted(b.name for b in self.builds), ['E', 'F'])
self.executor_server.release('F')
self.waitUntilSettled()
self.assertEqual(sorted(b.name for b in self.builds), ['E', 'G'])
self.executor_server.release('G')
self.waitUntilSettled()
self.assertEqual([b.name for b in self.builds], ['E'])
self.executor_server.release('E')
self.waitUntilSettled()
self.assertEqual(len(self.builds), 0)
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(len(self.builds), 0)
self.assertEqual(len(self.history), 7)
self.assertEqual(change.data['status'], 'MERGED')
self.assertEqual(change.reported, 2)
def test_jobs_launched_only_if_all_dependencies_are_successful(self):
"Test that a job waits till all dependencies are successful"
# Job dependencies, starting with A
# A
# / \
# B C*
# / \ / \
# D F E
# |
# G
self.executor_server.hold_jobs_in_build = True
change = self.fake_gerrit.addFakeChange(
'org/project', 'master', 'change')
change.addApproval('code-review', 2)
self.executor_server.failJob('C', change)
self.fake_gerrit.addEvent(change.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual([b.name for b in self.builds], ['A'])
self.executor_server.release('A')
self.waitUntilSettled()
self.assertEqual(sorted(b.name for b in self.builds), ['B', 'C'])
self.executor_server.release('B')
self.waitUntilSettled()
self.assertEqual(sorted(b.name for b in self.builds), ['C', 'D'])
self.executor_server.release('D')
self.waitUntilSettled()
self.assertEqual([b.name for b in self.builds], ['C'])
self.executor_server.release('C')
self.waitUntilSettled()
self.assertEqual(len(self.builds), 0)
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(len(self.builds), 0)
self.assertEqual(len(self.history), 4)
self.assertEqual(change.data['status'], 'NEW')
self.assertEqual(change.reported, 2)
class TestDuplicatePipeline(ZuulTestCase):
tenant_config_file = 'config/duplicate-pipeline/main.yaml'

36
zuul/configloader.py
View File

@ -193,6 +193,7 @@ class JobParser(object):
'roles': to_list(role),
'repos': to_list(str),
'vars': dict,
'dependencies': to_list(str),
}
return vs.Schema(job)
@ -276,6 +277,8 @@ class JobParser(object):
# accumulate onto any previously applied tags.
job.tags = job.tags.union(set(tags))
job.dependencies = frozenset(as_list(conf.get('dependencies')))
roles = []
for role in conf.get('roles', []):
if 'zuul' in role:
@ -364,45 +367,33 @@ class ProjectTemplateParser(object):
project_pipeline = model.ProjectPipelineConfig()
project_template.pipelines[pipeline.name] = project_pipeline
project_pipeline.queue_name = conf_pipeline.get('queue')
project_pipeline.job_tree = ProjectTemplateParser._parseJobTree(
ProjectTemplateParser._parseJobList(
tenant, layout, conf_pipeline.get('jobs', []),
source_context, start_mark)
source_context, start_mark, project_pipeline.job_list)
return project_template
@staticmethod
def _parseJobTree(tenant, layout, conf, source_context,
start_mark, tree=None):
if not tree:
tree = model.JobTree(None)
def _parseJobList(tenant, layout, conf, source_context,
start_mark, job_list):
for conf_job in conf:
if isinstance(conf_job, six.string_types):
job = model.Job(conf_job)
tree.addJob(job)
job_list.addJob(job)
elif isinstance(conf_job, dict):
# A dictionary in a job tree may override params, or
# be the root of a sub job tree, or both.
# A dictionary in a job tree may override params
jobname, attrs = conf_job.items()[0]
jobs = attrs.pop('jobs', None)
if attrs:
# We are overriding params, so make a new job def
attrs['name'] = jobname
attrs['_source_context'] = source_context
attrs['_start_mark'] = start_mark
subtree = tree.addJob(JobParser.fromYaml(
tenant, layout, attrs))
job_list.addJob(JobParser.fromYaml(tenant, layout, attrs))
else:
# Not overriding, so add a blank job
job = model.Job(jobname)
subtree = tree.addJob(job)
if jobs:
# This is the root of a sub tree
ProjectTemplateParser._parseJobTree(
tenant, layout, jobs, source_context,
start_mark, subtree)
job_list.addJob(job)
else:
raise Exception("Job must be a string or dictionary")
return tree
class ProjectParser(object):
@ -455,7 +446,6 @@ class ProjectParser(object):
project.merge_mode = model.MERGER_MAP['merge-resolve']
for pipeline in layout.pipelines.values():
project_pipeline = model.ProjectPipelineConfig()
project_pipeline.job_tree = model.JobTree(None)
queue_name = None
# For every template, iterate over the job tree and replace or
# create the jobs in the final definition as needed.
@ -467,8 +457,8 @@ class ProjectParser(object):
(template.name, pipeline.name))
pipeline_defined = True
template_pipeline = template.pipelines[pipeline.name]
project_pipeline.job_tree.inheritFrom(
template_pipeline.job_tree)
project_pipeline.job_list.inheritFrom(
template_pipeline.job_list)
if template_pipeline.queue_name:
queue_name = template_pipeline.queue_name
if queue_name:

66
zuul/manager/__init__.py
View File

@ -64,31 +64,29 @@ class PipelineManager(object):
self.log.info(" %s" % e)
self.log.info(" Projects:")
def log_jobs(tree, indent=0):
istr = ' ' + ' ' * indent
if tree.job:
# TODOv3(jeblair): represent matchers
efilters = ''
# for b in tree.job._branches:
# efilters += str(b)
# for f in tree.job._files:
# efilters += str(f)
# if tree.job.skip_if_matcher:
# efilters += str(tree.job.skip_if_matcher)
# if efilters:
# efilters = ' ' + efilters
tags = []
if tree.job.hold_following_changes:
tags.append('[hold]')
if not tree.job.voting:
tags.append('[nonvoting]')
if tree.job.mutex:
tags.append('[mutex: %s]' % tree.job.mutex)
tags = ' '.join(tags)
self.log.info("%s%s%s %s" % (istr, repr(tree.job),
efilters, tags))
for x in tree.job_trees:
log_jobs(x, indent + 2)
def log_jobs(job_list):
for job_name, job_variants in job_list.jobs.items():
for variant in job_variants:
# TODOv3(jeblair): represent matchers
efilters = ''
# for b in tree.job._branches:
# efilters += str(b)
# for f in tree.job._files:
# efilters += str(f)
# if tree.job.skip_if_matcher:
# efilters += str(tree.job.skip_if_matcher)
# if efilters:
# efilters = ' ' + efilters
tags = []
if variant.hold_following_changes:
tags.append('[hold]')
if not variant.voting:
tags.append('[nonvoting]')
if variant.mutex:
tags.append('[mutex: %s]' % variant.mutex)
tags = ' '.join(tags)
self.log.info(" %s%s %s" % (repr(variant),
efilters, tags))
for project_name in layout.project_configs.keys():
project_config = layout.project_configs.get(project_name)
@ -97,7 +95,7 @@ class PipelineManager(object):
self.pipeline.name)
if project_pipeline_config:
self.log.info(" %s" % project_name)
log_jobs(project_pipeline_config.job_tree)
log_jobs(project_pipeline_config.job_list)
self.log.info(" On start:")
self.log.info(" %s" % self.pipeline.start_actions)
self.log.info(" On success:")
@ -257,7 +255,7 @@ class PipelineManager(object):
# Rebuild the frozen job tree from the new layout, if
# we have one. If not, it will be built later.
if item.current_build_set.layout:
item.freezeJobTree()
item.freezeJobGraph()
# Re-set build results in case any new jobs have been
# added to the tree.
@ -540,8 +538,18 @@ class PipelineManager(object):
item.current_build_set.layout = self.getLayout(item)
if not item.current_build_set.layout:
return False
if not item.job_tree:
item.freezeJobTree()
if item.current_build_set.config_error:
return False
if not item.job_graph:
try:
item.freezeJobGraph()
except Exception as e:
# TODOv3(jeblair): nicify this exception as it will be reported
self.log.exception("Error freezing job graph for %s" %
item)
item.setConfigError("Unable to freeze job graph: %s" %
(str(e)))
return False
return True
def _processOneItem(self, item, nnfi):

348
zuul/model.py
View File

@ -675,6 +675,7 @@ class Job(object):
file_matcher=None,
irrelevant_file_matcher=None, # skip-if
tags=frozenset(),
dependencies=frozenset(),
)
# These attributes affect how the job is actually run and more
@ -851,60 +852,100 @@ class Job(object):
return True
class JobTree(object):
"""A JobTree holds one or more Jobs to represent Job dependencies.
class JobList(object):
""" A list of jobs in a project's pipeline. """
If Job foo should only execute if Job bar succeeds, then there will
be a JobTree for foo, which will contain a JobTree for bar. A JobTree
can hold more than one dependent JobTrees, such that jobs bar and bang
both depend on job foo being successful.
A root node of a JobTree will have no associated Job."""
def __init__(self, job):
self.job = job
self.job_trees = []
def __repr__(self):
return '<JobTree %s %s>' % (self.job, self.job_trees)
def __init__(self):
self.jobs = OrderedDict() # job.name -> [job, ...]
def addJob(self, job):
if job not in [x.job for x in self.job_trees]:
t = JobTree(job)
self.job_trees.append(t)
return t
for tree in self.job_trees:
if tree.job == job:
return tree
def getJobs(self):
jobs = []
for x in self.job_trees:
jobs.append(x.job)
jobs.extend(x.getJobs())
return jobs
def getJobTreeForJob(self, job):
if self.job == job:
return self
for tree in self.job_trees:
ret = tree.getJobTreeForJob(job)
if ret:
return ret
return None
if job.name in self.jobs:
self.jobs[job.name].append(job)
else:
self.jobs[job.name] = [job]
def inheritFrom(self, other):
if other.job:
if not self.job:
self.job = other.job.copy()
for jobname, jobs in other.jobs.items():
if jobname in self.jobs:
self.jobs[jobname].append(jobs)
else:
self.job.applyVariant(other.job)
for other_tree in other.job_trees:
this_tree = self.getJobTreeForJob(other_tree.job)
if not this_tree:
this_tree = JobTree(None)
self.job_trees.append(this_tree)
this_tree.inheritFrom(other_tree)
self.jobs[jobname] = jobs
class JobGraph(object):
""" A JobGraph represents the dependency graph between Job."""
def __init__(self):
self.jobs = OrderedDict() # job_name -> Job
self._dependencies = {} # dependent_job_name -> set(parent_job_names)
def __repr__(self):
return '<JobGraph %s>' % (self.jobs)
def addJob(self, job):
# A graph must be created after the job list is frozen,
# therefore we should only get one job with the same name.
if job.name in self.jobs:
raise Exception("Job %s already added" % (job.name,))
self.jobs[job.name] = job
# Append the dependency information
self._dependencies.setdefault(job.name, set())
try:
for dependency in job.dependencies:
# Make sure a circular dependency is never created
ancestor_jobs = self._getParentJobNamesRecursively(
dependency, soft=True)
ancestor_jobs.add(dependency)
if any((job.name == anc_job) for anc_job in ancestor_jobs):
raise Exception("Dependency cycle detected in job %s" %
(job.name,))
self._dependencies[job.name].add(dependency)
except Exception:
del self.jobs[job.name]
del self._dependencies[job.name]
raise
def getJobs(self):
return self.jobs.values() # Report in the order of the layout config
def _getDirectDependentJobs(self, parent_job):
ret = set()
for dependent_name, parent_names in self._dependencies.items():
if parent_job in parent_names:
ret.add(dependent_name)
return ret
def getDependentJobsRecursively(self, parent_job):
all_dependent_jobs = set()
jobs_to_iterate = set([parent_job])
while len(jobs_to_iterate) > 0:
current_job = jobs_to_iterate.pop()
current_dependent_jobs = self._getDirectDependentJobs(current_job)
new_dependent_jobs = current_dependent_jobs - all_dependent_jobs
jobs_to_iterate |= new_dependent_jobs
all_dependent_jobs |= new_dependent_jobs
return [self.jobs[name] for name in all_dependent_jobs]
def getParentJobsRecursively(self, dependent_job):
return [self.jobs[name] for name in
self._getParentJobNamesRecursively(dependent_job)]
def _getParentJobNamesRecursively(self, dependent_job, soft=False):
all_parent_jobs = set()
jobs_to_iterate = set([dependent_job])
while len(jobs_to_iterate) > 0:
current_job = jobs_to_iterate.pop()
current_parent_jobs = self._dependencies.get(current_job)
if current_parent_jobs is None:
if soft:
current_parent_jobs = set()
else:
raise Exception("Dependent job %s not found: " %
(dependent_job,))
new_parent_jobs = current_parent_jobs - all_parent_jobs
jobs_to_iterate |= new_parent_jobs
all_parent_jobs |= new_parent_jobs
return all_parent_jobs
class Build(object):
@ -1137,7 +1178,7 @@ class QueueItem(object):
self.active = False # Whether an item is within an active window
self.live = True # Whether an item is intended to be processed at all
self.layout = None # This item's shadow layout
self.job_tree = None
self.job_graph = None
def __repr__(self):
if self.pipeline:
@ -1165,23 +1206,33 @@ class QueueItem(object):
def setReportedResult(self, result):
self.current_build_set.result = result
def freezeJobTree(self):
def freezeJobGraph(self):
"""Find or create actual matching jobs for this item's change and
store the resulting job tree."""
layout = self.current_build_set.layout
self.job_tree = layout.createJobTree(self)
job_graph = layout.createJobGraph(self)
for job in job_graph.getJobs():
# Ensure that each jobs's dependencies are fully
# accessible. This will raise an exception if not.
job_graph.getParentJobsRecursively(job.name)
self.job_graph = job_graph
def hasJobTree(self):
"""Returns True if the item has a job tree."""
return self.job_tree is not None
def hasJobGraph(self):
"""Returns True if the item has a job graph."""
return self.job_graph is not None
def getJobs(self):
if not self.live or not self.job_tree:
if not self.live or not self.job_graph:
return []
return self.job_tree.getJobs()
return self.job_graph.getJobs()
def getJob(self, name):
if not self.job_graph:
return None
return self.job_graph.jobs.get(name)
def haveAllJobsStarted(self):
if not self.hasJobTree():
if not self.hasJobGraph():
return False
for job in self.getJobs():
build = self.current_build_set.getBuild(job.name)
@ -1193,7 +1244,7 @@ class QueueItem(object):
if (self.current_build_set.config_error or
self.current_build_set.unable_to_merge):
return True
if not self.hasJobTree():
if not self.hasJobGraph():
return False
for job in self.getJobs():
build = self.current_build_set.getBuild(job.name)
@ -1202,7 +1253,7 @@ class QueueItem(object):
return True
def didAllJobsSucceed(self):
if not self.hasJobTree():
if not self.hasJobGraph():
return False
for job in self.getJobs():
if not job.voting:
@ -1215,7 +1266,7 @@ class QueueItem(object):
return True
def didAnyJobFail(self):
if not self.hasJobTree():
if not self.hasJobGraph():
return False
for job in self.getJobs():
if not job.voting:
@ -1234,7 +1285,7 @@ class QueueItem(object):
def isHoldingFollowingChanges(self):
if not self.live:
return False
if not self.hasJobTree():
if not self.hasJobGraph():
return False
for job in self.getJobs():
if not job.hold_following_changes:
@ -1249,88 +1300,96 @@ class QueueItem(object):
return False
return self.item_ahead.isHoldingFollowingChanges()
def _findJobsToRun(self, job_trees, mutex):
def findJobsToRun(self, mutex):
torun = []
if not self.live:
return []
if not self.job_graph:
return []
if self.item_ahead:
# Only run jobs if any 'hold' jobs on the change ahead
# have completed successfully.
if self.item_ahead.isHoldingFollowingChanges():
return []
for tree in job_trees:
job = tree.job
result = None
if job:
if not job.changeMatches(self.change):
successful_job_names = set()
jobs_not_started = set()
for job in self.job_graph.getJobs():
build = self.current_build_set.getBuild(job.name)
if build:
if build.result == 'SUCCESS':
successful_job_names.add(job.name)
else:
jobs_not_started.add(job)
# Attempt to request nodes for jobs in the order jobs appear
# in configuration.
for job in self.job_graph.getJobs():
if job not in jobs_not_started:
continue
all_parent_jobs_successful = True
for parent_job in self.job_graph.getParentJobsRecursively(
job.name):
if parent_job.name not in successful_job_names:
all_parent_jobs_successful = False
break
if all_parent_jobs_successful:
nodeset = self.current_build_set.getJobNodeSet(job.name)
if nodeset is None:
# The nodes for this job are not ready, skip
# it for now.
continue
build = self.current_build_set.getBuild(job.name)
if build:
result = build.result
else:
# There is no build for the root of this job tree,
# so it has not run yet.
nodeset = self.current_build_set.getJobNodeSet(job.name)
if nodeset is None:
# The nodes for this job are not ready, skip
# it for now.
continue
if mutex.acquire(self, job):
# If this job needs a mutex, either acquire it or make
# sure that we have it before running the job.
torun.append(job)
# If there is no job, this is a null job tree, and we should
# run all of its jobs.
if result == 'SUCCESS' or not job:
torun.extend(self._findJobsToRun(tree.job_trees, mutex))
if mutex.acquire(self, job):
# If this job needs a mutex, either acquire it or make
# sure that we have it before running the job.
torun.append(job)
return torun
def findJobsToRun(self, mutex):
if not self.live:
return []
tree = self.job_tree
if not tree:
return []
return self._findJobsToRun(tree.job_trees, mutex)
def _findJobsToRequest(self, job_trees):
def findJobsToRequest(self):
build_set = self.current_build_set
toreq = []
if not self.live:
return []
if not self.job_graph:
return []
if self.item_ahead:
if self.item_ahead.isHoldingFollowingChanges():
return []
for tree in job_trees:
job = tree.job
result = None
if job:
if not job.changeMatches(self.change):
continue
build = build_set.getBuild(job.name)
if build:
result = build.result
else:
nodeset = build_set.getJobNodeSet(job.name)
if nodeset is None:
req = build_set.getJobNodeRequest(job.name)
if req is None:
toreq.append(job)
if result == 'SUCCESS' or not job:
toreq.extend(self._findJobsToRequest(tree.job_trees))
return toreq
def findJobsToRequest(self):
if not self.live:
return []
tree = self.job_tree
if not tree:
return []
return self._findJobsToRequest(tree.job_trees)
successful_job_names = set()
jobs_not_requested = set()
for job in self.job_graph.getJobs():
build = build_set.getBuild(job.name)
if build and build.result == 'SUCCESS':
successful_job_names.add(job.name)
else:
nodeset = build_set.getJobNodeSet(job.name)
if nodeset is None:
req = build_set.getJobNodeRequest(job.name)
if req is None:
jobs_not_requested.add(job)
# Attempt to request nodes for jobs in the order jobs appear
# in configuration.
for job in self.job_graph.getJobs():
if job not in jobs_not_requested:
continue
all_parent_jobs_successful = True
for parent_job in self.job_graph.getParentJobsRecursively(
job.name):
if parent_job.name not in successful_job_names:
all_parent_jobs_successful = False
break
if all_parent_jobs_successful:
toreq.append(job)
return toreq
def setResult(self, build):
if build.retry:
self.removeBuild(build)
elif build.result != 'SUCCESS':
# Get a JobTree from a Job so we can find only its dependent jobs
tree = self.job_tree.getJobTreeForJob(build.job)
for job in tree.getJobs():
for job in self.job_graph.getDependentJobsRecursively(
build.job.name):
fakebuild = Build(job, None)
fakebuild.result = 'SKIPPED'
self.addBuild(fakebuild)
@ -2014,7 +2073,7 @@ class ChangeishFilter(BaseFilter):
class ProjectPipelineConfig(object):
# Represents a project cofiguration in the context of a pipeline
def __init__(self):
self.job_tree = None
self.job_list = JobList()
self.queue_name = None
self.merge_mode = None
@ -2182,14 +2241,13 @@ class Layout(object):
def addProjectConfig(self, project_config):
self.project_configs[project_config.name] = project_config
def _createJobTree(self, change, job_trees, parent):
for tree in job_trees:
job = tree.job
if not job.changeMatches(change):
continue
def _createJobGraph(self, change, job_list, job_graph):
for jobname in job_list.jobs:
# This is the final job we are constructing
frozen_job = None
# Whether the change matches any globally defined variant
matched = False
for variant in self.getJobs(job.name):
for variant in self.getJobs(jobname):
if variant.changeMatches(change):
if frozen_job is None:
frozen_job = variant.copy()
@ -2203,25 +2261,33 @@ class Layout(object):
# the job that is defined in the tree).
continue
# If the job does not allow auth inheritance, do not allow
# the project-pipeline variant to update its execution
# the project-pipeline variants to update its execution
# attributes.
if frozen_job.auth and not frozen_job.auth.get('inherit'):
frozen_job.final = True
frozen_job.applyVariant(job)
frozen_tree = JobTree(frozen_job)
parent.job_trees.append(frozen_tree)
self._createJobTree(change, tree.job_trees, frozen_tree)
# Whether the change matches any of the project pipeline
# variants
matched = False
for variant in job_list.jobs[jobname]:
if variant.changeMatches(change):
frozen_job.applyVariant(variant)
matched = True
if not matched:
# A change must match at least one project pipeline
# job variant.
continue
job_graph.addJob(frozen_job)
def createJobTree(self, item):
def createJobGraph(self, item):
project_config = self.project_configs.get(
item.change.project.name, None)
ret = JobTree(None)
ret = JobGraph()
# NOTE(pabelanger): It is possible for a foreign project not to have a
# configured pipeline, if so return an empty JobTree.
# configured pipeline, if so return an empty JobGraph.
if project_config and item.pipeline.name in project_config.pipelines:
project_tree = \
project_config.pipelines[item.pipeline.name].job_tree
self._createJobTree(item.change, project_tree.job_trees, ret)
project_job_list = \
project_config.pipelines[item.pipeline.name].job_list
self._createJobGraph(item.change, project_job_list, ret)
return ret

7
zuul/scheduler.py
View File

@ -555,11 +555,10 @@ class Scheduler(threading.Thread):
project_name)
if new_pipeline.manager.reEnqueueItem(item,
last_head):
new_jobs = item.getJobs()
for build in item.current_build_set.getBuilds():
jobtree = item.job_tree.getJobTreeForJob(build.job)
if jobtree and jobtree.job in new_jobs:
build.job = jobtree.job
new_job = item.getJob(build.job.name)
if new_job:
build.job = new_job
else:
item.removeBuild(build)
builds_to_cancel.append(build)