Improve job dependencies using graph instead of tree

This replaces the job dependency tree with a graph so that we can
indicate that a job should wait until one or more jobs are complete
before starting.

Project pipeline job definitions are now a flat list, with each job
specifying its dependencies as the job attribute 'dependencies'.

Fixes bug #1166937.

Signed-off-by: Fredrik Medley <fredrik.medley@autoliv.com>
Signed-off-by: Fredrik Medley <fredrik.medley@gmail.com>
Signed-off-by: James E. Blair <jeblair@redhat.com>
Co-Authored-By: James E. Blair <jeblair@redhat.com>
Change-Id: I921940cafeea0738c39deb99357cfd7c91592359
This commit is contained in:
Fredrik Medley 2015-09-28 13:40:20 +02:00 committed by James E. Blair
parent e06a03bb45
commit f8aec83b3b
54 changed files with 726 additions and 444 deletions

View File

@ -771,8 +771,11 @@ given pipeline. Within the pipeline section, the jobs that should be
executed are listed. If a job is entered as a dictionary key, then executed are listed. If a job is entered as a dictionary key, then
jobs contained within that key are only executed if the key job jobs contained within that key are only executed if the key job
succeeds. In the above example, project-unittest, project-pep8, and succeeds. In the above example, project-unittest, project-pep8, and
project-pyflakes are only executed if project-merge succeeds. This project-pyflakes are only executed if project-merge succeeds.
can help avoid running unnecessary jobs. Furthermore, project-finaltest is executed only if project-unittest,
project-pep8 and project-pyflakes all succeed. This can help avoid
running unnecessary jobs while maximizing parallelism. It is also
useful when distributing results between jobs.
The special job named ``noop`` is internal to Zuul and will always The special job named ``noop`` is internal to Zuul and will always
return ``SUCCESS`` immediately. This can be useful if you require return ``SUCCESS`` immediately. This can be useful if you require

View File

@ -1,8 +1,7 @@
- pipeline: - pipeline:
name: check name: check
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: patchset-created - event: patchset-created
@ -17,8 +16,7 @@
name: gate name: gate
manager: dependent manager: dependent
success-message: Build succeeded (gate). success-message: Build succeeded (gate).
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: comment-added - event: comment-added
@ -41,7 +39,7 @@
pre-run: pre pre-run: pre
post-run: post post-run: post
vars: vars:
flagpath: "{{zuul._test.test_root}}/{{zuul.uuid}}.flag" flagpath: '{{zuul._test.test_root}}/{{zuul.uuid}}.flag'
roles: roles:
- zuul: bare-role - zuul: bare-role

View File

@ -4,7 +4,6 @@
- project: - project:
name: org/project name: org/project
check: check:
jobs: jobs:
- python27 - python27

View File

@ -0,0 +1,2 @@
- hosts: all
tasks: []

View File

@ -0,0 +1,2 @@
- hosts: all
tasks: []

View File

@ -0,0 +1,2 @@
- hosts: all
tasks: []

View File

@ -0,0 +1,2 @@
- hosts: all
tasks: []

View File

@ -0,0 +1,2 @@
- hosts: all
tasks: []

View File

@ -0,0 +1,2 @@
- hosts: all
tasks: []

View File

@ -0,0 +1,2 @@
- hosts: all
tasks: []

View File

@ -0,0 +1,2 @@
- hosts: all
tasks: []

View File

@ -0,0 +1,73 @@
- pipeline:
name: gate
manager: dependent
success-message: Build succeeded (gate).
source: gerrit
trigger:
gerrit:
- event: comment-added
approval:
- approved: 1
success:
gerrit:
verified: 2
submit: true
failure:
gerrit:
verified: -2
start:
gerrit:
verified: 0
precedence: high
- job:
name: A
- job:
name: B
- job:
name: C
- job:
name: D
- job:
name: E
- job:
name: F
- job:
name: G
- project:
name: org/project
gate:
jobs:
# Job dependencies, starting with A
# A
# / \
# B C
# / \ / \
# D F E
# |
# G
# This is intentionally not listed in the natural order to
# ensure that we can reference dependencies before they are
# defined.
- E:
dependencies: C
- A
- B:
dependencies: A
- C:
dependencies: A
- F:
dependencies:
- B
- C
- D:
dependencies: B
- G:
dependencies: F

View File

@ -0,0 +1 @@
test

View File

@ -0,0 +1,8 @@
- tenant:
name: tenant-one
source:
gerrit:
config-repos:
- common-config
project-repos:
- org/project

View File

@ -2,8 +2,7 @@
name: dup1 name: dup1
manager: independent manager: independent
success-message: Build succeeded (dup1). success-message: Build succeeded (dup1).
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: change-restored - event: change-restored
@ -18,8 +17,7 @@
name: dup2 name: dup2
manager: independent manager: independent
success-message: Build succeeded (dup2). success-message: Build succeeded (dup2).
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: change-restored - event: change-restored
@ -39,7 +37,6 @@
queue: integrated queue: integrated
jobs: jobs:
- project-test1 - project-test1
dup2: dup2:
queue: integrated queue: integrated
jobs: jobs:

View File

@ -1,8 +1,7 @@
- pipeline: - pipeline:
name: check name: check
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: patchset-created - event: patchset-created
@ -17,8 +16,7 @@
name: tenant-one-gate name: tenant-one-gate
manager: dependent manager: dependent
success-message: Build succeeded (tenant-one-gate). success-message: Build succeeded (tenant-one-gate).
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: comment-added - event: comment-added

View File

@ -1,8 +1,7 @@
- pipeline: - pipeline:
name: check name: check
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: patchset-created - event: patchset-created
@ -17,8 +16,7 @@
name: gate name: gate
manager: dependent manager: dependent
success-message: Build succeeded (gate). success-message: Build succeeded (gate).
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: comment-added - event: comment-added
@ -37,16 +35,13 @@
precedence: high precedence: high
- job: - job:
name: name: project-test1
project-test1
- job: - job:
name: name: project-test2
project-test2
- job: - job:
name: name: project-merge
project-merge
hold-following-changes: true hold-following-changes: true
- project: - project:
@ -75,6 +70,6 @@
merge-mode: cherry-pick merge-mode: cherry-pick
gate: gate:
jobs: jobs:
- project-merge: - project-merge
jobs: - project-test1:
- project-test1 dependencies: project-merge

View File

@ -1,8 +1,7 @@
- pipeline: - pipeline:
name: check name: check
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: patchset-created - event: patchset-created
@ -14,8 +13,7 @@
verified: -1 verified: -1
- job: - job:
name: name: python27
python27
nodes: nodes:
- name: controller - name: controller
image: ubuntu-trusty image: ubuntu-trusty

View File

@ -2,8 +2,7 @@
name: tenant-one-gate name: tenant-one-gate
manager: dependent manager: dependent
success-message: Build succeeded (tenant-one-gate). success-message: Build succeeded (tenant-one-gate).
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: comment-added - event: comment-added
@ -28,8 +27,7 @@
image: controller-image image: controller-image
- job: - job:
name: name: project1-test1
project1-test1
- project: - project:
name: org/project1 name: org/project1

View File

@ -2,8 +2,7 @@
name: tenant-two-gate name: tenant-two-gate
manager: dependent manager: dependent
success-message: Build succeeded (tenant-two-gate). success-message: Build succeeded (tenant-two-gate).
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: comment-added - event: comment-added
@ -28,8 +27,7 @@
image: controller-image image: controller-image
- job: - job:
name: name: project2-test1
project2-test1
- project: - project:
name: org/project2 name: org/project2

View File

@ -1,8 +1,7 @@
- pipeline: - pipeline:
name: check name: check
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: patchset-created - event: patchset-created
@ -17,8 +16,7 @@
name: gate name: gate
manager: dependent manager: dependent
success-message: Build succeeded (gate). success-message: Build succeeded (gate).
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: comment-added - event: comment-added
@ -39,8 +37,7 @@
- pipeline: - pipeline:
name: post name: post
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: ref-updated - event: ref-updated

View File

@ -1,11 +1,8 @@
# Pipeline definitions
- pipeline: - pipeline:
name: check name: check
manager: independent manager: independent
success-message: Build succeeded (check). success-message: Build succeeded (check).
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: patchset-created - event: patchset-created
@ -20,8 +17,7 @@
name: gate name: gate
manager: dependent manager: dependent
success-message: Build succeeded (gate). success-message: Build succeeded (gate).
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: comment-added - event: comment-added
@ -39,8 +35,6 @@
verified: 0 verified: 0
precedence: high precedence: high
# Job definitions
- job: - job:
name: base name: base
timeout: 30 timeout: 30
@ -78,8 +72,6 @@
- openstack/keystone - openstack/keystone
- openstack/nova - openstack/nova
# Project definitions
- project: - project:
name: openstack/nova name: openstack/nova
templates: templates:

View File

@ -1,8 +1,7 @@
- pipeline: - pipeline:
name: pipeline name: pipeline
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: comment-added - event: comment-added
@ -19,8 +18,7 @@
- pipeline: - pipeline:
name: trigger name: trigger
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: comment-added - event: comment-added

View File

@ -1,8 +1,7 @@
- pipeline: - pipeline:
name: pipeline name: pipeline
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: comment-added - event: comment-added
@ -20,8 +19,7 @@
- pipeline: - pipeline:
name: trigger name: trigger
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: comment-added - event: comment-added

View File

@ -1,8 +1,7 @@
- pipeline: - pipeline:
name: pipeline name: pipeline
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: comment-added - event: comment-added
@ -20,8 +19,7 @@
- pipeline: - pipeline:
name: trigger name: trigger
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: comment-added - event: comment-added

View File

@ -1,11 +1,10 @@
- pipeline: - pipeline:
name: pipeline name: pipeline
manager: independent manager: independent
source: source: gerrit
gerrit
reject: reject:
approval: approval:
- username: 'jenkins' - username: jenkins
trigger: trigger:
gerrit: gerrit:
- event: comment-added - event: comment-added
@ -19,13 +18,12 @@
- pipeline: - pipeline:
name: trigger name: trigger
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: comment-added - event: comment-added
reject-approval: reject-approval:
- username: 'jenkins' - username: jenkins
success: success:
gerrit: gerrit:
verified: 1 verified: 1

View File

@ -1,15 +1,18 @@
- pipeline: - pipeline:
name: pipeline name: pipeline
manager: independent manager: independent
source: source: gerrit
gerrit
require: require:
approval: approval:
- username: jenkins - username: jenkins
verified: [1, 2] verified:
- 1
- 2
reject: reject:
approval: approval:
- verified: [-1, -2] - verified:
- -1
- -2
trigger: trigger:
gerrit: gerrit:
- event: comment-added - event: comment-added
@ -23,16 +26,19 @@
- pipeline: - pipeline:
name: trigger name: trigger
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: comment-added - event: comment-added
require-approval: require-approval:
- username: jenkins - username: jenkins
verified: [1, 2] verified:
- 1
- 2
reject-approval: reject-approval:
- verified: [-1, -2] - verified:
- -1
- -2
success: success:
gerrit: gerrit:
verified: 1 verified: 1

View File

@ -1,10 +1,9 @@
- pipeline: - pipeline:
name: current-check name: current-check
manager: independent manager: independent
source: source: gerrit
gerrit
require: require:
current-patchset: True current-patchset: true
trigger: trigger:
gerrit: gerrit:
- event: patchset-created - event: patchset-created
@ -19,10 +18,9 @@
- pipeline: - pipeline:
name: open-check name: open-check
manager: independent manager: independent
source: source: gerrit
gerrit
require: require:
open: True open: true
trigger: trigger:
gerrit: gerrit:
- event: patchset-created - event: patchset-created
@ -37,8 +35,7 @@
- pipeline: - pipeline:
name: status-check name: status-check
manager: independent manager: independent
source: source: gerrit
gerrit
require: require:
status: NEW status: NEW
trigger: trigger:

View File

@ -1,8 +1,7 @@
- pipeline: - pipeline:
name: pipeline name: pipeline
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: comment-added - event: comment-added
@ -19,8 +18,7 @@
- pipeline: - pipeline:
name: trigger name: trigger
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: comment-added - event: comment-added

View File

@ -1,4 +1,3 @@
- pipeline: - pipeline:
name: pipeline name: pipeline
manager: independent manager: independent
@ -6,8 +5,7 @@
approval: approval:
- username: jenkins - username: jenkins
verified: 1 verified: 1
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: comment-added - event: comment-added
@ -21,8 +19,7 @@
- pipeline: - pipeline:
name: trigger name: trigger
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: comment-added - event: comment-added

View File

@ -4,9 +4,10 @@
require: require:
approval: approval:
- username: jenkins - username: jenkins
verified: [1, 2] verified:
source: - 1
gerrit - 2
source: gerrit
trigger: trigger:
gerrit: gerrit:
- event: comment-added - event: comment-added
@ -20,14 +21,15 @@
- pipeline: - pipeline:
name: trigger name: trigger
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: comment-added - event: comment-added
require-approval: require-approval:
- username: jenkins - username: jenkins
verified: [1, 2] verified:
- 1
- 2
success: success:
gerrit: gerrit:
verified: 1 verified: 1

View File

@ -1,8 +1,7 @@
- pipeline: - pipeline:
name: check name: check
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: patchset-created - event: patchset-created
@ -17,8 +16,7 @@
name: gate name: gate
manager: dependent manager: dependent
success-message: Build succeeded (gate). success-message: Build succeeded (gate).
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: comment-added - event: comment-added
@ -39,8 +37,7 @@
- pipeline: - pipeline:
name: post name: post
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: ref-updated - event: ref-updated
@ -49,8 +46,7 @@
- pipeline: - pipeline:
name: experimental name: experimental
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: patchset-created - event: patchset-created
@ -107,23 +103,26 @@
- job: - job:
name: project-testfile name: project-testfile
files: files:
- '.*-requires' - .*-requires
- project: - project:
name: org/project name: org/project
check: check:
jobs: jobs:
- project-merge: - project-merge
jobs: - project-test1:
- project-test1 dependencies: project-merge
- project-test2 - project-test2:
dependencies: project-merge
gate: gate:
jobs: jobs:
- project-merge: - project-merge
jobs: - project-test1:
- project-test1 dependencies: project-merge
- project-test2 - project-test2:
- project-testfile dependencies: project-merge
- project-testfile:
dependencies: project-merge
post: post:
jobs: jobs:
- project-post - project-post
@ -132,48 +131,58 @@
name: org/project1 name: org/project1
check: check:
jobs: jobs:
- project-merge: - project-merge
jobs: - project-test1:
- project-test1 dependencies: project-merge
- project-test2 - project-test2:
- project1-project2-integration dependencies: project-merge
- project1-project2-integration:
dependencies: project-merge
gate: gate:
queue: integrated queue: integrated
jobs: jobs:
- project-merge: - project-merge
jobs: - project-test1:
- project-test1 dependencies: project-merge
- project-test2 - project-test2:
- project1-project2-integration dependencies: project-merge
- project1-project2-integration:
dependencies: project-merge
- project: - project:
name: org/project2 name: org/project2
gate: gate:
queue: integrated queue: integrated
jobs: jobs:
- project-merge: - project-merge
jobs: - project-test1:
- project-test1 dependencies: project-merge
- project-test2 - project-test2:
- project1-project2-integration dependencies: project-merge
- project1-project2-integration:
dependencies: project-merge
- project: - project:
name: org/project3 name: org/project3
check: check:
jobs: jobs:
- project-merge: - project-merge
jobs: - project-test1:
- project-test1 dependencies: project-merge
- project-test2 - project-test2:
- project1-project2-integration dependencies: project-merge
- project1-project2-integration:
dependencies: project-merge
gate: gate:
queue: integrated queue: integrated
jobs: jobs:
- project-merge: - project-merge
jobs: - project-test1:
- project-test1 dependencies: project-merge
- project-test2 - project-test2:
- project1-project2-integration dependencies: project-merge
- project1-project2-integration:
dependencies: project-merge
post: post:
jobs: jobs:
- project-post - project-post
@ -182,9 +191,9 @@
name: org/experimental-project name: org/experimental-project
experimental: experimental:
jobs: jobs:
- project-merge: - project-merge
jobs: - experimental-project-test:
- experimental-project-test dependencies: project-merge
- project: - project:
name: org/noop-project name: org/noop-project
@ -199,16 +208,18 @@
name: org/nonvoting-project name: org/nonvoting-project
check: check:
jobs: jobs:
- nonvoting-project-merge: - nonvoting-project-merge
jobs: - nonvoting-project-test1:
- nonvoting-project-test1 dependencies: nonvoting-project-merge
- nonvoting-project-test2 - nonvoting-project-test2:
dependencies: nonvoting-project-merge
gate: gate:
jobs: jobs:
- nonvoting-project-merge: - nonvoting-project-merge
jobs: - nonvoting-project-test1:
- nonvoting-project-test1 dependencies: nonvoting-project-merge
- nonvoting-project-test2 - nonvoting-project-test2:
dependencies: nonvoting-project-merge
- project: - project:
name: org/no-jobs-project name: org/no-jobs-project

View File

@ -1,8 +1,7 @@
- pipeline: - pipeline:
name: check name: check
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: patchset-created - event: patchset-created

View File

@ -1,13 +1,12 @@
- pipeline: - pipeline:
name: post name: post
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: ref-updated - event: ref-updated
ref: ^(?!refs/).*$ ref: ^(?!refs/).*$
ignore-deletes: False ignore-deletes: false
- job: - job:
name: project-post name: project-post
@ -20,4 +19,3 @@
post: post:
jobs: jobs:
- project-post - project-post

View File

@ -2,8 +2,7 @@
name: gate name: gate
manager: dependent manager: dependent
success-message: Build succeeded (gate). success-message: Build succeeded (gate).
source: source: gerrit
gerrit
failure-message: Build failed. For information on how to proceed, see http://wiki.example.org/Test_Failures failure-message: Build failed. For information on how to proceed, see http://wiki.example.org/Test_Failures
footer-message: For CI problems and help debugging, contact ci@example.org footer-message: For CI problems and help debugging, contact ci@example.org
trigger: trigger:
@ -35,4 +34,3 @@
gate: gate:
jobs: jobs:
- project-test1 - project-test1

View File

@ -1,8 +1,7 @@
- pipeline: - pipeline:
name: periodic name: periodic
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
timer: timer:
- time: '* * * * * */1' - time: '* * * * * */1'

View File

@ -1,8 +1,7 @@
- pipeline: - pipeline:
name: check name: check
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: patchset-created - event: patchset-created
@ -13,7 +12,6 @@
gerrit: gerrit:
verified: -1 verified: -1
- job: - job:
name: project-test-irrelevant-starts-empty name: project-test-irrelevant-starts-empty

View File

@ -1,8 +1,7 @@
- pipeline: - pipeline:
name: check name: check
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: patchset-created - event: patchset-created
@ -13,7 +12,6 @@
gerrit: gerrit:
verified: -1 verified: -1
- job: - job:
name: project-test-irrelevant-files name: project-test-irrelevant-files

View File

@ -1,8 +1,7 @@
- pipeline: - pipeline:
name: check name: check
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: patchset-created - event: patchset-created

View File

@ -1,8 +1,7 @@
- pipeline: - pipeline:
name: check name: check
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: patchset-created - event: patchset-created

View File

@ -1,8 +1,7 @@
- pipeline: - pipeline:
name: check name: check
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: patchset-created - event: patchset-created
@ -18,8 +17,7 @@
manager: independent manager: independent
# Trigger is required, set it to one that is a noop # Trigger is required, set it to one that is a noop
# during tests that check the timer trigger. # during tests that check the timer trigger.
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: ref-updated - event: ref-updated

View File

@ -1,8 +1,7 @@
- pipeline: - pipeline:
name: check name: check
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: patchset-created - event: patchset-created
@ -17,8 +16,7 @@
name: gate name: gate
manager: dependent manager: dependent
success-message: Build succeeded (gate). success-message: Build succeeded (gate).
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: comment-added - event: comment-added
@ -60,13 +58,15 @@
name: org/delete-project name: org/delete-project
check: check:
jobs: jobs:
- project-merge: - project-merge
jobs: - project-test1:
- project-test1 dependencies: project-merge
- project-test2 - project-test2:
dependencies: project-merge
gate: gate:
jobs: jobs:
- project-merge: - project-merge
jobs: - project-test1:
- project-test1 dependencies: project-merge
- project-test2 - project-test2:
dependencies: project-merge

View File

@ -1,8 +1,7 @@
- pipeline: - pipeline:
name: check name: check
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: patchset-created - event: patchset-created
@ -23,8 +22,7 @@
name: gate name: gate
manager: dependent manager: dependent
success-message: Build succeeded (gate). success-message: Build succeeded (gate).
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: comment-added - event: comment-added
@ -69,13 +67,15 @@
name: org/project name: org/project
check: check:
jobs: jobs:
- project-merge: - project-merge
jobs: - project-test1:
- project-test1 dependencies: project-merge
- project-test2 - project-test2:
dependencies: project-merge
gate: gate:
jobs: jobs:
- project-merge: - project-merge
jobs: - project-test1:
- project-test1 dependencies: project-merge
- project-test2 - project-test2:
dependencies: project-merge

View File

@ -1,8 +1,7 @@
- pipeline: - pipeline:
name: check name: check
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: patchset-created - event: patchset-created
@ -34,19 +33,23 @@
check: check:
jobs: jobs:
- merge: - merge:
jobs:
- test1
- test2
- integration
tags: tags:
- extratag - extratag
- test1:
dependencies: merge
- test2:
dependencies: merge
- integration:
dependencies: merge
- project: - project:
name: org/project2 name: org/project2
check: check:
jobs: jobs:
- merge: - merge
jobs: - test1:
- test1 dependencies: merge
- test2 - test2:
- integration dependencies: merge
- integration:
dependencies: merge

View File

@ -1,8 +1,7 @@
- pipeline: - pipeline:
name: periodic name: periodic
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
timer: timer:
- time: '* * * * * */1' - time: '* * * * * */1'
@ -10,7 +9,7 @@
smtp: smtp:
to: alternative_me@example.com to: alternative_me@example.com
from: zuul_from@example.com from: zuul_from@example.com
subject: 'Periodic check for {change.project} succeeded' subject: Periodic check for {change.project} succeeded
- job: - job:
name: project-bitrot-stable-old name: project-bitrot-stable-old

View File

@ -1,8 +1,7 @@
- pipeline: - pipeline:
name: check name: check
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: patchset-created - event: patchset-created
@ -16,8 +15,7 @@
- pipeline: - pipeline:
name: periodic name: periodic
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
timer: timer:
- time: '* * * * * */1' - time: '* * * * * */1'

View File

@ -1,8 +1,7 @@
- pipeline: - pipeline:
name: check name: check
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: patchset-created - event: patchset-created
@ -18,7 +17,6 @@
gerrit: gerrit:
verified: -1 verified: -1
- job: - job:
name: docs-draft-test name: docs-draft-test
success-url: http://docs-draft.example.org/{build.parameters[LOG_PATH]}/publish-docs/ success-url: http://docs-draft.example.org/{build.parameters[LOG_PATH]}/publish-docs/

View File

@ -1,8 +1,7 @@
- pipeline: - pipeline:
name: check name: check
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: patchset-created - event: patchset-created
@ -17,8 +16,7 @@
name: gate name: gate
manager: dependent manager: dependent
success-message: Build succeeded (gate). success-message: Build succeeded (gate).
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: comment-added - event: comment-added
@ -39,8 +37,7 @@
- pipeline: - pipeline:
name: post name: post
manager: independent manager: independent
source: source: gerrit
gerrit
trigger: trigger:
gerrit: gerrit:
- event: ref-updated - event: ref-updated
@ -56,15 +53,15 @@
- project-template: - project-template:
name: test-three-and-four name: test-three-and-four
check: check:
jobs: jobs:
- layered-project-test3 - layered-project-test3
- layered-project-test4 - layered-project-test4
- project-template: - project-template:
name: test-five name: test-five
check: check:
jobs: jobs:
- layered-project-foo-test5 - layered-project-foo-test5
- job: - job:
name: project-test1 name: project-test1

View File

@ -225,7 +225,7 @@ class TestJob(BaseTestCase):
self.assertFalse(python27diablo.changeMatches(change)) self.assertFalse(python27diablo.changeMatches(change))
self.assertFalse(python27essex.changeMatches(change)) self.assertFalse(python27essex.changeMatches(change))
item.freezeJobTree() item.freezeJobGraph()
self.assertEqual(len(item.getJobs()), 1) self.assertEqual(len(item.getJobs()), 1)
job = item.getJobs()[0] job = item.getJobs()[0]
self.assertEqual(job.name, 'python27') self.assertEqual(job.name, 'python27')
@ -253,7 +253,7 @@ class TestJob(BaseTestCase):
self.assertTrue(python27diablo.changeMatches(change)) self.assertTrue(python27diablo.changeMatches(change))
self.assertFalse(python27essex.changeMatches(change)) self.assertFalse(python27essex.changeMatches(change))
item.freezeJobTree() item.freezeJobGraph()
self.assertEqual(len(item.getJobs()), 1) self.assertEqual(len(item.getJobs()), 1)
job = item.getJobs()[0] job = item.getJobs()[0]
self.assertEqual(job.name, 'python27') self.assertEqual(job.name, 'python27')
@ -282,7 +282,7 @@ class TestJob(BaseTestCase):
self.assertFalse(python27diablo.changeMatches(change)) self.assertFalse(python27diablo.changeMatches(change))
self.assertTrue(python27essex.changeMatches(change)) self.assertTrue(python27essex.changeMatches(change))
item.freezeJobTree() item.freezeJobGraph()
self.assertEqual(len(item.getJobs()), 1) self.assertEqual(len(item.getJobs()), 1)
job = item.getJobs()[0] job = item.getJobs()[0]
self.assertEqual(job.name, 'python27') self.assertEqual(job.name, 'python27')
@ -439,7 +439,7 @@ class TestJob(BaseTestCase):
self.assertTrue(python27.changeMatches(change)) self.assertTrue(python27.changeMatches(change))
self.assertFalse(python27diablo.changeMatches(change)) self.assertFalse(python27diablo.changeMatches(change))
item.freezeJobTree() item.freezeJobGraph()
self.assertEqual(len(item.getJobs()), 1) self.assertEqual(len(item.getJobs()), 1)
job = item.getJobs()[0] job = item.getJobs()[0]
self.assertEqual(job.name, 'python27') self.assertEqual(job.name, 'python27')
@ -453,7 +453,7 @@ class TestJob(BaseTestCase):
self.assertTrue(python27.changeMatches(change)) self.assertTrue(python27.changeMatches(change))
self.assertTrue(python27diablo.changeMatches(change)) self.assertTrue(python27diablo.changeMatches(change))
item.freezeJobTree() item.freezeJobGraph()
self.assertEqual(len(item.getJobs()), 1) self.assertEqual(len(item.getJobs()), 1)
job = item.getJobs()[0] job = item.getJobs()[0]
self.assertEqual(job.name, 'python27') self.assertEqual(job.name, 'python27')
@ -506,7 +506,7 @@ class TestJob(BaseTestCase):
self.assertTrue(base.changeMatches(change)) self.assertTrue(base.changeMatches(change))
self.assertFalse(python27.changeMatches(change)) self.assertFalse(python27.changeMatches(change))
item.freezeJobTree() item.freezeJobGraph()
self.assertEqual([], item.getJobs()) self.assertEqual([], item.getJobs())
def test_job_source_project(self): def test_job_source_project(self):
@ -609,3 +609,56 @@ class TestTimeDataBase(BaseTestCase):
for x in range(10): for x in range(10):
self.db.update('job-name', 100, 'SUCCESS') self.db.update('job-name', 100, 'SUCCESS')
self.assertEqual(self.db.getEstimatedTime('job-name'), 100) self.assertEqual(self.db.getEstimatedTime('job-name'), 100)
class TestGraph(BaseTestCase):
def test_job_graph_disallows_multiple_jobs_with_same_name(self):
graph = model.JobGraph()
job1 = model.Job('job')
job2 = model.Job('job')
graph.addJob(job1)
with testtools.ExpectedException(Exception,
"Job job already added"):
graph.addJob(job2)
def test_job_graph_disallows_circular_dependencies(self):
graph = model.JobGraph()
jobs = [model.Job('job%d' % i) for i in range(0, 10)]
prevjob = None
for j in jobs[:3]:
if prevjob:
j.dependencies = frozenset([prevjob.name])
graph.addJob(j)
prevjob = j
# 0 triggers 1 triggers 2 triggers 3...
# Cannot depend on itself
with testtools.ExpectedException(
Exception,
"Dependency cycle detected in job jobX"):
j = model.Job('jobX')
j.dependencies = frozenset([j.name])
graph.addJob(j)
# Disallow circular dependencies
with testtools.ExpectedException(
Exception,
"Dependency cycle detected in job job3"):
jobs[4].dependencies = frozenset([jobs[3].name])
graph.addJob(jobs[4])
jobs[3].dependencies = frozenset([jobs[4].name])
graph.addJob(jobs[3])
jobs[5].dependencies = frozenset([jobs[4].name])
graph.addJob(jobs[5])
with testtools.ExpectedException(
Exception,
"Dependency cycle detected in job job3"):
jobs[3].dependencies = frozenset([jobs[5].name])
graph.addJob(jobs[3])
jobs[3].dependencies = frozenset([jobs[2].name])
graph.addJob(jobs[3])
jobs[6].dependencies = frozenset([jobs[2].name])
graph.addJob(jobs[6])

View File

@ -4478,6 +4478,117 @@ For CI problems and help debugging, contact ci@example.org"""
self.assertIn('project-test2 : SKIPPED', A.messages[1]) self.assertIn('project-test2 : SKIPPED', A.messages[1])
class TestDependencyGraph(ZuulTestCase):
tenant_config_file = 'config/dependency-graph/main.yaml'
def test_dependeny_graph_dispatch_jobs_once(self):
"Test a job in a dependency graph is queued only once"
# Job dependencies, starting with A
# A
# / \
# B C
# / \ / \
# D F E
# |
# G
self.executor_server.hold_jobs_in_build = True
change = self.fake_gerrit.addFakeChange(
'org/project', 'master', 'change')
change.addApproval('code-review', 2)
self.fake_gerrit.addEvent(change.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual([b.name for b in self.builds], ['A'])
self.executor_server.release('A')
self.waitUntilSettled()
self.assertEqual(sorted(b.name for b in self.builds), ['B', 'C'])
self.executor_server.release('B')
self.waitUntilSettled()
self.assertEqual(sorted(b.name for b in self.builds), ['C', 'D'])
self.executor_server.release('D')
self.waitUntilSettled()
self.assertEqual([b.name for b in self.builds], ['C'])
self.executor_server.release('C')
self.waitUntilSettled()
self.assertEqual(sorted(b.name for b in self.builds), ['E', 'F'])
self.executor_server.release('F')
self.waitUntilSettled()
self.assertEqual(sorted(b.name for b in self.builds), ['E', 'G'])
self.executor_server.release('G')
self.waitUntilSettled()
self.assertEqual([b.name for b in self.builds], ['E'])
self.executor_server.release('E')
self.waitUntilSettled()
self.assertEqual(len(self.builds), 0)
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(len(self.builds), 0)
self.assertEqual(len(self.history), 7)
self.assertEqual(change.data['status'], 'MERGED')
self.assertEqual(change.reported, 2)
def test_jobs_launched_only_if_all_dependencies_are_successful(self):
"Test that a job waits till all dependencies are successful"
# Job dependencies, starting with A
# A
# / \
# B C*
# / \ / \
# D F E
# |
# G
self.executor_server.hold_jobs_in_build = True
change = self.fake_gerrit.addFakeChange(
'org/project', 'master', 'change')
change.addApproval('code-review', 2)
self.executor_server.failJob('C', change)
self.fake_gerrit.addEvent(change.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual([b.name for b in self.builds], ['A'])
self.executor_server.release('A')
self.waitUntilSettled()
self.assertEqual(sorted(b.name for b in self.builds), ['B', 'C'])
self.executor_server.release('B')
self.waitUntilSettled()
self.assertEqual(sorted(b.name for b in self.builds), ['C', 'D'])
self.executor_server.release('D')
self.waitUntilSettled()
self.assertEqual([b.name for b in self.builds], ['C'])
self.executor_server.release('C')
self.waitUntilSettled()
self.assertEqual(len(self.builds), 0)
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(len(self.builds), 0)
self.assertEqual(len(self.history), 4)
self.assertEqual(change.data['status'], 'NEW')
self.assertEqual(change.reported, 2)
class TestDuplicatePipeline(ZuulTestCase): class TestDuplicatePipeline(ZuulTestCase):
tenant_config_file = 'config/duplicate-pipeline/main.yaml' tenant_config_file = 'config/duplicate-pipeline/main.yaml'

View File

@ -193,6 +193,7 @@ class JobParser(object):
'roles': to_list(role), 'roles': to_list(role),
'repos': to_list(str), 'repos': to_list(str),
'vars': dict, 'vars': dict,
'dependencies': to_list(str),
} }
return vs.Schema(job) return vs.Schema(job)
@ -276,6 +277,8 @@ class JobParser(object):
# accumulate onto any previously applied tags. # accumulate onto any previously applied tags.
job.tags = job.tags.union(set(tags)) job.tags = job.tags.union(set(tags))
job.dependencies = frozenset(as_list(conf.get('dependencies')))
roles = [] roles = []
for role in conf.get('roles', []): for role in conf.get('roles', []):
if 'zuul' in role: if 'zuul' in role:
@ -364,45 +367,33 @@ class ProjectTemplateParser(object):
project_pipeline = model.ProjectPipelineConfig() project_pipeline = model.ProjectPipelineConfig()
project_template.pipelines[pipeline.name] = project_pipeline project_template.pipelines[pipeline.name] = project_pipeline
project_pipeline.queue_name = conf_pipeline.get('queue') project_pipeline.queue_name = conf_pipeline.get('queue')
project_pipeline.job_tree = ProjectTemplateParser._parseJobTree( ProjectTemplateParser._parseJobList(
tenant, layout, conf_pipeline.get('jobs', []), tenant, layout, conf_pipeline.get('jobs', []),
source_context, start_mark) source_context, start_mark, project_pipeline.job_list)
return project_template return project_template
@staticmethod @staticmethod
def _parseJobTree(tenant, layout, conf, source_context, def _parseJobList(tenant, layout, conf, source_context,
start_mark, tree=None): start_mark, job_list):
if not tree:
tree = model.JobTree(None)
for conf_job in conf: for conf_job in conf:
if isinstance(conf_job, six.string_types): if isinstance(conf_job, six.string_types):
job = model.Job(conf_job) job = model.Job(conf_job)
tree.addJob(job) job_list.addJob(job)
elif isinstance(conf_job, dict): elif isinstance(conf_job, dict):
# A dictionary in a job tree may override params, or # A dictionary in a job tree may override params
# be the root of a sub job tree, or both.
jobname, attrs = conf_job.items()[0] jobname, attrs = conf_job.items()[0]
jobs = attrs.pop('jobs', None)
if attrs: if attrs:
# We are overriding params, so make a new job def # We are overriding params, so make a new job def
attrs['name'] = jobname attrs['name'] = jobname
attrs['_source_context'] = source_context attrs['_source_context'] = source_context
attrs['_start_mark'] = start_mark attrs['_start_mark'] = start_mark
subtree = tree.addJob(JobParser.fromYaml( job_list.addJob(JobParser.fromYaml(tenant, layout, attrs))
tenant, layout, attrs))
else: else:
# Not overriding, so add a blank job # Not overriding, so add a blank job
job = model.Job(jobname) job = model.Job(jobname)
subtree = tree.addJob(job) job_list.addJob(job)
if jobs:
# This is the root of a sub tree
ProjectTemplateParser._parseJobTree(
tenant, layout, jobs, source_context,
start_mark, subtree)
else: else:
raise Exception("Job must be a string or dictionary") raise Exception("Job must be a string or dictionary")
return tree
class ProjectParser(object): class ProjectParser(object):
@ -455,7 +446,6 @@ class ProjectParser(object):
project.merge_mode = model.MERGER_MAP['merge-resolve'] project.merge_mode = model.MERGER_MAP['merge-resolve']
for pipeline in layout.pipelines.values(): for pipeline in layout.pipelines.values():
project_pipeline = model.ProjectPipelineConfig() project_pipeline = model.ProjectPipelineConfig()
project_pipeline.job_tree = model.JobTree(None)
queue_name = None queue_name = None
# For every template, iterate over the job tree and replace or # For every template, iterate over the job tree and replace or
# create the jobs in the final definition as needed. # create the jobs in the final definition as needed.
@ -467,8 +457,8 @@ class ProjectParser(object):
(template.name, pipeline.name)) (template.name, pipeline.name))
pipeline_defined = True pipeline_defined = True
template_pipeline = template.pipelines[pipeline.name] template_pipeline = template.pipelines[pipeline.name]
project_pipeline.job_tree.inheritFrom( project_pipeline.job_list.inheritFrom(
template_pipeline.job_tree) template_pipeline.job_list)
if template_pipeline.queue_name: if template_pipeline.queue_name:
queue_name = template_pipeline.queue_name queue_name = template_pipeline.queue_name
if queue_name: if queue_name:

View File

@ -64,31 +64,29 @@ class PipelineManager(object):
self.log.info(" %s" % e) self.log.info(" %s" % e)
self.log.info(" Projects:") self.log.info(" Projects:")
def log_jobs(tree, indent=0): def log_jobs(job_list):
istr = ' ' + ' ' * indent for job_name, job_variants in job_list.jobs.items():
if tree.job: for variant in job_variants:
# TODOv3(jeblair): represent matchers # TODOv3(jeblair): represent matchers
efilters = '' efilters = ''
# for b in tree.job._branches: # for b in tree.job._branches:
# efilters += str(b) # efilters += str(b)
# for f in tree.job._files: # for f in tree.job._files:
# efilters += str(f) # efilters += str(f)
# if tree.job.skip_if_matcher: # if tree.job.skip_if_matcher:
# efilters += str(tree.job.skip_if_matcher) # efilters += str(tree.job.skip_if_matcher)
# if efilters: # if efilters:
# efilters = ' ' + efilters # efilters = ' ' + efilters
tags = [] tags = []
if tree.job.hold_following_changes: if variant.hold_following_changes:
tags.append('[hold]') tags.append('[hold]')
if not tree.job.voting: if not variant.voting:
tags.append('[nonvoting]') tags.append('[nonvoting]')
if tree.job.mutex: if variant.mutex:
tags.append('[mutex: %s]' % tree.job.mutex) tags.append('[mutex: %s]' % variant.mutex)
tags = ' '.join(tags) tags = ' '.join(tags)
self.log.info("%s%s%s %s" % (istr, repr(tree.job), self.log.info(" %s%s %s" % (repr(variant),
efilters, tags)) efilters, tags))
for x in tree.job_trees:
log_jobs(x, indent + 2)
for project_name in layout.project_configs.keys(): for project_name in layout.project_configs.keys():
project_config = layout.project_configs.get(project_name) project_config = layout.project_configs.get(project_name)
@ -97,7 +95,7 @@ class PipelineManager(object):
self.pipeline.name) self.pipeline.name)
if project_pipeline_config: if project_pipeline_config:
self.log.info(" %s" % project_name) self.log.info(" %s" % project_name)
log_jobs(project_pipeline_config.job_tree) log_jobs(project_pipeline_config.job_list)
self.log.info(" On start:") self.log.info(" On start:")
self.log.info(" %s" % self.pipeline.start_actions) self.log.info(" %s" % self.pipeline.start_actions)
self.log.info(" On success:") self.log.info(" On success:")
@ -257,7 +255,7 @@ class PipelineManager(object):
# Rebuild the frozen job tree from the new layout, if # Rebuild the frozen job tree from the new layout, if
# we have one. If not, it will be built later. # we have one. If not, it will be built later.
if item.current_build_set.layout: if item.current_build_set.layout:
item.freezeJobTree() item.freezeJobGraph()
# Re-set build results in case any new jobs have been # Re-set build results in case any new jobs have been
# added to the tree. # added to the tree.
@ -540,8 +538,18 @@ class PipelineManager(object):
item.current_build_set.layout = self.getLayout(item) item.current_build_set.layout = self.getLayout(item)
if not item.current_build_set.layout: if not item.current_build_set.layout:
return False return False
if not item.job_tree: if item.current_build_set.config_error:
item.freezeJobTree() return False
if not item.job_graph:
try:
item.freezeJobGraph()
except Exception as e:
# TODOv3(jeblair): nicify this exception as it will be reported
self.log.exception("Error freezing job graph for %s" %
item)
item.setConfigError("Unable to freeze job graph: %s" %
(str(e)))
return False
return True return True
def _processOneItem(self, item, nnfi): def _processOneItem(self, item, nnfi):

View File

@ -675,6 +675,7 @@ class Job(object):
file_matcher=None, file_matcher=None,
irrelevant_file_matcher=None, # skip-if irrelevant_file_matcher=None, # skip-if
tags=frozenset(), tags=frozenset(),
dependencies=frozenset(),
) )
# These attributes affect how the job is actually run and more # These attributes affect how the job is actually run and more
@ -851,60 +852,100 @@ class Job(object):
return True return True
class JobTree(object): class JobList(object):
"""A JobTree holds one or more Jobs to represent Job dependencies. """ A list of jobs in a project's pipeline. """
If Job foo should only execute if Job bar succeeds, then there will def __init__(self):
be a JobTree for foo, which will contain a JobTree for bar. A JobTree self.jobs = OrderedDict() # job.name -> [job, ...]
can hold more than one dependent JobTrees, such that jobs bar and bang
both depend on job foo being successful.
A root node of a JobTree will have no associated Job."""
def __init__(self, job):
self.job = job
self.job_trees = []
def __repr__(self):
return '<JobTree %s %s>' % (self.job, self.job_trees)
def addJob(self, job): def addJob(self, job):
if job not in [x.job for x in self.job_trees]: if job.name in self.jobs:
t = JobTree(job) self.jobs[job.name].append(job)
self.job_trees.append(t) else:
return t self.jobs[job.name] = [job]
for tree in self.job_trees:
if tree.job == job:
return tree
def getJobs(self):
jobs = []
for x in self.job_trees:
jobs.append(x.job)
jobs.extend(x.getJobs())
return jobs
def getJobTreeForJob(self, job):
if self.job == job:
return self
for tree in self.job_trees:
ret = tree.getJobTreeForJob(job)
if ret:
return ret
return None
def inheritFrom(self, other): def inheritFrom(self, other):
if other.job: for jobname, jobs in other.jobs.items():
if not self.job: if jobname in self.jobs:
self.job = other.job.copy() self.jobs[jobname].append(jobs)
else: else:
self.job.applyVariant(other.job) self.jobs[jobname] = jobs
for other_tree in other.job_trees:
this_tree = self.getJobTreeForJob(other_tree.job)
if not this_tree: class JobGraph(object):
this_tree = JobTree(None) """ A JobGraph represents the dependency graph between Job."""
self.job_trees.append(this_tree)
this_tree.inheritFrom(other_tree) def __init__(self):
self.jobs = OrderedDict() # job_name -> Job
self._dependencies = {} # dependent_job_name -> set(parent_job_names)
def __repr__(self):
return '<JobGraph %s>' % (self.jobs)
def addJob(self, job):
# A graph must be created after the job list is frozen,
# therefore we should only get one job with the same name.
if job.name in self.jobs:
raise Exception("Job %s already added" % (job.name,))
self.jobs[job.name] = job
# Append the dependency information
self._dependencies.setdefault(job.name, set())
try:
for dependency in job.dependencies:
# Make sure a circular dependency is never created
ancestor_jobs = self._getParentJobNamesRecursively(
dependency, soft=True)
ancestor_jobs.add(dependency)
if any((job.name == anc_job) for anc_job in ancestor_jobs):
raise Exception("Dependency cycle detected in job %s" %
(job.name,))
self._dependencies[job.name].add(dependency)
except Exception:
del self.jobs[job.name]
del self._dependencies[job.name]
raise
def getJobs(self):
return self.jobs.values() # Report in the order of the layout config
def _getDirectDependentJobs(self, parent_job):
ret = set()
for dependent_name, parent_names in self._dependencies.items():
if parent_job in parent_names:
ret.add(dependent_name)
return ret
def getDependentJobsRecursively(self, parent_job):
all_dependent_jobs = set()
jobs_to_iterate = set([parent_job])
while len(jobs_to_iterate) > 0:
current_job = jobs_to_iterate.pop()
current_dependent_jobs = self._getDirectDependentJobs(current_job)
new_dependent_jobs = current_dependent_jobs - all_dependent_jobs
jobs_to_iterate |= new_dependent_jobs
all_dependent_jobs |= new_dependent_jobs
return [self.jobs[name] for name in all_dependent_jobs]
def getParentJobsRecursively(self, dependent_job):
return [self.jobs[name] for name in
self._getParentJobNamesRecursively(dependent_job)]
def _getParentJobNamesRecursively(self, dependent_job, soft=False):
all_parent_jobs = set()
jobs_to_iterate = set([dependent_job])
while len(jobs_to_iterate) > 0:
current_job = jobs_to_iterate.pop()
current_parent_jobs = self._dependencies.get(current_job)
if current_parent_jobs is None:
if soft:
current_parent_jobs = set()
else:
raise Exception("Dependent job %s not found: " %
(dependent_job,))
new_parent_jobs = current_parent_jobs - all_parent_jobs
jobs_to_iterate |= new_parent_jobs
all_parent_jobs |= new_parent_jobs
return all_parent_jobs
class Build(object): class Build(object):
@ -1137,7 +1178,7 @@ class QueueItem(object):
self.active = False # Whether an item is within an active window self.active = False # Whether an item is within an active window
self.live = True # Whether an item is intended to be processed at all self.live = True # Whether an item is intended to be processed at all
self.layout = None # This item's shadow layout self.layout = None # This item's shadow layout
self.job_tree = None self.job_graph = None
def __repr__(self): def __repr__(self):
if self.pipeline: if self.pipeline:
@ -1165,23 +1206,33 @@ class QueueItem(object):
def setReportedResult(self, result): def setReportedResult(self, result):
self.current_build_set.result = result self.current_build_set.result = result
def freezeJobTree(self): def freezeJobGraph(self):
"""Find or create actual matching jobs for this item's change and """Find or create actual matching jobs for this item's change and
store the resulting job tree.""" store the resulting job tree."""
layout = self.current_build_set.layout layout = self.current_build_set.layout
self.job_tree = layout.createJobTree(self) job_graph = layout.createJobGraph(self)
for job in job_graph.getJobs():
# Ensure that each jobs's dependencies are fully
# accessible. This will raise an exception if not.
job_graph.getParentJobsRecursively(job.name)
self.job_graph = job_graph
def hasJobTree(self): def hasJobGraph(self):
"""Returns True if the item has a job tree.""" """Returns True if the item has a job graph."""
return self.job_tree is not None return self.job_graph is not None
def getJobs(self): def getJobs(self):
if not self.live or not self.job_tree: if not self.live or not self.job_graph:
return [] return []
return self.job_tree.getJobs() return self.job_graph.getJobs()
def getJob(self, name):
if not self.job_graph:
return None
return self.job_graph.jobs.get(name)
def haveAllJobsStarted(self): def haveAllJobsStarted(self):
if not self.hasJobTree(): if not self.hasJobGraph():
return False return False
for job in self.getJobs(): for job in self.getJobs():
build = self.current_build_set.getBuild(job.name) build = self.current_build_set.getBuild(job.name)
@ -1193,7 +1244,7 @@ class QueueItem(object):
if (self.current_build_set.config_error or if (self.current_build_set.config_error or
self.current_build_set.unable_to_merge): self.current_build_set.unable_to_merge):
return True return True
if not self.hasJobTree(): if not self.hasJobGraph():
return False return False
for job in self.getJobs(): for job in self.getJobs():
build = self.current_build_set.getBuild(job.name) build = self.current_build_set.getBuild(job.name)
@ -1202,7 +1253,7 @@ class QueueItem(object):
return True return True
def didAllJobsSucceed(self): def didAllJobsSucceed(self):
if not self.hasJobTree(): if not self.hasJobGraph():
return False return False
for job in self.getJobs(): for job in self.getJobs():
if not job.voting: if not job.voting:
@ -1215,7 +1266,7 @@ class QueueItem(object):
return True return True
def didAnyJobFail(self): def didAnyJobFail(self):
if not self.hasJobTree(): if not self.hasJobGraph():
return False return False
for job in self.getJobs(): for job in self.getJobs():
if not job.voting: if not job.voting:
@ -1234,7 +1285,7 @@ class QueueItem(object):
def isHoldingFollowingChanges(self): def isHoldingFollowingChanges(self):
if not self.live: if not self.live:
return False return False
if not self.hasJobTree(): if not self.hasJobGraph():
return False return False
for job in self.getJobs(): for job in self.getJobs():
if not job.hold_following_changes: if not job.hold_following_changes:
@ -1249,88 +1300,96 @@ class QueueItem(object):
return False return False
return self.item_ahead.isHoldingFollowingChanges() return self.item_ahead.isHoldingFollowingChanges()
def _findJobsToRun(self, job_trees, mutex): def findJobsToRun(self, mutex):
torun = [] torun = []
if not self.live:
return []
if not self.job_graph:
return []
if self.item_ahead: if self.item_ahead:
# Only run jobs if any 'hold' jobs on the change ahead # Only run jobs if any 'hold' jobs on the change ahead
# have completed successfully. # have completed successfully.
if self.item_ahead.isHoldingFollowingChanges(): if self.item_ahead.isHoldingFollowingChanges():
return [] return []
for tree in job_trees:
job = tree.job successful_job_names = set()
result = None jobs_not_started = set()
if job: for job in self.job_graph.getJobs():
if not job.changeMatches(self.change): build = self.current_build_set.getBuild(job.name)
if build:
if build.result == 'SUCCESS':
successful_job_names.add(job.name)
else:
jobs_not_started.add(job)
# Attempt to request nodes for jobs in the order jobs appear
# in configuration.
for job in self.job_graph.getJobs():
if job not in jobs_not_started:
continue
all_parent_jobs_successful = True
for parent_job in self.job_graph.getParentJobsRecursively(
job.name):
if parent_job.name not in successful_job_names:
all_parent_jobs_successful = False
break
if all_parent_jobs_successful:
nodeset = self.current_build_set.getJobNodeSet(job.name)
if nodeset is None:
# The nodes for this job are not ready, skip
# it for now.
continue continue
build = self.current_build_set.getBuild(job.name) if mutex.acquire(self, job):
if build: # If this job needs a mutex, either acquire it or make
result = build.result # sure that we have it before running the job.
else: torun.append(job)
# There is no build for the root of this job tree,
# so it has not run yet.
nodeset = self.current_build_set.getJobNodeSet(job.name)
if nodeset is None:
# The nodes for this job are not ready, skip
# it for now.
continue
if mutex.acquire(self, job):
# If this job needs a mutex, either acquire it or make
# sure that we have it before running the job.
torun.append(job)
# If there is no job, this is a null job tree, and we should
# run all of its jobs.
if result == 'SUCCESS' or not job:
torun.extend(self._findJobsToRun(tree.job_trees, mutex))
return torun return torun
def findJobsToRun(self, mutex): def findJobsToRequest(self):
if not self.live:
return []
tree = self.job_tree
if not tree:
return []
return self._findJobsToRun(tree.job_trees, mutex)
def _findJobsToRequest(self, job_trees):
build_set = self.current_build_set build_set = self.current_build_set
toreq = [] toreq = []
if not self.live:
return []
if not self.job_graph:
return []
if self.item_ahead: if self.item_ahead:
if self.item_ahead.isHoldingFollowingChanges(): if self.item_ahead.isHoldingFollowingChanges():
return [] return []
for tree in job_trees:
job = tree.job
result = None
if job:
if not job.changeMatches(self.change):
continue
build = build_set.getBuild(job.name)
if build:
result = build.result
else:
nodeset = build_set.getJobNodeSet(job.name)
if nodeset is None:
req = build_set.getJobNodeRequest(job.name)
if req is None:
toreq.append(job)
if result == 'SUCCESS' or not job:
toreq.extend(self._findJobsToRequest(tree.job_trees))
return toreq
def findJobsToRequest(self): successful_job_names = set()
if not self.live: jobs_not_requested = set()
return [] for job in self.job_graph.getJobs():
tree = self.job_tree build = build_set.getBuild(job.name)
if not tree: if build and build.result == 'SUCCESS':
return [] successful_job_names.add(job.name)
return self._findJobsToRequest(tree.job_trees) else:
nodeset = build_set.getJobNodeSet(job.name)
if nodeset is None:
req = build_set.getJobNodeRequest(job.name)
if req is None:
jobs_not_requested.add(job)
# Attempt to request nodes for jobs in the order jobs appear
# in configuration.
for job in self.job_graph.getJobs():
if job not in jobs_not_requested:
continue
all_parent_jobs_successful = True
for parent_job in self.job_graph.getParentJobsRecursively(
job.name):
if parent_job.name not in successful_job_names:
all_parent_jobs_successful = False
break
if all_parent_jobs_successful:
toreq.append(job)
return toreq
def setResult(self, build): def setResult(self, build):
if build.retry: if build.retry:
self.removeBuild(build) self.removeBuild(build)
elif build.result != 'SUCCESS': elif build.result != 'SUCCESS':
# Get a JobTree from a Job so we can find only its dependent jobs for job in self.job_graph.getDependentJobsRecursively(
tree = self.job_tree.getJobTreeForJob(build.job) build.job.name):
for job in tree.getJobs():
fakebuild = Build(job, None) fakebuild = Build(job, None)
fakebuild.result = 'SKIPPED' fakebuild.result = 'SKIPPED'
self.addBuild(fakebuild) self.addBuild(fakebuild)
@ -2014,7 +2073,7 @@ class ChangeishFilter(BaseFilter):
class ProjectPipelineConfig(object): class ProjectPipelineConfig(object):
# Represents a project cofiguration in the context of a pipeline # Represents a project cofiguration in the context of a pipeline
def __init__(self): def __init__(self):
self.job_tree = None self.job_list = JobList()
self.queue_name = None self.queue_name = None
self.merge_mode = None self.merge_mode = None
@ -2182,14 +2241,13 @@ class Layout(object):
def addProjectConfig(self, project_config): def addProjectConfig(self, project_config):
self.project_configs[project_config.name] = project_config self.project_configs[project_config.name] = project_config
def _createJobTree(self, change, job_trees, parent): def _createJobGraph(self, change, job_list, job_graph):
for tree in job_trees: for jobname in job_list.jobs:
job = tree.job # This is the final job we are constructing
if not job.changeMatches(change):
continue
frozen_job = None frozen_job = None
# Whether the change matches any globally defined variant
matched = False matched = False
for variant in self.getJobs(job.name): for variant in self.getJobs(jobname):
if variant.changeMatches(change): if variant.changeMatches(change):
if frozen_job is None: if frozen_job is None:
frozen_job = variant.copy() frozen_job = variant.copy()
@ -2203,25 +2261,33 @@ class Layout(object):
# the job that is defined in the tree). # the job that is defined in the tree).
continue continue
# If the job does not allow auth inheritance, do not allow # If the job does not allow auth inheritance, do not allow
# the project-pipeline variant to update its execution # the project-pipeline variants to update its execution
# attributes. # attributes.
if frozen_job.auth and not frozen_job.auth.get('inherit'): if frozen_job.auth and not frozen_job.auth.get('inherit'):
frozen_job.final = True frozen_job.final = True
frozen_job.applyVariant(job) # Whether the change matches any of the project pipeline
frozen_tree = JobTree(frozen_job) # variants
parent.job_trees.append(frozen_tree) matched = False
self._createJobTree(change, tree.job_trees, frozen_tree) for variant in job_list.jobs[jobname]:
if variant.changeMatches(change):
frozen_job.applyVariant(variant)
matched = True
if not matched:
# A change must match at least one project pipeline
# job variant.
continue
job_graph.addJob(frozen_job)
def createJobTree(self, item): def createJobGraph(self, item):
project_config = self.project_configs.get( project_config = self.project_configs.get(
item.change.project.name, None) item.change.project.name, None)
ret = JobTree(None) ret = JobGraph()
# NOTE(pabelanger): It is possible for a foreign project not to have a # NOTE(pabelanger): It is possible for a foreign project not to have a
# configured pipeline, if so return an empty JobTree. # configured pipeline, if so return an empty JobGraph.
if project_config and item.pipeline.name in project_config.pipelines: if project_config and item.pipeline.name in project_config.pipelines:
project_tree = \ project_job_list = \
project_config.pipelines[item.pipeline.name].job_tree project_config.pipelines[item.pipeline.name].job_list
self._createJobTree(item.change, project_tree.job_trees, ret) self._createJobGraph(item.change, project_job_list, ret)
return ret return ret

View File

@ -555,11 +555,10 @@ class Scheduler(threading.Thread):
project_name) project_name)
if new_pipeline.manager.reEnqueueItem(item, if new_pipeline.manager.reEnqueueItem(item,
last_head): last_head):
new_jobs = item.getJobs()
for build in item.current_build_set.getBuilds(): for build in item.current_build_set.getBuilds():
jobtree = item.job_tree.getJobTreeForJob(build.job) new_job = item.getJob(build.job.name)
if jobtree and jobtree.job in new_jobs: if new_job:
build.job = jobtree.job build.job = new_job
else: else:
item.removeBuild(build) item.removeBuild(build)
builds_to_cancel.append(build) builds_to_cancel.append(build)