Browse Source

Allow soft job dependencies

A "soft" dependency can be used to indicate that a job must run
after another completes, but only if it runs at all.  For example,
a deployment job which depends on a build job with different file
matcher criteria.

Change-Id: I4d7fc2b40942569323da273c4529fdb365a3b11a
tags/3.6.0^2
James E. Blair 2 months ago
parent
commit
db3388688a

+ 21
- 0
doc/source/user/config.rst View File

@@ -1070,6 +1070,27 @@ Here is an example of two job definitions:
1070 1070
       completed successfully, and if one or more of them fail, this
1071 1071
       job will not be run.
1072 1072
 
1073
+      The format for this attribute is either a list of strings or
1074
+      dictionaries.  Strings are interpreted as job names,
1075
+      dictionaries, if used, may have the following attributes:
1076
+
1077
+      .. attr:: name
1078
+         :required:
1079
+
1080
+         The name of the required job.
1081
+
1082
+      .. attr:: soft
1083
+         :default: false
1084
+
1085
+         A boolean value which indicates whether this job is a *hard*
1086
+         or *soft* dependency.  A *hard* dependency will cause an
1087
+         error if the specified job is not run.  That is, if job B
1088
+         depends on job A, but job A is not run for any reason (for
1089
+         example, it containes a file matcher which does not match),
1090
+         then Zuul will not run any jobs and report an error.  A
1091
+         *soft* dependency will simply be ignored if the dependent job
1092
+         is not run.
1093
+
1073 1094
    .. attr:: allowed-projects
1074 1095
 
1075 1096
       A list of Zuul projects which may use this job.  By default, a

+ 8
- 0
releasenotes/notes/soft-dependencies-08b02bf3133a6c57.yaml View File

@@ -0,0 +1,8 @@
1
+---
2
+features:
3
+  - The :attr:`job.dependencies` attribute may now be used to express
4
+    "soft" dependencies -- that is, to indicate a job should run
5
+    after another completes, but only if it runs at all.  For example,
6
+    a deployment job which should always run, but depends on a build
7
+    job which only runs if the source code is changed.
8
+

+ 34
- 0
tests/fixtures/layouts/soft-dependencies-error.yaml View File

@@ -0,0 +1,34 @@
1
+- pipeline:
2
+    name: check
3
+    manager: independent
4
+    trigger:
5
+      gerrit:
6
+        - event: patchset-created
7
+    success:
8
+      gerrit:
9
+        Verified: 1
10
+    failure:
11
+      gerrit:
12
+        Verified: -1
13
+
14
+- job:
15
+    name: base
16
+    parent: null
17
+    run: playbooks/base.yaml
18
+
19
+- job:
20
+    name: build
21
+    files: main.c
22
+
23
+- job:
24
+    name: deploy
25
+
26
+- project:
27
+    name: org/project
28
+    check:
29
+      jobs:
30
+        - build
31
+        - deploy:
32
+            dependencies:
33
+              - name: project-merge
34
+                soft: true

+ 34
- 0
tests/fixtures/layouts/soft-dependencies.yaml View File

@@ -0,0 +1,34 @@
1
+- pipeline:
2
+    name: check
3
+    manager: independent
4
+    trigger:
5
+      gerrit:
6
+        - event: patchset-created
7
+    success:
8
+      gerrit:
9
+        Verified: 1
10
+    failure:
11
+      gerrit:
12
+        Verified: -1
13
+
14
+- job:
15
+    name: base
16
+    parent: null
17
+    run: playbooks/base.yaml
18
+
19
+- job:
20
+    name: build
21
+    files: main.c
22
+
23
+- job:
24
+    name: deploy
25
+
26
+- project:
27
+    name: org/project
28
+    check:
29
+      jobs:
30
+        - build
31
+        - deploy:
32
+            dependencies:
33
+              - name: build
34
+                soft: true

+ 72
- 8
tests/unit/test_model.py View File

@@ -441,7 +441,8 @@ class TestGraph(BaseTestCase):
441 441
         prevjob = None
442 442
         for j in jobs[:3]:
443 443
             if prevjob:
444
-                j.dependencies = frozenset([prevjob.name])
444
+                j.dependencies = frozenset([
445
+                    model.JobDependency(prevjob.name)])
445 446
             graph.addJob(j)
446 447
             prevjob = j
447 448
         # 0 triggers 1 triggers 2 triggers 3...
@@ -451,32 +452,95 @@ class TestGraph(BaseTestCase):
451 452
                 Exception,
452 453
                 "Dependency cycle detected in job jobX"):
453 454
             j = model.Job('jobX')
454
-            j.dependencies = frozenset([j.name])
455
+            j.dependencies = frozenset([model.JobDependency(j.name)])
455 456
             graph.addJob(j)
456 457
 
457 458
         # Disallow circular dependencies
458 459
         with testtools.ExpectedException(
459 460
                 Exception,
460 461
                 "Dependency cycle detected in job job3"):
461
-            jobs[4].dependencies = frozenset([jobs[3].name])
462
+            jobs[4].dependencies = frozenset([
463
+                model.JobDependency(jobs[3].name)])
462 464
             graph.addJob(jobs[4])
463
-            jobs[3].dependencies = frozenset([jobs[4].name])
465
+            jobs[3].dependencies = frozenset([
466
+                model.JobDependency(jobs[4].name)])
464 467
             graph.addJob(jobs[3])
465 468
 
466
-        jobs[5].dependencies = frozenset([jobs[4].name])
469
+        jobs[5].dependencies = frozenset([model.JobDependency(jobs[4].name)])
467 470
         graph.addJob(jobs[5])
468 471
 
469 472
         with testtools.ExpectedException(
470 473
                 Exception,
471 474
                 "Dependency cycle detected in job job3"):
472
-            jobs[3].dependencies = frozenset([jobs[5].name])
475
+            jobs[3].dependencies = frozenset([
476
+                model.JobDependency(jobs[5].name)])
473 477
             graph.addJob(jobs[3])
474 478
 
475
-        jobs[3].dependencies = frozenset([jobs[2].name])
479
+        jobs[3].dependencies = frozenset([
480
+            model.JobDependency(jobs[2].name)])
476 481
         graph.addJob(jobs[3])
477
-        jobs[6].dependencies = frozenset([jobs[2].name])
482
+        jobs[6].dependencies = frozenset([
483
+            model.JobDependency(jobs[2].name)])
478 484
         graph.addJob(jobs[6])
479 485
 
486
+    def test_job_graph_allows_soft_dependencies(self):
487
+        parent = model.Job('parent')
488
+        child = model.Job('child')
489
+        child.dependencies = frozenset([
490
+            model.JobDependency(parent.name, True)])
491
+
492
+        # With the parent
493
+        graph = model.JobGraph()
494
+        graph.addJob(parent)
495
+        graph.addJob(child)
496
+        self.assertEqual(graph.getParentJobsRecursively(child.name),
497
+                         [parent])
498
+
499
+        # Skip the parent
500
+        graph = model.JobGraph()
501
+        graph.addJob(child)
502
+        self.assertEqual(graph.getParentJobsRecursively(child.name), [])
503
+
504
+    def test_job_graph_allows_soft_dependencies4(self):
505
+        # A more complex scenario with multiple parents at each level
506
+        parents = [model.Job('parent%i' % i) for i in range(6)]
507
+        child = model.Job('child')
508
+        child.dependencies = frozenset([
509
+            model.JobDependency(parents[0].name, True),
510
+            model.JobDependency(parents[1].name)])
511
+        parents[0].dependencies = frozenset([
512
+            model.JobDependency(parents[2].name),
513
+            model.JobDependency(parents[3].name, True)])
514
+        parents[1].dependencies = frozenset([
515
+            model.JobDependency(parents[4].name),
516
+            model.JobDependency(parents[5].name)])
517
+        # Run them all
518
+        graph = model.JobGraph()
519
+        for j in parents:
520
+            graph.addJob(j)
521
+        graph.addJob(child)
522
+        self.assertEqual(set(graph.getParentJobsRecursively(child.name)),
523
+                         set(parents))
524
+
525
+        # Skip first parent, therefore its recursive dependencies don't appear
526
+        graph = model.JobGraph()
527
+        for j in parents:
528
+            if j is not parents[0]:
529
+                graph.addJob(j)
530
+        graph.addJob(child)
531
+        self.assertEqual(set(graph.getParentJobsRecursively(child.name)),
532
+                         set(parents) -
533
+                         set([parents[0], parents[2], parents[3]]))
534
+
535
+        # Skip a leaf node
536
+        graph = model.JobGraph()
537
+        for j in parents:
538
+            if j is not parents[3]:
539
+                graph.addJob(j)
540
+        graph.addJob(child)
541
+        self.assertEqual(set(graph.getParentJobsRecursively(child.name)),
542
+                         set(parents) - set([parents[3]]))
543
+
480 544
 
481 545
 class TestTenant(BaseTestCase):
482 546
     def test_add_project(self):

+ 19
- 0
tests/unit/test_scheduler.py View File

@@ -5673,6 +5673,25 @@ class TestDependencyGraph(ZuulTestCase):
5673 5673
         self.assertEqual(change.data['status'], 'NEW')
5674 5674
         self.assertEqual(change.reported, 2)
5675 5675
 
5676
+    @simple_layout('layouts/soft-dependencies-error.yaml')
5677
+    def test_soft_dependencies_error(self):
5678
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
5679
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5680
+        self.waitUntilSettled()
5681
+        self.assertHistory([])
5682
+        self.assertEqual(len(A.messages), 1)
5683
+        self.assertTrue('Job project-merge not defined' in A.messages[0])
5684
+        print(A.messages)
5685
+
5686
+    @simple_layout('layouts/soft-dependencies.yaml')
5687
+    def test_soft_dependencies(self):
5688
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
5689
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5690
+        self.waitUntilSettled()
5691
+        self.assertHistory([
5692
+            dict(name='deploy', result='SUCCESS', changes='1,1'),
5693
+        ], ordered=False)
5694
+
5676 5695
 
5677 5696
 class TestDuplicatePipeline(ZuulTestCase):
5678 5697
     tenant_config_file = 'config/duplicate-pipeline/main.yaml'

+ 6
- 3
tests/unit/test_web.py View File

@@ -517,7 +517,8 @@ class TestWeb(BaseTestWeb):
517 517
                 [{'abstract': False,
518 518
                   'attempts': 3,
519 519
                   'branches': [],
520
-                  'dependencies': ['project-merge'],
520
+                  'dependencies': [{'name': 'project-merge',
521
+                                    'soft': False}],
521 522
                   'description': None,
522 523
                   'files': [],
523 524
                   'final': False,
@@ -547,7 +548,8 @@ class TestWeb(BaseTestWeb):
547 548
                 [{'abstract': False,
548 549
                   'attempts': 3,
549 550
                   'branches': [],
550
-                  'dependencies': ['project-merge'],
551
+                  'dependencies': [{'name': 'project-merge',
552
+                                    'soft': False}],
551 553
                   'description': None,
552 554
                   'files': [],
553 555
                   'final': False,
@@ -577,7 +579,8 @@ class TestWeb(BaseTestWeb):
577 579
                 [{'abstract': False,
578 580
                   'attempts': 3,
579 581
                   'branches': [],
580
-                  'dependencies': ['project-merge'],
582
+                  'dependencies': [{'name': 'project-merge',
583
+                                    'soft': False}],
581 584
                   'description': None,
582 585
                   'files': [],
583 586
                   'final': False,

+ 19
- 2
zuul/configloader.py View File

@@ -533,6 +533,9 @@ class JobParser(object):
533 533
                    'override-branch': str,
534 534
                    'override-checkout': str}
535 535
 
536
+    job_dependency = {vs.Required('name'): str,
537
+                      'soft': bool}
538
+
536 539
     secret = {vs.Required('name'): str,
537 540
               vs.Required('secret'): str,
538 541
               'pass-to-parent': bool}
@@ -575,7 +578,7 @@ class JobParser(object):
575 578
                       'extra-vars': dict,
576 579
                       'host-vars': {str: dict},
577 580
                       'group-vars': {str: dict},
578
-                      'dependencies': to_list(str),
581
+                      'dependencies': to_list(vs.Any(job_dependency, str)),
579 582
                       'allowed-projects': to_list(str),
580 583
                       'override-branch': str,
581 584
                       'override-checkout': str,
@@ -764,6 +767,20 @@ class JobParser(object):
764 767
                 new_projects[project.canonical_name] = job_project
765 768
             job.required_projects = new_projects
766 769
 
770
+        if 'dependencies' in conf:
771
+            new_dependencies = []
772
+            dependencies = as_list(conf.get('dependencies', []))
773
+            for dep in dependencies:
774
+                if isinstance(dep, dict):
775
+                    dep_name = dep['name']
776
+                    dep_soft = dep.get('soft', False)
777
+                else:
778
+                    dep_name = dep
779
+                    dep_soft = False
780
+                job_dependency = model.JobDependency(dep_name, dep_soft)
781
+                new_dependencies.append(job_dependency)
782
+            job.dependencies = new_dependencies
783
+
767 784
         if 'semaphore' in conf:
768 785
             semaphore = conf.get('semaphore')
769 786
             if isinstance(semaphore, str):
@@ -773,7 +790,7 @@ class JobParser(object):
773 790
                     semaphore.get('name'),
774 791
                     semaphore.get('resources-first', False))
775 792
 
776
-        for k in ('tags', 'requires', 'provides', 'dependencies'):
793
+        for k in ('tags', 'requires', 'provides'):
777 794
             v = frozenset(as_list(conf.get(k)))
778 795
             if v:
779 796
                 setattr(job, k, v)

+ 41
- 18
zuul/model.py View File

@@ -1213,7 +1213,7 @@ class Job(ConfigObject):
1213 1213
         d['tags'] = list(self.tags)
1214 1214
         d['provides'] = list(self.provides)
1215 1215
         d['requires'] = list(self.requires)
1216
-        d['dependencies'] = list(self.dependencies)
1216
+        d['dependencies'] = list(map(lambda x: x.toDict(), self.dependencies))
1217 1217
         d['attempts'] = self.attempts
1218 1218
         d['roles'] = list(map(lambda x: x.toDict(), self.roles))
1219 1219
         d['run'] = list(map(lambda x: x.toSchemaDict(), self.run))
@@ -1649,12 +1649,25 @@ class JobList(ConfigObject):
1649 1649
                     joblist.append(job)
1650 1650
 
1651 1651
 
1652
+class JobDependency(ConfigObject):
1653
+    """ A reference to another job in the project-pipeline-config. """
1654
+    def __init__(self, name, soft=False):
1655
+        super(JobDependency, self).__init__()
1656
+        self.name = name
1657
+        self.soft = soft
1658
+
1659
+    def toDict(self):
1660
+        return {'name': self.name,
1661
+                'soft': self.soft}
1662
+
1663
+
1652 1664
 class JobGraph(object):
1653 1665
     """ A JobGraph represents the dependency graph between Job."""
1654 1666
 
1655 1667
     def __init__(self):
1656 1668
         self.jobs = OrderedDict()  # job_name -> Job
1657
-        self._dependencies = {}  # dependent_job_name -> set(parent_job_names)
1669
+        # dependent_job_name -> dict(parent_job_name -> soft)
1670
+        self._dependencies = {}
1658 1671
 
1659 1672
     def __repr__(self):
1660 1673
         return '<JobGraph %s>' % (self.jobs)
@@ -1666,17 +1679,18 @@ class JobGraph(object):
1666 1679
             raise Exception("Job %s already added" % (job.name,))
1667 1680
         self.jobs[job.name] = job
1668 1681
         # Append the dependency information
1669
-        self._dependencies.setdefault(job.name, set())
1682
+        self._dependencies.setdefault(job.name, {})
1670 1683
         try:
1671 1684
             for dependency in job.dependencies:
1672 1685
                 # Make sure a circular dependency is never created
1673 1686
                 ancestor_jobs = self._getParentJobNamesRecursively(
1674
-                    dependency, soft=True)
1675
-                ancestor_jobs.add(dependency)
1687
+                    dependency.name, soft=True)
1688
+                ancestor_jobs.add(dependency.name)
1676 1689
                 if any((job.name == anc_job) for anc_job in ancestor_jobs):
1677 1690
                     raise Exception("Dependency cycle detected in job %s" %
1678 1691
                                     (job.name,))
1679
-                self._dependencies[job.name].add(dependency)
1692
+                self._dependencies[job.name][dependency.name] = \
1693
+                    dependency.soft
1680 1694
         except Exception:
1681 1695
             del self.jobs[job.name]
1682 1696
             del self._dependencies[job.name]
@@ -1703,25 +1717,34 @@ class JobGraph(object):
1703 1717
             all_dependent_jobs |= new_dependent_jobs
1704 1718
         return [self.jobs[name] for name in all_dependent_jobs]
1705 1719
 
1706
-    def getParentJobsRecursively(self, dependent_job, soft=False):
1720
+    def getParentJobsRecursively(self, dependent_job, layout=None):
1707 1721
         return [self.jobs[name] for name in
1708
-                self._getParentJobNamesRecursively(dependent_job, soft)]
1722
+                self._getParentJobNamesRecursively(dependent_job,
1723
+                                                   layout=layout)]
1709 1724
 
1710
-    def _getParentJobNamesRecursively(self, dependent_job, soft=False):
1725
+    def _getParentJobNamesRecursively(self, dependent_job, soft=False,
1726
+                                      layout=None):
1711 1727
         all_parent_jobs = set()
1712
-        jobs_to_iterate = set([dependent_job])
1728
+        jobs_to_iterate = set([(dependent_job, False)])
1713 1729
         while len(jobs_to_iterate) > 0:
1714
-            current_job = jobs_to_iterate.pop()
1730
+            (current_job, current_soft) = jobs_to_iterate.pop()
1715 1731
             current_parent_jobs = self._dependencies.get(current_job)
1716 1732
             if current_parent_jobs is None:
1717
-                if soft:
1718
-                    current_parent_jobs = set()
1733
+                if soft or current_soft:
1734
+                    if layout:
1735
+                        # If the caller supplied a layout, verify that
1736
+                        # the job exists to provide a helpful error
1737
+                        # message.  Called for exception side effect:
1738
+                        layout.getJob(current_job)
1739
+                    current_parent_jobs = {}
1719 1740
                 else:
1720 1741
                     raise Exception("Job %s depends on %s which was not run." %
1721 1742
                                     (dependent_job, current_job))
1722
-            new_parent_jobs = current_parent_jobs - all_parent_jobs
1723
-            jobs_to_iterate |= new_parent_jobs
1724
-            all_parent_jobs |= new_parent_jobs
1743
+            elif dependent_job != current_job:
1744
+                all_parent_jobs.add(current_job)
1745
+            new_parent_jobs = set(current_parent_jobs.keys()) - all_parent_jobs
1746
+            for j in new_parent_jobs:
1747
+                jobs_to_iterate.add((j, current_parent_jobs[j]))
1725 1748
         return all_parent_jobs
1726 1749
 
1727 1750
 
@@ -2066,7 +2089,7 @@ class QueueItem(object):
2066 2089
             for job in job_graph.getJobs():
2067 2090
                 # Ensure that each jobs's dependencies are fully
2068 2091
                 # accessible.  This will raise an exception if not.
2069
-                job_graph.getParentJobsRecursively(job.name)
2092
+                job_graph.getParentJobsRecursively(job.name, self.layout)
2070 2093
             self.job_graph = job_graph
2071 2094
         except Exception:
2072 2095
             self.project_pipeline_config = None
@@ -2645,7 +2668,7 @@ class QueueItem(object):
2645 2668
 
2646 2669
             ret['jobs'].append({
2647 2670
                 'name': job.name,
2648
-                'dependencies': list(job.dependencies),
2671
+                'dependencies': [x.name for x in job.dependencies],
2649 2672
                 'elapsed_time': elapsed,
2650 2673
                 'remaining_time': remaining,
2651 2674
                 'url': build_url,

Loading…
Cancel
Save