Browse Source

Run jobs when their own config changes

This causes file matchers to automatically match when the
configuration of the job itself changes.  This can be used instead
of matching "^.zuul.yaml$" which may cause too many jobs to run
in larger repos.

Change-Id: Ieddaead91b597282c5674ba99b0c0f387843c722
changes/52/669752/3
James E. Blair 1 week ago
parent
commit
7fba932e5d

+ 9
- 0
doc/source/user/config.rst View File

@@ -1252,6 +1252,15 @@ Here is an example of two job definitions:
1252 1252
       are in the docs directory.  A regular expression or list of
1253 1253
       regular expressions.
1254 1254
 
1255
+   .. attr:: match-on-config-updates
1256
+      :default: true
1257
+
1258
+      If this is set to ``true`` (the default), then the job's file
1259
+      matchers are ignored if a change alters the job's configuration.
1260
+      This means that changes to jobs with file matchers will be
1261
+      self-testing without requiring that the file matchers include
1262
+      the Zuul configuration file defining the job.
1263
+
1255 1264
 .. _project:
1256 1265
 
1257 1266
 Project

+ 10
- 0
releasenotes/notes/match-on-config-updates-1c6621885bd3e1c9.yaml View File

@@ -0,0 +1,10 @@
1
+---
2
+upgrade:
3
+  - |
4
+    Jobs with file matchers will now automatically match if the configuration
5
+    of the job is changed.  This means that the Zuul configuration file no
6
+    longer needs to be included in the list of files to match in order for
7
+    changes to job configuration to be self-testing.
8
+
9
+    To keep the old behavior, set :attr:`job.match-on-config-updates`
10
+    to ``False``.

+ 1
- 0
tests/fixtures/config/job-update/git/common-config/playbooks/run.yaml View File

@@ -0,0 +1 @@
1
+---

+ 17
- 0
tests/fixtures/config/job-update/git/common-config/zuul.yaml View File

@@ -0,0 +1,17 @@
1
+- pipeline:
2
+    name: check
3
+    manager: independent
4
+    trigger:
5
+      gerrit:
6
+        - event: patchset-created
7
+    success:
8
+      gerrit:
9
+        Verified: 1
10
+    failure:
11
+      gerrit:
12
+        Verified: -1
13
+
14
+- job:
15
+    name: base
16
+    parent: null
17
+    run: playbooks/run.yaml

+ 1
- 0
tests/fixtures/config/job-update/git/org_project/README View File

@@ -0,0 +1 @@
1
+test

+ 19
- 0
tests/fixtures/config/job-update/git/org_project/zuul.d/existing.yaml View File

@@ -0,0 +1,19 @@
1
+# Every fake change in the unit tests modifies "README"
2
+
3
+- job:
4
+    name: existing-files
5
+    files:
6
+      - README.txt
7
+
8
+- job:
9
+    name: existing-irr
10
+    irrelevant-files:
11
+      - README
12
+      - ^zuul.d/.*$
13
+
14
+- project:
15
+    name: org/project
16
+    check:
17
+      jobs:
18
+        - existing-files
19
+        - existing-irr

+ 8
- 0
tests/fixtures/config/job-update/main.yaml View File

@@ -0,0 +1,8 @@
1
+- tenant:
2
+    name: tenant-one
3
+    source:
4
+      gerrit:
5
+        config-projects:
6
+          - common-config
7
+        untrusted-projects:
8
+          - org/project

+ 92
- 0
tests/unit/test_scheduler.py View File

@@ -5712,6 +5712,98 @@ For CI problems and help debugging, contact ci@example.org"""
5712 5712
         ], ordered=False)
5713 5713
 
5714 5714
 
5715
+class TestJobUpdateFileMatcher(ZuulTestCase):
5716
+    tenant_config_file = 'config/job-update/main.yaml'
5717
+
5718
+    def test_matchers(self):
5719
+        "Test matchers work as expected with no change"
5720
+        file_dict = {'README.txt': ''}
5721
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
5722
+                                           files=file_dict)
5723
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5724
+        self.waitUntilSettled()
5725
+
5726
+        file_dict = {'something_else': ''}
5727
+        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B',
5728
+                                           files=file_dict)
5729
+        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
5730
+        self.waitUntilSettled()
5731
+
5732
+        self.assertHistory([
5733
+            dict(name='existing-files', result='SUCCESS', changes='1,1'),
5734
+            dict(name='existing-irr', result='SUCCESS', changes='2,1'),
5735
+        ])
5736
+
5737
+    def test_job_update(self):
5738
+        "Test matchers are overridden with a config update"
5739
+        in_repo_conf = textwrap.dedent(
5740
+            """
5741
+            - job:
5742
+                name: existing-files
5743
+                tags: foo
5744
+            - job:
5745
+                name: existing-irr
5746
+                tags: foo
5747
+            """)
5748
+
5749
+        file_dict = {'zuul.d/new.yaml': in_repo_conf}
5750
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
5751
+                                           files=file_dict)
5752
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5753
+        self.waitUntilSettled()
5754
+
5755
+        self.assertHistory([
5756
+            dict(name='existing-files', result='SUCCESS', changes='1,1'),
5757
+            dict(name='existing-irr', result='SUCCESS', changes='1,1'),
5758
+        ], ordered=False)
5759
+
5760
+    def test_new_job(self):
5761
+        "Test matchers are overridden when creating a new job"
5762
+        in_repo_conf = textwrap.dedent(
5763
+            """
5764
+            - job:
5765
+                name: new-files
5766
+                parent: existing-files
5767
+
5768
+            - project:
5769
+                check:
5770
+                  jobs:
5771
+                    - new-files
5772
+            """)
5773
+
5774
+        file_dict = {'zuul.d/new.yaml': in_repo_conf}
5775
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
5776
+                                           files=file_dict)
5777
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5778
+        self.waitUntilSettled()
5779
+
5780
+        self.assertHistory([
5781
+            dict(name='new-files', result='SUCCESS', changes='1,1'),
5782
+        ])
5783
+
5784
+    def test_disable_match(self):
5785
+        "Test matchers are not overridden if we say so"
5786
+        in_repo_conf = textwrap.dedent(
5787
+            """
5788
+            - job:
5789
+                name: new-files
5790
+                parent: existing-files
5791
+                match-on-config-updates: false
5792
+
5793
+            - project:
5794
+                check:
5795
+                  jobs:
5796
+                    - new-files
5797
+            """)
5798
+
5799
+        file_dict = {'zuul.d/new.yaml': in_repo_conf}
5800
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
5801
+                                           files=file_dict)
5802
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5803
+        self.waitUntilSettled()
5804
+        self.assertHistory([])
5805
+
5806
+
5715 5807
 class TestAmbiguousProjectNames(ZuulTestCase):
5716 5808
     config_file = 'zuul-connections-multiple-gerrits.conf'
5717 5809
     tenant_config_file = 'config/ambiguous-names/main.yaml'

+ 8
- 0
tests/unit/test_web.py View File

@@ -324,6 +324,7 @@ class TestWeb(BaseTestWeb):
324 324
                 'description': None,
325 325
                 'files': [],
326 326
                 'irrelevant_files': [],
327
+                'match_on_config_updates': True,
327 328
                 'final': False,
328 329
                 'implied_branch': None,
329 330
                 'nodeset': {
@@ -364,6 +365,7 @@ class TestWeb(BaseTestWeb):
364 365
                 'description': None,
365 366
                 'files': [],
366 367
                 'irrelevant_files': [],
368
+                'match_on_config_updates': True,
367 369
                 'final': False,
368 370
                 'implied_branch': None,
369 371
                 'nodeset': {
@@ -410,6 +412,7 @@ class TestWeb(BaseTestWeb):
410 412
                 'final': False,
411 413
                 'implied_branch': None,
412 414
                 'irrelevant_files': [],
415
+                'match_on_config_updates': True,
413 416
                 'name': 'test-job',
414 417
                 'parent': 'base',
415 418
                 'post_review': None,
@@ -527,6 +530,7 @@ class TestWeb(BaseTestWeb):
527 530
                   'final': False,
528 531
                   'implied_branch': None,
529 532
                   'irrelevant_files': [],
533
+                  'match_on_config_updates': True,
530 534
                   'name': 'project-merge',
531 535
                   'parent': 'base',
532 536
                   'post_review': None,
@@ -560,6 +564,7 @@ class TestWeb(BaseTestWeb):
560 564
                   'final': False,
561 565
                   'implied_branch': None,
562 566
                   'irrelevant_files': [],
567
+                  'match_on_config_updates': True,
563 568
                   'name': 'project-test1',
564 569
                   'parent': 'base',
565 570
                   'post_review': None,
@@ -593,6 +598,7 @@ class TestWeb(BaseTestWeb):
593 598
                   'final': False,
594 599
                   'implied_branch': None,
595 600
                   'irrelevant_files': [],
601
+                  'match_on_config_updates': True,
596 602
                   'name': 'project-test2',
597 603
                   'parent': 'base',
598 604
                   'post_review': None,
@@ -626,6 +632,7 @@ class TestWeb(BaseTestWeb):
626 632
                   'final': False,
627 633
                   'implied_branch': None,
628 634
                   'irrelevant_files': [],
635
+                  'match_on_config_updates': True,
629 636
                   'name': 'project1-project2-integration',
630 637
                   'parent': 'base',
631 638
                   'post_review': None,
@@ -679,6 +686,7 @@ class TestWeb(BaseTestWeb):
679 686
                              'final': False,
680 687
                              'implied_branch': None,
681 688
                              'irrelevant_files': [],
689
+                             'match_on_config_updates': True,
682 690
                              'name': 'project-post',
683 691
                              'parent': 'base',
684 692
                              'post_review': None,

+ 4
- 1
zuul/configloader.py View File

@@ -621,7 +621,9 @@ class JobParser(object):
621 621
                       'override-checkout': str,
622 622
                       'description': str,
623 623
                       'variant-description': str,
624
-                      'post-review': bool}
624
+                      'post-review': bool,
625
+                      'match-on-config-updates': bool,
626
+    }
625 627
 
626 628
     job_name = {vs.Required('name'): str}
627 629
 
@@ -645,6 +647,7 @@ class JobParser(object):
645 647
         'success-url',
646 648
         'override-branch',
647 649
         'override-checkout',
650
+        'match-on-config-updates',
648 651
     ]
649 652
 
650 653
     def __init__(self, pcontext):

+ 62
- 15
zuul/model.py View File

@@ -1141,6 +1141,7 @@ class Job(ConfigObject):
1141 1141
             branch_matcher=None,
1142 1142
             file_matcher=None,
1143 1143
             irrelevant_file_matcher=None,  # skip-if
1144
+            match_on_config_updates=True,
1144 1145
             tags=frozenset(),
1145 1146
             provides=frozenset(),
1146 1147
             requires=frozenset(),
@@ -1253,6 +1254,7 @@ class Job(ConfigObject):
1253 1254
         d['cleanup_run'] = list(map(lambda x: x.toSchemaDict(),
1254 1255
                                     self.cleanup_run))
1255 1256
         d['post_review'] = self.post_review
1257
+        d['match_on_config_updates'] = self.match_on_config_updates
1256 1258
         if self.isBase():
1257 1259
             d['parent'] = None
1258 1260
         elif self.parent:
@@ -2111,6 +2113,7 @@ class QueueItem(object):
2111 2113
         self.layout = None
2112 2114
         self.project_pipeline_config = None
2113 2115
         self.job_graph = None
2116
+        self._old_job_graph = None  # Cached job graph of previous layout
2114 2117
         self._cached_sql_results = {}
2115 2118
         self.event = event  # The trigger event that lead to this queue item
2116 2119
 
@@ -2131,6 +2134,7 @@ class QueueItem(object):
2131 2134
         self.layout = None
2132 2135
         self.project_pipeline_config = None
2133 2136
         self.job_graph = None
2137
+        self._old_job_graph = None
2134 2138
 
2135 2139
     def addBuild(self, build):
2136 2140
         self.current_build_set.addBuild(build)
@@ -2176,6 +2180,7 @@ class QueueItem(object):
2176 2180
         except Exception:
2177 2181
             self.project_pipeline_config = None
2178 2182
             self.job_graph = None
2183
+            self._old_job_graph = None
2179 2184
             raise
2180 2185
 
2181 2186
     def hasJobGraph(self):
@@ -2864,6 +2869,31 @@ class QueueItem(object):
2864 2869
                     newrev=newrev,
2865 2870
                     )
2866 2871
 
2872
+    def updatesJobConfig(self, job):
2873
+        log = self.annotateLogger(self.log)
2874
+        layout_ahead = self.pipeline.tenant.layout
2875
+        if self.item_ahead and self.item_ahead.layout:
2876
+            layout_ahead = self.item_ahead.layout
2877
+        if layout_ahead and self.layout and self.layout is not layout_ahead:
2878
+            # This change updates the layout.  Calculate the job as it
2879
+            # would be if the layout had not changed.
2880
+            if self._old_job_graph is None:
2881
+                ppc = layout_ahead.getProjectPipelineConfig(self)
2882
+                log.debug("Creating job graph for config change detecction")
2883
+                self._old_job_graph = layout_ahead.createJobGraph(
2884
+                    self, ppc, skip_file_matcher=True)
2885
+                log.debug("Done creating job graph for "
2886
+                          "config change detecction")
2887
+            old_job = self._old_job_graph.jobs.get(job.name)
2888
+            if old_job is None:
2889
+                log.debug("Found a newly created job")
2890
+                return True  # A newly created job
2891
+            if (job.toDict(self.pipeline.tenant) !=
2892
+                old_job.toDict(self.pipeline.tenant)):
2893
+                log.debug("Found an updated job")
2894
+                return True  # This job's configuration has changed
2895
+        return False
2896
+
2867 2897
 
2868 2898
 class Ref(object):
2869 2899
     """An existing state of a Project."""
@@ -3942,6 +3972,17 @@ class Layout(object):
3942 3972
                     frozen_job.applyVariant(variant, item.layout)
3943 3973
                     frozen_job.name = variant.name
3944 3974
             frozen_job.name = jobname
3975
+
3976
+            # Now merge variables set from this parent ppc
3977
+            # (i.e. project+templates) directly into the job vars
3978
+            frozen_job.updateProjectVariables(ppc.variables)
3979
+
3980
+            # If the job does not specify an ansible version default to the
3981
+            # tenant default.
3982
+            if not frozen_job.ansible_version:
3983
+                frozen_job.ansible_version = \
3984
+                    item.layout.tenant.default_ansible_version
3985
+
3945 3986
             log.debug("Froze job %s for %s", jobname, change)
3946 3987
             # Whether the change matches any of the project pipeline
3947 3988
             # variants
@@ -3965,13 +4006,29 @@ class Layout(object):
3965 4006
                 item.debug("No matching pipeline variants for {jobname}".
3966 4007
                            format(jobname=jobname), indent=2)
3967 4008
                 continue
4009
+            updates_job_config = False
3968 4010
             if not skip_file_matcher and \
3969 4011
                not frozen_job.changeMatchesFiles(change):
3970
-                log.debug("Job %s did not match files in %s",
3971
-                          repr(frozen_job), change)
3972
-                item.debug("Job {jobname} did not match files".
3973
-                           format(jobname=jobname), indent=2)
3974
-                continue
4012
+                matched_files = False
4013
+                if frozen_job.match_on_config_updates:
4014
+                    updates_job_config = item.updatesJobConfig(frozen_job)
4015
+            else:
4016
+                matched_files = True
4017
+            if not matched_files:
4018
+                if updates_job_config:
4019
+                    # Log the reason we're ignoring the file matcher
4020
+                    log.debug("The configuration of job %s is "
4021
+                              "changed by %s; ignoring file matcher",
4022
+                              repr(frozen_job), change)
4023
+                    item.debug("The configuration of job {jobname} is "
4024
+                               "changed; ignoring file matcher".
4025
+                               format(jobname=jobname), indent=2)
4026
+                else:
4027
+                    log.debug("Job %s did not match files in %s",
4028
+                              repr(frozen_job), change)
4029
+                    item.debug("Job {jobname} did not match files".
4030
+                               format(jobname=jobname), indent=2)
4031
+                    continue
3975 4032
             if frozen_job.abstract:
3976 4033
                 raise Exception("Job %s is abstract and may not be "
3977 4034
                                 "directly run" %
@@ -3989,16 +4046,6 @@ class Layout(object):
3989 4046
                 raise Exception("Job %s does not specify a run playbook" % (
3990 4047
                     frozen_job.name,))
3991 4048
 
3992
-            # Now merge variables set from this parent ppc
3993
-            # (i.e. project+templates) directly into the job vars
3994
-            frozen_job.updateProjectVariables(ppc.variables)
3995
-
3996
-            # If the job does not specify an ansible version default to the
3997
-            # tenant default.
3998
-            if not frozen_job.ansible_version:
3999
-                frozen_job.ansible_version = \
4000
-                    item.layout.tenant.default_ansible_version
4001
-
4002 4049
             job_graph.addJob(frozen_job)
4003 4050
 
4004 4051
     def createJobGraph(self, item, ppc, skip_file_matcher=False):

Loading…
Cancel
Save