Browse Source

Merge "Run jobs when their own config changes"

tags/3.10.0
Zuul 3 months ago
parent
commit
86f071464d

+ 9
- 0
doc/source/user/config.rst View File

@@ -1252,6 +1252,15 @@ Here is an example of two job definitions:
1252 1252
       are in the docs directory.  A regular expression or list of
1253 1253
       regular expressions.
1254 1254
 
1255
+   .. attr:: match-on-config-updates
1256
+      :default: true
1257
+
1258
+      If this is set to ``true`` (the default), then the job's file
1259
+      matchers are ignored if a change alters the job's configuration.
1260
+      This means that changes to jobs with file matchers will be
1261
+      self-testing without requiring that the file matchers include
1262
+      the Zuul configuration file defining the job.
1263
+
1255 1264
 .. _project:
1256 1265
 
1257 1266
 Project

+ 10
- 0
releasenotes/notes/match-on-config-updates-1c6621885bd3e1c9.yaml View File

@@ -0,0 +1,10 @@
1
+---
2
+upgrade:
3
+  - |
4
+    Jobs with file matchers will now automatically match if the configuration
5
+    of the job is changed.  This means that the Zuul configuration file no
6
+    longer needs to be included in the list of files to match in order for
7
+    changes to job configuration to be self-testing.
8
+
9
+    To keep the old behavior, set :attr:`job.match-on-config-updates`
10
+    to ``False``.

+ 1
- 0
tests/fixtures/config/job-update/git/common-config/playbooks/run.yaml View File

@@ -0,0 +1 @@
1
+---

+ 17
- 0
tests/fixtures/config/job-update/git/common-config/zuul.yaml View File

@@ -0,0 +1,17 @@
1
+- pipeline:
2
+    name: check
3
+    manager: independent
4
+    trigger:
5
+      gerrit:
6
+        - event: patchset-created
7
+    success:
8
+      gerrit:
9
+        Verified: 1
10
+    failure:
11
+      gerrit:
12
+        Verified: -1
13
+
14
+- job:
15
+    name: base
16
+    parent: null
17
+    run: playbooks/run.yaml

+ 1
- 0
tests/fixtures/config/job-update/git/org_project/README View File

@@ -0,0 +1 @@
1
+test

+ 19
- 0
tests/fixtures/config/job-update/git/org_project/zuul.d/existing.yaml View File

@@ -0,0 +1,19 @@
1
+# Every fake change in the unit tests modifies "README"
2
+
3
+- job:
4
+    name: existing-files
5
+    files:
6
+      - README.txt
7
+
8
+- job:
9
+    name: existing-irr
10
+    irrelevant-files:
11
+      - README
12
+      - ^zuul.d/.*$
13
+
14
+- project:
15
+    name: org/project
16
+    check:
17
+      jobs:
18
+        - existing-files
19
+        - existing-irr

+ 8
- 0
tests/fixtures/config/job-update/main.yaml View File

@@ -0,0 +1,8 @@
1
+- tenant:
2
+    name: tenant-one
3
+    source:
4
+      gerrit:
5
+        config-projects:
6
+          - common-config
7
+        untrusted-projects:
8
+          - org/project

+ 92
- 0
tests/unit/test_scheduler.py View File

@@ -5712,6 +5712,98 @@ For CI problems and help debugging, contact ci@example.org"""
5712 5712
         ], ordered=False)
5713 5713
 
5714 5714
 
5715
+class TestJobUpdateFileMatcher(ZuulTestCase):
5716
+    tenant_config_file = 'config/job-update/main.yaml'
5717
+
5718
+    def test_matchers(self):
5719
+        "Test matchers work as expected with no change"
5720
+        file_dict = {'README.txt': ''}
5721
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
5722
+                                           files=file_dict)
5723
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5724
+        self.waitUntilSettled()
5725
+
5726
+        file_dict = {'something_else': ''}
5727
+        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B',
5728
+                                           files=file_dict)
5729
+        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
5730
+        self.waitUntilSettled()
5731
+
5732
+        self.assertHistory([
5733
+            dict(name='existing-files', result='SUCCESS', changes='1,1'),
5734
+            dict(name='existing-irr', result='SUCCESS', changes='2,1'),
5735
+        ])
5736
+
5737
+    def test_job_update(self):
5738
+        "Test matchers are overridden with a config update"
5739
+        in_repo_conf = textwrap.dedent(
5740
+            """
5741
+            - job:
5742
+                name: existing-files
5743
+                tags: foo
5744
+            - job:
5745
+                name: existing-irr
5746
+                tags: foo
5747
+            """)
5748
+
5749
+        file_dict = {'zuul.d/new.yaml': in_repo_conf}
5750
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
5751
+                                           files=file_dict)
5752
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5753
+        self.waitUntilSettled()
5754
+
5755
+        self.assertHistory([
5756
+            dict(name='existing-files', result='SUCCESS', changes='1,1'),
5757
+            dict(name='existing-irr', result='SUCCESS', changes='1,1'),
5758
+        ], ordered=False)
5759
+
5760
+    def test_new_job(self):
5761
+        "Test matchers are overridden when creating a new job"
5762
+        in_repo_conf = textwrap.dedent(
5763
+            """
5764
+            - job:
5765
+                name: new-files
5766
+                parent: existing-files
5767
+
5768
+            - project:
5769
+                check:
5770
+                  jobs:
5771
+                    - new-files
5772
+            """)
5773
+
5774
+        file_dict = {'zuul.d/new.yaml': in_repo_conf}
5775
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
5776
+                                           files=file_dict)
5777
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5778
+        self.waitUntilSettled()
5779
+
5780
+        self.assertHistory([
5781
+            dict(name='new-files', result='SUCCESS', changes='1,1'),
5782
+        ])
5783
+
5784
+    def test_disable_match(self):
5785
+        "Test matchers are not overridden if we say so"
5786
+        in_repo_conf = textwrap.dedent(
5787
+            """
5788
+            - job:
5789
+                name: new-files
5790
+                parent: existing-files
5791
+                match-on-config-updates: false
5792
+
5793
+            - project:
5794
+                check:
5795
+                  jobs:
5796
+                    - new-files
5797
+            """)
5798
+
5799
+        file_dict = {'zuul.d/new.yaml': in_repo_conf}
5800
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
5801
+                                           files=file_dict)
5802
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5803
+        self.waitUntilSettled()
5804
+        self.assertHistory([])
5805
+
5806
+
5715 5807
 class TestAmbiguousProjectNames(ZuulTestCase):
5716 5808
     config_file = 'zuul-connections-multiple-gerrits.conf'
5717 5809
     tenant_config_file = 'config/ambiguous-names/main.yaml'

+ 8
- 0
tests/unit/test_web.py View File

@@ -324,6 +324,7 @@ class TestWeb(BaseTestWeb):
324 324
                 'description': None,
325 325
                 'files': [],
326 326
                 'irrelevant_files': [],
327
+                'match_on_config_updates': True,
327 328
                 'final': False,
328 329
                 'implied_branch': None,
329 330
                 'nodeset': {
@@ -364,6 +365,7 @@ class TestWeb(BaseTestWeb):
364 365
                 'description': None,
365 366
                 'files': [],
366 367
                 'irrelevant_files': [],
368
+                'match_on_config_updates': True,
367 369
                 'final': False,
368 370
                 'implied_branch': None,
369 371
                 'nodeset': {
@@ -410,6 +412,7 @@ class TestWeb(BaseTestWeb):
410 412
                 'final': False,
411 413
                 'implied_branch': None,
412 414
                 'irrelevant_files': [],
415
+                'match_on_config_updates': True,
413 416
                 'name': 'test-job',
414 417
                 'parent': 'base',
415 418
                 'post_review': None,
@@ -527,6 +530,7 @@ class TestWeb(BaseTestWeb):
527 530
                   'final': False,
528 531
                   'implied_branch': None,
529 532
                   'irrelevant_files': [],
533
+                  'match_on_config_updates': True,
530 534
                   'name': 'project-merge',
531 535
                   'parent': 'base',
532 536
                   'post_review': None,
@@ -560,6 +564,7 @@ class TestWeb(BaseTestWeb):
560 564
                   'final': False,
561 565
                   'implied_branch': None,
562 566
                   'irrelevant_files': [],
567
+                  'match_on_config_updates': True,
563 568
                   'name': 'project-test1',
564 569
                   'parent': 'base',
565 570
                   'post_review': None,
@@ -593,6 +598,7 @@ class TestWeb(BaseTestWeb):
593 598
                   'final': False,
594 599
                   'implied_branch': None,
595 600
                   'irrelevant_files': [],
601
+                  'match_on_config_updates': True,
596 602
                   'name': 'project-test2',
597 603
                   'parent': 'base',
598 604
                   'post_review': None,
@@ -626,6 +632,7 @@ class TestWeb(BaseTestWeb):
626 632
                   'final': False,
627 633
                   'implied_branch': None,
628 634
                   'irrelevant_files': [],
635
+                  'match_on_config_updates': True,
629 636
                   'name': 'project1-project2-integration',
630 637
                   'parent': 'base',
631 638
                   'post_review': None,
@@ -679,6 +686,7 @@ class TestWeb(BaseTestWeb):
679 686
                              'final': False,
680 687
                              'implied_branch': None,
681 688
                              'irrelevant_files': [],
689
+                             'match_on_config_updates': True,
682 690
                              'name': 'project-post',
683 691
                              'parent': 'base',
684 692
                              'post_review': None,

+ 4
- 1
zuul/configloader.py View File

@@ -621,7 +621,9 @@ class JobParser(object):
621 621
                       'override-checkout': str,
622 622
                       'description': str,
623 623
                       'variant-description': str,
624
-                      'post-review': bool}
624
+                      'post-review': bool,
625
+                      'match-on-config-updates': bool,
626
+    }
625 627
 
626 628
     job_name = {vs.Required('name'): str}
627 629
 
@@ -645,6 +647,7 @@ class JobParser(object):
645 647
         'success-url',
646 648
         'override-branch',
647 649
         'override-checkout',
650
+        'match-on-config-updates',
648 651
     ]
649 652
 
650 653
     def __init__(self, pcontext):

+ 62
- 15
zuul/model.py View File

@@ -1141,6 +1141,7 @@ class Job(ConfigObject):
1141 1141
             branch_matcher=None,
1142 1142
             file_matcher=None,
1143 1143
             irrelevant_file_matcher=None,  # skip-if
1144
+            match_on_config_updates=True,
1144 1145
             tags=frozenset(),
1145 1146
             provides=frozenset(),
1146 1147
             requires=frozenset(),
@@ -1253,6 +1254,7 @@ class Job(ConfigObject):
1253 1254
         d['cleanup_run'] = list(map(lambda x: x.toSchemaDict(),
1254 1255
                                     self.cleanup_run))
1255 1256
         d['post_review'] = self.post_review
1257
+        d['match_on_config_updates'] = self.match_on_config_updates
1256 1258
         if self.isBase():
1257 1259
             d['parent'] = None
1258 1260
         elif self.parent:
@@ -2111,6 +2113,7 @@ class QueueItem(object):
2111 2113
         self.layout = None
2112 2114
         self.project_pipeline_config = None
2113 2115
         self.job_graph = None
2116
+        self._old_job_graph = None  # Cached job graph of previous layout
2114 2117
         self._cached_sql_results = {}
2115 2118
         self.event = event  # The trigger event that lead to this queue item
2116 2119
 
@@ -2131,6 +2134,7 @@ class QueueItem(object):
2131 2134
         self.layout = None
2132 2135
         self.project_pipeline_config = None
2133 2136
         self.job_graph = None
2137
+        self._old_job_graph = None
2134 2138
 
2135 2139
     def addBuild(self, build):
2136 2140
         self.current_build_set.addBuild(build)
@@ -2176,6 +2180,7 @@ class QueueItem(object):
2176 2180
         except Exception:
2177 2181
             self.project_pipeline_config = None
2178 2182
             self.job_graph = None
2183
+            self._old_job_graph = None
2179 2184
             raise
2180 2185
 
2181 2186
     def hasJobGraph(self):
@@ -2864,6 +2869,31 @@ class QueueItem(object):
2864 2869
                     newrev=newrev,
2865 2870
                     )
2866 2871
 
2872
+    def updatesJobConfig(self, job):
2873
+        log = self.annotateLogger(self.log)
2874
+        layout_ahead = self.pipeline.tenant.layout
2875
+        if self.item_ahead and self.item_ahead.layout:
2876
+            layout_ahead = self.item_ahead.layout
2877
+        if layout_ahead and self.layout and self.layout is not layout_ahead:
2878
+            # This change updates the layout.  Calculate the job as it
2879
+            # would be if the layout had not changed.
2880
+            if self._old_job_graph is None:
2881
+                ppc = layout_ahead.getProjectPipelineConfig(self)
2882
+                log.debug("Creating job graph for config change detecction")
2883
+                self._old_job_graph = layout_ahead.createJobGraph(
2884
+                    self, ppc, skip_file_matcher=True)
2885
+                log.debug("Done creating job graph for "
2886
+                          "config change detecction")
2887
+            old_job = self._old_job_graph.jobs.get(job.name)
2888
+            if old_job is None:
2889
+                log.debug("Found a newly created job")
2890
+                return True  # A newly created job
2891
+            if (job.toDict(self.pipeline.tenant) !=
2892
+                old_job.toDict(self.pipeline.tenant)):
2893
+                log.debug("Found an updated job")
2894
+                return True  # This job's configuration has changed
2895
+        return False
2896
+
2867 2897
 
2868 2898
 class Ref(object):
2869 2899
     """An existing state of a Project."""
@@ -3942,6 +3972,17 @@ class Layout(object):
3942 3972
                     frozen_job.applyVariant(variant, item.layout)
3943 3973
                     frozen_job.name = variant.name
3944 3974
             frozen_job.name = jobname
3975
+
3976
+            # Now merge variables set from this parent ppc
3977
+            # (i.e. project+templates) directly into the job vars
3978
+            frozen_job.updateProjectVariables(ppc.variables)
3979
+
3980
+            # If the job does not specify an ansible version default to the
3981
+            # tenant default.
3982
+            if not frozen_job.ansible_version:
3983
+                frozen_job.ansible_version = \
3984
+                    item.layout.tenant.default_ansible_version
3985
+
3945 3986
             log.debug("Froze job %s for %s", jobname, change)
3946 3987
             # Whether the change matches any of the project pipeline
3947 3988
             # variants
@@ -3965,13 +4006,29 @@ class Layout(object):
3965 4006
                 item.debug("No matching pipeline variants for {jobname}".
3966 4007
                            format(jobname=jobname), indent=2)
3967 4008
                 continue
4009
+            updates_job_config = False
3968 4010
             if not skip_file_matcher and \
3969 4011
                not frozen_job.changeMatchesFiles(change):
3970
-                log.debug("Job %s did not match files in %s",
3971
-                          repr(frozen_job), change)
3972
-                item.debug("Job {jobname} did not match files".
3973
-                           format(jobname=jobname), indent=2)
3974
-                continue
4012
+                matched_files = False
4013
+                if frozen_job.match_on_config_updates:
4014
+                    updates_job_config = item.updatesJobConfig(frozen_job)
4015
+            else:
4016
+                matched_files = True
4017
+            if not matched_files:
4018
+                if updates_job_config:
4019
+                    # Log the reason we're ignoring the file matcher
4020
+                    log.debug("The configuration of job %s is "
4021
+                              "changed by %s; ignoring file matcher",
4022
+                              repr(frozen_job), change)
4023
+                    item.debug("The configuration of job {jobname} is "
4024
+                               "changed; ignoring file matcher".
4025
+                               format(jobname=jobname), indent=2)
4026
+                else:
4027
+                    log.debug("Job %s did not match files in %s",
4028
+                              repr(frozen_job), change)
4029
+                    item.debug("Job {jobname} did not match files".
4030
+                               format(jobname=jobname), indent=2)
4031
+                    continue
3975 4032
             if frozen_job.abstract:
3976 4033
                 raise Exception("Job %s is abstract and may not be "
3977 4034
                                 "directly run" %
@@ -3989,16 +4046,6 @@ class Layout(object):
3989 4046
                 raise Exception("Job %s does not specify a run playbook" % (
3990 4047
                     frozen_job.name,))
3991 4048
 
3992
-            # Now merge variables set from this parent ppc
3993
-            # (i.e. project+templates) directly into the job vars
3994
-            frozen_job.updateProjectVariables(ppc.variables)
3995
-
3996
-            # If the job does not specify an ansible version default to the
3997
-            # tenant default.
3998
-            if not frozen_job.ansible_version:
3999
-                frozen_job.ansible_version = \
4000
-                    item.layout.tenant.default_ansible_version
4001
-
4002 4049
             job_graph.addJob(frozen_job)
4003 4050
 
4004 4051
     def createJobGraph(self, item, ppc, skip_file_matcher=False):

Loading…
Cancel
Save