Browse Source

Fix wedged scheduler on force-merge of non-existing template

If there is already a post pipeline defined a force-merge of a change
referencing a non-existent project template currently can wedge the
scheduler. In this case an exception is raised during reporting which
throws the run handler completely out of its loop and the loop starts
over from the beginning. However the item to be reported is not
consumed in this case so zuul is stuck in an exception loop [1] and
can only be recovered by a restart.

This can be fixed by catching the exception and continuing the
reporting.

[1] Traceback:
2019-01-28 07:33:57,304 ERROR zuul.Scheduler: Exception in run handler:
Traceback (most recent call last):
  File "/opt/zuul/lib/python3.6/site-packages/zuul/scheduler.py", line 1033, in run
    while (pipeline.manager.processQueue() and
  File "/opt/zuul/lib/python3.6/site-packages/zuul/manager/__init__.py", line 768, in processQueue
    item, nnfi)
  File "/opt/zuul/lib/python3.6/site-packages/zuul/manager/__init__.py", line 735, in _processOneItem
    self.reportItem(item)
  File "/opt/zuul/lib/python3.6/site-packages/zuul/manager/__init__.py", line 880, in reportItem
    item.reported = not self._reportItem(item)
  File "/opt/zuul/lib/python3.6/site-packages/zuul/manager/__init__.py", line 920, in _reportItem
    if not layout.getProjectPipelineConfig(item):
  File "/opt/zuul/lib/python3.6/site-packages/zuul/model.py", line 3435, in getProjectPipelineConfig
    templates = self.getProjectTemplates(template_name)
  File "/opt/zuul/lib/python3.6/site-packages/zuul/model.py", line 3374, in getProjectTemplates
    raise TemplateNotFoundError("Project template %s not found" % name)
zuul.model.TemplateNotFoundError: Project template foo not found

Change-Id: I1a3b59dadbd9337a8ba5b146f09ad093a0123ce8
tags/3.6.0
Tobias Henkel 5 months ago
parent
commit
b999b7c862
No account linked to committer's email address

+ 2
- 0
tests/fixtures/config/force-merge-template/git/common-config/playbooks/test.yaml View File

@@ -0,0 +1,2 @@
1
+- hosts: all
2
+  tasks: []

+ 35
- 0
tests/fixtures/config/force-merge-template/git/common-config/zuul.yaml View File

@@ -0,0 +1,35 @@
1
+- pipeline:
2
+    name: check
3
+    manager: independent
4
+    trigger:
5
+      gerrit:
6
+        - event: patchset-created
7
+    success:
8
+      gerrit:
9
+        Verified: 1
10
+    failure:
11
+      gerrit:
12
+        Verified: -1
13
+
14
+- pipeline:
15
+    name: post
16
+    manager: independent
17
+    trigger:
18
+      gerrit:
19
+        - event: ref-updated
20
+          ref: ^(?!refs/).*$
21
+    precedence: low
22
+
23
+
24
+- job:
25
+    name: base
26
+    parent: null
27
+
28
+- job:
29
+    name: post-job
30
+    run: playbooks/test.yaml
31
+
32
+- job:
33
+    name: other-job
34
+    run: playbooks/test.yaml
35
+

+ 1
- 0
tests/fixtures/config/force-merge-template/git/org_project/README View File

@@ -0,0 +1 @@
1
+test

+ 7
- 0
tests/fixtures/config/force-merge-template/git/org_project/zuul.yaml View File

@@ -0,0 +1,7 @@
1
+- project:
2
+    check:
3
+      jobs:
4
+        - noop
5
+    post:
6
+      jobs:
7
+        - post-job

+ 1
- 0
tests/fixtures/config/force-merge-template/git/org_project2/README View File

@@ -0,0 +1 @@
1
+test

+ 4
- 0
tests/fixtures/config/force-merge-template/git/org_project2/zuul.yaml View File

@@ -0,0 +1,4 @@
1
+- project:
2
+    check:
3
+      jobs:
4
+        - other-job

+ 17
- 0
tests/fixtures/config/force-merge-template/main.yaml View File

@@ -0,0 +1,17 @@
1
+- tenant:
2
+    name: tenant-one
3
+    source:
4
+      gerrit:
5
+        config-projects:
6
+          - common-config
7
+        untrusted-projects:
8
+          - org/project
9
+
10
+- tenant:
11
+    name: tenant-two
12
+    source:
13
+      gerrit:
14
+        config-projects:
15
+          - common-config
16
+        untrusted-projects:
17
+          - org/project2

+ 44
- 0
tests/unit/test_v3.py View File

@@ -5002,3 +5002,47 @@ class TestProvidesRequires(ZuulDBTestCase):
5002 5002
         ])
5003 5003
         self.assertIn('image-user : SKIPPED', B.messages[0])
5004 5004
         self.assertIn('not met by build', B.messages[0])
5005
+
5006
+
5007
+class TestForceMergeMissingTemplate(ZuulTestCase):
5008
+    tenant_config_file = "config/force-merge-template/main.yaml"
5009
+
5010
+    def test_force_merge_missing_template(self):
5011
+        """
5012
+        Tests that force merging a change using a non-existent project
5013
+        template triggering a post job doesn't wedge zuul on reporting.
5014
+        """
5015
+
5016
+        # Create change that adds uses a non-existent project template
5017
+        conf = textwrap.dedent(
5018
+            """
5019
+            - project:
5020
+                templates:
5021
+                  - non-existent
5022
+                check:
5023
+                  jobs:
5024
+                    - noop
5025
+                post:
5026
+                  jobs:
5027
+                    - post-job
5028
+            """)
5029
+
5030
+        file_dict = {'zuul.yaml': conf}
5031
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
5032
+                                           files=file_dict)
5033
+
5034
+        # Now force merge the change
5035
+        A.setMerged()
5036
+        self.fake_gerrit.addEvent(A.getChangeMergedEvent())
5037
+        self.waitUntilSettled()
5038
+        self.fake_gerrit.addEvent(A.getRefUpdatedEvent())
5039
+        self.waitUntilSettled()
5040
+
5041
+        B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
5042
+        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
5043
+        self.waitUntilSettled()
5044
+
5045
+        self.assertEqual(B.reported, 1)
5046
+        self.assertHistory([
5047
+            dict(name='other-job', result='SUCCESS', changes='2,1'),
5048
+        ])

+ 6
- 1
zuul/manager/__init__.py View File

@@ -912,7 +912,12 @@ class PipelineManager(object):
912 912
         layout = (item.layout or self.pipeline.tenant.layout)
913 913
 
914 914
         project_in_pipeline = True
915
-        if not layout.getProjectPipelineConfig(item):
915
+        ppc = None
916
+        try:
917
+            ppc = layout.getProjectPipelineConfig(item)
918
+        except Exception:
919
+            self.log.exception("Invalid config for change %s" % item.change)
920
+        if not ppc:
916 921
             self.log.debug("Project %s not in pipeline %s for change %s" % (
917 922
                 item.change.project, self.pipeline, item.change))
918 923
             project_in_pipeline = False

Loading…
Cancel
Save