zuul
/
zuul
Code Issues Proposed changes

Report dequeued changes via Github checks API 

This patch provides the general functionality to allow reporting of
dequeued items and makes use of that in the Github checks API.
This reporting will only apply if the item wasn't a success or failure.

Change-Id: I1297da4d1708908c6b179110479fe0450e5550fe
This commit is contained in:
Felix Edel 2020-03-03 08:04:29 +01:00
parent 98a8ebe890
commit af2c919ca7
No known key found for this signature in database
GPG Key ID: E95717A102DD3030
12 changed files with 348 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
doc/source/reference/drivers/github.rst
View File

@ -366,8 +366,8 @@ itself. Status name, description, and context is taken from the pipeline.
.. attr:: check
If the reporter should utilize github's checks API to set the commit
status, this must be set to ``in_progress``, ``success`` or ``failure``
(depending on which status the reporter should report).
status, this must be set to ``in_progress``, ``success``, ``failure``
or ``cancelled`` (depending on which status the reporter should report).
.. attr:: comment
:default: true

13
doc/source/reference/pipeline_def.rst
View File

@ -219,6 +219,13 @@ success, the pipeline reports back to Gerrit with ``Verified`` vote of
The introductory text in reports when an item is dequeued
without running any jobs. Empty by default.
.. attr:: dequeue-message
:default: Build canceled.
The introductory text in reports when an item is dequeued.
The dequeue message only applies if the item was dequeued without
a result.
.. attr:: footer-message
Supplies additional information after test results. Useful for
@ -354,6 +361,12 @@ success, the pipeline reports back to Gerrit with ``Verified`` vote of
These reporters describe what Zuul should do when a pipeline is
disabled. See ``disable-after-consecutive-failures``.
.. attr:: dequeue
These reporters describe what Zuul should do if an item is
dequeued. The dequeue reporters will only apply, if the item
was dequeued without a result.
The following options can be used to alter Zuul's behavior to
mitigate situations in which jobs are failing frequently (perhaps
due to a problem with an external dependency, or unusually high

6
releasenotes/notes/dequeue-reporting-620f364309587304.yaml Normal file
View File

@ -0,0 +1,6 @@
---
features:
- |
Pipelines now provide a :attr:`pipeline.dequeue` reporter action so that
reporters may run whenever an item is dequeued. The dequeue reporters will
only apply if the item wasn't a success or failure.

85
tests/fixtures/layouts/dequeue-reporting.yaml vendored Normal file
View File

@ -0,0 +1,85 @@
- pipeline:
name: check
manager: independent
failure-message: Build failed (check)
success-message: Build succeeded (check)
dequeue-message: Build canceled (check)
start-message: Build started (check)
trigger:
gerrit:
- event: patchset-created
success:
gerrit:
Verified: 1
failure:
gerrit:
Verified: -1
start:
gerrit:
Verified: 0
dequeue:
gerrit:
Verified: 0
- pipeline:
name: gate
manager: dependent
supercedes: check
failure-message: Build failed (gate)
success-message: Build succeeded (gate)
dequeue-message: Build canceled (gate)
start-message: Build started (gate)
trigger:
gerrit:
- event: comment-added
approval:
- Approved: 1
success:
gerrit:
Verified: 2
submit: true
failure:
gerrit:
Verified: -2
start:
gerrit:
Verified: 0
dequeue:
gerrit:
Verified: 0
precedence: high
- job:
name: base
parent: null
run: playbooks/base.yaml
- job:
name: project-test1
run: playbooks/project-test1.yaml
- job:
name: project-test2
run: playbooks/project-test2.yaml
- job:
name: project-merge
hold-following-changes: true
run: playbooks/project-merge.yaml
- project:
name: org/project
check:
jobs:
- project-merge
- project-test1:
dependencies: project-merge
- project-test2:
dependencies: project-merge
gate:
jobs:
- project-merge
- project-test1:
dependencies: project-merge
- project-test2:
dependencies: project-merge

3
tests/fixtures/layouts/reporting-github.yaml vendored
View File

@ -96,6 +96,9 @@
failure:
github:
check: failure
dequeue:
github:
check: cancelled
- pipeline:
name: gate

55
tests/unit/test_github_driver.py
View File

@ -1684,6 +1684,61 @@ class TestGithubAppDriver(ZuulGithubAppTestCase):
self.waitUntilSettled()
self.assertTrue(A.is_merged)
@simple_layout("layouts/reporting-github.yaml", driver="github")
def test_reporting_checks_api_dequeue(self):
"Test that a dequeued change will be reported back to the check run"
project = "org/project3"
github = self.fake_github.getGithubClient(None)
client = zuul.rpcclient.RPCClient(
"127.0.0.1", self.gearman_server.port
)
self.addCleanup(client.shutdown)
self.executor_server.hold_jobs_in_build = True
A = self.fake_github.openFakePullRequest(project, "master", "A")
self.fake_github.emitEvent(A.getPullRequestOpenedEvent())
self.waitUntilSettled()
# We should have a pending check for the head sha
self.assertIn(
A.head_sha, github.repo_from_project(project)._commits.keys())
check_runs = self.fake_github.getCommitChecks(project, A.head_sha)
self.assertEqual(1, len(check_runs))
check_run = check_runs[0]
self.assertEqual("tenant-one/checks-api-reporting", check_run["name"])
self.assertEqual("in_progress", check_run["status"])
self.assertThat(
check_run["output"]["summary"],
MatchesRegex(r'.*Starting checks-api-reporting jobs.*', re.DOTALL)
)
# Use the client to dequeue the pending change
client.dequeue(
tenant="tenant-one",
pipeline="checks-api-reporting",
project="org/project3",
change="{},{}".format(A.number, A.head_sha),
ref=None,
)
self.waitUntilSettled()
# We should now have a cancelled check run for the head sha
check_runs = self.fake_github.getCommitChecks(project, A.head_sha)
self.assertEqual(1, len(check_runs))
check_run = check_runs[0]
self.assertEqual("tenant-one/checks-api-reporting", check_run["name"])
self.assertEqual("completed", check_run["status"])
self.assertEqual("cancelled", check_run["conclusion"])
self.assertThat(
check_run["output"]["summary"],
MatchesRegex(r'.*Build canceled.*', re.DOTALL)
)
self.assertIsNotNone(check_run["completed_at"])
@simple_layout("layouts/reporting-github.yaml", driver="github")
def test_update_non_existing_check_run(self):
project = "org/project3"

137
tests/unit/test_reporting.py Normal file
View File

@ -0,0 +1,137 @@
# Copyright 2020 BMW Group
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import zuul.rpcclient
from tests.base import ZuulTestCase, simple_layout
class TestReporting(ZuulTestCase):
tenant_config_file = "config/single-tenant/main.yaml"
@simple_layout("layouts/dequeue-reporting.yaml")
def test_dequeue_reporting(self):
"""Check that explicitly dequeued items are reported as dequeued"""
# We use the rpcclient to explicitly dequeue the item
client = zuul.rpcclient.RPCClient(
"127.0.0.1", self.gearman_server.port
)
self.addCleanup(client.shutdown)
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
A.addApproval("Code-Review", 2)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
client.dequeue(
tenant="tenant-one",
pipeline="check",
project="org/project",
change="1,1",
ref=None,
)
self.waitUntilSettled()
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
check_pipeline = tenant.layout.pipelines['check']
# A should have been reported two times: start, cancel
self.assertEqual(2, A.reported)
self.assertEqual(2, len(A.messages))
self.assertIn("Build started (check)", A.messages[0])
self.assertIn("Build canceled (check)", A.messages[1])
# There shouldn't be any successful items
self.assertEqual(len(check_pipeline.getAllItems()), 0)
# But one canceled
self.assertEqual(self.countJobResults(self.history, "ABORTED"), 1)
@simple_layout("layouts/dequeue-reporting.yaml")
def test_dequeue_reporting_gate_reset(self):
"""Check that a gate reset is not reported as dequeued"""
A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
B = self.fake_gerrit.addFakeChange("org/project", "master", "B")
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
self.executor_server.failJob("project-test1", A)
self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
self.fake_gerrit.addEvent(B.addApproval("Approved", 1))
self.waitUntilSettled()
# None of the items should be reported as dequeued, only success or
# failure
self.assertEqual(A.data["status"], "NEW")
self.assertEqual(B.data["status"], "MERGED")
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 2)
self.assertIn("Build started (gate)", A.messages[0])
self.assertIn("Build failed (gate)", A.messages[1])
self.assertIn("Build started (gate)", B.messages[0])
self.assertIn("Build succeeded (gate)", B.messages[1])
@simple_layout("layouts/dequeue-reporting.yaml")
def test_dequeue_reporting_supercedes(self):
"""Test that a superceeded change is reported as dequeued"""
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
A.addApproval("Code-Review", 2)
self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(4, A.reported)
self.assertIn("Build started (check)", A.messages[0])
self.assertIn("Build canceled (check)", A.messages[1])
self.assertIn("Build started (gate)", A.messages[2])
self.assertIn("Build succeeded (gate)", A.messages[3])
@simple_layout("layouts/dequeue-reporting.yaml")
def test_dequeue_reporting_new_patchset(self):
"Test that change superceeded by a new patchset is reported as deqeued"
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(1, len(self.builds))
A.addPatchset()
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(4, A.reported)
self.assertIn("Build started (check)", A.messages[0])
self.assertIn("Build canceled (check)", A.messages[1])
self.assertIn("Build started (check)", A.messages[2])
self.assertIn("Build succeeded (check)", A.messages[3])

7
zuul/configloader.py
View File

@ -1151,6 +1151,7 @@ class PipelineParser(object):
'merge-failure': 'merge_failure_actions',
'no-jobs': 'no_jobs_actions',
'disabled': 'disabled_actions',
'dequeue': 'dequeue_actions',
}
def __init__(self, pcontext):
@ -1200,6 +1201,7 @@ class PipelineParser(object):
'merge-failure-message': str,
'no-jobs-message': str,
'footer-message': str,
'dequeue-message': str,
'dequeue-on-new-patchset': bool,
'ignore-dependencies': bool,
'post-review': bool,
@ -1218,7 +1220,7 @@ class PipelineParser(object):
pipeline['reject'] = self.getDriverSchema('reject')
pipeline['trigger'] = vs.Required(self.getDriverSchema('trigger'))
for action in ['enqueue', 'start', 'success', 'failure',
'merge-failure', 'no-jobs', 'disabled']:
'merge-failure', 'no-jobs', 'disabled', 'dequeue']:
pipeline[action] = self.getDriverSchema('reporter')
return vs.Schema(pipeline)
@ -1247,6 +1249,9 @@ class PipelineParser(object):
"Starting {pipeline.name} jobs.")
pipeline.enqueue_message = conf.get('enqueue-message', "")
pipeline.no_jobs_message = conf.get('no-jobs-message', "")
pipeline.dequeue_message = conf.get(
"dequeue-message", "Build canceled."
)
pipeline.dequeue_on_new_patchset = conf.get(
'dequeue-on-new-patchset', True)
pipeline.ignore_dependencies = conf.get(

15
zuul/driver/github/githubreporter.py
View File

@ -213,11 +213,16 @@ class GithubReporter(BaseReporter):
pr_number = item.change.number
sha = item.change.patchset
# Check if the buildset is finished or not. In case it's finished, we
# must provide additional parameters when updating the check_run via
# the Github API later on.
completed = item.current_build_set.result is not None
status = self._check
# We declare a item as completed if it either has a result
# (success|failure) or a dequeue reporter is called (cancelled in case
# of Github checks API). For the latter one, the item might or might
# not have a result, but we still must set a conclusion on the check
# run. Thus, we cannot rely on the buildset's result only, but also
# check the state the reporter is going to report.
completed = (
item.current_build_set.result is not None or status == "cancelled"
)
log.debug(
"Updating check for change %s, params %s, context %s, message: %s",
@ -313,6 +318,6 @@ def getSchema():
'unlabel': scalar_or_list(str),
'review': v.Any('approve', 'request-changes', 'comment'),
'review-body': str,
'check': v.Any("in_progress", "success", "failure"),
'check': v.Any("in_progress", "success", "failure", "cancelled"),
})
return github_reporter

19
zuul/manager/__init__.py
View File

@ -169,6 +169,19 @@ class PipelineManager(metaclass=ABCMeta):
self.log.error("Reporting item start %s received: %s" %
(item, ret))
def reportDequeue(self, item):
if not self.pipeline._disabled:
self.log.info(
"Reporting dequeue, action %s item%s",
self.pipeline.dequeue_actions,
item,
)
ret = self.sendReport(self.pipeline.dequeue_actions, item)
if ret:
self.log.error(
"Reporting item dequeue %s received: %s", item, ret
)
def sendReport(self, action_reporters, item, message=None):
"""Sends the built message off to configured reporters.
@ -371,6 +384,12 @@ class PipelineManager(metaclass=ABCMeta):
log = get_annotated_logger(self.log, item.event)
log.debug("Removing change %s from queue", item.change)
item.queue.dequeueItem(item)
# In case a item is dequeued that doesn't have a result yet
# (success/failed/...) we report it as dequeued.
# Without this check, all items with a valid result would be reported
# twice.
if not item.current_build_set.result and item.live:
self.reportDequeue(item)
def removeItem(self, item):
log = get_annotated_logger(self.log, item.event)

5
zuul/model.py
View File

@ -261,6 +261,7 @@ class Pipeline(object):
self.footer_message = None
self.enqueue_message = None
self.start_message = None
self.dequeue_message = None
self.post_review = False
self.dequeue_on_new_patchset = True
self.ignore_dependencies = False
@ -276,6 +277,7 @@ class Pipeline(object):
self.merge_failure_actions = []
self.no_jobs_actions = []
self.disabled_actions = []
self.dequeue_actions = []
self.disable_at = None
self._consecutive_failures = 0
self._disabled = False
@ -295,7 +297,8 @@ class Pipeline(object):
self.failure_actions +
self.merge_failure_actions +
self.no_jobs_actions +
self.disabled_actions
self.disabled_actions +
self.dequeue_actions
)
def __repr__(self):

9
zuul/reporter/__init__.py
View File

@ -123,7 +123,8 @@ class BaseReporter(object, metaclass=abc.ABCMeta):
'failure': self._formatItemReportFailure,
'merge-failure': self._formatItemReportMergeFailure,
'no-jobs': self._formatItemReportNoJobs,
'disabled': self._formatItemReportDisabled
'disabled': self._formatItemReportDisabled,
'dequeue': self._formatItemReportDequeue,
}
return format_methods[self._action]
@ -208,6 +209,12 @@ class BaseReporter(object, metaclass=abc.ABCMeta):
else:
return self._formatItemReport(item)
def _formatItemReportDequeue(self, item, with_jobs=True):
msg = item.pipeline.dequeue_message
if with_jobs:
msg += '\n\n' + self._formatItemReportJobs(item)
return msg
def _getItemReportJobsFields(self, item):
# Extract the report elements from an item
config = self.connection.sched.config