zuul/tests/unit/test_github_requirements.py
Tobias Henkel 0093aabd4e
Fix multiple prs found when commit is not head
When receiving a status or checks event we need to find the
corresponding pull request to that event. Github stores the status on
the head commit of a pr and not on the pr itself which leads to
problems if multiple prs exist with the same head but different target
branches. Therefore zuul errors out when more than one pr is found for
that event. However the github search also returns all prs that
contain the sha we search for but have a different head which leads to
the same error as if we had two conflicting prs. This can be solved by
delaying the error and filtering the pr bodies for the head sha first.

Change-Id: Iadafd3cf68a742941e9189e84fca594bc3394084
2020-09-04 13:47:54 +02:00

639 lines
26 KiB
Python

# Copyright (c) 2017 IBM Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import time
from tests.base import ZuulGithubAppTestCase, ZuulTestCase, simple_layout
class TestGithubRequirements(ZuulTestCase):
"""Test pipeline and trigger requirements"""
config_file = 'zuul-github-driver.conf'
@simple_layout('layouts/requirements-github.yaml', driver='github')
def test_pipeline_require_status(self):
"Test pipeline requirement: status"
project = 'org/project1'
A = self.fake_github.openFakePullRequest(project, 'master', 'A')
# A comment event that we will keep submitting to trigger
comment = A.getCommentAddedEvent('test me')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
# No status from zuul so should not be enqueued
self.assertEqual(len(self.history), 0)
# An error status should not cause it to be enqueued
self.fake_github.setCommitStatus(project, A.head_sha, 'error',
context='tenant-one/check')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 0)
# A success status goes in
self.fake_github.setCommitStatus(project, A.head_sha, 'success',
context='tenant-one/check')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 1)
self.assertEqual(self.history[0].name, 'project1-pipeline')
# Trigger regex matched status
self.fake_github.emitEvent(A.getCommentAddedEvent('test regex'))
self.waitUntilSettled()
self.assertEqual(len(self.history), 2)
self.assertEqual(self.history[1].name, 'project1-pipeline')
@simple_layout('layouts/requirements-github.yaml', driver='github')
def test_trigger_require_status(self):
"Test trigger requirement: status"
project = 'org/project1'
A = self.fake_github.openFakePullRequest(project, 'master', 'A')
# A comment event that we will keep submitting to trigger
comment = A.getCommentAddedEvent('trigger me')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
# No status from zuul so should not be enqueued
self.assertEqual(len(self.history), 0)
# An error status should not cause it to be enqueued
self.fake_github.setCommitStatus(project, A.head_sha, 'error',
context='tenant-one/check')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 0)
# A success status goes in
self.fake_github.setCommitStatus(project, A.head_sha, 'success',
context='tenant-one/check')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 1)
self.assertEqual(self.history[0].name, 'project1-pipeline')
self.fake_github.emitEvent(A.getCommentAddedEvent('trigger regex'))
self.waitUntilSettled()
self.assertEqual(len(self.history), 2)
self.assertEqual(self.history[1].name, 'project1-pipeline')
@simple_layout('layouts/requirements-github.yaml', driver='github')
def test_trigger_on_status(self):
"Test trigger on: status"
project = 'org/project2'
A = self.fake_github.openFakePullRequest(project, 'master', 'A')
# Create second PR which contains the head of A in its history. Zuul
# should not get disturbed by the existence of this one.
self.fake_github.openFakePullRequest(
project, 'master', 'A', base_sha=A.head_sha)
# An error status should not cause it to be enqueued
self.fake_github.setCommitStatus(project, A.head_sha, 'error',
context='tenant-one/check')
self.fake_github.emitEvent(A.getCommitStatusEvent('tenant-one/check',
state='error'))
self.waitUntilSettled()
self.assertEqual(len(self.history), 0)
# A success status from unknown user should not cause it to be
# enqueued
self.fake_github.setCommitStatus(project, A.head_sha, 'success',
context='tenant-one/check',
user='foo')
self.fake_github.emitEvent(A.getCommitStatusEvent('tenant-one/check',
state='success',
user='foo'))
self.waitUntilSettled()
self.assertEqual(len(self.history), 0)
# A success status from zuul goes in
self.fake_github.setCommitStatus(project, A.head_sha, 'success',
context='tenant-one/check')
self.fake_github.emitEvent(A.getCommitStatusEvent('tenant-one/check'))
self.waitUntilSettled()
self.assertEqual(len(self.history), 1)
self.assertEqual(self.history[0].name, 'project2-trigger')
# An error status for a different context should not cause it to be
# enqueued
self.fake_github.setCommitStatus(project, A.head_sha, 'error',
context='tenant-one/gate')
self.fake_github.emitEvent(A.getCommitStatusEvent('tenant-one/gate',
state='error'))
self.waitUntilSettled()
self.assertEqual(len(self.history), 1)
# A success status with a regex match goes in
self.fake_github.emitEvent(A.getCommitStatusEvent('cooltest',
user='other-ci'))
self.waitUntilSettled()
self.assertEqual(len(self.history), 2)
self.assertEqual(self.history[1].name, 'project2-trigger')
@simple_layout("layouts/requirements-github.yaml", driver="github")
def test_trigger_on_check_run(self):
"""Test trigger on: check_run"""
project = "org/project15"
A = self.fake_github.openFakePullRequest(project, "master", "A")
# A check_run request with a different name should not cause it to be
# enqueued.
self.fake_github.emitEvent(
A.getCheckRunRequestedEvent("tenant-one/different-check")
)
self.waitUntilSettled()
self.assertEqual(len(self.history), 0)
# A check_run request with the correct name, but for a different app
# should not cause it to be enqueued.
self.fake_github.emitEvent(
A.getCheckRunRequestedEvent("tenant-one/check", app="other-ci")
)
self.waitUntilSettled()
self.assertEqual(len(self.history), 0)
# A check_run request with the correct name for the correct app should
# cause it to be enqueued.
self.fake_github.emitEvent(
A.getCheckRunRequestedEvent("tenant-one/check"))
self.waitUntilSettled()
self.assertEqual(len(self.history), 1)
self.assertEqual(self.history[0].name, "project15-check-run")
@simple_layout('layouts/requirements-github.yaml', driver='github')
def test_pipeline_require_review_username(self):
"Test pipeline requirement: review username"
A = self.fake_github.openFakePullRequest('org/project3', 'master', 'A')
# A comment event that we will keep submitting to trigger
comment = A.getCommentAddedEvent('test me')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
# No approval from derp so should not be enqueued
self.assertEqual(len(self.history), 0)
# Add an approved review from derp
A.addReview('derp', 'APPROVED')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 1)
self.assertEqual(self.history[0].name, 'project3-reviewusername')
@simple_layout('layouts/requirements-github.yaml', driver='github')
def test_pipeline_require_review_state(self):
"Test pipeline requirement: review state"
A = self.fake_github.openFakePullRequest('org/project4', 'master', 'A')
# Add derp to writers
A.writers.extend(('derp', 'werp'))
# A comment event that we will keep submitting to trigger
comment = A.getCommentAddedEvent('test me')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
# No positive review from derp so should not be enqueued
self.assertEqual(len(self.history), 0)
# A negative review from derp should not cause it to be enqueued
A.addReview('derp', 'CHANGES_REQUESTED')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 0)
# A negative review from werp should not cause it to be enqueued
A.addReview('werp', 'CHANGES_REQUESTED')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 0)
# A positive from nobody should not cause it to be enqueued
A.addReview('nobody', 'APPROVED')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 0)
# A positive review from derp should still be blocked by the
# negative review from werp
A.addReview('derp', 'APPROVED')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 0)
# A positive review from werp should cause it to be enqueued
A.addReview('werp', 'APPROVED')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 1)
self.assertEqual(self.history[0].name, 'project4-reviewreq')
@simple_layout('layouts/requirements-github.yaml', driver='github')
def test_pipeline_require_review_user_state(self):
"Test pipeline requirement: review state from user"
A = self.fake_github.openFakePullRequest('org/project5', 'master', 'A')
# Add derp and herp to writers
A.writers.extend(('derp', 'herp'))
# A comment event that we will keep submitting to trigger
comment = A.getCommentAddedEvent('test me')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
# No positive review from derp so should not be enqueued
self.assertEqual(len(self.history), 0)
# A negative review from derp should not cause it to be enqueued
A.addReview('derp', 'CHANGES_REQUESTED')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 0)
# A positive from nobody should not cause it to be enqueued
A.addReview('nobody', 'APPROVED')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 0)
# A positive review from herp (a writer) should not cause it to be
# enqueued
A.addReview('herp', 'APPROVED')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 0)
# A positive review from derp should cause it to be enqueued
A.addReview('derp', 'APPROVED')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 1)
self.assertEqual(self.history[0].name, 'project5-reviewuserstate')
# TODO: Implement reject on approval username/state
@simple_layout('layouts/requirements-github.yaml', driver='github')
def test_pipeline_require_review_latest_user_state(self):
"Test pipeline requirement: review state from user"
A = self.fake_github.openFakePullRequest('org/project5', 'master', 'A')
# Add derp and herp to writers
A.writers.extend(('derp', 'herp'))
# A comment event that we will keep submitting to trigger
comment = A.getCommentAddedEvent('test me')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
# No positive review from derp so should not be enqueued
self.assertEqual(len(self.history), 0)
# The first negative review from derp should not cause it to be
# enqueued
A.addReview('derp', 'CHANGES_REQUESTED')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 0)
# A positive review from derp should cause it to be enqueued
A.addReview('derp', 'APPROVED')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 1)
self.assertEqual(self.history[0].name, 'project5-reviewuserstate')
@simple_layout('layouts/requirements-github.yaml', driver='github')
def test_pipeline_require_review_write_perms(self):
"Test pipeline requirement: review from user with write"
A = self.fake_github.openFakePullRequest('org/project4', 'master', 'A')
# Add herp to admins
A.admins.append('herp')
# A comment event that we will keep submitting to trigger
comment = A.getCommentAddedEvent('test me')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
# No positive review from derp so should not be enqueued
self.assertEqual(len(self.history), 0)
# The first review is from a reader, and thus should not be enqueued
A.addReview('derp', 'APPROVED')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 0)
# A positive review from herp should cause it to be enqueued
A.addReview('herp', 'APPROVED')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 1)
self.assertEqual(self.history[0].name, 'project4-reviewreq')
@simple_layout('layouts/requirements-github.yaml', driver='github')
def test_pipeline_require_review_comment_masked(self):
"Test pipeline requirement: review comments on top of votes"
A = self.fake_github.openFakePullRequest('org/project5', 'master', 'A')
# Add derp to writers
A.writers.append('derp')
# A comment event that we will keep submitting to trigger
comment = A.getCommentAddedEvent('test me')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
# No positive review from derp so should not be enqueued
self.assertEqual(len(self.history), 0)
# The first negative review from derp should not cause it to be
# enqueued
A.addReview('derp', 'CHANGES_REQUESTED')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 0)
# A positive review is required, so provide it
A.addReview('derp', 'APPROVED')
# Add a comment review on top to make sure we can still enqueue
A.addReview('derp', 'COMMENTED')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 1)
self.assertEqual(self.history[0].name, 'project5-reviewuserstate')
@simple_layout('layouts/requirements-github.yaml', driver='github')
def test_require_review_newer_than(self):
A = self.fake_github.openFakePullRequest('org/project6', 'master', 'A')
# Add derp and herp to writers
A.writers.extend(('derp', 'herp'))
# A comment event that we will keep submitting to trigger
comment = A.getCommentAddedEvent('test me')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
# No positive review from derp so should not be enqueued
self.assertEqual(len(self.history), 0)
# Add a too-old positive review, should not be enqueued
submitted_at = time.time() - 72 * 60 * 60
A.addReview('derp', 'APPROVED',
submitted_at)
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 0)
# Add a recent positive review
submitted_at = time.time() - 12 * 60 * 60
A.addReview('derp', 'APPROVED', submitted_at)
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 1)
self.assertEqual(self.history[0].name, 'project6-newerthan')
@simple_layout('layouts/requirements-github.yaml', driver='github')
def test_require_review_older_than(self):
A = self.fake_github.openFakePullRequest('org/project7', 'master', 'A')
# Add derp and herp to writers
A.writers.extend(('derp', 'herp'))
# A comment event that we will keep submitting to trigger
comment = A.getCommentAddedEvent('test me')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
# No positive review from derp so should not be enqueued
self.assertEqual(len(self.history), 0)
# Add a too-new positive, should not be enqueued
submitted_at = time.time() - 12 * 60 * 60
A.addReview('derp', 'APPROVED',
submitted_at)
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 0)
# Add an old enough positive, should enqueue
submitted_at = time.time() - 72 * 60 * 60
A.addReview('herp', 'APPROVED', submitted_at)
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 1)
self.assertEqual(self.history[0].name, 'project7-olderthan')
@simple_layout('layouts/requirements-github.yaml', driver='github')
def test_require_open(self):
A = self.fake_github.openFakePullRequest('org/project8', 'master', 'A')
# A comment event that we will keep submitting to trigger
comment = A.getCommentAddedEvent('test me')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
# PR is open, we should have enqueued
self.assertEqual(len(self.history), 1)
# close the PR and try again
A.state = 'closed'
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
# PR is closed, should not trigger
self.assertEqual(len(self.history), 1)
@simple_layout('layouts/requirements-github.yaml', driver='github')
def test_reject_open(self):
A = self.fake_github.openFakePullRequest('org/project13', 'master',
'A')
# A comment event that we will keep submitting to trigger
comment = A.getCommentAddedEvent('test me')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
# PR is open, we should not have enqueued
self.assertEqual(len(self.history), 0)
# close the PR and try again
A.state = 'closed'
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
# PR is closed, should trigger
self.assertEqual(len(self.history), 1)
@simple_layout('layouts/requirements-github.yaml', driver='github')
def test_require_current(self):
A = self.fake_github.openFakePullRequest('org/project9', 'master',
'A')
# A sync event that we will keep submitting to trigger
sync = A.getPullRequestSynchronizeEvent()
self.fake_github.emitEvent(sync)
self.waitUntilSettled()
# PR head is current should enqueue
self.assertEqual(len(self.history), 1)
# Add a commit to the PR, re-issue the original comment event
A.addCommit()
self.fake_github.emitEvent(sync)
self.waitUntilSettled()
# Event hash is not current, should not trigger
self.assertEqual(len(self.history), 1)
@simple_layout('layouts/requirements-github.yaml', driver='github')
def test_reject_current(self):
A = self.fake_github.openFakePullRequest('org/project14', 'master',
'A')
# A sync event that we will keep submitting to trigger
sync = A.getPullRequestSynchronizeEvent()
self.fake_github.emitEvent(sync)
self.waitUntilSettled()
# PR head is current, should not enqueue
self.assertEqual(len(self.history), 0)
# Add a commit to the PR, re-issue the original comment event
A.addCommit()
self.fake_github.emitEvent(sync)
self.waitUntilSettled()
# Event hash is not current, should trigger
self.assertEqual(len(self.history), 1)
@simple_layout('layouts/requirements-github.yaml', driver='github')
def test_pipeline_require_label(self):
"Test pipeline requirement: label"
A = self.fake_github.openFakePullRequest('org/project10', 'master',
'A')
# A comment event that we will keep submitting to trigger
comment = A.getCommentAddedEvent('test me')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
# No label so should not be enqueued
self.assertEqual(len(self.history), 0)
# A derp label should not cause it to be enqueued
A.addLabel('derp')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 0)
# An approved label goes in
A.addLabel('approved')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 1)
self.assertEqual(self.history[0].name, 'project10-label')
@simple_layout('layouts/requirements-github.yaml', driver='github')
def test_pipeline_reject_label(self):
"Test pipeline reject: label"
A = self.fake_github.openFakePullRequest('org/project11', 'master',
'A')
# A comment event that we will keep submitting to trigger
comment = A.getCommentAddedEvent('test me')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
# No label so should not be enqueued
self.assertEqual(len(self.history), 0)
# A do-not-merge label should not cause it to be enqueued
A.addLabel('do-not-merge')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 0)
# An approved label should still not enqueue due to d-n-m
A.addLabel('approved')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 0)
# Remove do-not-merge should enqueue
A.removeLabel('do-not-merge')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 1)
self.assertEqual(self.history[0].name, 'project11-label')
@simple_layout('layouts/requirements-github.yaml', driver='github')
def test_pipeline_reject_status(self):
"Test pipeline reject: status"
project = 'org/project12'
A = self.fake_github.openFakePullRequest(project, 'master', 'A')
# Set rejected error status
self.fake_github.setCommitStatus(project, A.head_sha, 'error',
context='tenant-one/check')
# A comment event that we will keep submitting to trigger
comment = A.getCommentAddedEvent('test me')
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
# Status should cause it to be rejected
self.assertEqual(len(self.history), 0)
# Test that also the regex matched pipeline doesn't trigger
self.fake_github.emitEvent(A.getCommentAddedEvent('test regex'))
self.waitUntilSettled()
# Status should cause it to be rejected
self.assertEqual(len(self.history), 0)
self.fake_github.setCommitStatus(project, A.head_sha, 'success',
context='tenant-one/check')
# Now that status is not error, it should be enqueued
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 1)
self.assertEqual(self.history[0].name, 'project12-status')
# Test that also the regex matched pipeline triggers now
self.fake_github.emitEvent(A.getCommentAddedEvent('test regex'))
self.waitUntilSettled()
self.assertEqual(len(self.history), 2)
self.assertEqual(self.history[1].name, 'project12-status')
class TestGithubAppRequirements(ZuulGithubAppTestCase):
"""Test pipeline and trigger requirements with app authentication"""
config_file = 'zuul-github-driver.conf'
@simple_layout("layouts/requirements-github.yaml", driver="github")
def test_pipeline_require_check_run(self):
"Test pipeline requirement: status (reported via a check run)"
project = "org/project16"
github = self.fake_github.getGithubClient()
repo = github.repo_from_project(project)
A = self.fake_github.openFakePullRequest(project, "master", "A")
# A comment event that we will keep submitting to trigger
comment = A.getCommentAddedEvent("trigger me")
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
# No status from zuul, so nothing should be enqueued
self.assertEqual(len(self.history), 0)
# An error check run should also not cause it to be enqueued
repo.create_check_run(
A.head_sha,
"tenant-one/check",
conclusion="failure",
app="check-run",
)
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 0)
# A success check run goes in, ready to be enqueued
repo.create_check_run(
A.head_sha,
"tenant-one/check",
conclusion="success",
app="check-run",
)
self.fake_github.emitEvent(comment)
self.waitUntilSettled()
self.assertEqual(len(self.history), 1)