# Copyright 2019 BMW Group
# Copyright 2023-2024 Acme Gating, LLC
#
# This module is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This software is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this software. If not, see .
from collections import Counter
import fixtures
import re
import textwrap
import threading
import json
from zuul.model import PromoteEvent
import zuul.scheduler
from tests.base import (
iterate_timeout,
simple_layout,
gerrit_config,
ZuulGithubAppTestCase,
ZuulTestCase,
)
class TestGerritCircularDependencies(ZuulTestCase):
config_file = "zuul-gerrit-github.conf"
tenant_config_file = "config/circular-dependencies/main.yaml"
def _test_simple_cycle(self, project1, project2):
A = self.fake_gerrit.addFakeChange(project1, "master", "A")
B = self.fake_gerrit.addFakeChange(project2, "master", "B")
# A <-> B (via commit-depends)
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(len(A.patchsets[-1]["approvals"]), 1)
self.assertEqual(A.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(A.patchsets[-1]["approvals"][0]["value"], "1")
self.assertEqual(len(B.patchsets[-1]["approvals"]), 1)
self.assertEqual(B.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(B.patchsets[-1]["approvals"][0]["value"], "1")
# We're about to add approvals to changes without adding the
# triggering events to Zuul, so that we can be sure that it is
# enqueing the changes based on dependencies, not because of
# triggering events. Since it will have the changes cached
# already (without approvals), we need to clear the cache
# first.
for connection in self.scheds.first.connections.connections.values():
connection.maintainCache([], max_age=0)
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
A.addApproval("Approved", 1)
self.fake_gerrit.addEvent(B.addApproval("Approved", 1))
self.waitUntilSettled()
self.assertEqual(A.reported, 3)
self.assertEqual(B.reported, 3)
self.assertEqual(A.data["status"], "MERGED")
self.assertEqual(B.data["status"], "MERGED")
def _test_transitive_cycle(self, project1, project2, project3):
A = self.fake_gerrit.addFakeChange(project1, "master", "A")
B = self.fake_gerrit.addFakeChange(project2, "master", "B")
C = self.fake_gerrit.addFakeChange(project3, "master", "C")
# A -> B -> C -> A (via commit-depends)
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, C.data["url"]
)
C.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
C.subject, A.data["url"]
)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(len(A.patchsets[-1]["approvals"]), 1)
self.assertEqual(A.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(A.patchsets[-1]["approvals"][0]["value"], "1")
self.assertEqual(len(B.patchsets[-1]["approvals"]), 1)
self.assertEqual(B.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(B.patchsets[-1]["approvals"][0]["value"], "1")
self.assertEqual(len(C.patchsets[-1]["approvals"]), 1)
self.assertEqual(C.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(C.patchsets[-1]["approvals"][0]["value"], "1")
# We're about to add approvals to changes without adding the
# triggering events to Zuul, so that we can be sure that it is
# enqueing the changes based on dependencies, not because of
# triggering events. Since it will have the changes cached
# already (without approvals), we need to clear the cache
# first.
for connection in self.scheds.first.connections.connections.values():
connection.maintainCache([], max_age=0)
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
C.addApproval("Code-Review", 2)
C.addApproval("Approved", 1)
A.addApproval("Approved", 1)
self.fake_gerrit.addEvent(B.addApproval("Approved", 1))
self.waitUntilSettled()
self.assertEqual(A.reported, 3)
self.assertEqual(B.reported, 3)
self.assertEqual(C.reported, 3)
self.assertEqual(A.data["status"], "MERGED")
self.assertEqual(B.data["status"], "MERGED")
self.assertEqual(C.data["status"], "MERGED")
def test_single_project_cycle(self):
self._test_simple_cycle("org/project", "org/project")
def test_crd_cycle(self):
self._test_simple_cycle("org/project1", "org/project2")
def test_single_project_transitive_cycle(self):
self._test_transitive_cycle(
"org/project1", "org/project1", "org/project1"
)
def test_crd_transitive_cycle(self):
self._test_transitive_cycle(
"org/project", "org/project1", "org/project2"
)
def test_enqueue_order(self):
A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
B = self.fake_gerrit.addFakeChange("org/project1", "master", "B")
C = self.fake_gerrit.addFakeChange("org/project2", "master", "C")
# A <-> B and A -> C (via commit-depends)
A.data[
"commitMessage"
] = "{}\n\nDepends-On: {}\nDepends-On: {}\n".format(
A.subject, B.data["url"], C.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(len(A.patchsets[-1]["approvals"]), 1)
self.assertEqual(A.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(A.patchsets[-1]["approvals"][0]["value"], "1")
self.assertEqual(len(B.patchsets[-1]["approvals"]), 1)
self.assertEqual(B.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(B.patchsets[-1]["approvals"][0]["value"], "1")
self.assertEqual(len(C.patchsets[-1]["approvals"]), 1)
self.assertEqual(C.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(C.patchsets[-1]["approvals"][0]["value"], "1")
# We're about to add approvals to changes without adding the
# triggering events to Zuul, so that we can be sure that it is
# enqueuing the changes based on dependencies, not because of
# triggering events. Since it will have the changes cached
# already (without approvals), we need to clear the cache
# first.
for connection in self.scheds.first.connections.connections.values():
connection.maintainCache([], max_age=0)
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
C.addApproval("Code-Review", 2)
B.addApproval("Approved", 1)
C.addApproval("Approved", 1)
self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
self.waitUntilSettled()
self.assertEqual(A.reported, 3)
self.assertEqual(B.reported, 3)
self.assertEqual(C.reported, 3)
self.assertEqual(A.data["status"], "MERGED")
self.assertEqual(B.data["status"], "MERGED")
self.assertEqual(C.data["status"], "MERGED")
self.assertHistory([
# Change A (check + gate)
dict(name="project1-job", result="SUCCESS", changes="3,1 2,1 1,1"),
dict(name="project-vars-job", result="SUCCESS",
changes="3,1 2,1 1,1"),
dict(name="project1-job", result="SUCCESS", changes="3,1 2,1 1,1"),
dict(name="project-vars-job", result="SUCCESS",
changes="3,1 2,1 1,1"),
# Change B (check + gate)
dict(name="project-job", result="SUCCESS", changes="3,1 2,1 1,1"),
dict(name="project-job", result="SUCCESS", changes="3,1 2,1 1,1"),
# Change C (check + gate)
dict(name="project2-job", result="SUCCESS", changes="3,1"),
dict(name="project2-job", result="SUCCESS", changes="3,1"),
], ordered=False)
def test_forbidden_cycle(self):
A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
B = self.fake_gerrit.addFakeChange("org/project3", "master", "B")
# A <-> B (via commit-depends)
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(len(A.patchsets[-1]["approvals"]), 1)
self.assertEqual(A.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(A.patchsets[-1]["approvals"][0]["value"], "-1")
self.assertEqual(len(B.patchsets[-1]["approvals"]), 1)
self.assertEqual(B.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(B.patchsets[-1]["approvals"][0]["value"], "-1")
# We're about to add approvals to changes without adding the
# triggering events to Zuul, so that we can be sure that it is
# enqueing the changes based on dependencies, not because of
# triggering events. Since it will have the changes cached
# already (without approvals), we need to clear the cache
# first.
for connection in self.scheds.first.connections.connections.values():
connection.maintainCache([], max_age=0)
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
B.addApproval("Approved", 1)
self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
self.waitUntilSettled()
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 1)
self.assertEqual(A.data["status"], "NEW")
self.assertEqual(B.data["status"], "NEW")
def test_git_dependency_with_cycle(self):
A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
B = self.fake_gerrit.addFakeChange("org/project", "master", "B")
C = self.fake_gerrit.addFakeChange("org/project1", "master", "C")
# A -> B (git) -> C -> A
A.setDependsOn(B, 1)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, C.data["url"]
)
C.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
C.subject, A.data["url"]
)
self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(len(A.patchsets[-1]["approvals"]), 1)
self.assertEqual(A.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(A.patchsets[-1]["approvals"][0]["value"], "1")
self.assertEqual(len(B.patchsets[-1]["approvals"]), 1)
self.assertEqual(B.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(B.patchsets[-1]["approvals"][0]["value"], "1")
self.assertEqual(len(C.patchsets[-1]["approvals"]), 1)
self.assertEqual(C.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(C.patchsets[-1]["approvals"][0]["value"], "1")
# We're about to add approvals to changes without adding the
# triggering events to Zuul, so that we can be sure that it is
# enqueing the changes based on dependencies, not because of
# triggering events. Since it will have the changes cached
# already (without approvals), we need to clear the cache
# first.
for connection in self.scheds.first.connections.connections.values():
connection.maintainCache([], max_age=0)
self.executor_server.hold_jobs_in_build = True
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
C.addApproval("Code-Review", 2)
A.addApproval("Approved", 1)
B.addApproval("Approved", 1)
self.fake_gerrit.addEvent(C.addApproval("Approved", 1))
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.reported, 3)
self.assertEqual(B.reported, 3)
self.assertEqual(C.reported, 3)
self.assertEqual(A.data["status"], "MERGED")
self.assertEqual(B.data["status"], "MERGED")
self.assertEqual(C.data["status"], "MERGED")
def test_dependency_on_cycle(self):
A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
B = self.fake_gerrit.addFakeChange("org/project1", "master", "B")
C = self.fake_gerrit.addFakeChange("org/project2", "master", "C")
# A -> B -> C -> B (via commit-depends)
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, C.data["url"]
)
C.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
C.subject, B.data["url"]
)
self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(len(A.patchsets[-1]["approvals"]), 1)
self.assertEqual(A.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(A.patchsets[-1]["approvals"][0]["value"], "1")
self.assertEqual(len(B.patchsets[-1]["approvals"]), 1)
self.assertEqual(B.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(B.patchsets[-1]["approvals"][0]["value"], "1")
self.assertEqual(len(C.patchsets[-1]["approvals"]), 1)
self.assertEqual(C.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(C.patchsets[-1]["approvals"][0]["value"], "1")
# We're about to add approvals to changes without adding the
# triggering events to Zuul, so that we can be sure that it is
# enqueing the changes based on dependencies, not because of
# triggering events. Since it will have the changes cached
# already (without approvals), we need to clear the cache
# first.
for connection in self.scheds.first.connections.connections.values():
connection.maintainCache([], max_age=0)
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
C.addApproval("Code-Review", 2)
B.addApproval("Approved", 1)
C.addApproval("Approved", 1)
self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
self.waitUntilSettled()
self.assertEqual(A.reported, 3)
self.assertEqual(B.reported, 3)
self.assertEqual(C.reported, 3)
self.assertEqual(A.data["status"], "MERGED")
self.assertEqual(B.data["status"], "MERGED")
self.assertEqual(C.data["status"], "MERGED")
def test_dependency_on_merged_cycle(self):
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
B = self.fake_gerrit.addFakeChange("org/project1", "master", "B")
C = self.fake_gerrit.addFakeChange("org/project2", "master", "C")
# A -> B -> C -> B (via commit-depends)
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, C.data["url"]
)
C.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
C.subject, B.data["url"]
)
# Start jobs for A while B + C are still open so they get
# enqueued as a non-live item ahead of A.
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
# Fake merge of change B + C while those changes are
# still part of a non-live item as dependency for A.
B.setMerged()
C.setMerged()
self.fake_gerrit.addEvent(B.getChangeMergedEvent())
self.fake_gerrit.addEvent(C.getChangeMergedEvent())
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertHistory([
dict(name="project-job", result="SUCCESS", changes="3,1 2,1 1,1"),
], ordered=False)
def test_dependent_change_on_cycle(self):
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
B = self.fake_gerrit.addFakeChange("org/project", "master", "B")
C = self.fake_gerrit.addFakeChange("org/project2", "master", "C")
A.setDependsOn(B, 1)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, C.data["url"]
)
C.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
C.subject, B.data["url"]
)
A.addApproval("Code-Review", 2)
A.addApproval("Approved", 1)
B.addApproval("Code-Review", 2)
B.addApproval("Approved", 1)
C.addApproval("Code-Review", 2)
self.fake_gerrit.addEvent(C.addApproval("Approved", 1))
self.waitUntilSettled()
self.assertEqual(len(self.builds), 3)
# Make sure the out-of-cycle change (A) is enqueued after the cycle.
tenant = self.scheds.first.sched.abide.tenants.get("tenant-one")
queue_change_numbers = []
for queue in tenant.layout.pipelines["gate"].queues:
for item in queue.queue:
for change in item.changes:
queue_change_numbers.append(change.number)
self.assertEqual(queue_change_numbers, ['2', '3', '1'])
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 2)
self.assertEqual(C.reported, 2)
self.assertEqual(A.data["status"], "MERGED")
self.assertEqual(B.data["status"], "MERGED")
self.assertEqual(C.data["status"], "MERGED")
def test_cycle_dependency_on_cycle(self):
A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
B = self.fake_gerrit.addFakeChange("org/project1", "master", "B")
C = self.fake_gerrit.addFakeChange("org/project1", "master", "C")
D = self.fake_gerrit.addFakeChange("org/project2", "master", "D")
# A -> B -> A + C
# C -> D -> C
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data[
"commitMessage"
] = "{}\n\nDepends-On: {}\nDepends-On: {}\n".format(
B.subject, A.data["url"], C.data["url"]
)
C.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
C.subject, D.data["url"]
)
D.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
D.subject, C.data["url"]
)
self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(len(A.patchsets[-1]["approvals"]), 1)
self.assertEqual(A.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(A.patchsets[-1]["approvals"][0]["value"], "1")
self.assertEqual(len(B.patchsets[-1]["approvals"]), 1)
self.assertEqual(B.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(B.patchsets[-1]["approvals"][0]["value"], "1")
self.assertEqual(len(C.patchsets[-1]["approvals"]), 1)
self.assertEqual(C.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(C.patchsets[-1]["approvals"][0]["value"], "1")
self.assertEqual(len(D.patchsets[-1]["approvals"]), 1)
self.assertEqual(D.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(D.patchsets[-1]["approvals"][0]["value"], "1")
# We're about to add approvals to changes without adding the
# triggering events to Zuul, so that we can be sure that it is
# enqueing the changes based on dependencies, not because of
# triggering events. Since it will have the changes cached
# already (without approvals), we need to clear the cache
# first.
for connection in self.scheds.first.connections.connections.values():
connection.maintainCache([], max_age=0)
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
C.addApproval("Code-Review", 2)
D.addApproval("Code-Review", 2)
B.addApproval("Approved", 1)
C.addApproval("Approved", 1)
D.addApproval("Approved", 1)
self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
self.waitUntilSettled()
self.assertEqual(A.reported, 3)
self.assertEqual(B.reported, 3)
self.assertEqual(C.reported, 3)
self.assertEqual(D.reported, 3)
self.assertEqual(A.data["status"], "MERGED")
self.assertEqual(B.data["status"], "MERGED")
self.assertEqual(C.data["status"], "MERGED")
self.assertEqual(D.data["status"], "MERGED")
def test_cycle_failure(self):
A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
B = self.fake_gerrit.addFakeChange("org/project1", "master", "B")
# A <-> B (via commit-depends)
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.executor_server.failJob("project-job", A)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(len(A.patchsets[-1]["approvals"]), 1)
self.assertEqual(A.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(A.patchsets[-1]["approvals"][0]["value"], "-1")
# We're about to add approvals to changes without adding the
# triggering events to Zuul, so that we can be sure that it is
# enqueing the changes based on dependencies, not because of
# triggering events. Since it will have the changes cached
# already (without approvals), we need to clear the cache
# first.
for connection in self.scheds.first.connections.connections.values():
connection.maintainCache([], max_age=0)
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
B.addApproval("Approved", 1)
self.executor_server.failJob("project-job", A)
self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
self.waitUntilSettled()
self.assertEqual(A.reported, 3)
self.assertEqual(B.reported, 3)
self.assertIn("cycle that failed", A.messages[-1])
self.assertIn("cycle that failed", B.messages[-1])
self.assertEqual(A.data["status"], "NEW")
self.assertEqual(B.data["status"], "NEW")
@simple_layout('layouts/circular-deps-node-failure.yaml')
def test_cycle_failed_node_request(self):
# Test a node request failure as part of a dependency cycle
# Pause nodepool so we can fail the node request later
self.fake_nodepool.pause()
A = self.fake_gerrit.addFakeChange("org/project1", "master", "A")
B = self.fake_gerrit.addFakeChange("org/project2", "master", "B")
# A <-> B (via commit-depends)
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
B.addApproval("Approved", 1)
self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
self.waitUntilSettled()
# Fail the node request and unpause
req = self.fake_nodepool.getNodeRequests()
self.fake_nodepool.addFailRequest(req[0])
self.fake_nodepool.unpause()
self.waitUntilSettled()
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 2)
self.assertIn("cycle that failed", A.messages[-1])
self.assertIn("cycle that failed", B.messages[-1])
self.assertEqual(A.data["status"], "NEW")
self.assertEqual(B.data["status"], "NEW")
def test_failing_cycle_behind_failing_change(self):
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
B = self.fake_gerrit.addFakeChange("org/project", "master", "B")
C = self.fake_gerrit.addFakeChange("org/project", "master", "C")
D = self.fake_gerrit.addFakeChange("org/project", "master", "D")
E = self.fake_gerrit.addFakeChange("org/project", "master", "E")
# C <-> D (via commit-depends)
C.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
C.subject, D.data["url"]
)
D.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
D.subject, C.data["url"]
)
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
C.addApproval("Code-Review", 2)
D.addApproval("Code-Review", 2)
E.addApproval("Code-Review", 2)
self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
self.fake_gerrit.addEvent(B.addApproval("Approved", 1))
# Make sure we enqueue C as part of the circular dependency with D, so
# we end up with the following queue state: A, B, C, ...
C.addApproval("Approved", 1)
self.fake_gerrit.addEvent(D.addApproval("Approved", 1))
self.fake_gerrit.addEvent(E.addApproval("Approved", 1))
self.waitUntilSettled()
# Fail a job of the circular dependency
self.executor_server.failJob("project-job", D)
self.executor_server.release("project-job", change="4 1")
# Fail job for item B ahead of the circular dependency so that this
# causes a gate reset and item C and D are moved behind item A.
self.executor_server.failJob("project-job", B)
self.executor_server.release("project-job", change="2 1")
self.waitUntilSettled()
# Don't fail any other jobs
self.executor_server.fail_tests.clear()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.data["status"], "MERGED")
self.assertEqual(B.data["status"], "NEW")
self.assertEqual(C.data["status"], "MERGED")
self.assertEqual(D.data["status"], "MERGED")
self.assertEqual(E.data["status"], "MERGED")
self.assertHistory([
dict(name="project-job", result="SUCCESS", changes="1,1"),
dict(name="project-job", result="FAILURE", changes="1,1 2,1"),
# First attempt of change C and D before gate reset due to change B
dict(name="project-job", result="FAILURE",
changes="1,1 2,1 3,1 4,1"),
dict(name="project-job", result="FAILURE",
changes="1,1 2,1 3,1 4,1"),
dict(name="project-job", result="ABORTED",
changes="1,1 2,1 3,1 4,1 5,1"),
dict(name="project-job", result="SUCCESS", changes="1,1 3,1 4,1"),
dict(name="project-job", result="SUCCESS", changes="1,1 3,1 4,1"),
dict(name="project-job", result="SUCCESS",
changes="1,1 3,1 4,1 5,1"),
], ordered=False)
def test_dependency_on_cycle_failure(self):
A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
B = self.fake_gerrit.addFakeChange("org/project1", "master", "B")
C = self.fake_gerrit.addFakeChange("org/project2", "master", "C")
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
B.addApproval("Approved", 1)
C.addApproval("Code-Review", 2)
C.addApproval("Approved", 1)
# A -> B -> C -> B (via commit-depends)
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, C.data["url"]
)
C.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
C.subject, B.data["url"]
)
self.executor_server.failJob("project2-job", C)
self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
self.waitUntilSettled()
self.assertIn("depends on a change that failed to merge",
A.messages[-1])
self.assertTrue(re.search(r'Change http://localhost:\d+/2 is needed',
A.messages[-1]))
self.assertFalse(re.search('Change .*? can not be merged',
A.messages[-1]))
self.assertIn("cycle that failed.", B.messages[-1])
self.assertFalse(re.search('Change http://localhost:.*? is needed',
B.messages[-1]))
self.assertFalse(re.search('Change .*? can not be merged',
B.messages[-1]))
self.assertIn("cycle that failed.", C.messages[-1])
self.assertFalse(re.search('Change http://localhost:.*? is needed',
C.messages[-1]))
self.assertFalse(re.search('Change .*? can not be merged',
C.messages[-1]))
self.assertEqual(A.data["status"], "NEW")
self.assertEqual(B.data["status"], "NEW")
self.assertEqual(C.data["status"], "NEW")
def test_cycle_dependency_on_change(self):
A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
B = self.fake_gerrit.addFakeChange("org/project1", "master", "B")
C = self.fake_gerrit.addFakeChange("org/project2", "master", "C")
# A -> B -> A + C (git)
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
B.setDependsOn(C, 1)
self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(len(A.patchsets[-1]["approvals"]), 1)
self.assertEqual(A.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(A.patchsets[-1]["approvals"][0]["value"], "1")
self.assertEqual(len(B.patchsets[-1]["approvals"]), 1)
self.assertEqual(B.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(B.patchsets[-1]["approvals"][0]["value"], "1")
self.assertEqual(len(C.patchsets[-1]["approvals"]), 1)
self.assertEqual(C.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(C.patchsets[-1]["approvals"][0]["value"], "1")
# We're about to add approvals to changes without adding the
# triggering events to Zuul, so that we can be sure that it is
# enqueing the changes based on dependencies, not because of
# triggering events. Since it will have the changes cached
# already (without approvals), we need to clear the cache
# first.
for connection in self.scheds.first.connections.connections.values():
connection.maintainCache([], max_age=0)
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
C.addApproval("Code-Review", 2)
A.addApproval("Approved", 1)
B.addApproval("Approved", 1)
self.fake_gerrit.addEvent(C.addApproval("Approved", 1))
self.waitUntilSettled()
self.assertEqual(A.reported, 3)
self.assertEqual(B.reported, 3)
self.assertEqual(C.reported, 3)
self.assertEqual(A.data["status"], "MERGED")
self.assertEqual(B.data["status"], "MERGED")
self.assertEqual(C.data["status"], "MERGED")
def test_failing_cycle_dependency_on_change(self):
A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
B = self.fake_gerrit.addFakeChange("org/project1", "master", "B")
C = self.fake_gerrit.addFakeChange("org/project2", "master", "C")
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
B.addApproval("Approved", 1)
C.addApproval("Code-Review", 2)
C.addApproval("Approved", 1)
# A -> B -> A + C (via commit-depends)
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data[
"commitMessage"
] = "{}\n\nDepends-On: {}\nDepends-On: {}\n".format(
B.subject, A.data["url"], C.data["url"]
)
self.executor_server.failJob("project-job", A)
self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
self.waitUntilSettled()
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 2)
self.assertEqual(C.reported, 2)
self.assertEqual(A.data["status"], "NEW")
self.assertEqual(B.data["status"], "NEW")
self.assertEqual(C.data["status"], "MERGED")
def test_reopen_cycle(self):
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
B = self.fake_gerrit.addFakeChange("org/project2", "master", "B")
# A <-> B (via commit-depends)
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
A.addApproval("Approved", 1)
self.fake_gerrit.addEvent(B.addApproval("Approved", 1))
self.waitUntilSettled()
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
items_before = tenant.layout.pipelines['gate'].getAllItems()
# Trigger a re-enqueue of change B
self.fake_gerrit.addEvent(B.getChangeAbandonedEvent())
self.fake_gerrit.addEvent(B.getChangeRestoredEvent())
self.fake_gerrit.addEvent(B.addApproval("Approved", 1))
self.waitUntilSettled()
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
items_after = tenant.layout.pipelines['gate'].getAllItems()
# Make sure the complete cycle was re-enqueued
for before, after in zip(items_before, items_after):
self.assertNotEqual(before, after)
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
# Start, Dequeue (cycle abandoned), Start, Success
self.assertEqual(A.reported, 4)
self.assertEqual(B.reported, 4)
self.assertEqual(A.data["status"], "MERGED")
self.assertEqual(B.data["status"], "MERGED")
def test_cycle_larger_pipeline_window(self):
tenant = self.scheds.first.sched.abide.tenants.get("tenant-one")
# Make the gate window smaller than the length of the cycle
for queue in tenant.layout.pipelines["gate"].queues:
if any("org/project" in p.name for p in queue.projects):
queue.window = 1
self._test_simple_cycle("org/project", "org/project")
def test_cycle_reporting_failure(self):
A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
B = self.fake_gerrit.addFakeChange("org/project1", "master", "B")
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
B.addApproval("Approved", 1)
B.fail_merge = True
# A <-> B (via commit-depends)
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
self.waitUntilSettled()
self.assertEqual(A.reported, 3)
self.assertEqual(B.reported, 3)
self.assertEqual(A.patchsets[-1]["approvals"][-1]["value"], "-2")
self.assertEqual(B.patchsets[-1]["approvals"][-1]["value"], "-2")
self.assertIn("cycle that failed", A.messages[-1])
self.assertIn("cycle that failed", B.messages[-1])
self.assertEqual(A.data["status"], "MERGED")
self.assertEqual(B.data["status"], "NEW")
buildsets = self.scheds.first.connections.connections[
'database'].getBuildsets()
self.assertEqual(len(buildsets), 1)
self.assertEqual(buildsets[0].result, 'MERGE_FAILURE')
def test_cycle_reporting_partial_failure(self):
A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
B = self.fake_gerrit.addFakeChange("org/project1", "master", "B")
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
B.addApproval("Approved", 1)
A.fail_merge = True
# A <-> B (via commit-depends)
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
self.waitUntilSettled()
self.assertEqual(A.reported, 3)
self.assertEqual(B.reported, 3)
self.assertIn("cycle that failed", A.messages[-1])
self.assertIn("cycle that failed", B.messages[-1])
self.assertEqual(A.data["status"], "NEW")
self.assertEqual(B.data["status"], "MERGED")
def test_gate_reset_with_cycle(self):
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
B = self.fake_gerrit.addFakeChange("org/project1", "master", "B")
C = self.fake_gerrit.addFakeChange("org/project1", "master", "C")
# A <-> B (via depends-on)
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
C.addApproval("Code-Review", 2)
C.addApproval("Approved", 1)
B.addApproval("Approved", 1)
self.fake_gerrit.addEvent(C.addApproval("Approved", 1))
self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
self.waitUntilSettled()
self.executor_server.failJob("project1-job", C)
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(len(self.builds), 3)
for build in self.builds:
self.assertTrue(build.hasChanges(A, B))
self.assertFalse(build.hasChanges(C))
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 2)
self.assertEqual(C.reported, 2)
self.assertEqual(A.data["status"], "MERGED")
self.assertEqual(B.data["status"], "MERGED")
self.assertEqual(C.data["status"], "NEW")
def test_independent_cycle_items(self):
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
B = self.fake_gerrit.addFakeChange("org/project", "master", "B")
# A <-> B (via commit-depends)
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
tenant = self.scheds.first.sched.abide.tenants.get("tenant-one")
self.assertEqual(len(tenant.layout.pipelines["check"].queues), 1)
queue = tenant.layout.pipelines["check"].queues[0].queue
self.assertEqual(len(queue), 1)
self.assertEqual(len(self.builds), 2)
for build in self.builds:
self.assertTrue(build.hasChanges(A, B))
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(len(A.patchsets[-1]["approvals"]), 1)
self.assertEqual(A.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(A.patchsets[-1]["approvals"][0]["value"], "1")
self.assertEqual(len(B.patchsets[-1]["approvals"]), 1)
self.assertEqual(B.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(B.patchsets[-1]["approvals"][0]["value"], "1")
def test_gate_correct_commits(self):
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
B = self.fake_gerrit.addFakeChange("org/project1", "master", "B")
C = self.fake_gerrit.addFakeChange("org/project1", "master", "C")
D = self.fake_gerrit.addFakeChange("org/project", "master", "D")
# A <-> B (via depends-on)
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
D.setDependsOn(A, 1)
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
C.addApproval("Code-Review", 2)
D.addApproval("Code-Review", 2)
C.addApproval("Approved", 1)
B.addApproval("Approved", 1)
self.fake_gerrit.addEvent(C.addApproval("Approved", 1))
self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
self.fake_gerrit.addEvent(D.addApproval("Approved", 1))
self.waitUntilSettled()
for build in self.builds:
if build.change in ("1 1", "2 1"):
self.assertTrue(build.hasChanges(C, B, A))
self.assertFalse(build.hasChanges(D))
elif build.change == "3 1":
self.assertTrue(build.hasChanges(C))
self.assertFalse(build.hasChanges(A))
self.assertFalse(build.hasChanges(B))
self.assertFalse(build.hasChanges(D))
else:
self.assertTrue(build.hasChanges(C, B, A, D))
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 2)
self.assertEqual(C.reported, 2)
self.assertEqual(D.reported, 2)
self.assertEqual(A.data["status"], "MERGED")
self.assertEqual(B.data["status"], "MERGED")
self.assertEqual(C.data["status"], "MERGED")
self.assertEqual(D.data["status"], "MERGED")
def test_cycle_git_dependency(self):
A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
B = self.fake_gerrit.addFakeChange("org/project", "master", "B")
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
B.addApproval("Approved", 1)
# A -> B (via commit-depends)
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
# B -> A (via parent-child dependency)
B.setDependsOn(A, 1)
self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
self.waitUntilSettled()
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 2)
self.assertEqual(A.data["status"], "MERGED")
self.assertEqual(B.data["status"], "MERGED")
def test_cycle_git_dependency2(self):
# Reverse the enqueue order to make sure both cases are
# tested.
A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
B = self.fake_gerrit.addFakeChange("org/project", "master", "B")
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
A.addApproval("Approved", 1)
# A -> B (via commit-depends)
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
# B -> A (via parent-child dependency)
B.setDependsOn(A, 1)
self.fake_gerrit.addEvent(B.addApproval("Approved", 1))
self.waitUntilSettled()
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 2)
self.assertEqual(A.data["status"], "MERGED")
self.assertEqual(B.data["status"], "MERGED")
def test_cycle_git_dependency_failure(self):
A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
B = self.fake_gerrit.addFakeChange("org/project", "master", "B")
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
B.addApproval("Approved", 1)
# A -> B (via commit-depends)
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
# B -> A (via parent-child dependency)
B.setDependsOn(A, 1)
self.executor_server.failJob("project-job", A)
self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
self.waitUntilSettled()
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 2)
self.assertEqual(A.data["status"], "NEW")
self.assertEqual(B.data["status"], "NEW")
def test_independent_reporting(self):
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
B = self.fake_gerrit.addFakeChange("org/project", "master", "B")
# A <-> B (via commit-depends)
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.fake_gerrit.addEvent(B.getChangeAbandonedEvent())
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(len(A.patchsets[-1]["approvals"]), 1)
self.assertEqual(A.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(A.patchsets[-1]["approvals"][0]["value"], "-1")
self.assertEqual(len(B.patchsets[-1]["approvals"]), 1)
self.assertEqual(B.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(B.patchsets[-1]["approvals"][0]["value"], "-1")
def test_abandon_cancel_jobs(self):
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange("org/project7", "master", "A")
B = self.fake_gerrit.addFakeChange("org/project7", "master", "B")
# A <-> B (via commit-depends)
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.fake_gerrit.addEvent(B.getChangeAbandonedEvent())
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertHistory([
dict(name="project7-parent-job", result="ABORTED",
changes="2,1 1,1"),
], ordered=False)
def test_cycle_merge_conflict(self):
self.hold_merge_jobs_in_queue = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
# A <-> B (via commit-depends)
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
B.addApproval("Approved", 1)
self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
self.waitUntilSettled()
# We only want to have a merge failure for the first item in the queue
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
items = tenant.layout.pipelines['gate'].getAllItems()
with self.createZKContext() as context:
items[0].current_build_set.updateAttributes(context,
unable_to_merge=True)
self.waitUntilSettled()
self.hold_merge_jobs_in_queue = False
self.merger_api.release()
self.waitUntilSettled()
self.assertEqual(A.reported, 1)
self.assertEqual(B.reported, 1)
self.assertEqual(A.data["status"], "NEW")
self.assertEqual(B.data["status"], "NEW")
def test_circular_config_change(self):
define_job = textwrap.dedent(
"""
- job:
name: new-job
""")
use_job = textwrap.dedent(
"""
- project:
queue: integrated
check:
jobs:
- new-job
gate:
jobs:
- new-job
""")
A = self.fake_gerrit.addFakeChange("org/project", "master", "A",
files={"zuul.yaml": define_job})
B = self.fake_gerrit.addFakeChange("org/project1", "master", "B",
files={"zuul.yaml": use_job})
# A <-> B (via commit-depends)
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(len(A.patchsets[-1]["approvals"]), 1)
self.assertEqual(A.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(A.patchsets[-1]["approvals"][0]["value"], "1")
self.assertEqual(len(B.patchsets[-1]["approvals"]), 1)
self.assertEqual(B.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(B.patchsets[-1]["approvals"][0]["value"], "1")
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
self.fake_gerrit.addEvent(B.addApproval("Approved", 1))
self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
self.waitUntilSettled()
self.assertEqual(A.reported, 3)
self.assertEqual(B.reported, 3)
self.assertEqual(A.data["status"], "MERGED")
self.assertEqual(B.data["status"], "MERGED")
def test_circular_config_change_job_vars(self):
org_project_files = {
"zuul.yaml": textwrap.dedent(
"""
- job:
name: project-vars-job
deduplicate: false
vars:
test_var: pass
- project:
queue: integrated
check:
jobs:
- project-vars-job
gate:
jobs:
- project-vars-job
""")
}
A = self.fake_gerrit.addFakeChange("org/project2", "master", "A",
files=org_project_files)
B = self.fake_gerrit.addFakeChange("org/project1", "master", "B")
C = self.fake_gerrit.addFakeChange("org/project1", "master", "C")
# C <-> A <-> B (via commit-depends)
A.data["commitMessage"] = (
"{}\n\nDepends-On: {}\nDepends-On: {}\n".format(
A.subject, B.data["url"], C.data["url"]
)
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
C.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
C.subject, A.data["url"]
)
self.executor_server.hold_jobs_in_build = True
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
vars_builds = [b for b in self.builds if b.name == "project-vars-job"]
self.assertEqual(len(vars_builds), 3)
self.assertEqual(vars_builds[0].job.variables["test_var"], "pass")
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(len(A.patchsets[-1]["approvals"]), 1)
self.assertEqual(A.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(A.patchsets[-1]["approvals"][0]["value"], "1")
self.assertEqual(len(B.patchsets[-1]["approvals"]), 1)
self.assertEqual(B.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(B.patchsets[-1]["approvals"][0]["value"], "1")
self.assertEqual(len(C.patchsets[-1]["approvals"]), 1)
self.assertEqual(C.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(C.patchsets[-1]["approvals"][0]["value"], "1")
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
C.addApproval("Code-Review", 2)
self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
self.fake_gerrit.addEvent(B.addApproval("Approved", 1))
self.fake_gerrit.addEvent(C.addApproval("Approved", 1))
self.waitUntilSettled()
vars_builds = [b for b in self.builds if b.name == "project-vars-job"]
self.assertEqual(len(vars_builds), 3)
for build in vars_builds:
self.assertEqual(build.job.variables["test_var"], "pass")
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.reported, 3)
self.assertEqual(B.reported, 3)
self.assertEqual(C.reported, 3)
self.assertEqual(A.data["status"], "MERGED")
self.assertEqual(B.data["status"], "MERGED")
self.assertEqual(C.data["status"], "MERGED")
def test_circular_config_change_single_merge_job(self):
"""Regression tests to make sure that a cycle with non-live
config changes only spawns one merge job (so that we avoid
problems with multiple jobs arriving in the wrong order)."""
define_job = textwrap.dedent(
"""
- job:
name: new-job
""")
use_job = textwrap.dedent(
"""
- project:
check:
jobs:
- new-job
gate:
jobs:
- new-job
""")
A = self.fake_gerrit.addFakeChange("org/project", "master", "A",
files={"zuul.yaml": define_job})
B = self.fake_gerrit.addFakeChange("org/project1", "master", "B",
files={"zuul.yaml": use_job})
# A <-> B (via commit-depends)
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.waitUntilSettled()
self.hold_merge_jobs_in_queue = True
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
# Assert that there is a single merge job for the cycle.
self.assertEqual(len(self.merger_api.queued()), 1)
self.hold_merge_jobs_in_queue = False
self.merger_api.release()
self.waitUntilSettled()
self.assertEqual(len(A.patchsets[-1]["approvals"]), 1)
self.assertEqual(A.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(A.patchsets[-1]["approvals"][0]["value"], "1")
def test_bundle_id_in_zuul_var(self):
A = self.fake_gerrit.addFakeChange("org/project1", "master", "A")
B = self.fake_gerrit.addFakeChange("org/project1", "master", "B")
C = self.fake_gerrit.addFakeChange("org/project1", "master", "C")
# A <-> B (via commit-depends)
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.executor_server.hold_jobs_in_build = True
# bundle_id should be in check build of A,B
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
var_zuul_items = self.builds[0].parameters["zuul"]["items"]
self.assertEqual(len(var_zuul_items), 2)
self.assertIn("bundle_id", var_zuul_items[0])
bundle_id_0 = var_zuul_items[0]["bundle_id"]
self.assertIn("bundle_id", var_zuul_items[1])
bundle_id_1 = var_zuul_items[1]["bundle_id"]
self.assertEqual(bundle_id_0, bundle_id_1)
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(len(A.patchsets[-1]["approvals"]), 1)
self.assertEqual(A.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(A.patchsets[-1]["approvals"][0]["value"], "1")
self.assertEqual(len(B.patchsets[-1]["approvals"]), 1)
self.assertEqual(B.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(B.patchsets[-1]["approvals"][0]["value"], "1")
# bundle_id should not be in check build of C
self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
var_zuul_items = self.builds[0].parameters["zuul"]["items"]
self.assertEqual(len(var_zuul_items), 1)
self.assertNotIn("bundle_id", var_zuul_items[0])
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(len(C.patchsets[-1]["approvals"]), 1)
self.assertEqual(C.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(C.patchsets[-1]["approvals"][0]["value"], "1")
# bundle_id should be in gate jobs of A and B, but not in C
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
C.addApproval("Code-Review", 2)
self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
self.fake_gerrit.addEvent(B.addApproval("Approved", 1))
self.fake_gerrit.addEvent(C.addApproval("Approved", 1))
self.waitUntilSettled()
var_zuul_items = self.builds[-1].parameters["zuul"]["items"]
self.assertEqual(len(var_zuul_items), 3)
self.assertIn("bundle_id", var_zuul_items[0])
bundle_id_0 = var_zuul_items[0]["bundle_id"]
self.assertIn("bundle_id", var_zuul_items[1])
bundle_id_1 = var_zuul_items[1]["bundle_id"]
self.assertEqual(bundle_id_0, bundle_id_1)
self.assertNotIn("bundle_id", var_zuul_items[2])
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.reported, 3)
self.assertEqual(B.reported, 3)
self.assertEqual(C.reported, 3)
self.assertEqual(A.data["status"], "MERGED")
self.assertEqual(B.data["status"], "MERGED")
self.assertEqual(C.data["status"], "MERGED")
def test_cross_tenant_cycle(self):
org_project_files = {
"zuul.yaml": textwrap.dedent(
"""
- job:
name: project-vars-job
vars:
test_var: pass
- project:
queue: integrated
check:
jobs:
- project-vars-job
gate:
jobs:
- project-vars-job
""")
}
# Change zuul config so the cycle is considered updating config
A = self.fake_gerrit.addFakeChange("org/project2", "master", "A",
files=org_project_files)
B = self.fake_gerrit.addFakeChange("org/project1", "master", "B")
C = self.fake_gerrit.addFakeChange("org/project1", "master", "C")
D = self.fake_gerrit.addFakeChange("org/project4", "master", "D",)
# C <-> A <-> B (via commit-depends)
A.data["commitMessage"] = (
"{}\n\nDepends-On: {}\nDepends-On: {}\n".format(
A.subject, B.data["url"], C.data["url"]
)
)
# A <-> B (via commit-depends)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
# A <-> C <-> D (via commit-depends)
C.data["commitMessage"] = (
"{}\n\nDepends-On: {}\nDepends-On: {}\n".format(
C.subject, A.data["url"], D.data["url"]
)
)
# D <-> C (via commit-depends)
D.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
D.subject, C.data["url"]
)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(len(A.patchsets[-1]["approvals"]), 1)
self.assertEqual(A.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(A.patchsets[-1]["approvals"][0]["value"], "-1")
self.assertEqual(len(B.patchsets[-1]["approvals"]), 1)
self.assertEqual(B.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(B.patchsets[-1]["approvals"][0]["value"], "-1")
self.assertEqual(len(C.patchsets[-1]["approvals"]), 1)
self.assertEqual(C.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(C.patchsets[-1]["approvals"][0]["value"], "-1")
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
C.addApproval("Code-Review", 2)
D.addApproval("Code-Review", 2)
# Add the approvals first, then add the events to ensure we
# are not racing gerrit approval changes.
events = [c.addApproval("Approved", 1) for c in [A, B, C, D]]
for event in events:
self.fake_gerrit.addEvent(event)
self.waitUntilSettled()
self.assertEqual(A.data["status"], "NEW")
self.assertEqual(B.data["status"], "NEW")
self.assertEqual(C.data["status"], "NEW")
D.setMerged()
for connection in self.scheds.first.connections.connections.values():
connection.maintainCache([], max_age=0)
# Pretend D was merged so we can gate the cycle
self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
self.waitUntilSettled()
# Two failures, and one success (start and end)
self.assertEqual(A.reported, 4)
self.assertEqual(B.reported, 4)
self.assertEqual(C.reported, 4)
self.assertEqual(A.data["status"], "MERGED")
self.assertEqual(B.data["status"], "MERGED")
self.assertEqual(C.data["status"], "MERGED")
def test_cycle_unknown_repo(self):
self.init_repo("org/unknown", tag='init')
A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
B = self.fake_gerrit.addFakeChange("org/unknown", "master", "B")
# A <-> B (via commit-depends)
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(len(A.patchsets[-1]["approvals"]), 1)
self.assertEqual(A.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(A.patchsets[-1]["approvals"][0]["value"], "-1")
for connection in self.scheds.first.connections.connections.values():
connection.maintainCache([], max_age=0)
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
B.addApproval("Approved", 1)
self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
self.waitUntilSettled()
self.assertEqual(A.reported, 2)
self.assertEqual(A.data["status"], "NEW")
self.assertEqual(B.data["status"], "NEW")
for connection in self.scheds.first.connections.connections.values():
connection.maintainCache([], max_age=0)
B.setMerged()
self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
self.waitUntilSettled()
self.assertEqual(A.reported, 4)
self.assertEqual(A.data["status"], "MERGED")
def test_promote_cycle(self):
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
B = self.fake_gerrit.addFakeChange("org/project1", "master", "B")
C = self.fake_gerrit.addFakeChange("org/project2", "master", "C")
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
C.addApproval("Code-Review", 2)
B.addApproval("Approved", 1)
self.fake_gerrit.addEvent(C.addApproval("Approved", 1))
self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
self.waitUntilSettled()
event = PromoteEvent('tenant-one', 'gate', ["2,1"])
self.scheds.first.sched.pipeline_management_events['tenant-one'][
'gate'].put(event)
self.waitUntilSettled()
self.assertEqual(len(self.builds), 4)
self.assertTrue(self.builds[0].hasChanges(A))
self.assertTrue(self.builds[0].hasChanges(B))
self.assertFalse(self.builds[0].hasChanges(C))
self.assertTrue(self.builds[1].hasChanges(A))
self.assertTrue(self.builds[1].hasChanges(B))
self.assertFalse(self.builds[0].hasChanges(C))
self.assertTrue(self.builds[3].hasChanges(B))
self.assertTrue(self.builds[3].hasChanges(C))
self.assertTrue(self.builds[3].hasChanges(A))
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 2)
self.assertEqual(C.reported, 2)
self.assertEqual(A.data["status"], "MERGED")
self.assertEqual(B.data["status"], "MERGED")
self.assertEqual(C.data["status"], "MERGED")
def test_shared_queue_removed(self):
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
A.addApproval('Code-Review', 2)
B.addApproval('Code-Review', 2)
C.addApproval('Code-Review', 2)
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
# Remove the shared queue.
self.commitConfigUpdate(
'common-config',
'layouts/circular-dependency-shared-queue-removed.yaml')
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
self.assertEqual(C.data['status'], 'MERGED')
def _test_job_deduplication(self):
# We make a second scheduler here so that the first scheduler
# can freeze the jobs for the first item, and the second
# scheduler freeze jobs for the second. This forces the
# scheduler to compare a desiralized FrozenJob with a newly
# created one and therefore show difference-in-serialization
# issues.
self.hold_merge_jobs_in_queue = True
app = self.createScheduler()
app.start()
self.assertEqual(len(self.scheds), 2)
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
A.addApproval('Code-Review', 2)
B.addApproval('Code-Review', 2)
with app.sched.run_handler_lock:
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
self.waitUntilSettled(matcher=[self.scheds.first])
self.merger_api.release(self.merger_api.queued()[0])
self.waitUntilSettled(matcher=[self.scheds.first])
# Hold the lock on the first scheduler so that only the second
# will act.
with self.scheds.first.sched.run_handler_lock:
self.merger_api.release()
self.waitUntilSettled(matcher=[app])
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(B.data['status'], 'MERGED')
@simple_layout('layouts/job-dedup-auto-shared.yaml')
def test_job_deduplication_auto_shared(self):
self._test_job_deduplication()
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
# This is deduplicated
# dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self.assertEqual(len(self.fake_nodepool.history), 3)
@simple_layout('layouts/job-dedup-auto-unshared.yaml')
def test_job_deduplication_auto_unshared(self):
self._test_job_deduplication()
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1",
ref='refs/changes/02/2/1'),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
# This is not deduplicated
dict(name="common-job", result="SUCCESS", changes="2,1 1,1",
ref='refs/changes/01/1/1'),
], ordered=False)
self.assertEqual(len(self.fake_nodepool.history), 4)
@simple_layout('layouts/job-dedup-true.yaml')
def test_job_deduplication_true(self):
self._test_job_deduplication()
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
# This is deduplicated
# dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self.assertEqual(len(self.fake_nodepool.history), 3)
@simple_layout('layouts/job-dedup-false.yaml')
def test_job_deduplication_false(self):
self._test_job_deduplication()
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1",
ref='refs/changes/02/2/1'),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
# This is not deduplicated, though it would be under auto
dict(name="common-job", result="SUCCESS", changes="2,1 1,1",
ref='refs/changes/01/1/1'),
], ordered=False)
self.assertEqual(len(self.fake_nodepool.history), 4)
@simple_layout('layouts/job-dedup-empty-nodeset.yaml')
def test_job_deduplication_empty_nodeset(self):
# Make sure that jobs with empty nodesets can still be
# deduplicated
self._test_job_deduplication()
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
# This is deduplicated
# dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self.assertEqual(len(self.fake_nodepool.history), 0)
@simple_layout('layouts/job-dedup-child-jobs.yaml')
def test_job_deduplication_child_jobs(self):
# Test that child jobs of deduplicated parents are
# deduplicated, and also that supplying child_jobs to
# zuul_return filters correctly. child1-job should not run,
# but child2 should run and be deduplicated. This uses auto
# deduplication.
self.executor_server.returnData(
'common-job', 'refs/changes/02/2/1',
{'zuul': {'child_jobs': ['child2-job']}}
)
self.executor_server.returnData(
'common-job', 'refs/changes/01/1/1',
{'zuul': {'child_jobs': ['child2-job']}}
)
self._test_job_deduplication()
self.assertHistory([
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="child2-job", result="SUCCESS", changes="2,1 1,1"),
# This is deduplicated
# dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
# dict(name="child2-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
@simple_layout('layouts/job-dedup-mismatched-child-jobs.yaml')
def test_job_deduplication_mismatched_child_jobs(self):
# Test that a parent job with different child jobs is
# deduplicated. This uses auto-deduplication.
self._test_job_deduplication()
self.assertHistory([
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="child1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="child2-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
# This is deduplicated
# dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
@simple_layout('layouts/job-dedup-child-of-diff-parent.yaml')
def test_job_deduplication_child_of_diff_parent_diff_data(self):
# The common job is forced to not deduplicate, and the child
# job is deduplicated. The child job treats each of the
# common jobs as a parent.
self.executor_server.returnData(
'common-job', 'refs/changes/02/2/1',
{'foo': 'a'}
)
self.executor_server.returnData(
'common-job', 'refs/changes/01/1/1',
{'bar': 'b'}
)
self._test_job_deduplication()
self.assertHistory([
dict(name="common-job", result="SUCCESS", changes="2,1 1,1",
ref='refs/changes/02/2/1'),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1",
ref='refs/changes/01/1/1'),
dict(name="child-job", result="SUCCESS", changes="2,1 1,1",
ref='refs/changes/01/1/1'),
], ordered=False)
# child-job for 2 is deduplicated into 1 (the triggering change).
job = self.getJobFromHistory('child-job')
self.assertEqual({'foo': 'a', 'bar': 'b'},
job.parameters['parent_data'])
@simple_layout('layouts/job-dedup-paused-parent.yaml')
def test_job_deduplication_paused_parent(self):
# Pause a parent job
# Ensure it waits for all children before continuing
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.executor_server.returnData(
'common-job', A,
{'zuul': {'pause': True}}
)
self.executor_server.returnData(
'common-job', B,
{'zuul': {'pause': True}}
)
A.addApproval('Code-Review', 2)
B.addApproval('Code-Review', 2)
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
self.waitUntilSettled()
self.executor_server.release('common-job')
self.waitUntilSettled()
self.executor_server.release('project1-job')
self.waitUntilSettled()
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self.assertEqual(len(self.builds), 2)
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertHistory([
dict(name="common-job", result="SUCCESS", changes="2,1 1,1",),
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
@simple_layout('layouts/job-dedup-auto-shared.yaml')
def test_job_deduplication_failed_node_request(self):
# Pause nodepool so we can fail the node request later
self.fake_nodepool.pause()
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
A.addApproval('Code-Review', 2)
B.addApproval('Code-Review', 2)
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
self.waitUntilSettled()
# Fail the node request and unpause
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
pipeline = tenant.layout.pipelines['gate']
items = pipeline.getAllItems()
job = items[0].current_build_set.job_graph.getJob(
'common-job', items[0].changes[0].cache_key)
for req in self.fake_nodepool.getNodeRequests():
if req['requestor_data']['job_uuid'] == job.uuid:
self.fake_nodepool.addFailRequest(req)
self.fake_nodepool.unpause()
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
# This would previously fail both items in the bundle as soon
# as the bundle as a whole started failing. The new/current
# behavior is more like non-bundle items, in that the item
# will continue running jobs even after one job fails.
# common-job does not appear in history due to the node failure
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self.assertEqual(len(self.fake_nodepool.history), 3)
@simple_layout('layouts/job-dedup-auto-shared.yaml')
def test_job_deduplication_failed_job(self):
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
A.addApproval('Code-Review', 2)
B.addApproval('Code-Review', 2)
self.executor_server.failJob("common-job", A)
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
# If we don't make sure these jobs finish first, then one of
# the items may complete before the other and cause Zuul to
# abort the project*-job on the other item (with a "bundle
# failed to merge" error).
self.waitUntilSettled()
self.executor_server.release('project1-job')
self.executor_server.release('project2-job')
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result="FAILURE", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
# This is deduplicated
# dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self.assertEqual(len(self.fake_nodepool.history), 3)
@simple_layout('layouts/job-dedup-false.yaml')
def test_job_deduplication_false_failed_job(self):
# Test that if we are *not* deduplicating jobs, we don't
# duplicate the result on two different builds.
# The way we check that is to retry the common-job between two
# items, but only once, and only on one item. The other item
# should be unaffected.
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
A.addApproval('Code-Review', 2)
B.addApproval('Code-Review', 2)
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
# If we don't make sure these jobs finish first, then one of
# the items may complete before the other and cause Zuul to
# abort the project*-job on the other item (with a "bundle
# failed to merge" error).
self.waitUntilSettled()
for build in self.builds:
if build.name == 'common-job' and build.project == 'org/project1':
break
else:
raise Exception("Unable to find build")
build.should_retry = True
self.executor_server.release('project1-job')
self.executor_server.release('project2-job')
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(B.data['status'], 'MERGED')
self.assertHistory([
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result=None, changes="2,1 1,1"),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self.assertEqual(len(self.fake_nodepool.history), 5)
@simple_layout('layouts/job-dedup-auto-shared.yaml')
def test_job_deduplication_multi_scheduler(self):
# Test that a second scheduler can correctly refresh
# deduplicated builds
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
A.addApproval('Code-Review', 2)
B.addApproval('Code-Review', 2)
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
self.waitUntilSettled()
app = self.createScheduler()
app.start()
self.assertEqual(len(self.scheds), 2)
# Hold the lock on the first scheduler so that only the second
# will act.
with self.scheds.first.sched.run_handler_lock:
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled(matcher=[app])
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(B.data['status'], 'MERGED')
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
@simple_layout('layouts/job-dedup-multi-sched-complete.yaml')
def test_job_deduplication_multi_scheduler_complete(self):
# Test that a second scheduler can correctly refresh
# deduplicated builds after the first scheduler has completed
# the builds for one item.
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
A.addApproval('Code-Review', 2)
B.addApproval('Code-Review', 2)
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
self.executor_server.release('common-job')
self.executor_server.release('project1-job')
self.waitUntilSettled()
app = self.createScheduler()
app.start()
self.assertEqual(len(self.scheds), 2)
# Hold the lock on the first scheduler so that only the second
# will act.
with self.scheds.first.sched.run_handler_lock:
self.executor_server.hold_jobs_in_build = False
# Make sure we go through an extra pipeline cycle so that
# the cleanup method runs.
self.executor_server.release('project2-job2')
self.waitUntilSettled(matcher=[app])
self.executor_server.release()
self.waitUntilSettled(matcher=[app])
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job2", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
@simple_layout('layouts/job-dedup-multi-sched-complete.yaml')
def test_job_deduplication_multi_scheduler_complete2(self):
# Test that a second scheduler can correctly refresh
# deduplicated builds after the first scheduler has completed
# the builds for one item.
# This is similar to the previous test but starts the second
# scheduler slightly later.
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
A.addApproval('Code-Review', 2)
B.addApproval('Code-Review', 2)
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
self.waitUntilSettled()
self.executor_server.release('common-job')
self.waitUntilSettled()
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
self.waitUntilSettled()
self.executor_server.release('project1-job')
self.waitUntilSettled()
app = self.createScheduler()
app.start()
self.assertEqual(len(self.scheds), 2)
# Hold the lock on the first scheduler so that only the second
# will act.
with self.scheds.first.sched.run_handler_lock:
self.executor_server.hold_jobs_in_build = False
self.executor_server.release('project2-job2')
self.waitUntilSettled(matcher=[app])
self.executor_server.release()
self.waitUntilSettled(matcher=[app])
# We expect common-job to be "un-deduplicated".
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="1,1 2,1"),
dict(name="common-job", result="SUCCESS", changes="1,1 2,1"),
dict(name="project2-job", result="SUCCESS", changes="1,1 2,1"),
dict(name="project2-job2", result="SUCCESS", changes="1,1 2,1"),
], ordered=False)
@simple_layout('layouts/job-dedup-noop.yaml')
def test_job_deduplication_noop(self):
# Test that we deduplicate noop (there's no good reason not
# to do so)
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
A.addApproval('Code-Review', 2)
B.addApproval('Code-Review', 2)
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(B.data['status'], 'MERGED')
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
# It's tricky to get info about a noop build, but the jobs in
# the report have the build UUID, so we make sure it's
# the same and we only have one noop job.
a_noop = [l for l in A.messages[-1].split('\n') if 'noop' in l]
b_noop = [l for l in B.messages[-1].split('\n') if 'noop' in l]
self.assertEqual(a_noop, b_noop)
self.assertEqual(len(a_noop), 1)
@simple_layout('layouts/job-dedup-retry.yaml')
def test_job_deduplication_retry(self):
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.executor_server.retryJob('common-job', A)
A.addApproval('Code-Review', 2)
B.addApproval('Code-Review', 2)
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
# There should be exactly 3 runs of the job (not 6)
dict(name="common-job", result=None, changes="2,1 1,1"),
dict(name="common-job", result=None, changes="2,1 1,1"),
dict(name="common-job", result=None, changes="2,1 1,1"),
], ordered=False)
self.assertEqual(len(self.fake_nodepool.history), 5)
@simple_layout('layouts/job-dedup-retry-child.yaml')
def test_job_deduplication_retry_child(self):
# This tests retrying a paused build (simulating an executor restart)
# See test_data_return_child_from_retried_paused_job
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
# A is the triggering change for the deduplicated parent-job
self.executor_server.returnData(
'parent-job', A,
{'zuul': {'pause': True}}
)
A.addApproval('Code-Review', 2)
B.addApproval('Code-Review', 2)
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
self.waitUntilSettled()
self.executor_server.release('parent-job')
self.waitUntilSettled("till job is paused")
paused_job = self.builds[0]
self.assertTrue(paused_job.paused)
# Stop the job worker to simulate an executor restart
for job_worker in self.executor_server.job_workers.values():
if job_worker.build_request.uuid == paused_job.uuid:
job_worker.stop()
self.waitUntilSettled("stop job worker")
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled("all jobs are done")
# The "pause" job might be paused during the waitUntilSettled
# call and appear settled; it should automatically resume
# though, so just wait for it.
for x in iterate_timeout(60, 'paused job'):
if not self.builds:
break
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(B.data['status'], 'MERGED')
self.assertHistory([
dict(name="parent-job", result="ABORTED", changes="2,1 1,1"),
dict(name="project1-job", result="ABORTED", changes="2,1 1,1"),
dict(name="project2-job", result="ABORTED", changes="2,1 1,1"),
dict(name="parent-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self.assertEqual(len(self.fake_nodepool.history), 6)
@simple_layout('layouts/job-dedup-parent-data.yaml')
def test_job_deduplication_parent_data(self):
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
# The parent job returns data
self.executor_server.returnData(
'parent-job', A,
{'zuul':
{'artifacts': [
{'name': 'image',
'url': 'http://example.com/image',
'metadata': {
'type': 'container_image'
}},
]}}
)
A.addApproval('Code-Review', 2)
B.addApproval('Code-Review', 2)
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(B.data['status'], 'MERGED')
self.assertHistory([
dict(name="parent-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
# Only one run of the common job since it's the same
dict(name="common-child-job", result="SUCCESS", changes="2,1 1,1"),
# The forked job depends on different parents
# so it should run twice
dict(name="forked-child-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="forked-child-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self.assertEqual(len(self.fake_nodepool.history), 6)
def _test_job_deduplication_semaphore(self):
"Test semaphores with max=1 (mutex) and get resources first"
self.executor_server.hold_jobs_in_build = True
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
self.assertEqual(
len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
0)
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
A.addApproval('Code-Review', 2)
B.addApproval('Code-Review', 2)
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
self.waitUntilSettled()
self.assertEqual(
len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
1)
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
# This is deduplicated
# dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self.assertEqual(len(self.fake_nodepool.history), 3)
self.assertEqual(
len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
0)
@simple_layout('layouts/job-dedup-semaphore.yaml')
def test_job_deduplication_semaphore(self):
self._test_job_deduplication_semaphore()
@simple_layout('layouts/job-dedup-semaphore-first.yaml')
def test_job_deduplication_semaphore_resources_first(self):
self._test_job_deduplication_semaphore()
def _merge_noop_pair(self):
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
A.addApproval('Code-Review', 2)
B.addApproval('Code-Review', 2)
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(B.data['status'], 'MERGED')
@simple_layout('layouts/job-dedup-noop.yaml')
def test_job_deduplication_buildsets(self):
self._merge_noop_pair()
self._merge_noop_pair()
self._merge_noop_pair()
buildsets = self.scheds.first.connections.connections[
'database'].getBuildsets()
self.assertEqual(len(buildsets), 3)
# If we accidentally limit by rows, we should get fewer than 2
# buildsets returned here.
buildsets = self.scheds.first.connections.connections[
'database'].getBuildsets(limit=2)
self.assertEqual(len(buildsets), 2)
# Start check tests
def _test_job_deduplication_check(self):
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.executor_server.release('common-job')
self.executor_server.release('project1-job')
self.waitUntilSettled()
# We do this even though it results in no changes to force an
# extra pipeline processing run to make sure we don't garbage
# collect the item early.
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.executor_server.release('project2-job')
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
def _assert_job_deduplication_check(self):
# Make sure there are no leaked queue items
tenant = self.scheds.first.sched.abide.tenants.get("tenant-one")
pipeline = tenant.layout.pipelines["check"]
pipeline_path = pipeline.state.getPath()
all_items = set(self.zk_client.client.get_children(
f"{pipeline_path}/item"))
self.assertEqual(len(all_items), 0)
self.assertEqual(len(self.fake_nodepool.history), len(self.history))
@simple_layout('layouts/job-dedup-auto-shared.yaml')
def test_job_deduplication_check_auto_shared(self):
self._test_job_deduplication_check()
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
# This is deduplicated
# dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self._assert_job_deduplication_check()
@simple_layout('layouts/job-dedup-auto-unshared.yaml')
def test_job_deduplication_check_auto_unshared(self):
self._test_job_deduplication_check()
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1",
ref='refs/changes/01/1/1'),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
# This is not deduplicated
dict(name="common-job", result="SUCCESS", changes="2,1 1,1",
ref='refs/changes/02/2/1'),
], ordered=False)
self._assert_job_deduplication_check()
@simple_layout('layouts/job-dedup-true.yaml')
def test_job_deduplication_check_true(self):
self._test_job_deduplication_check()
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
# This is deduplicated
# dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self._assert_job_deduplication_check()
@simple_layout('layouts/job-dedup-false.yaml')
def test_job_deduplication_check_false(self):
self._test_job_deduplication_check()
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1",
ref='refs/changes/02/2/1'),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
# This is not deduplicated, though it would be under auto
dict(name="common-job", result="SUCCESS", changes="2,1 1,1",
ref='refs/changes/01/1/1'),
], ordered=False)
self._assert_job_deduplication_check()
@simple_layout('layouts/job-dedup-empty-nodeset.yaml')
def test_job_deduplication_check_empty_nodeset(self):
# Make sure that jobs with empty nodesets can still be
# deduplicated
self._test_job_deduplication_check()
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
# This is deduplicated
# dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
tenant = self.scheds.first.sched.abide.tenants.get("tenant-one")
pipeline = tenant.layout.pipelines["check"]
pipeline_path = pipeline.state.getPath()
all_items = set(self.zk_client.client.get_children(
f"{pipeline_path}/item"))
self.assertEqual(len(all_items), 0)
@simple_layout('layouts/job-dedup-child-jobs.yaml')
def test_job_deduplication_check_child_jobs(self):
# Test that child jobs of deduplicated parents are
# deduplicated, and also that supplying child_jobs to
# zuul_return filters correctly. child1-job should not run,
# but child2 should run and be deduplicated. This uses auto
# deduplication.
self.executor_server.returnData(
'common-job', 'refs/changes/02/2/1',
{'zuul': {'child_jobs': ['child2-job']}}
)
self.executor_server.returnData(
'common-job', 'refs/changes/01/1/1',
{'zuul': {'child_jobs': ['child2-job']}}
)
self._test_job_deduplication_check()
self.assertHistory([
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="child2-job", result="SUCCESS", changes="2,1 1,1"),
# This is deduplicated
# dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
# dict(name="child2-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self._assert_job_deduplication_check()
@simple_layout('layouts/job-dedup-child-jobs.yaml')
def test_job_deduplication_check_child_jobs_paused_parent(self):
# Test that child jobs returned from deduplicated paused
# parents are deduplicated and correctly skipped.
# child1-job should not run, but child2 should run and be
# deduplicated. This uses auto deduplication.
self.executor_server.returnData(
'common-job', 'refs/changes/02/2/1',
{'zuul': {
'child_jobs': ['child2-job'],
'pause': True,
}}
)
self.executor_server.returnData(
'common-job', 'refs/changes/01/1/1',
{'zuul': {
'child_jobs': ['child2-job'],
'pause': True,
}}
)
self._test_job_deduplication_check()
self.assertHistory([
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="child2-job", result="SUCCESS", changes="2,1 1,1"),
# This is deduplicated
# dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
# dict(name="child2-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self._assert_job_deduplication_check()
@simple_layout('layouts/job-dedup-mismatched-child-jobs.yaml')
def test_job_deduplication_check_mismatched_child_jobs(self):
# Test that a parent job with different child jobs is
# deduplicated. This uses auto-deduplication.
self._test_job_deduplication_check()
self.assertHistory([
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="child1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="child2-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
# This is deduplicated
# dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self._assert_job_deduplication_check()
@simple_layout('layouts/job-dedup-child-of-diff-parent.yaml')
def test_job_deduplication_check_child_of_diff_parent_diff_data(self):
# The common job is forced to not deduplicate, and the child
# job is deduplicated. The child job treats each of the
# common jobs as a parent.
self.executor_server.returnData(
'common-job', 'refs/changes/02/2/1',
{'foo': 'a'}
)
self.executor_server.returnData(
'common-job', 'refs/changes/01/1/1',
{'bar': 'b'}
)
self._test_job_deduplication_check()
self.assertHistory([
dict(name="common-job", result="SUCCESS", changes="2,1 1,1",
ref='refs/changes/01/1/1'),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1",
ref='refs/changes/02/2/1'),
dict(name="child-job", result="SUCCESS", changes="2,1 1,1",
ref='refs/changes/01/1/1'),
], ordered=False)
# child-job for 2 is deduplicated into 1 (the triggering change).
self._assert_job_deduplication_check()
job = self.getJobFromHistory('child-job')
self.assertEqual({'foo': 'a', 'bar': 'b'},
job.parameters['parent_data'])
@simple_layout('layouts/job-dedup-paused-parent.yaml')
def test_job_deduplication_check_paused_parent(self):
# Pause a parent job
# Ensure it waits for all children before continuing
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.executor_server.returnData(
'common-job', A,
{'zuul': {'pause': True}}
)
self.executor_server.returnData(
'common-job', B,
{'zuul': {'pause': True}}
)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.executor_server.release('common-job')
self.waitUntilSettled()
self.executor_server.release('project1-job')
self.waitUntilSettled()
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self.assertEqual(len(self.builds), 2)
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertHistory([
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self._assert_job_deduplication_check()
@simple_layout('layouts/job-dedup-auto-shared.yaml')
def test_job_deduplication_check_failed_node_request(self):
# Pause nodepool so we can fail the node request later
self.fake_nodepool.pause()
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
# Fail the node request and unpause
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
pipeline = tenant.layout.pipelines['check']
items = pipeline.getAllItems()
job = items[0].current_build_set.job_graph.getJob(
'common-job', items[0].changes[0].cache_key)
for req in self.fake_nodepool.getNodeRequests():
if req['requestor_data']['job_uuid'] == job.uuid:
self.fake_nodepool.addFailRequest(req)
self.fake_nodepool.unpause()
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self.assertEqual(len(self.fake_nodepool.history), 3)
@simple_layout('layouts/job-dedup-auto-shared.yaml')
def test_job_deduplication_check_failed_job(self):
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.executor_server.failJob("common-job", A)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
# If we don't make sure these jobs finish first, then one of
# the items may complete before the other and cause Zuul to
# abort the project*-job on the other item (with a "bundle
# failed to merge" error).
self.waitUntilSettled()
self.executor_server.release('project1-job')
self.executor_server.release('project2-job')
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result="FAILURE", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
# This is deduplicated
# dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self.assertEqual(len(self.fake_nodepool.history), 3)
@simple_layout('layouts/job-dedup-false.yaml')
def test_job_deduplication_check_false_failed_job(self):
# Test that if we are *not* deduplicating jobs, we don't
# duplicate the result on two different builds. The way we
# check that is to retry the common-job between two changes,
# but only once, and only on one change. The other change
# should be unaffected.
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
for build in self.builds:
if build.name == 'common-job' and build.project == 'org/project1':
break
else:
raise Exception("Unable to find build")
build.should_retry = True
self.executor_server.release('project1-job')
self.executor_server.release('project2-job')
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
self.assertHistory([
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result=None, changes="2,1 1,1",
ref='refs/changes/01/1/1'),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1",
ref='refs/changes/01/1/1'),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1",
ref='refs/changes/02/2/1'),
], ordered=False)
self.assertEqual(len(self.fake_nodepool.history), 5)
@simple_layout('layouts/job-dedup-auto-shared.yaml')
def test_job_deduplication_check_multi_scheduler(self):
# Test that a second scheduler can correctly refresh
# deduplicated builds
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
app = self.createScheduler()
app.start()
self.assertEqual(len(self.scheds), 2)
# Hold the lock on the first scheduler so that only the second
# will act.
with self.scheds.first.sched.run_handler_lock:
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled(matcher=[app])
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
@simple_layout('layouts/job-dedup-multi-sched-complete.yaml')
def test_job_deduplication_check_multi_scheduler_complete(self):
# Test that a second scheduler can correctly refresh
# deduplicated builds after the first scheduler has completed
# the builds for one item.
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.executor_server.release('common-job')
self.executor_server.release('project1-job')
self.waitUntilSettled()
app = self.createScheduler()
app.start()
self.assertEqual(len(self.scheds), 2)
# Hold the lock on the first scheduler so that only the second
# will act.
with self.scheds.first.sched.run_handler_lock:
self.executor_server.hold_jobs_in_build = False
# Make sure we go through an extra pipeline cycle so that
# the cleanup method runs.
self.executor_server.release('project2-job2')
self.waitUntilSettled(matcher=[app])
self.executor_server.release()
self.waitUntilSettled(matcher=[app])
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job2", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
@simple_layout('layouts/job-dedup-multi-sched-complete.yaml')
def test_job_deduplication_check_multi_scheduler_complete2(self):
# Test that a second scheduler can correctly refresh
# deduplicated builds after the first scheduler has completed
# the builds for one item.
# This is similar to the previous test but starts the second
# scheduler slightly later.
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.executor_server.release('common-job')
self.waitUntilSettled()
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.executor_server.release('project1-job')
self.waitUntilSettled()
app = self.createScheduler()
app.start()
self.assertEqual(len(self.scheds), 2)
# Hold the lock on the first scheduler so that only the second
# will act.
with self.scheds.first.sched.run_handler_lock:
self.executor_server.hold_jobs_in_build = False
self.executor_server.release('project2-job2')
self.waitUntilSettled(matcher=[app])
self.executor_server.release()
self.waitUntilSettled(matcher=[app])
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job2", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
@simple_layout('layouts/job-dedup-noop.yaml')
def test_job_deduplication_check_noop(self):
# Test that we deduplicate noop (there's no good reason not
# to do so)
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
# It's tricky to get info about a noop build, but the jobs in
# the report have the build UUID, so we make sure it's
# the same and we only have one noop job.
a_noop = [l for l in A.messages[-1].split('\n') if 'noop' in l]
b_noop = [l for l in B.messages[-1].split('\n') if 'noop' in l]
self.assertEqual(a_noop, b_noop)
self.assertEqual(len(a_noop), 1)
@simple_layout('layouts/job-dedup-retry.yaml')
def test_job_deduplication_check_retry(self):
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.executor_server.retryJob('common-job', A)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
# There should be exactly 3 runs of the job (not 6)
dict(name="common-job", result=None, changes="2,1 1,1"),
dict(name="common-job", result=None, changes="2,1 1,1"),
dict(name="common-job", result=None, changes="2,1 1,1"),
], ordered=False)
self.assertEqual(len(self.fake_nodepool.history), 5)
@simple_layout('layouts/job-dedup-retry-child.yaml')
def test_job_deduplication_check_retry_child(self):
# This tests retrying a paused build (simulating an executor restart)
# See test_data_return_child_from_retried_paused_job
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.executor_server.returnData(
'parent-job', A,
{'zuul': {'pause': True}}
)
self.executor_server.returnData(
'parent-job', B,
{'zuul': {'pause': True}}
)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.executor_server.release('parent-job')
self.waitUntilSettled("till job is paused")
paused_job = self.builds[0]
self.assertTrue(paused_job.paused)
# Stop the job worker to simulate an executor restart
for job_worker in self.executor_server.job_workers.values():
if job_worker.build_request.uuid == paused_job.uuid:
job_worker.stop()
self.waitUntilSettled("stop job worker")
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled("all jobs are done")
# The "pause" job might be paused during the waitUntilSettled
# call and appear settled; it should automatically resume
# though, so just wait for it.
for x in iterate_timeout(60, 'paused job'):
if not self.builds:
break
self.waitUntilSettled()
self.assertHistory([
dict(name="parent-job", result="ABORTED", changes="2,1 1,1"),
dict(name="project1-job", result="ABORTED", changes="2,1 1,1"),
dict(name="project2-job", result="ABORTED", changes="2,1 1,1"),
dict(name="parent-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self.assertEqual(len(self.fake_nodepool.history), 6)
@simple_layout('layouts/job-dedup-parent-data.yaml')
def test_job_deduplication_check_parent_data(self):
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
# The parent job returns data
self.executor_server.returnData(
'parent-job', A,
{'zuul':
{'artifacts': [
{'name': 'image',
'url': 'http://example.com/image',
'metadata': {
'type': 'container_image'
}},
]}}
)
self.executor_server.returnData(
'parent-job', B,
{'zuul':
{'artifacts': [
{'name': 'image',
'url': 'http://example.com/image',
'metadata': {
'type': 'container_image'
}},
]}}
)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertHistory([
dict(name="parent-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
# Only one run of the common job since it's the same
dict(name="common-child-job", result="SUCCESS", changes="2,1 1,1"),
# The forked job depends on different parents
# so it should run twice
dict(name="forked-child-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="forked-child-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self.assertEqual(len(self.fake_nodepool.history), 6)
@simple_layout('layouts/job-dedup-parent.yaml')
def test_job_deduplication_check_parent_data2(self):
# This is similar to the parent_data test, but it waits longer
# to add the second change so that the deduplication of both
# parent and child happen at the same time. This verifies
# that when parent data is supplied to child jobs
# deduplication still works.
# This has no equivalent for dependent pipelines since it's
# not possible to delay enqueing the second change.
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
self.executor_server.returnData(
'common-job', A,
{'zuul':
{'artifacts': [
{'name': 'image',
'url': 'http://example.com/image',
'metadata': {
'type': 'container_image'
}},
]}}
)
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.executor_server.release('common-job')
self.waitUntilSettled()
self.executor_server.release('child-job')
self.waitUntilSettled()
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertHistory([
dict(name="common-job", result="SUCCESS", changes="2,1 1,1",
ref='refs/changes/01/1/1'),
dict(name="child-job", result="SUCCESS", changes="2,1 1,1",
ref='refs/changes/01/1/1'),
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1",
ref='refs/changes/01/1/1'),
], ordered=False)
# All jobs for 2 are deduplicated into 1 (the triggering change).
self._assert_job_deduplication_check()
def _test_job_deduplication_check_semaphore(self):
"Test semaphores with max=1 (mutex) and get resources first"
self.executor_server.hold_jobs_in_build = True
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
self.assertEqual(
len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
0)
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
# A <-> B
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(
len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
1)
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertHistory([
dict(name="project1-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="project2-job", result="SUCCESS", changes="2,1 1,1"),
# This is deduplicated
# dict(name="common-job", result="SUCCESS", changes="2,1 1,1"),
], ordered=False)
self.assertEqual(len(self.fake_nodepool.history), 3)
self.assertEqual(
len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
0)
@simple_layout('layouts/job-dedup-semaphore.yaml')
def test_job_deduplication_check_semaphore(self):
self._test_job_deduplication_check_semaphore()
@simple_layout('layouts/job-dedup-semaphore-first.yaml')
def test_job_deduplication_check_semaphore_resources_first(self):
self._test_job_deduplication_check_semaphore()
# End check tests
@gerrit_config(submit_whole_topic=True)
def test_submitted_together(self):
A = self.fake_gerrit.addFakeChange('org/project1', "master", "A",
topic='test-topic')
B = self.fake_gerrit.addFakeChange('org/project2', "master", "B",
topic='test-topic')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(len(A.patchsets[-1]["approvals"]), 1)
self.assertEqual(A.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(A.patchsets[-1]["approvals"][0]["value"], "1")
self.assertEqual(len(B.patchsets[-1]["approvals"]), 1)
self.assertEqual(B.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(B.patchsets[-1]["approvals"][0]["value"], "1")
# We're about to add approvals to changes without adding the
# triggering events to Zuul, so that we can be sure that it is
# enqueing the changes based on dependencies, not because of
# triggering events. Since it will have the changes cached
# already (without approvals), we need to clear the cache
# first.
for connection in self.scheds.first.connections.connections.values():
connection.maintainCache([], max_age=0)
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
A.addApproval("Approved", 1)
self.fake_gerrit.addEvent(B.addApproval("Approved", 1))
self.waitUntilSettled()
self.assertEqual(A.reported, 3)
self.assertEqual(B.reported, 3)
self.assertEqual(A.data["status"], "MERGED")
self.assertEqual(B.data["status"], "MERGED")
@gerrit_config(submit_whole_topic=True)
def test_submitted_together_storm(self):
# Test that if many changes are uploaded with the same topic,
# we handle queries efficiently.
self.waitUntilSettled()
A = self.fake_gerrit.addFakeChange('org/project', "master", "A",
topic='test-topic')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
B = self.fake_gerrit.addFakeChange('org/project1', "master", "B",
topic='test-topic')
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
C = self.fake_gerrit.addFakeChange('org/project2', "master", "C",
topic='test-topic')
self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
# Output all the queries seen for debugging
for q in self.fake_gerrit.api_calls:
self.log.debug("Query: %s", q)
gets = [q[1] for q in self.fake_gerrit.api_calls if q[0] == 'GET']
counters = Counter()
for q in gets:
parts = q.split('/')[2:]
if len(parts) > 2 and parts[2] == 'revisions':
parts.pop(3)
if 'q=message' in parts[1]:
parts[1] = 'message'
counters[tuple(parts)] += 1
# Ensure that we don't run these queries more than once for each change
qstring = ('o=DETAILED_ACCOUNTS&o=CURRENT_REVISION&'
'o=CURRENT_COMMIT&o=CURRENT_FILES&o=LABELS&'
'o=DETAILED_LABELS&o=ALL_REVISIONS')
self.assertEqual(1, counters[('changes', f'1?{qstring}')])
self.assertEqual(1, counters[('changes', f'2?{qstring}')])
self.assertEqual(1, counters[('changes', f'3?{qstring}')])
self.assertEqual(1, counters[('changes', '1', 'revisions', 'related')])
self.assertEqual(1, counters[('changes', '2', 'revisions', 'related')])
self.assertEqual(1, counters[('changes', '3', 'revisions', 'related')])
self.assertEqual(1, counters[
('changes', '1', 'revisions', 'files?parent=1')])
self.assertEqual(1, counters[
('changes', '2', 'revisions', 'files?parent=1')])
self.assertEqual(1, counters[
('changes', '3', 'revisions', 'files?parent=1')])
self.assertEqual(3, counters[('changes', 'message')])
# These queries are no longer used
self.assertEqual(0, counters[('changes', '1', 'submitted_together')])
self.assertEqual(0, counters[('changes', '2', 'submitted_together')])
self.assertEqual(0, counters[('changes', '3', 'submitted_together')])
# This query happens once for each event in the scheduler.
qstring = ('?n=500&o=CURRENT_REVISION&o=CURRENT_COMMIT&'
'q=status%3Aopen%20topic%3A%22test-topic%22')
self.assertEqual(3, counters[('changes', qstring)])
self.assertHistory([
dict(name="project-job", changes="1,1"),
dict(name="project-job", changes="1,1 2,1"),
dict(name="project1-job", changes="1,1 2,1"),
dict(name="project-vars-job", changes="1,1 2,1"),
dict(name="project-job", changes="2,1 1,1 3,1"),
dict(name="project1-job", changes="2,1 1,1 3,1"),
dict(name="project-vars-job", changes="2,1 1,1 3,1"),
dict(name="project2-job", changes="2,1 1,1 3,1"),
], ordered=False)
@gerrit_config(submit_whole_topic=True)
def test_submitted_together_storm_fast(self):
# Test that if many changes are uploaded with the same topic,
# we handle queries efficiently.
def waitForEvents():
for x in iterate_timeout(60, 'empty event queue'):
if not len(self.fake_gerrit.event_queue):
return
self.log.debug("Empty event queue")
# This mimics the changes being uploaded in rapid succession.
self.waitUntilSettled()
with self.scheds.first.sched.run_handler_lock:
A = self.fake_gerrit.addFakeChange('org/project', "master", "A",
topic='test-topic')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
waitForEvents()
B = self.fake_gerrit.addFakeChange('org/project1', "master", "B",
topic='test-topic')
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
waitForEvents()
C = self.fake_gerrit.addFakeChange('org/project2', "master", "C",
topic='test-topic')
self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
waitForEvents()
self.waitUntilSettled()
# Output all the queries seen for debugging
for q in self.fake_gerrit.api_calls:
self.log.debug("Query: %s", q)
gets = [q[1] for q in self.fake_gerrit.api_calls if q[0] == 'GET']
counters = Counter()
for q in gets:
parts = q.split('/')[2:]
if len(parts) > 2 and parts[2] == 'revisions':
parts.pop(3)
if 'q=message' in parts[1]:
parts[1] = 'message'
counters[tuple(parts)] += 1
# Ensure that we don't run these queries more than once for each change
qstring = ('o=DETAILED_ACCOUNTS&o=CURRENT_REVISION&'
'o=CURRENT_COMMIT&o=CURRENT_FILES&o=LABELS&'
'o=DETAILED_LABELS&o=ALL_REVISIONS')
self.assertEqual(1, counters[('changes', f'1?{qstring}')])
self.assertEqual(1, counters[('changes', f'2?{qstring}')])
self.assertEqual(1, counters[('changes', f'3?{qstring}')])
self.assertEqual(1, counters[('changes', '1', 'revisions', 'related')])
self.assertEqual(1, counters[('changes', '2', 'revisions', 'related')])
self.assertEqual(1, counters[('changes', '3', 'revisions', 'related')])
self.assertEqual(1, counters[
('changes', '1', 'revisions', 'files?parent=1')])
self.assertEqual(1, counters[
('changes', '2', 'revisions', 'files?parent=1')])
self.assertEqual(1, counters[
('changes', '3', 'revisions', 'files?parent=1')])
self.assertEqual(3, counters[('changes', 'message')])
# These queries are no longer used
self.assertEqual(0, counters[('changes', '1', 'submitted_together')])
self.assertEqual(0, counters[('changes', '2', 'submitted_together')])
self.assertEqual(0, counters[('changes', '3', 'submitted_together')])
# This query happens once for each event (but is cached).
# * A+B+C: 3x scheduler, 0x pipeline
qstring = ('?n=500&o=CURRENT_REVISION&o=CURRENT_COMMIT&'
'q=status%3Aopen%20topic%3A%22test-topic%22')
self.assertEqual(1, counters[('changes', qstring)])
self.assertHistory([
dict(name="project-job", changes="3,1 2,1 1,1"),
dict(name="project1-job", changes="3,1 2,1 1,1"),
dict(name="project-vars-job", changes="3,1 2,1 1,1"),
dict(name="project2-job", changes="3,1 2,1 1,1"),
], ordered=False)
@gerrit_config(submit_whole_topic=True)
def test_submitted_together_git(self):
A = self.fake_gerrit.addFakeChange('org/project1', "master", "A")
B = self.fake_gerrit.addFakeChange('org/project1', "master", "B")
C = self.fake_gerrit.addFakeChange('org/project1', "master", "C")
D = self.fake_gerrit.addFakeChange('org/project1', "master", "D")
E = self.fake_gerrit.addFakeChange('org/project1', "master", "E")
F = self.fake_gerrit.addFakeChange('org/project1', "master", "F")
G = self.fake_gerrit.addFakeChange('org/project1', "master", "G")
G.setDependsOn(F, 1)
F.setDependsOn(E, 1)
E.setDependsOn(D, 1)
D.setDependsOn(C, 1)
C.setDependsOn(B, 1)
B.setDependsOn(A, 1)
self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(len(C.patchsets[-1]["approvals"]), 1)
self.assertEqual(C.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(C.patchsets[-1]["approvals"][0]["value"], "1")
self.assertEqual(A.queried, 1)
self.assertEqual(B.queried, 1)
self.assertEqual(C.queried, 1)
self.assertEqual(D.queried, 1)
self.assertEqual(E.queried, 1)
self.assertEqual(F.queried, 1)
self.assertEqual(G.queried, 1)
self.assertHistory([
dict(name="project1-job", result="SUCCESS",
changes="1,1 2,1 3,1"),
dict(name="project-vars-job", result="SUCCESS",
changes="1,1 2,1 3,1"),
], ordered=False)
@gerrit_config(submit_whole_topic=True)
def test_submitted_together_git_topic(self):
A = self.fake_gerrit.addFakeChange('org/project1', "master", "A",
topic='test-topic')
B = self.fake_gerrit.addFakeChange('org/project1', "master", "B",
topic='test-topic')
C = self.fake_gerrit.addFakeChange('org/project1', "master", "C",
topic='test-topic')
D = self.fake_gerrit.addFakeChange('org/project1', "master", "D",
topic='test-topic')
E = self.fake_gerrit.addFakeChange('org/project1', "master", "E",
topic='test-topic')
F = self.fake_gerrit.addFakeChange('org/project1', "master", "F",
topic='test-topic')
G = self.fake_gerrit.addFakeChange('org/project1', "master", "G",
topic='test-topic')
G.setDependsOn(F, 1)
F.setDependsOn(E, 1)
E.setDependsOn(D, 1)
D.setDependsOn(C, 1)
C.setDependsOn(B, 1)
B.setDependsOn(A, 1)
self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(len(C.patchsets[-1]["approvals"]), 1)
self.assertEqual(C.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(C.patchsets[-1]["approvals"][0]["value"], "1")
self.assertEqual(A.queried, 2)
self.assertEqual(B.queried, 2)
self.assertEqual(C.queried, 2)
self.assertEqual(D.queried, 2)
self.assertEqual(E.queried, 2)
self.assertEqual(F.queried, 2)
self.assertEqual(G.queried, 2)
self.assertHistory([
dict(name="project1-job", result="SUCCESS",
changes="1,1 2,1 3,1 4,1 5,1 6,1 7,1",
ref="refs/changes/01/1/1"),
dict(name="project1-job", result="SUCCESS",
changes="1,1 2,1 3,1 4,1 5,1 6,1 7,1",
ref="refs/changes/02/2/1"),
dict(name="project1-job", result="SUCCESS",
changes="1,1 2,1 3,1 4,1 5,1 6,1 7,1",
ref="refs/changes/03/3/1"),
dict(name="project1-job", result="SUCCESS",
changes="1,1 2,1 3,1 4,1 5,1 6,1 7,1",
ref="refs/changes/04/4/1"),
dict(name="project1-job", result="SUCCESS",
changes="1,1 2,1 3,1 4,1 5,1 6,1 7,1",
ref="refs/changes/05/5/1"),
dict(name="project1-job", result="SUCCESS",
changes="1,1 2,1 3,1 4,1 5,1 6,1 7,1",
ref="refs/changes/06/6/1"),
dict(name="project1-job", result="SUCCESS",
changes="1,1 2,1 3,1 4,1 5,1 6,1 7,1",
ref="refs/changes/07/7/1"),
dict(name="project-vars-job", result="SUCCESS",
changes="1,1 2,1 3,1 4,1 5,1 6,1 7,1",
ref="refs/changes/01/1/1"),
dict(name="project-vars-job", result="SUCCESS",
changes="1,1 2,1 3,1 4,1 5,1 6,1 7,1",
ref="refs/changes/02/2/1"),
dict(name="project-vars-job", result="SUCCESS",
changes="1,1 2,1 3,1 4,1 5,1 6,1 7,1",
ref="refs/changes/03/3/1"),
dict(name="project-vars-job", result="SUCCESS",
changes="1,1 2,1 3,1 4,1 5,1 6,1 7,1",
ref="refs/changes/04/4/1"),
dict(name="project-vars-job", result="SUCCESS",
changes="1,1 2,1 3,1 4,1 5,1 6,1 7,1",
ref="refs/changes/05/5/1"),
dict(name="project-vars-job", result="SUCCESS",
changes="1,1 2,1 3,1 4,1 5,1 6,1 7,1",
ref="refs/changes/06/6/1"),
dict(name="project-vars-job", result="SUCCESS",
changes="1,1 2,1 3,1 4,1 5,1 6,1 7,1",
ref="refs/changes/07/7/1"),
], ordered=False)
@simple_layout('layouts/submitted-together-per-branch.yaml')
@gerrit_config(submit_whole_topic=True)
def test_submitted_together_per_branch(self):
self.create_branch('org/project2', 'stable/foo')
A = self.fake_gerrit.addFakeChange('org/project1', "master", "A",
topic='test-topic')
B = self.fake_gerrit.addFakeChange('org/project2', "stable/foo", "B",
topic='test-topic')
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
A.addApproval("Approved", 1)
self.fake_gerrit.addEvent(B.addApproval("Approved", 1))
self.waitUntilSettled()
self.assertEqual(A.reported, 0)
self.assertEqual(B.reported, 1)
self.assertEqual(A.data["status"], "NEW")
self.assertEqual(B.data["status"], "NEW")
self.assertIn("does not share a change queue", B.messages[-1])
@simple_layout('layouts/deps-by-topic.yaml')
def test_deps_by_topic(self):
A = self.fake_gerrit.addFakeChange('org/project1', "master", "A",
topic='test topic')
B = self.fake_gerrit.addFakeChange('org/project2', "master", "B",
topic='test topic')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(len(A.patchsets[-1]["approvals"]), 1)
self.assertEqual(A.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(A.patchsets[-1]["approvals"][0]["value"], "1")
self.assertEqual(len(B.patchsets[-1]["approvals"]), 1)
self.assertEqual(B.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(B.patchsets[-1]["approvals"][0]["value"], "1")
# We're about to add approvals to changes without adding the
# triggering events to Zuul, so that we can be sure that it is
# enqueing the changes based on dependencies, not because of
# triggering events. Since it will have the changes cached
# already (without approvals), we need to clear the cache
# first.
for connection in self.scheds.first.connections.connections.values():
connection.maintainCache([], max_age=0)
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
A.addApproval("Approved", 1)
self.fake_gerrit.addEvent(B.addApproval("Approved", 1))
self.waitUntilSettled()
self.assertEqual(A.reported, 3)
self.assertEqual(B.reported, 3)
self.assertEqual(A.data["status"], "MERGED")
self.assertEqual(B.data["status"], "MERGED")
@simple_layout('layouts/deps-by-topic.yaml')
def test_deps_by_topic_git_needs(self):
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', "master", "A",
topic='test-topic')
B = self.fake_gerrit.addFakeChange('org/project2', "master", "B",
topic='test-topic')
C = self.fake_gerrit.addFakeChange('org/project2', "master", "C",
topic='other-topic')
D = self.fake_gerrit.addFakeChange('org/project1', "master", "D",
topic='other-topic')
# Git level dependency between B and C
B.setDependsOn(C, 1)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(len(A.patchsets[-1]["approvals"]), 1)
self.assertEqual(A.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(A.patchsets[-1]["approvals"][0]["value"], "1")
self.assertEqual(len(B.patchsets[-1]["approvals"]), 1)
self.assertEqual(B.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(B.patchsets[-1]["approvals"][0]["value"], "1")
self.assertEqual(len(C.patchsets[-1]["approvals"]), 1)
self.assertEqual(C.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(C.patchsets[-1]["approvals"][0]["value"], "1")
self.assertEqual(len(D.patchsets[-1]["approvals"]), 1)
self.assertEqual(D.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(D.patchsets[-1]["approvals"][0]["value"], "1")
# We're about to add approvals to changes without adding the
# triggering events to Zuul, so that we can be sure that it is
# enqueing the changes based on dependencies, not because of
# triggering events. Since it will have the changes cached
# already (without approvals), we need to clear the cache
# first.
for connection in self.scheds.first.connections.connections.values():
connection.maintainCache([], max_age=0)
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
C.addApproval("Code-Review", 2)
D.addApproval("Code-Review", 2)
A.addApproval("Approved", 1)
C.addApproval("Approved", 1)
D.addApproval("Approved", 1)
self.fake_gerrit.addEvent(B.addApproval("Approved", 1))
self.waitUntilSettled()
self.assertEqual(A.reported, 3)
self.assertEqual(B.reported, 3)
self.assertEqual(C.reported, 3)
self.assertEqual(D.reported, 3)
self.assertEqual(A.data["status"], "MERGED")
self.assertEqual(B.data["status"], "MERGED")
self.assertEqual(C.data["status"], "MERGED")
self.assertEqual(D.data["status"], "MERGED")
@simple_layout('layouts/deps-by-topic.yaml')
def test_deps_by_topic_git_needs_outdated_patchset(self):
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', "master", "A",
topic='test-topic')
B = self.fake_gerrit.addFakeChange('org/project1', "master", "B",
topic='other-topic')
C = self.fake_gerrit.addFakeChange('org/project2', "master", "C",
topic='other-topic')
D = self.fake_gerrit.addFakeChange('org/project2', "master", "D",
topic='test-topic')
B.addPatchset()
D.addPatchset()
# Git level dependency between A and B + C and D on outdated
# patchset.
A.setDependsOn(B, 1)
C.setDependsOn(D, 1)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))
self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(2))
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
# This used to run jobs with some very unlikely changes
# merged. That passed the test because we didn't actually get
# a ref for the outdated commits, so when the merger merged
# them, that was a noop. Since we now have correct refs for
# the outdated changes, we hit a merge conflict, which is a
# reasonable and likely error. It's not likely that we can
# make this test run a job with correct data.
# self.assertHistory([
# dict(name="check-job", result="SUCCESS",
# changes="4,2 4,1 3,1 2,2 2,1 1,1"),
# dict(name="check-job", result="SUCCESS",
# changes="4,2 2,2 2,1 1,1 4,1 3,1"),
# ], ordered=False)
self.assertHistory([], ordered=False)
@simple_layout('layouts/deps-by-topic.yaml')
def test_deps_by_topic_new_patchset(self):
# Make sure that we correctly update the change cache on new
# patchsets.
A = self.fake_gerrit.addFakeChange('org/project1', "master", "A",
topic='test-topic')
B = self.fake_gerrit.addFakeChange('org/project2', "master", "B",
topic='test-topic')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(len(A.patchsets[-1]["approvals"]), 1)
self.assertEqual(A.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(A.patchsets[-1]["approvals"][0]["value"], "1")
self.assertEqual(len(B.patchsets[-1]["approvals"]), 1)
self.assertEqual(B.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(B.patchsets[-1]["approvals"][0]["value"], "1")
self.assertHistory([
dict(name="check-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="check-job", result="SUCCESS", changes="2,1 1,1"),
])
A.addPatchset()
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
self.waitUntilSettled()
self.assertHistory([
# Original check run
dict(name="check-job", result="SUCCESS", changes="2,1 1,1"),
dict(name="check-job", result="SUCCESS", changes="2,1 1,1"),
# Second check run
dict(name="check-job", result="SUCCESS", changes="2,1 1,2"),
dict(name="check-job", result="SUCCESS", changes="2,1 1,2"),
], ordered=False)
def test_deps_by_topic_multi_tenant(self):
A = self.fake_gerrit.addFakeChange('org/project5', "master", "A",
topic='test-topic')
B = self.fake_gerrit.addFakeChange('org/project6', "master", "B",
topic='test-topic')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(len(A.patchsets[-1]["approvals"]), 1)
self.assertEqual(A.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(A.patchsets[-1]["approvals"][0]["value"], "1")
self.assertEqual(len(B.patchsets[-1]["approvals"]), 1)
self.assertEqual(B.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(B.patchsets[-1]["approvals"][0]["value"], "1")
# We're about to add approvals to changes without adding the
# triggering events to Zuul, so that we can be sure that it is
# enqueuing the changes based on dependencies, not because of
# triggering events. Since it will have the changes cached
# already (without approvals), we need to clear the cache
# first.
for connection in self.scheds.first.connections.connections.values():
connection.maintainCache([], max_age=0)
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
A.addApproval("Approved", 1)
self.fake_gerrit.addEvent(B.addApproval("Approved", 1))
self.waitUntilSettled()
self.assertEqual(A.reported, 4)
self.assertEqual(B.reported, 4)
self.assertEqual(A.data["status"], "MERGED")
self.assertEqual(B.data["status"], "MERGED")
self.assertHistory([
# Check
dict(name="project5-job-t1", result="SUCCESS", changes="1,1"),
dict(name="project6-job-t1", result="SUCCESS", changes="2,1"),
dict(name="project5-job-t2", result="SUCCESS", changes="2,1 1,1"),
dict(name="project6-job-t2", result="SUCCESS", changes="2,1 1,1"),
# Gate
dict(name="project5-job-t2", result="SUCCESS", changes="1,1 2,1"),
dict(name="project6-job-t2", result="SUCCESS", changes="1,1 2,1"),
], ordered=False)
def test_dependency_refresh(self):
# Test that when two changes are put into a cycle, the
# dependencies are refreshed and items already in pipelines
# are updated.
self.executor_server.hold_jobs_in_build = True
# This simulates the typical workflow where a developer only
# knows the change id of changes one at a time.
# The first change:
A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled("Stage 1")
# Now that it has been uploaded, upload the second change and
# point it at the first.
# B -> A
B = self.fake_gerrit.addFakeChange("org/project", "master", "B")
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled("Stage 2")
# Now that the second change is known, update the first change
# B <-> A
A.addPatchset()
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
self.waitUntilSettled("Stage 3")
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled("Stage 4")
self.assertHistory([
dict(name="project-job", result="ABORTED", changes="1,1"),
dict(name="project-job", result="ABORTED", changes="1,1 2,1"),
dict(name="project-job", result="SUCCESS", changes="2,1 1,2"),
dict(name="project-job", result="SUCCESS", changes="2,1 1,2"),
], ordered=False)
@simple_layout('layouts/deps-by-topic.yaml')
def test_dependency_refresh_by_topic_check(self):
# Test that when two changes are put into a cycle, the
# dependencies are refreshed and items already in pipelines
# are updated.
self.executor_server.hold_jobs_in_build = True
# This simulates the typical workflow where a developer
# uploads changes one at a time.
# The first change:
A = self.fake_gerrit.addFakeChange('org/project1', "master", "A",
topic='test-topic')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
# Now that it has been uploaded, upload the second change
# in the same topic.
B = self.fake_gerrit.addFakeChange('org/project2', "master", "B",
topic='test-topic')
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertHistory([
dict(name="check-job", result="ABORTED", changes="1,1"),
dict(name="check-job", result="SUCCESS", changes="1,1 2,1",
ref="refs/changes/01/1/1"),
dict(name="check-job", result="SUCCESS", changes="1,1 2,1",
ref="refs/changes/02/2/1"),
], ordered=False)
@simple_layout('layouts/deps-by-topic.yaml')
def test_dependency_refresh_by_topic_gate(self):
# Test that when two changes are put into a cycle, the
# dependencies are refreshed and items already in pipelines
# are updated.
self.executor_server.hold_jobs_in_build = True
# This simulates a workflow where a developer adds a change to
# a cycle already in gate.
A = self.fake_gerrit.addFakeChange('org/project1', "master", "A",
topic='test-topic')
B = self.fake_gerrit.addFakeChange('org/project2', "master", "B",
topic='test-topic')
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
A.addApproval("Approved", 1)
self.fake_gerrit.addEvent(B.addApproval("Approved", 1))
self.waitUntilSettled()
# Add a new change to the cycle.
C = self.fake_gerrit.addFakeChange('org/project1', "master", "C",
topic='test-topic')
self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
# At the end of this process, the gate jobs should be aborted
# because the new dpendency showed up.
self.assertEqual(A.data["status"], "NEW")
self.assertEqual(B.data["status"], "NEW")
self.assertEqual(C.data["status"], "NEW")
self.assertHistory([
dict(name="gate-job", result="ABORTED", changes="1,1 2,1",
ref="refs/changes/01/1/1"),
dict(name="gate-job", result="ABORTED", changes="1,1 2,1",
ref="refs/changes/02/2/1"),
dict(name="check-job", result="SUCCESS", changes="2,1 1,1 3,1",
ref="refs/changes/02/2/1"),
dict(name="check-job", result="SUCCESS", changes="2,1 1,1 3,1",
ref="refs/changes/03/3/1"),
# check-job for change 1 is deduplicated into change 3
# because they are the same repo; 3 is the triggering change.
], ordered=False)
def test_dependency_refresh_config_error(self):
# Test that when two changes are put into a cycle, the
# dependencies are refreshed and items already in pipelines
# are updated.
self.executor_server.hold_jobs_in_build = True
# This simulates the typical workflow where a developer only
# knows the change id of changes one at a time.
# The first change:
A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
# Now that it has been uploaded, upload the second change and
# point it at the first.
# B -> A
B = self.fake_gerrit.addFakeChange("org/project", "master", "B")
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
# Enforce a broken project pipeline config. The project stanza
# for org/project references an non-existing project-template
# which triggers the corner case we are looking for in this test.
self.commitConfigUpdate(
'common-config',
'config/circular-dependencies/zuul-reconfiguration-broken.yaml')
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
self.waitUntilSettled()
# Now that the second change is known, update the first change
# B <-> A
A.addPatchset()
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
# Since the tenant configuration got broken, zuul won't attempt to
# re-enqueue those changes.
self.assertHistory([
dict(name="project-job", result="ABORTED", changes="1,1"),
dict(name="project-job", result="ABORTED", changes="1,1 2,1"),
], ordered=False)
def test_circular_dependency_config_error(self):
# Use an error that does not leave a file comment to better
# exercise the change matching function.
use_job = textwrap.dedent(
"""
- foo:
project:
""")
A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
B = self.fake_gerrit.addFakeChange("org/project1", "master", "B",
files={"zuul.yaml": use_job})
# A <-> B (via commit-depends)
A.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
A.subject, B.data["url"]
)
B.data["commitMessage"] = "{}\n\nDepends-On: {}\n".format(
B.subject, A.data["url"]
)
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertHistory([])
self.assertEqual(len(A.patchsets[-1]["approvals"]), 1)
self.assertEqual(A.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(A.patchsets[-1]["approvals"][0]["value"], "-1")
self.assertEqual(len(B.patchsets[-1]["approvals"]), 1)
self.assertEqual(B.patchsets[-1]["approvals"][0]["type"], "Verified")
self.assertEqual(B.patchsets[-1]["approvals"][0]["value"], "-1")
# Only report the error message to B
self.assertNotIn("more than one key", A.messages[-1])
self.assertIn("more than one key", B.messages[-1])
# Indicate that B has an error
self.assertIn("/2 (config error)", A.messages[-1])
self.assertIn("/2 (config error)", B.messages[-1])
@gerrit_config(submit_whole_topic=True)
def test_abandoned_change(self):
# Test that we can re-enqueue a topic cycle after abandoning a
# change (out of band).
A = self.fake_gerrit.addFakeChange('org/project1', "master", "A",
topic='test-topic')
B = self.fake_gerrit.addFakeChange('org/project2', "master", "B",
topic='test-topic',
files={'conflict': 'B'})
X = self.fake_gerrit.addFakeChange('org/project2', "master", "X",
topic='test-topic',
files={'conflict': 'X'})
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
X.addApproval("Code-Review", 2)
X.addApproval("Approved", 1)
B.addApproval("Approved", 1)
self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
self.waitUntilSettled()
X.setAbandoned()
self.waitUntilSettled("abandoned")
self.log.debug("add reapproval")
self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
self.waitUntilSettled("reapproved")
self.assertEqual(A.data["status"], "MERGED")
self.assertEqual(B.data["status"], "MERGED")
self.assertEqual(X.data["status"], "ABANDONED")
@gerrit_config(submit_whole_topic=True)
def test_abandoned_change_refresh_changes(self):
# Test that we can re-enqueue a topic cycle after abandoning a
# change (out of band). This adds extra events to refresh all
# of the changes while the re-enqueue is in progress in order
# to trigger a race condition.
A = self.fake_gerrit.addFakeChange('org/project1', "master", "A",
topic='test-topic')
B = self.fake_gerrit.addFakeChange('org/project2', "master", "B",
topic='test-topic',
files={'conflict': 'B'})
X = self.fake_gerrit.addFakeChange('org/project2', "master", "X",
topic='test-topic',
files={'conflict': 'X'})
A.addApproval("Code-Review", 2)
B.addApproval("Code-Review", 2)
X.addApproval("Code-Review", 2)
X.addApproval("Approved", 1)
B.addApproval("Approved", 1)
self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
self.waitUntilSettled()
orig_forward = zuul.scheduler.Scheduler._forward_trigger_event
stop_event = threading.Event()
go_event = threading.Event()
def patched_forward(obj, *args, **kw):
stop_event.set()
go_event.wait()
return orig_forward(obj, *args, **kw)
self.useFixture(fixtures.MonkeyPatch(
'zuul.scheduler.Scheduler._forward_trigger_event',
patched_forward))
X.setAbandoned()
X.addApproval("Approved", -1)
self.waitUntilSettled("abandoned")
self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
stop_event.wait()
stop_event.clear()
# The scheduler is waiting to forward the approved event; send
# another event that refreshes the cache:
self.fake_gerrit.addEvent(A.getChangeCommentEvent(1, 'testcomment'))
self.fake_gerrit.addEvent(B.getChangeCommentEvent(1, 'testcomment'))
self.fake_gerrit.addEvent(X.getChangeCommentEvent(1, 'testcomment'))
for x in iterate_timeout(30, 'events to be submitted'):
if len(self.scheds.first.sched.trigger_events['tenant-one']) == 4:
break
go_event.set()
stop_event.wait()
self.waitUntilSettled("reapproved")
self.assertEqual(A.data["status"], "MERGED")
self.assertEqual(B.data["status"], "MERGED")
self.assertEqual(X.data["status"], "ABANDONED")
class TestGithubCircularDependencies(ZuulTestCase):
config_file = "zuul-gerrit-github.conf"
tenant_config_file = "config/circular-dependencies/main.yaml"
scheduler_count = 1
def test_cycle_not_ready(self):
# C is missing the approved label
A = self.fake_github.openFakePullRequest("gh/project", "master", "A")
B = self.fake_github.openFakePullRequest("gh/project1", "master", "B")
C = self.fake_github.openFakePullRequest("gh/project1", "master", "C")
A.addReview('derp', 'APPROVED')
B.addReview('derp', 'APPROVED')
B.addLabel("approved")
C.addReview('derp', 'APPROVED')
# A -> B + C (via PR depends)
# B -> A
# C -> A
A.body = "{}\n\nDepends-On: {}\nDepends-On: {}\n".format(
A.subject, B.url, C.url
)
B.body = "{}\n\nDepends-On: {}\n".format(
B.subject, A.url
)
C.body = "{}\n\nDepends-On: {}\n".format(
C.subject, A.url
)
self.fake_github.emitEvent(A.addLabel("approved"))
self.waitUntilSettled()
self.assertEqual(len(A.comments), 0)
self.assertEqual(len(B.comments), 0)
self.assertEqual(len(C.comments), 0)
self.assertFalse(A.is_merged)
self.assertFalse(B.is_merged)
self.assertFalse(C.is_merged)
def test_complex_cycle_not_ready(self):
A = self.fake_github.openFakePullRequest("gh/project", "master", "A")
B = self.fake_github.openFakePullRequest("gh/project1", "master", "B")
C = self.fake_github.openFakePullRequest("gh/project1", "master", "C")
X = self.fake_github.openFakePullRequest("gh/project1", "master", "C")
Y = self.fake_github.openFakePullRequest("gh/project1", "master", "C")
A.addReview('derp', 'APPROVED')
A.addLabel("approved")
B.addReview('derp', 'APPROVED')
B.addLabel("approved")
C.addReview('derp', 'APPROVED')
Y.addReview('derp', 'APPROVED')
Y.addLabel("approved")
X.addReview('derp', 'APPROVED')
# A -> B + C (via PR depends)
# B -> A
# C -> A
# X -> A + Y
# Y -> X
A.body = "{}\n\nDepends-On: {}\nDepends-On: {}\n".format(
A.subject, B.url, C.url
)
B.body = "{}\n\nDepends-On: {}\n".format(
B.subject, A.url
)
C.body = "{}\n\nDepends-On: {}\n".format(
C.subject, A.url
)
X.body = "{}\n\nDepends-On: {}\nDepends-On: {}\n".format(
X.subject, Y.url, A.url
)
Y.body = "{}\n\nDepends-On: {}\n".format(
Y.subject, X.url
)
self.fake_github.emitEvent(X.addLabel("approved"))
self.waitUntilSettled()
self.assertEqual(len(A.comments), 0)
self.assertEqual(len(B.comments), 0)
self.assertEqual(len(C.comments), 0)
self.assertEqual(len(X.comments), 0)
self.assertEqual(len(Y.comments), 0)
self.assertFalse(A.is_merged)
self.assertFalse(B.is_merged)
self.assertFalse(C.is_merged)
self.assertFalse(X.is_merged)
self.assertFalse(Y.is_merged)
def test_filter_unprotected_branches(self):
"""
Tests that repo state filtering due to excluding unprotected branches
doesn't break builds if the items are targeted against different
branches.
"""
github = self.fake_github.getGithubClient()
self.create_branch('gh/project', 'stable/foo')
github.repo_from_project('gh/project')._set_branch_protection(
'master', True)
github.repo_from_project('gh/project')._set_branch_protection(
'stable/foo', True)
pevent = self.fake_github.getPushEvent(project='gh/project',
ref='refs/heads/stable/foo')
self.fake_github.emitEvent(pevent)
self.create_branch('gh/project1', 'stable/bar')
github.repo_from_project('gh/project1')._set_branch_protection(
'master', True)
github.repo_from_project('gh/project1')._set_branch_protection(
'stable/bar', True)
pevent = self.fake_github.getPushEvent(project='gh/project',
ref='refs/heads/stable/bar')
self.fake_github.emitEvent(pevent)
# Wait until push events are processed to pick up branch
# protection settings
self.waitUntilSettled()
A = self.fake_github.openFakePullRequest(
"gh/project", "stable/foo", "A")
B = self.fake_github.openFakePullRequest(
"gh/project1", "stable/bar", "B")
A.addReview('derp', 'APPROVED')
B.addReview('derp', 'APPROVED')
B.addLabel("approved")
# A <-> B
A.body = "{}\n\nDepends-On: {}\n".format(
A.subject, B.url
)
B.body = "{}\n\nDepends-On: {}\n".format(
B.subject, A.url
)
self.fake_github.emitEvent(A.addLabel("approved"))
self.waitUntilSettled()
self.assertEqual(len(A.comments), 2)
self.assertEqual(len(B.comments), 2)
self.assertTrue(A.is_merged)
self.assertTrue(B.is_merged)
def test_cycle_failed_reporting(self):
self.executor_server.hold_jobs_in_build = True
A = self.fake_github.openFakePullRequest("gh/project", "master", "A")
B = self.fake_github.openFakePullRequest("gh/project1", "master", "B")
C = self.fake_github.openFakePullRequest("gh/project1", "master", "B")
A.addReview('derp', 'APPROVED')
B.addReview('derp', 'APPROVED')
C.addReview('derp', 'APPROVED')
B.addLabel("approved")
# A <-> B
A.body = "{}\n\nDepends-On: {}\n".format(
A.subject, B.url
)
B.body = "{}\n\nDepends-On: {}\n".format(
B.subject, A.url
)
self.fake_github.emitEvent(A.addLabel("approved"))
self.waitUntilSettled()
# Put an indepedente change behind this pair to test that it
# gets reset.
self.fake_github.emitEvent(C.addLabel("approved"))
self.waitUntilSettled()
# Change draft status of A so it can no longer merge. Note that we
# don't send an event to test the "github doesn't send an event"
# case.
A.draft = True
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(len(A.comments), 2)
self.assertEqual(len(B.comments), 2)
self.assertFalse(A.is_merged)
self.assertFalse(B.is_merged)
self.assertTrue(C.is_merged)
self.assertIn("failed to merge",
A.comments[-1])
self.assertTrue(
re.search("Change https://github.com/gh/project/pull/1 "
"can not be merged",
A.comments[-1]))
self.assertFalse(re.search('Change .*? is needed',
A.comments[-1]))
self.assertIn("failed to merge",
B.comments[-1])
self.assertTrue(
re.search("Change https://github.com/gh/project/pull/1 "
"can not be merged",
B.comments[-1]))
self.assertFalse(re.search('Change .*? is needed',
B.comments[-1]))
self.assertHistory([
dict(name="project1-job", result="SUCCESS",
changes=f"{B.number},{B.head_sha} {A.number},{A.head_sha}"),
dict(name="project-vars-job", result="SUCCESS",
changes=f"{B.number},{B.head_sha} {A.number},{A.head_sha}"),
dict(name="project-job", result="SUCCESS",
changes=f"{B.number},{B.head_sha} {A.number},{A.head_sha}"),
dict(name="project1-job", result="SUCCESS",
changes=(f"{B.number},{B.head_sha} {A.number},{A.head_sha} "
f"{C.number},{C.head_sha}")),
dict(name="project-vars-job", result="SUCCESS",
changes=(f"{B.number},{B.head_sha} {A.number},{A.head_sha} "
f"{C.number},{C.head_sha}")),
dict(name="project1-job", result="SUCCESS",
changes=f"{C.number},{C.head_sha}"),
dict(name="project-vars-job", result="SUCCESS",
changes=f"{C.number},{C.head_sha}"),
], ordered=False)
def test_dependency_refresh(self):
# Test that when two changes are put into a cycle, the
# dependencies are refreshed and items already in pipelines
# are updated.
self.executor_server.hold_jobs_in_build = True
# This simulates the typical workflow where a developer only
# knows the PR id of changes one at a time.
# The first change:
A = self.fake_github.openFakePullRequest("gh/project", "master", "A")
self.fake_github.emitEvent(A.getPullRequestOpenedEvent())
self.waitUntilSettled("Stage 1")
# Now that it has been uploaded, upload the second change and
# point it at the first.
# B -> A
B = self.fake_github.openFakePullRequest("gh/project", "master", "B")
B.body = "{}\n\nDepends-On: {}\n".format(
B.subject, A.url
)
self.fake_github.emitEvent(B.getPullRequestOpenedEvent())
self.waitUntilSettled("Stage 2")
# Now that the second change is known, update the first change
# B <-> A
A.body = "{}\n\nDepends-On: {}\n".format(
A.subject, B.url
)
self.fake_github.emitEvent(A.getPullRequestEditedEvent(A.subject))
self.waitUntilSettled("Stage 3")
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled("Stage 4")
self.assertHistory([
dict(name="project-job", result="ABORTED",
changes=f"{A.number},{A.head_sha}"),
dict(name="project-job", result="ABORTED",
changes=f"{A.number},{A.head_sha} {B.number},{B.head_sha}"),
dict(name="project-job", result="SUCCESS",
changes=f"{B.number},{B.head_sha} {A.number},{A.head_sha}",
ref="refs/pull/1/head",
),
dict(name="project-job", result="SUCCESS",
changes=f"{B.number},{B.head_sha} {A.number},{A.head_sha}",
ref="refs/pull/2/head",
),
], ordered=False)
class TestGithubAppCircularDependencies(ZuulGithubAppTestCase):
config_file = "zuul-gerrit-github-app.conf"
tenant_config_file = "config/circular-dependencies/main.yaml"
scheduler_count = 1
def test_dependency_refresh_checks_api(self):
# Test that when two changes are put into a cycle, the
# dependencies are refreshed and items already in pipelines
# are updated and that the Github check-run is still
# in-progress.
self.executor_server.hold_jobs_in_build = True
# This simulates the typical workflow where a developer only
# knows the PR id of changes one at a time.
# The first change:
A = self.fake_github.openFakePullRequest("gh/project", "master", "A")
self.fake_github.emitEvent(A.getPullRequestOpenedEvent())
self.waitUntilSettled()
# Now that it has been uploaded, upload the second change and
# point it at the first.
# B -> A
B = self.fake_github.openFakePullRequest("gh/project", "master", "B")
B.body = "{}\n\nDepends-On: {}\n".format(
B.subject, A.url
)
self.fake_github.emitEvent(B.getPullRequestOpenedEvent())
self.waitUntilSettled()
# Now that the second change is known, update the first change
# B <-> A
A.body = "{}\n\nDepends-On: {}\n".format(
A.subject, B.url
)
self.fake_github.emitEvent(A.getPullRequestEditedEvent(A.subject))
self.waitUntilSettled()
# Validate that the Github check-run was cancelled and a new
# one restarted.
check_runs = self.fake_github.getCommitChecks("gh/project", A.head_sha)
self.assertEqual(len(check_runs), 2)
self.assertEqual(check_runs[0]["status"], "in_progress")
self.assertEqual(check_runs[1]["status"], "completed")
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertHistory([
dict(name="project-job", result="ABORTED",
changes=f"{A.number},{A.head_sha}"),
dict(name="project-job", result="ABORTED",
changes=f"{A.number},{A.head_sha} {B.number},{B.head_sha}"),
dict(name="project-job", result="SUCCESS",
changes=f"{B.number},{B.head_sha} {A.number},{A.head_sha}",
ref="refs/pull/1/head"),
dict(name="project-job", result="SUCCESS",
changes=f"{B.number},{B.head_sha} {A.number},{A.head_sha}",
ref="refs/pull/2/head"),
], ordered=False)
# Verify that since we are issuing two check runs, they both
# complete.
check_runs = self.fake_github.getCommitChecks("gh/project", A.head_sha)
self.assertEqual(len(check_runs), 2)
check_run = check_runs[0]
self.assertEqual(check_run["status"], "completed")
check_run = check_runs[1]
self.assertEqual(check_run["status"], "completed")
self.assertNotEqual(check_runs[0]["id"], check_runs[1]["id"])
check_runs = self.fake_github.getCommitChecks("gh/project", B.head_sha)
self.assertEqual(len(check_runs), 2)
check_run = check_runs[0]
self.assertEqual(check_run["status"], "completed")
check_run = check_runs[1]
self.assertEqual(check_run["status"], "completed")
self.assertNotEqual(check_runs[0]["id"], check_runs[1]["id"])
@simple_layout('layouts/dependency_removal_gate.yaml', driver='github')
def test_dependency_removal_gate(self):
# Dependency cycles can be updated without uploading new
# patchsets. This test exercises such changes.
self.executor_server.hold_jobs_in_build = True
tenant = self.scheds.first.sched.abide.tenants.get("tenant-one")
pipeline = tenant.layout.pipelines["gate"]
A = self.fake_github.openFakePullRequest("gh/project1", "master", "A")
B = self.fake_github.openFakePullRequest("gh/project2", "master", "B")
C = self.fake_github.openFakePullRequest("gh/project3", "master", "C")
D = self.fake_github.openFakePullRequest("gh/project", "master", "D")
E = self.fake_github.openFakePullRequest("gh/project", "master", "E")
# Because we call setConfiguration immediately upon removing a
# change, in order to fully exercise setConfiguration on a
# queue that has been reloaded from ZK with potentially stale
# bundle data, we need to remove a change from the bundle
# twice. ABC is the main bundle we're interested in. E is a
# sacrificial change to force a bundle update when it's
# removed.
# E->C, A->E, B->A, C->B
E.body = "{}\n\nDepends-On: {}\n".format(
E.subject, C.url
)
A.body = "{}\n\nDepends-On: {}\n".format(
A.subject, E.url
)
B.body = "{}\n\nDepends-On: {}\n".format(
B.subject, A.url
)
C.body = "{}\n\nDepends-On: {}\n".format(
C.subject, B.url
)
A.addLabel("approved")
B.addLabel("approved")
D.addLabel("approved")
E.addLabel("approved")
self.fake_github.emitEvent(C.addLabel("approved"))
self.waitUntilSettled()
expected_cycle = {A.number, B.number, C.number, E.number}
self.assertEqual(len(list(pipeline.getAllItems())), 1)
for item in pipeline.getAllItems():
cycle = {c.number for c in item.changes}
self.assertEqual(expected_cycle, cycle)
# Assert we get the same triggering change every time
self.assertEqual(json.loads(item.event.ref)['stable_id'], '3')
# Now we remove the dependency on E. This re-enqueues ABC and E.
# A->C, B->A, C->B
A.body = "{}\n\nDepends-On: {}\n".format(
A.subject, C.url
)
self.fake_github.emitEvent(A.getPullRequestEditedEvent(A.body))
self.waitUntilSettled()
self.assertEqual(len(list(pipeline.getAllItems())), 2)
# Now remove all dependencies for the three remaining changes,
# and also E so that it doesn't get pulled back in as an
# approved change behind C.
E.body = E.subject
A.body = A.subject
B.body = B.subject
C.body = C.subject
self.fake_github.emitEvent(A.getPullRequestEditedEvent(A.body))
self.waitUntilSettled()
self.fake_github.emitEvent(B.getPullRequestEditedEvent(B.body))
self.waitUntilSettled()
self.fake_github.emitEvent(C.getPullRequestEditedEvent(C.body))
self.waitUntilSettled()
self.assertEqual(len(list(pipeline.getAllItems())), 3)
for item in pipeline.getAllItems():
self.assertEqual(len(item.changes), 1)
# Assert we get the same triggering change every time
self.assertEqual(json.loads(item.event.ref)['stable_id'], '3')
# Remove the first change from the queue by forcing a
# dependency on D.
A.body = "{}\n\nDepends-On: {}\n".format(
A.subject, D.url
)
self.fake_github.emitEvent(A.getPullRequestEditedEvent(A.body))
self.waitUntilSettled()
self.assertEqual(len(list(pipeline.getAllItems())), 2)
for item in pipeline.getAllItems():
self.assertEqual(len(item.changes), 1)
# Assert we get the same triggering change every time
self.assertEqual(json.loads(item.event.ref)['stable_id'], '3')
# B and C are still in the queue. Put them
# back into a bundle.
C.body = "{}\n\nDepends-On: {}\n".format(
C.subject, B.url
)
self.fake_github.emitEvent(C.getPullRequestEditedEvent(C.body))
self.waitUntilSettled()
B.body = "{}\n\nDepends-On: {}\n".format(
B.subject, C.url
)
self.fake_github.emitEvent(B.getPullRequestEditedEvent(B.body))
self.waitUntilSettled()
expected_cycle = {B.number, C.number}
self.assertEqual(len(list(pipeline.getAllItems())), 1)
for item in pipeline.getAllItems():
cycle = {c.number for c in item.changes}
self.assertEqual(expected_cycle, cycle)
# Assert we get the same triggering change every time
self.assertEqual(json.loads(item.event.ref)['stable_id'], '3')
# All done.
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
for build in self.history[:-3]:
self.assertEqual(build.result, 'ABORTED')
# Changes B, C in the end
for build in self.history[-3:]:
self.assertEqual(build.result, 'SUCCESS')
@simple_layout('layouts/dependency_removal_gate.yaml', driver='github')
def test_dependency_addition_gate(self):
# Dependency cycles can be updated without uploading new
# patchsets. This test exercises such changes.
self.executor_server.hold_jobs_in_build = True
tenant = self.scheds.first.sched.abide.tenants.get("tenant-one")
pipeline = tenant.layout.pipelines["gate"]
A = self.fake_github.openFakePullRequest("gh/project1", "master", "A")
B = self.fake_github.openFakePullRequest("gh/project2", "master", "B")
B.addLabel("approved")
self.fake_github.emitEvent(A.addLabel("approved"))
self.waitUntilSettled()
self.assertEqual(len(list(pipeline.getAllItems())), 1)
for item in pipeline.getAllItems():
self.assertEqual(len(item.changes), 1)
# Assert we get the same triggering change every time
self.assertEqual(json.loads(item.event.ref)['stable_id'], '1')
B.body = "{}\n\nDepends-On: {}\n".format(
B.subject, A.url
)
self.fake_github.emitEvent(B.getPullRequestEditedEvent(B.body))
self.waitUntilSettled()
A.body = "{}\n\nDepends-On: {}\n".format(
A.subject, B.url
)
self.fake_github.emitEvent(A.getPullRequestEditedEvent(A.body))
self.waitUntilSettled()
self.assertEqual(len(list(pipeline.getAllItems())), 1)
expected_cycle = {A.number, B.number}
for item in pipeline.getAllItems():
cycle = {c.number for c in item.changes}
self.assertEqual(expected_cycle, cycle)
# Assert we get the same triggering change every time
self.assertEqual(json.loads(item.event.ref)['stable_id'], '1')
# All done.
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
for build in self.history[:-3]:
self.assertEqual(build.result, 'ABORTED')
# Changes A, B in the end
for build in self.history[-3:]:
self.assertEqual(build.result, 'SUCCESS')
def assertQueueCycles(self, pipeline, queue_index, bundles):
queue = pipeline.queues[queue_index]
self.assertEqual(len(queue.queue), len(bundles))
for x, item in enumerate(queue.queue):
cycle = {c.number for c in item.changes}
expected_cycle = {c.number for c in bundles[x]}
self.assertEqual(expected_cycle, cycle)
@simple_layout('layouts/dependency_removal_check.yaml', driver='github')
def test_dependency_removal_check(self):
# Dependency cycles can be updated without uploading new
# patchsets. This test exercises such changes.
self.executor_server.hold_jobs_in_build = True
tenant = self.scheds.first.sched.abide.tenants.get("tenant-one")
pipeline = tenant.layout.pipelines["check"]
A = self.fake_github.openFakePullRequest("gh/project1", "master", "A")
B = self.fake_github.openFakePullRequest("gh/project2", "master", "B")
C = self.fake_github.openFakePullRequest("gh/project3", "master", "C")
E = self.fake_github.openFakePullRequest("gh/project", "master", "E")
# Because we call setConfiguration immediately upon removing a
# change, in order to fully exercise setConfiguration on a
# queue that has been reloaded from ZK with potentially stale
# bundle data, we need to remove a change from the bundle
# twice. ABC is the main bundle we're interested in. E is a
# sacrificial change to force a bundle update when it's
# removed.
# E->C, A->E, B->A, C->B
E.body = "{}\n\nDepends-On: {}\n".format(
E.subject, C.url
)
A.body = "{}\n\nDepends-On: {}\n".format(
A.subject, E.url
)
B.body = "{}\n\nDepends-On: {}\n".format(
B.subject, A.url
)
C.body = "{}\n\nDepends-On: {}\n".format(
C.subject, B.url
)
self.fake_github.emitEvent(C.getPullRequestOpenedEvent())
self.waitUntilSettled()
abce = [A, B, C, E]
self.assertEqual(len(pipeline.queues), 1)
self.assertQueueCycles(pipeline, 0, [abce])
for item in pipeline.getAllItems():
# Assert we get the same triggering change every time
self.assertEqual(json.loads(item.event.ref)['stable_id'], '3')
# Now we remove the dependency on E.
# A->C, B->A, C->B
A.body = "{}\n\nDepends-On: {}\n".format(
A.subject, C.url
)
self.fake_github.emitEvent(A.getPullRequestEditedEvent(A.body))
self.waitUntilSettled()
abc = [A, B, C]
# ABC, E
self.assertQueueCycles(pipeline, 0, [abc, [E]])
for item in pipeline.getAllItems():
# Assert we get the same triggering change every time
self.assertEqual(json.loads(item.event.ref)['stable_id'], '3')
# Now remove all dependencies for the three remaining changes.
A.body = A.subject
B.body = B.subject
C.body = C.subject
self.fake_github.emitEvent(A.getPullRequestEditedEvent(A.body))
self.waitUntilSettled()
self.fake_github.emitEvent(B.getPullRequestEditedEvent(B.body))
self.waitUntilSettled()
self.fake_github.emitEvent(C.getPullRequestEditedEvent(C.body))
self.waitUntilSettled()
# A, B, C individually
self.assertEqual(len(pipeline.queues), 3)
self.assertQueueCycles(pipeline, 0, [[A], [B]])
self.assertQueueCycles(pipeline, 1, [[A], [B], [C]])
self.assertQueueCycles(pipeline, 2, [[A]])
for item in pipeline.getAllItems():
# Assert we get the same triggering change every time
self.assertEqual(json.loads(item.event.ref)['stable_id'], '3')
# Verify that we can put B and C into a bundle.
C.body = "{}\n\nDepends-On: {}\n".format(
C.subject, B.url
)
self.fake_github.emitEvent(C.getPullRequestEditedEvent(C.body))
self.waitUntilSettled()
for item in pipeline.getAllItems():
# Assert we get the same triggering change every time
self.assertEqual(json.loads(item.event.ref)['stable_id'], '3')
B.body = "{}\n\nDepends-On: {}\n".format(
B.subject, C.url
)
self.fake_github.emitEvent(B.getPullRequestEditedEvent(B.body))
self.waitUntilSettled()
for item in pipeline.getAllItems():
# Assert we get the same triggering change every time
self.assertEqual(json.loads(item.event.ref)['stable_id'], '3')
self.assertEqual(len(pipeline.queues), 2)
bc = [B, C]
self.assertQueueCycles(pipeline, 0, [[A]])
self.assertQueueCycles(pipeline, 1, [bc])
# All done.
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
for build in self.history[:-5]:
self.assertEqual(build.result, 'ABORTED')
# Changes A, B, C in the end
for build in self.history[-5:]:
self.assertEqual(build.result, 'SUCCESS')
@simple_layout('layouts/dependency_removal_check.yaml', driver='github')
def test_dependency_addition_check(self):
# Dependency cycles can be updated without uploading new
# patchsets. This test exercises such changes.
self.executor_server.hold_jobs_in_build = True
tenant = self.scheds.first.sched.abide.tenants.get("tenant-one")
pipeline = tenant.layout.pipelines["check"]
A = self.fake_github.openFakePullRequest("gh/project1", "master", "A")
B = self.fake_github.openFakePullRequest("gh/project2", "master", "B")
self.fake_github.emitEvent(A.getPullRequestOpenedEvent())
self.waitUntilSettled()
self.assertEqual(len(list(pipeline.getAllItems())), 1)
for item in pipeline.getAllItems():
self.assertEqual(len(item.changes), 1)
# Assert we get the same triggering change every time
self.assertEqual(json.loads(item.event.ref)['stable_id'], '1')
B.body = "{}\n\nDepends-On: {}\n".format(
B.subject, A.url
)
self.fake_github.emitEvent(B.getPullRequestEditedEvent(B.body))
self.waitUntilSettled()
A.body = "{}\n\nDepends-On: {}\n".format(
A.subject, B.url
)
self.fake_github.emitEvent(A.getPullRequestEditedEvent(A.body))
self.waitUntilSettled()
self.assertEqual(len(list(pipeline.getAllItems())), 1)
for item in pipeline.getAllItems():
self.assertEqual(len(item.changes), 2)
# Assert we get the same triggering change every time
self.assertEqual(json.loads(item.event.ref)['stable_id'], '1')
self.assertEqual(len(pipeline.queues), 1)
ab = [A, B]
self.assertQueueCycles(pipeline, 0, [ab])
# All done.
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
for build in self.history[:-4]:
self.assertEqual(build.result, 'ABORTED')
# A single change in the end
for build in self.history[-2:]:
self.assertEqual(build.result, 'SUCCESS')