zuul/zuul
Code Issues Proposed changes

Dependencies from undefined projects

Browse Source 
3rd party CI layout usually has only a few projects defined,
so it's possible that some changes depend on projects
which are unknown to Zuul scheduler.
These items had None as a "item.change.project", which
is not handled in many places, for ex. in reconfiguration.

These cases could be handled by defining these projects in layout
as "foreign" projects: no jobs, no other non-standard attributes.
Changes to those projects are also dropped, unless
they came as dependencies.

Change-Id: I7912197fb86c1a7becb7f43ca36078101f632715
This commit is contained in:
Evgeny Antyshev 2015-08-03 20:22:56 +00:00
parent e8aa3c6842
commit 0deaaadac7
5 changed files with 156 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
tests/fixtures/layout-live-reconfiguration-del-project.yaml vendored Normal file
View File

@ -0,0 +1,21 @@
pipelines:
- name: check
manager: IndependentPipelineManager
trigger:
gerrit:
- event: patchset-created
success:
gerrit:
verified: 1
failure:
gerrit:
verified: -1
projects:
- name: org/project
merge-mode: cherry-pick
check:
- project-merge:
- project-test1
- project-test2
- project-testfile

110
tests/test_scheduler.py
View File

@ -2494,6 +2494,57 @@ class TestScheduler(ZuulTestCase):
# Ensure the removed job was not included in the report.
self.assertNotIn('project1-project2-integration', A.messages[0])
def test_live_reconfiguration_del_project(self):
# Test project deletion from layout
# while changes are enqueued
self.worker.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project1', 'master', 'C')
# A Depends-On: B
A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
A.subject, B.data['id'])
self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.worker.release('.*-merge')
self.waitUntilSettled()
self.assertEqual(len(self.builds), 5)
# This layout defines only org/project, not org/project1
self.config.set('zuul', 'layout_config',
'tests/fixtures/layout-live-'
'reconfiguration-del-project.yaml')
self.sched.reconfigure(self.config)
self.waitUntilSettled()
# Builds for C aborted, builds for A succeed,
# and have change B applied ahead
job_c = self.getJobFromHistory('project1-test1')
self.assertEqual(job_c.changes, '3,1')
self.assertEqual(job_c.result, 'ABORTED')
self.worker.hold_jobs_in_build = False
self.worker.release()
self.waitUntilSettled()
self.assertEqual(self.getJobFromHistory('project-test1').changes,
'2,1 1,1')
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
self.assertEqual(C.data['status'], 'NEW')
self.assertEqual(A.reported, 1)
self.assertEqual(B.reported, 0)
self.assertEqual(C.reported, 0)
self.assertEqual(len(self.sched.layout.pipelines['check'].queues), 0)
self.assertIn('Build succeeded', A.messages[0])
def test_live_reconfiguration_functions(self):
"Test live reconfiguration with a custom function"
self.worker.registerFunction('build:node-project-test1:debian')
@ -3668,6 +3719,48 @@ For CI problems and help debugging, contact ci@example.org"""
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
def test_crd_gate_unknown(self):
"Test unknown projects in dependent pipeline"
self.init_repo("org/unknown")
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/unknown', 'master', 'B')
A.addApproval('CRVW', 2)
B.addApproval('CRVW', 2)
# A Depends-On: B
A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
A.subject, B.data['id'])
B.addApproval('APRV', 1)
self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
self.waitUntilSettled()
# Unknown projects cannot share a queue with any other
# since they don't have common jobs with any other (they have no jobs).
# Changes which depend on unknown project changes
# should not be processed in dependent pipeline
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
self.assertEqual(A.reported, 0)
self.assertEqual(B.reported, 0)
self.assertEqual(len(self.history), 0)
# Simulate change B being gated outside this layout
self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
B.setMerged()
self.waitUntilSettled()
self.assertEqual(len(self.history), 0)
# Now that B is merged, A should be able to be enqueued and
# merged.
self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(B.reported, 0)
def test_crd_check(self):
"Test cross-repo dependencies in independent pipelines"
@ -3782,12 +3875,12 @@ For CI problems and help debugging, contact ci@example.org"""
self.assertIn('Build succeeded', A.messages[0])
self.assertIn('Build succeeded', B.messages[0])
def test_crd_check_reconfiguration(self):
def _test_crd_check_reconfiguration(self, project1, project2):
"Test cross-repo dependencies re-enqueued in independent pipelines"
self.gearman_server.hold_jobs_in_queue = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
A = self.fake_gerrit.addFakeChange(project1, 'master', 'A')
B = self.fake_gerrit.addFakeChange(project2, 'master', 'B')
# A Depends-On: B
A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
@ -3820,6 +3913,17 @@ For CI problems and help debugging, contact ci@example.org"""
self.assertEqual(self.history[0].changes, '2,1 1,1')
self.assertEqual(len(self.sched.layout.pipelines['check'].queues), 0)
def test_crd_check_reconfiguration(self):
self._test_crd_check_reconfiguration('org/project1', 'org/project2')
def test_crd_undefined_project(self):
"""Test that undefined projects in dependencies are handled for
independent pipelines"""
# It's a hack for fake gerrit,
# as it implies repo creation upon the creation of any change
self.init_repo("org/unknown")
self._test_crd_check_reconfiguration('org/project1', 'org/unknown')
def test_crd_check_ignore_dependencies(self):
"Test cross-repo dependencies can be ignored"
self.config.set('zuul', 'layout_config',

6
zuul/model.py
View File

@ -431,9 +431,13 @@ class ChangeQueue(object):
class Project(object):
def __init__(self, name):
def __init__(self, name, foreign=False):
self.name = name
self.merge_mode = MERGER_MERGE_RESOLVE
# foreign projects are those referenced in dependencies
# of layout projects, this should matter
# when deciding whether to enqueue their changes
self.foreign = foreign
def __str__(self):
return self.name

29
zuul/scheduler.py
View File

@ -507,11 +507,15 @@ class Scheduler(threading.Thread):
name = reporter.name
self.reporters[name] = reporter
def getProject(self, name):
def getProject(self, name, create_foreign=False):
self.layout_lock.acquire()
p = None
try:
p = self.layout.projects.get(name)
if p is None and create_foreign:
self.log.info("Registering foreign project: %s" % name)
p = Project(name, foreign=True)
self.layout.projects[name] = p
finally:
self.layout_lock.release()
return p
@ -685,15 +689,15 @@ class Scheduler(threading.Thread):
item.items_behind = []
item.pipeline = None
item.queue = None
project = layout.projects.get(item.change.project.name)
if not project:
self.log.warning("Unable to find project for "
"change %s while reenqueueing" %
item.change)
item.change.project = None
items_to_remove.append(item)
continue
item.change.project = project
project_name = item.change.project.name
item.change.project = layout.projects.get(project_name)
if not item.change.project:
self.log.debug("Project %s not defined, "
"re-instantiating as foreign" %
project_name)
project = Project(project_name, foreign=True)
layout.projects[project_name] = project
item.change.project = project
item_jobs = new_pipeline.getJobs(item)
for build in item.current_build_set.getBuilds():
job = layout.jobs.get(build.job.name)
@ -861,7 +865,7 @@ class Scheduler(threading.Thread):
self.log.debug("Processing trigger event %s" % event)
try:
project = self.layout.projects.get(event.project_name)
if not project:
if not project or project.foreign:
self.log.debug("Project %s not found" % event.project_name)
return
@ -1797,10 +1801,11 @@ class IndependentPipelineManager(BasePipelineManager):
if existing:
return DynamicChangeQueueContextManager(existing)
if change.project not in self.pipeline.getProjects():
return DynamicChangeQueueContextManager(None)
self.pipeline.addProject(change.project)
change_queue = ChangeQueue(self.pipeline)
change_queue.addProject(change.project)
self.pipeline.addQueue(change_queue)
self.log.debug("Dynamically created queue %s", change_queue)
return DynamicChangeQueueContextManager(change_queue)
def enqueueChangesAhead(self, change, quiet, ignore_requirements,

8
zuul/trigger/gerrit.py
View File

@ -94,7 +94,7 @@ class GerritEventConnector(threading.Thread):
Can not get account information." % event.type)
event.account = None
if event.change_number:
if event.change_number and self.sched.getProject(event.project_name):
# Call _getChange for the side effect of updating the
# cache. Note that this modifies Change objects outside
# the main thread.
@ -404,7 +404,11 @@ class Gerrit(object):
if 'project' not in data:
raise Exception("Change %s,%s not found" % (change.number,
change.patchset))
change.project = self.sched.getProject(data['project'])
# If updated changed came as a dependent on
# and its project is not defined,
# then create a 'foreign' project for it in layout
change.project = self.sched.getProject(data['project'],
create_foreign=bool(history))
change.branch = data['branch']
change.url = data['url']
max_ps = 0