Support dynamic dependent pipeline additions

Previously one could not speculatively add a project to a dependent
pipeline if the desired queue did not exist. This change handles this
special case by creating a dynamic queue for the use of the speculative
change. If the change merges, then the queue will exist for any future
change.

Introduces a new ChangeQueue attribute, 'dynamic', to help
DependentPipelineManager objects determine whether a ChangeQueue was
created dynamically or not, during a clean up phase. Only used in
DependentPipelineManager pipelines.

Note that this doesn't necessarily support a newly added named queue, or
adding a project to an existing named queue. That will be follow up
work.

Change-Id: I51ab7fb113f1bcbcef5f4f6c96ca046d0c76fdd9
Story: 2000898
Task: 3528
This commit is contained in:
Jesse Keating 2017-07-13 14:27:40 -07:00
parent 6e9357d8d3
commit 78f544a813
4 changed files with 91 additions and 3 deletions

View File

@ -32,6 +32,26 @@
verified: 0
precedence: high
- pipeline:
name: gate
manager: dependent
trigger:
gerrit:
- event: comment-added
approval:
- code-review: 2
success:
gerrit:
verified: 2
submit: true
failure:
gerrit:
verified: -2
start:
gerrit:
verified: 0
precedence: high
- job:
name: common-config-test

View File

@ -199,6 +199,52 @@ class TestInRepoConfig(ZuulTestCase):
self.executor_server.release()
self.waitUntilSettled()
def test_dynamic_dependent_pipeline(self):
# Test dynamically adding a project to a
# dependent pipeline for the first time
self.executor_server.hold_jobs_in_build = True
tenant = self.sched.abide.tenants.get('tenant-one')
gate_pipeline = tenant.layout.pipelines['gate']
in_repo_conf = textwrap.dedent(
"""
- job:
name: project-test2
- project:
name: org/project
gate:
jobs:
- project-test2
""")
in_repo_playbook = textwrap.dedent(
"""
- hosts: all
tasks: []
""")
file_dict = {'.zuul.yaml': in_repo_conf,
'playbooks/project-test2.yaml': in_repo_playbook}
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
files=file_dict)
A.addApproval('approved', 1)
self.fake_gerrit.addEvent(A.addApproval('code-review', 2))
self.waitUntilSettled()
items = gate_pipeline.getAllItems()
self.assertEqual(items[0].change.number, '1')
self.assertEqual(items[0].change.patchset, '1')
self.assertTrue(items[0].live)
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
# Make sure the dynamic queue got cleaned up
self.assertEqual(gate_pipeline.queues, [])
def test_in_repo_branch(self):
in_repo_conf = textwrap.dedent(
"""

View File

@ -14,6 +14,7 @@ import logging
from zuul import model
from zuul.manager import PipelineManager, StaticChangeQueueContextManager
from zuul.manager import DynamicChangeQueueContextManager
class DependentPipelineManager(PipelineManager):
@ -75,8 +76,17 @@ class DependentPipelineManager(PipelineManager):
def getChangeQueue(self, change, existing=None):
if existing:
return StaticChangeQueueContextManager(existing)
return StaticChangeQueueContextManager(
self.pipeline.getQueue(change.project))
queue = self.pipeline.getQueue(change.project)
if queue:
return StaticChangeQueueContextManager(queue)
else:
# There is no existing queue for this change. Create a
# dynamic one for this one change's use
change_queue = model.ChangeQueue(self.pipeline, dynamic=True)
change_queue.addProject(change.project)
self.pipeline.addQueue(change_queue)
self.log.debug("Dynamically created queue %s", change_queue)
return DynamicChangeQueueContextManager(change_queue)
def isChangeReadyToBeEnqueued(self, change):
source = change.project.source
@ -201,3 +211,11 @@ class DependentPipelineManager(PipelineManager):
if failing_items:
return failing_items
return None
def dequeueItem(self, item):
super(DependentPipelineManager, self).dequeueItem(item)
# If this was a dynamic queue from a speculative change,
# remove the queue (if empty)
if item.queue.dynamic:
if not item.queue.queue:
self.pipeline.removeQueue(item.queue)

View File

@ -208,11 +208,14 @@ class ChangeQueue(object):
be processed. If a Change succeeds, the Window is increased by
`window_increase_factor`. If a Change fails, the Window is decreased by
`window_decrease_factor`.
A ChangeQueue may be a dynamically created queue, which may be removed
from a DependentPipelineManager once empty.
"""
def __init__(self, pipeline, window=0, window_floor=1,
window_increase_type='linear', window_increase_factor=1,
window_decrease_type='exponential', window_decrease_factor=2,
name=None):
name=None, dynamic=False):
self.pipeline = pipeline
if name:
self.name = name
@ -227,6 +230,7 @@ class ChangeQueue(object):
self.window_increase_factor = window_increase_factor
self.window_decrease_type = window_decrease_type
self.window_decrease_factor = window_decrease_factor
self.dynamic = dynamic
def __repr__(self):
return '<ChangeQueue %s: %s>' % (self.pipeline.name, self.name)