Merge "Move queue from pipeline to project"
This commit is contained in:
commit
87e5529ef7
|
@ -148,28 +148,9 @@ pipeline.
|
|||
all pipelines of this project. For more information see
|
||||
:ref:`variable inheritance <user_jobs_variable_inheritance>`.
|
||||
|
||||
.. attr:: <pipeline>
|
||||
|
||||
Each pipeline that the project participates in should have an
|
||||
entry in the project. The value for this key should be a
|
||||
dictionary with the following format:
|
||||
|
||||
.. attr:: jobs
|
||||
:required:
|
||||
|
||||
A list of jobs that should be run when items for this project
|
||||
are enqueued into the pipeline. Each item of this list may
|
||||
be a string, in which case it is treated as a job name, or it
|
||||
may be a dictionary, in which case it is treated as a job
|
||||
variant local to this project and pipeline. In that case,
|
||||
the format of the dictionary is the same as the top level
|
||||
:attr:`job` definition. Any attributes set on the job here
|
||||
will override previous versions of the job.
|
||||
|
||||
.. attr:: queue
|
||||
|
||||
If this pipeline is a :value:`dependent
|
||||
<pipeline.manager.dependent>` pipeline, this specifies the
|
||||
This specifies the
|
||||
name of the shared queue this project is in. Any projects
|
||||
which interact with each other in tests should be part of the
|
||||
same shared queue in order to ensure that they don't merge
|
||||
|
@ -192,6 +173,34 @@ pipeline.
|
|||
.. note:: This attribute is not evaluated speculatively and
|
||||
its setting shall be merged to be effective.
|
||||
|
||||
.. attr:: <pipeline>
|
||||
|
||||
Each pipeline that the project participates in should have an
|
||||
entry in the project. The value for this key should be a
|
||||
dictionary with the following format:
|
||||
|
||||
.. attr:: jobs
|
||||
:required:
|
||||
|
||||
A list of jobs that should be run when items for this project
|
||||
are enqueued into the pipeline. Each item of this list may
|
||||
be a string, in which case it is treated as a job name, or it
|
||||
may be a dictionary, in which case it is treated as a job
|
||||
variant local to this project and pipeline. In that case,
|
||||
the format of the dictionary is the same as the top level
|
||||
:attr:`job` definition. Any attributes set on the job here
|
||||
will override previous versions of the job.
|
||||
|
||||
.. attr:: queue
|
||||
|
||||
This is the same as :attr:`project.queue` but on per pipeline
|
||||
level for backwards compatibility reasons. If :attr:`project.queue`
|
||||
is defined this setting is ignored.
|
||||
|
||||
.. note:: It is deprecated to define the queue in the pipeline
|
||||
configuration. Configure it on :attr:`project.queue`
|
||||
instead.
|
||||
|
||||
.. attr:: debug
|
||||
|
||||
If this is set to `true`, Zuul will include debugging
|
||||
|
|
|
@ -5,7 +5,7 @@ Queue
|
|||
|
||||
Projects that interact with each other should share a ``queue``.
|
||||
This is especially used in a :value:`dependent <pipeline.manager.dependent>`
|
||||
pipeline. The :attr:`project.<pipeline>.queue` can optionally refer
|
||||
pipeline. The :attr:`project.queue` can optionally refer
|
||||
to a specific :attr:`queue` object that can further configure the
|
||||
behavior of the queue.
|
||||
|
||||
|
|
|
@ -0,0 +1,6 @@
|
|||
---
|
||||
deprecations:
|
||||
- |
|
||||
Shared ``queues`` should be configured per project now instead per
|
||||
pipeline. Specifying :attr:`project.<pipeline>.queue` is deprecated
|
||||
and will be removed in a future release.
|
|
@ -33,3 +33,10 @@
|
|||
queue: integrated
|
||||
jobs:
|
||||
- project-test
|
||||
|
||||
- project:
|
||||
name: org/project4
|
||||
queue: integrated
|
||||
gate:
|
||||
jobs:
|
||||
- project-test
|
||||
|
|
|
@ -0,0 +1 @@
|
|||
test
|
|
@ -8,3 +8,4 @@
|
|||
- org/project
|
||||
- org/project2
|
||||
- org/project3
|
||||
- org/project4
|
||||
|
|
|
@ -137,6 +137,7 @@
|
|||
|
||||
- project:
|
||||
name: org/project1
|
||||
queue: integrated
|
||||
check:
|
||||
jobs:
|
||||
- project-merge
|
||||
|
@ -147,7 +148,8 @@
|
|||
- project1-project2-integration:
|
||||
dependencies: project-merge
|
||||
gate:
|
||||
queue: integrated
|
||||
# This will be overridden on project level
|
||||
queue: integrated-overridden
|
||||
jobs:
|
||||
- project-merge
|
||||
- project-test1:
|
||||
|
|
|
@ -0,0 +1,27 @@
|
|||
- pipeline:
|
||||
name: gate
|
||||
manager: dependent
|
||||
trigger: {}
|
||||
|
||||
- job:
|
||||
name: base
|
||||
parent: null
|
||||
run: playbooks/base.yaml
|
||||
|
||||
- project-template:
|
||||
name: integrated-jobs
|
||||
gate:
|
||||
jobs:
|
||||
- base
|
||||
|
||||
- project:
|
||||
name: org/project1
|
||||
queue: integrated
|
||||
templates:
|
||||
- integrated-jobs
|
||||
|
||||
- project:
|
||||
name: org/project2
|
||||
queue: integrated
|
||||
templates:
|
||||
- integrated-jobs
|
|
@ -3203,6 +3203,27 @@ class TestScheduler(ZuulTestCase):
|
|||
self.assertEqual(q1.name, 'integrated')
|
||||
self.assertEqual(q2.name, 'integrated')
|
||||
|
||||
@simple_layout("layouts/template-project-queue.yaml")
|
||||
def test_template_project_queue(self):
|
||||
"Test a shared queue can be constructed from a project-template"
|
||||
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
|
||||
(trusted, project1) = tenant.getProject('org/project1')
|
||||
(trusted, project2) = tenant.getProject('org/project2')
|
||||
|
||||
# Change queues are created lazy by the dependent pipeline manager
|
||||
# so retrieve the queue first without having to really enqueue a
|
||||
# change first.
|
||||
gate = tenant.layout.pipelines['gate']
|
||||
FakeChange = namedtuple('FakeChange', ['project', 'branch'])
|
||||
fake_a = FakeChange(project1, 'master')
|
||||
fake_b = FakeChange(project2, 'master')
|
||||
gate.manager.getChangeQueue(fake_a, None)
|
||||
gate.manager.getChangeQueue(fake_b, None)
|
||||
q1 = gate.getQueue(project1, None)
|
||||
q2 = gate.getQueue(project2, None)
|
||||
self.assertEqual(q1.name, 'integrated')
|
||||
self.assertEqual(q2.name, 'integrated')
|
||||
|
||||
@simple_layout("layouts/regex-template-queue.yaml")
|
||||
def test_regex_template_queue(self):
|
||||
"Test a shared queue can be constructed from a regex project-template"
|
||||
|
@ -6461,6 +6482,15 @@ class TestChangeQueues(ZuulTestCase):
|
|||
'org/project3', queue_name='integrated-untrusted',
|
||||
queue_repo='org/project3')
|
||||
|
||||
def test_dependent_queues_per_branch_project_queue(self):
|
||||
"""
|
||||
Test that change queues can be different for different branches.
|
||||
|
||||
In this case we create changes for two branches in a repo that
|
||||
references the queue on project level instead of pipeline level.
|
||||
"""
|
||||
self._test_dependent_queues_per_branch('org/project4')
|
||||
|
||||
|
||||
class TestJobUpdateBrokenConfig(ZuulTestCase):
|
||||
tenant_config_file = 'config/job-update-broken/main.yaml'
|
||||
|
|
|
@ -708,6 +708,7 @@ class TestWeb(BaseTestWeb):
|
|||
'configs': [{
|
||||
'templates': [],
|
||||
'default_branch': 'master',
|
||||
'queue_name': 'integrated',
|
||||
'merge_mode': 'merge-resolve',
|
||||
'pipelines': [{
|
||||
'name': 'check',
|
||||
|
@ -715,7 +716,7 @@ class TestWeb(BaseTestWeb):
|
|||
'jobs': jobs,
|
||||
}, {
|
||||
'name': 'gate',
|
||||
'queue_name': 'integrated',
|
||||
'queue_name': 'integrated-overridden',
|
||||
'jobs': jobs,
|
||||
}, {'name': 'post',
|
||||
'queue_name': None,
|
||||
|
|
|
@ -972,13 +972,14 @@ class ProjectTemplateParser(object):
|
|||
self.schema = self.getSchema()
|
||||
self.not_pipelines = ['name', 'description', 'templates',
|
||||
'merge-mode', 'default-branch', 'vars',
|
||||
'_source_context', '_start_mark']
|
||||
'queue', '_source_context', '_start_mark']
|
||||
|
||||
def getSchema(self):
|
||||
job = {str: vs.Any(str, JobParser.job_attributes)}
|
||||
job_list = [vs.Any(str, job)]
|
||||
|
||||
pipeline_contents = {
|
||||
# TODO(tobiash): Remove pipeline specific queue after deprecation
|
||||
'queue': str,
|
||||
'debug': bool,
|
||||
'fail-fast': bool,
|
||||
|
@ -988,6 +989,7 @@ class ProjectTemplateParser(object):
|
|||
project = {
|
||||
'name': str,
|
||||
'description': str,
|
||||
'queue': str,
|
||||
'vars': ansible_vars_dict,
|
||||
str: pipeline_contents,
|
||||
'_source_context': model.SourceContext,
|
||||
|
@ -1004,11 +1006,13 @@ class ProjectTemplateParser(object):
|
|||
project_template = model.ProjectConfig(conf.get('name'))
|
||||
project_template.source_context = conf['_source_context']
|
||||
project_template.start_mark = conf['_start_mark']
|
||||
project_template.queue_name = conf.get('queue')
|
||||
for pipeline_name, conf_pipeline in conf.items():
|
||||
if pipeline_name in self.not_pipelines:
|
||||
continue
|
||||
project_pipeline = model.ProjectPipelineConfig()
|
||||
project_template.pipelines[pipeline_name] = project_pipeline
|
||||
# TODO(tobiash): Remove pipeline specific queue after deprecation
|
||||
project_pipeline.queue_name = conf_pipeline.get('queue')
|
||||
project_pipeline.debug = conf_pipeline.get('debug')
|
||||
project_pipeline.fail_fast = conf_pipeline.get(
|
||||
|
@ -1063,6 +1067,7 @@ class ProjectParser(object):
|
|||
job_list = [vs.Any(str, job)]
|
||||
|
||||
pipeline_contents = {
|
||||
# TODO(tobiash): Remove pipeline specific queue after deprecation
|
||||
'queue': str,
|
||||
'debug': bool,
|
||||
'fail-fast': bool,
|
||||
|
@ -1077,6 +1082,7 @@ class ProjectParser(object):
|
|||
'merge-mode': vs.Any('merge', 'merge-resolve',
|
||||
'cherry-pick', 'squash-merge'),
|
||||
'default-branch': str,
|
||||
'queue': str,
|
||||
str: pipeline_contents,
|
||||
'_source_context': model.SourceContext,
|
||||
'_start_mark': ZuulMark,
|
||||
|
@ -1142,6 +1148,8 @@ class ProjectParser(object):
|
|||
default_branch = conf.get('default-branch', 'master')
|
||||
project_config.default_branch = default_branch
|
||||
|
||||
project_config.queue_name = conf.get('queue', None)
|
||||
|
||||
variables = conf.get('vars', {})
|
||||
if variables:
|
||||
if 'zuul' in variables or 'nodepool' in variables:
|
||||
|
|
|
@ -72,7 +72,8 @@ class PipelineManager(metaclass=ABCMeta):
|
|||
|
||||
for project_name, project_configs in layout_project_configs.items():
|
||||
(trusted, project) = tenant.getProject(project_name)
|
||||
queue_name = None
|
||||
project_queue_name = None
|
||||
pipeline_queue_name = None
|
||||
project_in_pipeline = False
|
||||
for project_config in layout.getAllProjectConfigs(project_name):
|
||||
project_pipeline_config = project_config.pipelines.get(
|
||||
|
@ -80,11 +81,17 @@ class PipelineManager(metaclass=ABCMeta):
|
|||
if project_pipeline_config is None:
|
||||
continue
|
||||
project_in_pipeline = True
|
||||
queue_name = project_pipeline_config.queue_name
|
||||
if queue_name:
|
||||
break
|
||||
if not pipeline_queue_name:
|
||||
pipeline_queue_name = project_pipeline_config.queue_name
|
||||
if not project_queue_name:
|
||||
project_queue_name = project_config.queue_name
|
||||
if not project_in_pipeline:
|
||||
continue
|
||||
|
||||
# Note: we currently support queue name per pipeline and per
|
||||
# project while project has precedence.
|
||||
queue_name = project_queue_name or pipeline_queue_name
|
||||
|
||||
if not queue_name:
|
||||
continue
|
||||
if queue_name in change_queues:
|
||||
|
|
|
@ -71,20 +71,27 @@ class SharedQueuePipelineManager(PipelineManager, metaclass=ABCMeta):
|
|||
|
||||
for project_name, project_configs in layout_project_configs.items():
|
||||
(trusted, project) = tenant.getProject(project_name)
|
||||
queue_name = None
|
||||
project_queue_name = None
|
||||
pipeline_queue_name = None
|
||||
project_in_pipeline = False
|
||||
for project_config in layout.getAllProjectConfigs(project_name):
|
||||
project_pipeline_config = project_config.pipelines.get(
|
||||
self.pipeline.name)
|
||||
if not project_queue_name:
|
||||
project_queue_name = project_config.queue_name
|
||||
if project_pipeline_config is None:
|
||||
continue
|
||||
project_in_pipeline = True
|
||||
queue_name = project_pipeline_config.queue_name
|
||||
if queue_name:
|
||||
break
|
||||
# TODO(tobiash): Remove pipeline_queue_name after deprecation
|
||||
if not pipeline_queue_name:
|
||||
pipeline_queue_name = project_pipeline_config.queue_name
|
||||
if not project_in_pipeline:
|
||||
continue
|
||||
|
||||
# Note: we currently support queue name per pipeline and per
|
||||
# project while project has precedence.
|
||||
queue_name = project_queue_name or pipeline_queue_name
|
||||
|
||||
# Check if the queue is global or per branch
|
||||
queue = layout.queues.get(queue_name)
|
||||
per_branch = queue and queue.per_branch
|
||||
|
|
|
@ -3522,6 +3522,7 @@ class ProjectConfig(ConfigObject):
|
|||
# stanzas.
|
||||
self.merge_mode = None
|
||||
self.default_branch = None
|
||||
self.queue_name = None
|
||||
|
||||
def __repr__(self):
|
||||
return '<ProjectConfig %s source: %s %s>' % (
|
||||
|
@ -3537,6 +3538,7 @@ class ProjectConfig(ConfigObject):
|
|||
r.variables = self.variables
|
||||
r.merge_mode = self.merge_mode
|
||||
r.default_branch = self.default_branch
|
||||
r.queue_name = self.queue_name
|
||||
return r
|
||||
|
||||
def setImpliedBranchMatchers(self, branches):
|
||||
|
@ -3564,6 +3566,7 @@ class ProjectConfig(ConfigObject):
|
|||
else:
|
||||
d['merge_mode'] = None
|
||||
d['templates'] = self.templates
|
||||
d['queue_name'] = self.queue_name
|
||||
return d
|
||||
|
||||
|
||||
|
|
Loading…
Reference in New Issue