Merge "Move queue from pipeline to project"

This commit is contained in:
Zuul 2021-02-25 00:45:45 +00:00 committed by Gerrit Code Review
commit 87e5529ef7
14 changed files with 143 additions and 34 deletions

View File

@ -148,6 +148,31 @@ pipeline.
all pipelines of this project. For more information see all pipelines of this project. For more information see
:ref:`variable inheritance <user_jobs_variable_inheritance>`. :ref:`variable inheritance <user_jobs_variable_inheritance>`.
.. attr:: queue
This specifies the
name of the shared queue this project is in. Any projects
which interact with each other in tests should be part of the
same shared queue in order to ensure that they don't merge
changes which break the others. This is a free-form string;
just set the same value for each group of projects.
The name can refer to the name of a :attr:`queue` which allows
further configuration of the queue.
Each pipeline for a project can only belong to one queue,
therefore Zuul will use the first value that it encounters.
It need not appear in the first instance of a :attr:`project`
stanza; it may appear in secondary instances or even in a
:ref:`project-template` definition.
Pipeline managers other than `dependent` do not use this
attribute, however, it may still be used if
:attr:`scheduler.relative_priority` is enabled.
.. note:: This attribute is not evaluated speculatively and
its setting shall be merged to be effective.
.. attr:: <pipeline> .. attr:: <pipeline>
Each pipeline that the project participates in should have an Each pipeline that the project participates in should have an
@ -168,29 +193,13 @@ pipeline.
.. attr:: queue .. attr:: queue
If this pipeline is a :value:`dependent This is the same as :attr:`project.queue` but on per pipeline
<pipeline.manager.dependent>` pipeline, this specifies the level for backwards compatibility reasons. If :attr:`project.queue`
name of the shared queue this project is in. Any projects is defined this setting is ignored.
which interact with each other in tests should be part of the
same shared queue in order to ensure that they don't merge
changes which break the others. This is a free-form string;
just set the same value for each group of projects.
The name can refer to the name of a :attr:`queue` which allows .. note:: It is deprecated to define the queue in the pipeline
further configuration of the queue. configuration. Configure it on :attr:`project.queue`
instead.
Each pipeline for a project can only belong to one queue,
therefore Zuul will use the first value that it encounters.
It need not appear in the first instance of a :attr:`project`
stanza; it may appear in secondary instances or even in a
:ref:`project-template` definition.
Pipeline managers other than `dependent` do not use this
attribute, however, it may still be used if
:attr:`scheduler.relative_priority` is enabled.
.. note:: This attribute is not evaluated speculatively and
its setting shall be merged to be effective.
.. attr:: debug .. attr:: debug

View File

@ -5,7 +5,7 @@ Queue
Projects that interact with each other should share a ``queue``. Projects that interact with each other should share a ``queue``.
This is especially used in a :value:`dependent <pipeline.manager.dependent>` This is especially used in a :value:`dependent <pipeline.manager.dependent>`
pipeline. The :attr:`project.<pipeline>.queue` can optionally refer pipeline. The :attr:`project.queue` can optionally refer
to a specific :attr:`queue` object that can further configure the to a specific :attr:`queue` object that can further configure the
behavior of the queue. behavior of the queue.

View File

@ -0,0 +1,6 @@
---
deprecations:
- |
Shared ``queues`` should be configured per project now instead per
pipeline. Specifying :attr:`project.<pipeline>.queue` is deprecated
and will be removed in a future release.

View File

@ -33,3 +33,10 @@
queue: integrated queue: integrated
jobs: jobs:
- project-test - project-test
- project:
name: org/project4
queue: integrated
gate:
jobs:
- project-test

View File

@ -0,0 +1 @@
test

View File

@ -8,3 +8,4 @@
- org/project - org/project
- org/project2 - org/project2
- org/project3 - org/project3
- org/project4

View File

@ -137,6 +137,7 @@
- project: - project:
name: org/project1 name: org/project1
queue: integrated
check: check:
jobs: jobs:
- project-merge - project-merge
@ -147,7 +148,8 @@
- project1-project2-integration: - project1-project2-integration:
dependencies: project-merge dependencies: project-merge
gate: gate:
queue: integrated # This will be overridden on project level
queue: integrated-overridden
jobs: jobs:
- project-merge - project-merge
- project-test1: - project-test1:

View File

@ -0,0 +1,27 @@
- pipeline:
name: gate
manager: dependent
trigger: {}
- job:
name: base
parent: null
run: playbooks/base.yaml
- project-template:
name: integrated-jobs
gate:
jobs:
- base
- project:
name: org/project1
queue: integrated
templates:
- integrated-jobs
- project:
name: org/project2
queue: integrated
templates:
- integrated-jobs

View File

@ -3203,6 +3203,27 @@ class TestScheduler(ZuulTestCase):
self.assertEqual(q1.name, 'integrated') self.assertEqual(q1.name, 'integrated')
self.assertEqual(q2.name, 'integrated') self.assertEqual(q2.name, 'integrated')
@simple_layout("layouts/template-project-queue.yaml")
def test_template_project_queue(self):
"Test a shared queue can be constructed from a project-template"
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
(trusted, project1) = tenant.getProject('org/project1')
(trusted, project2) = tenant.getProject('org/project2')
# Change queues are created lazy by the dependent pipeline manager
# so retrieve the queue first without having to really enqueue a
# change first.
gate = tenant.layout.pipelines['gate']
FakeChange = namedtuple('FakeChange', ['project', 'branch'])
fake_a = FakeChange(project1, 'master')
fake_b = FakeChange(project2, 'master')
gate.manager.getChangeQueue(fake_a, None)
gate.manager.getChangeQueue(fake_b, None)
q1 = gate.getQueue(project1, None)
q2 = gate.getQueue(project2, None)
self.assertEqual(q1.name, 'integrated')
self.assertEqual(q2.name, 'integrated')
@simple_layout("layouts/regex-template-queue.yaml") @simple_layout("layouts/regex-template-queue.yaml")
def test_regex_template_queue(self): def test_regex_template_queue(self):
"Test a shared queue can be constructed from a regex project-template" "Test a shared queue can be constructed from a regex project-template"
@ -6461,6 +6482,15 @@ class TestChangeQueues(ZuulTestCase):
'org/project3', queue_name='integrated-untrusted', 'org/project3', queue_name='integrated-untrusted',
queue_repo='org/project3') queue_repo='org/project3')
def test_dependent_queues_per_branch_project_queue(self):
"""
Test that change queues can be different for different branches.
In this case we create changes for two branches in a repo that
references the queue on project level instead of pipeline level.
"""
self._test_dependent_queues_per_branch('org/project4')
class TestJobUpdateBrokenConfig(ZuulTestCase): class TestJobUpdateBrokenConfig(ZuulTestCase):
tenant_config_file = 'config/job-update-broken/main.yaml' tenant_config_file = 'config/job-update-broken/main.yaml'

View File

@ -708,6 +708,7 @@ class TestWeb(BaseTestWeb):
'configs': [{ 'configs': [{
'templates': [], 'templates': [],
'default_branch': 'master', 'default_branch': 'master',
'queue_name': 'integrated',
'merge_mode': 'merge-resolve', 'merge_mode': 'merge-resolve',
'pipelines': [{ 'pipelines': [{
'name': 'check', 'name': 'check',
@ -715,7 +716,7 @@ class TestWeb(BaseTestWeb):
'jobs': jobs, 'jobs': jobs,
}, { }, {
'name': 'gate', 'name': 'gate',
'queue_name': 'integrated', 'queue_name': 'integrated-overridden',
'jobs': jobs, 'jobs': jobs,
}, {'name': 'post', }, {'name': 'post',
'queue_name': None, 'queue_name': None,

View File

@ -972,13 +972,14 @@ class ProjectTemplateParser(object):
self.schema = self.getSchema() self.schema = self.getSchema()
self.not_pipelines = ['name', 'description', 'templates', self.not_pipelines = ['name', 'description', 'templates',
'merge-mode', 'default-branch', 'vars', 'merge-mode', 'default-branch', 'vars',
'_source_context', '_start_mark'] 'queue', '_source_context', '_start_mark']
def getSchema(self): def getSchema(self):
job = {str: vs.Any(str, JobParser.job_attributes)} job = {str: vs.Any(str, JobParser.job_attributes)}
job_list = [vs.Any(str, job)] job_list = [vs.Any(str, job)]
pipeline_contents = { pipeline_contents = {
# TODO(tobiash): Remove pipeline specific queue after deprecation
'queue': str, 'queue': str,
'debug': bool, 'debug': bool,
'fail-fast': bool, 'fail-fast': bool,
@ -988,6 +989,7 @@ class ProjectTemplateParser(object):
project = { project = {
'name': str, 'name': str,
'description': str, 'description': str,
'queue': str,
'vars': ansible_vars_dict, 'vars': ansible_vars_dict,
str: pipeline_contents, str: pipeline_contents,
'_source_context': model.SourceContext, '_source_context': model.SourceContext,
@ -1004,11 +1006,13 @@ class ProjectTemplateParser(object):
project_template = model.ProjectConfig(conf.get('name')) project_template = model.ProjectConfig(conf.get('name'))
project_template.source_context = conf['_source_context'] project_template.source_context = conf['_source_context']
project_template.start_mark = conf['_start_mark'] project_template.start_mark = conf['_start_mark']
project_template.queue_name = conf.get('queue')
for pipeline_name, conf_pipeline in conf.items(): for pipeline_name, conf_pipeline in conf.items():
if pipeline_name in self.not_pipelines: if pipeline_name in self.not_pipelines:
continue continue
project_pipeline = model.ProjectPipelineConfig() project_pipeline = model.ProjectPipelineConfig()
project_template.pipelines[pipeline_name] = project_pipeline project_template.pipelines[pipeline_name] = project_pipeline
# TODO(tobiash): Remove pipeline specific queue after deprecation
project_pipeline.queue_name = conf_pipeline.get('queue') project_pipeline.queue_name = conf_pipeline.get('queue')
project_pipeline.debug = conf_pipeline.get('debug') project_pipeline.debug = conf_pipeline.get('debug')
project_pipeline.fail_fast = conf_pipeline.get( project_pipeline.fail_fast = conf_pipeline.get(
@ -1063,6 +1067,7 @@ class ProjectParser(object):
job_list = [vs.Any(str, job)] job_list = [vs.Any(str, job)]
pipeline_contents = { pipeline_contents = {
# TODO(tobiash): Remove pipeline specific queue after deprecation
'queue': str, 'queue': str,
'debug': bool, 'debug': bool,
'fail-fast': bool, 'fail-fast': bool,
@ -1077,6 +1082,7 @@ class ProjectParser(object):
'merge-mode': vs.Any('merge', 'merge-resolve', 'merge-mode': vs.Any('merge', 'merge-resolve',
'cherry-pick', 'squash-merge'), 'cherry-pick', 'squash-merge'),
'default-branch': str, 'default-branch': str,
'queue': str,
str: pipeline_contents, str: pipeline_contents,
'_source_context': model.SourceContext, '_source_context': model.SourceContext,
'_start_mark': ZuulMark, '_start_mark': ZuulMark,
@ -1142,6 +1148,8 @@ class ProjectParser(object):
default_branch = conf.get('default-branch', 'master') default_branch = conf.get('default-branch', 'master')
project_config.default_branch = default_branch project_config.default_branch = default_branch
project_config.queue_name = conf.get('queue', None)
variables = conf.get('vars', {}) variables = conf.get('vars', {})
if variables: if variables:
if 'zuul' in variables or 'nodepool' in variables: if 'zuul' in variables or 'nodepool' in variables:

View File

@ -72,7 +72,8 @@ class PipelineManager(metaclass=ABCMeta):
for project_name, project_configs in layout_project_configs.items(): for project_name, project_configs in layout_project_configs.items():
(trusted, project) = tenant.getProject(project_name) (trusted, project) = tenant.getProject(project_name)
queue_name = None project_queue_name = None
pipeline_queue_name = None
project_in_pipeline = False project_in_pipeline = False
for project_config in layout.getAllProjectConfigs(project_name): for project_config in layout.getAllProjectConfigs(project_name):
project_pipeline_config = project_config.pipelines.get( project_pipeline_config = project_config.pipelines.get(
@ -80,11 +81,17 @@ class PipelineManager(metaclass=ABCMeta):
if project_pipeline_config is None: if project_pipeline_config is None:
continue continue
project_in_pipeline = True project_in_pipeline = True
queue_name = project_pipeline_config.queue_name if not pipeline_queue_name:
if queue_name: pipeline_queue_name = project_pipeline_config.queue_name
break if not project_queue_name:
project_queue_name = project_config.queue_name
if not project_in_pipeline: if not project_in_pipeline:
continue continue
# Note: we currently support queue name per pipeline and per
# project while project has precedence.
queue_name = project_queue_name or pipeline_queue_name
if not queue_name: if not queue_name:
continue continue
if queue_name in change_queues: if queue_name in change_queues:

View File

@ -71,20 +71,27 @@ class SharedQueuePipelineManager(PipelineManager, metaclass=ABCMeta):
for project_name, project_configs in layout_project_configs.items(): for project_name, project_configs in layout_project_configs.items():
(trusted, project) = tenant.getProject(project_name) (trusted, project) = tenant.getProject(project_name)
queue_name = None project_queue_name = None
pipeline_queue_name = None
project_in_pipeline = False project_in_pipeline = False
for project_config in layout.getAllProjectConfigs(project_name): for project_config in layout.getAllProjectConfigs(project_name):
project_pipeline_config = project_config.pipelines.get( project_pipeline_config = project_config.pipelines.get(
self.pipeline.name) self.pipeline.name)
if not project_queue_name:
project_queue_name = project_config.queue_name
if project_pipeline_config is None: if project_pipeline_config is None:
continue continue
project_in_pipeline = True project_in_pipeline = True
queue_name = project_pipeline_config.queue_name # TODO(tobiash): Remove pipeline_queue_name after deprecation
if queue_name: if not pipeline_queue_name:
break pipeline_queue_name = project_pipeline_config.queue_name
if not project_in_pipeline: if not project_in_pipeline:
continue continue
# Note: we currently support queue name per pipeline and per
# project while project has precedence.
queue_name = project_queue_name or pipeline_queue_name
# Check if the queue is global or per branch # Check if the queue is global or per branch
queue = layout.queues.get(queue_name) queue = layout.queues.get(queue_name)
per_branch = queue and queue.per_branch per_branch = queue and queue.per_branch

View File

@ -3522,6 +3522,7 @@ class ProjectConfig(ConfigObject):
# stanzas. # stanzas.
self.merge_mode = None self.merge_mode = None
self.default_branch = None self.default_branch = None
self.queue_name = None
def __repr__(self): def __repr__(self):
return '<ProjectConfig %s source: %s %s>' % ( return '<ProjectConfig %s source: %s %s>' % (
@ -3537,6 +3538,7 @@ class ProjectConfig(ConfigObject):
r.variables = self.variables r.variables = self.variables
r.merge_mode = self.merge_mode r.merge_mode = self.merge_mode
r.default_branch = self.default_branch r.default_branch = self.default_branch
r.queue_name = self.queue_name
return r return r
def setImpliedBranchMatchers(self, branches): def setImpliedBranchMatchers(self, branches):
@ -3564,6 +3566,7 @@ class ProjectConfig(ConfigObject):
else: else:
d['merge_mode'] = None d['merge_mode'] = None
d['templates'] = self.templates d['templates'] = self.templates
d['queue_name'] = self.queue_name
return d return d