Merge "Move queue from pipeline to project"

doc/source/reference/project_def.rst

@@ -148,6 +148,31 @@ pipeline.
       all pipelines of this project.  For more information see
       :ref:`variable inheritance <user_jobs_variable_inheritance>`.
 
+   .. attr:: queue
+
+      This specifies the
+      name of the shared queue this project is in.  Any projects
+      which interact with each other in tests should be part of the
+      same shared queue in order to ensure that they don't merge
+      changes which break the others.  This is a free-form string;
+      just set the same value for each group of projects.
+
+      The name can refer to the name of a :attr:`queue` which allows
+      further configuration of the queue.
+
+      Each pipeline for a project can only belong to one queue,
+      therefore Zuul will use the first value that it encounters.
+      It need not appear in the first instance of a :attr:`project`
+      stanza; it may appear in secondary instances or even in a
+      :ref:`project-template` definition.
+
+      Pipeline managers other than `dependent` do not use this
+      attribute, however, it may still be used if
+      :attr:`scheduler.relative_priority` is enabled.
+
+      .. note:: This attribute is not evaluated speculatively and
+                its setting shall be merged to be effective.
+
    .. attr:: <pipeline>
 
       Each pipeline that the project participates in should have an
@@ -168,29 +193,13 @@ pipeline.
 
       .. attr:: queue
 
-         If this pipeline is a :value:`dependent
-         <pipeline.manager.dependent>` pipeline, this specifies the
-         name of the shared queue this project is in.  Any projects
-         which interact with each other in tests should be part of the
-         same shared queue in order to ensure that they don't merge
-         changes which break the others.  This is a free-form string;
-         just set the same value for each group of projects.
-
-         The name can refer to the name of a :attr:`queue` which allows
-         further configuration of the queue.
-
-         Each pipeline for a project can only belong to one queue,
-         therefore Zuul will use the first value that it encounters.
-         It need not appear in the first instance of a :attr:`project`
-         stanza; it may appear in secondary instances or even in a
-         :ref:`project-template` definition.
-
-         Pipeline managers other than `dependent` do not use this
-         attribute, however, it may still be used if
-         :attr:`scheduler.relative_priority` is enabled.
-
-         .. note:: This attribute is not evaluated speculatively and
-                   its setting shall be merged to be effective.
+         This is the same as :attr:`project.queue` but on per pipeline
+         level for backwards compatibility reasons. If :attr:`project.queue`
+         is defined this setting is ignored.
+
+         .. note:: It is deprecated to define the queue in the pipeline
+                   configuration. Configure it on :attr:`project.queue`
+                   instead.
 
       .. attr:: debug
 

doc/source/reference/queue_def.rst

@@ -5,7 +5,7 @@ Queue
 
 Projects that interact with each other should share a ``queue``.
 This is especially used in a :value:`dependent <pipeline.manager.dependent>`
-pipeline. The :attr:`project.<pipeline>.queue` can optionally refer
+pipeline. The :attr:`project.queue` can optionally refer
 to a specific :attr:`queue` object that can further configure the
 behavior of the queue.
 

releasenotes/notes/change-queue-project-790553bd212b50eb.yaml

@@ -0,0 +1,6 @@
+---
+deprecations:
+  - |
+    Shared ``queues`` should be configured per project now instead per
+    pipeline. Specifying :attr:`project.<pipeline>.queue` is deprecated
+    and will be removed in a future release.

tests/fixtures/config/change-queues/git/common-config/zuul.d/config.yaml

@@ -33,3 +33,10 @@
       queue: integrated
       jobs:
         - project-test
+
+- project:
+    name: org/project4
+    queue: integrated
+    gate:
+      jobs:
+        - project-test

tests/fixtures/config/change-queues/git/org_project4/readme

@@ -0,0 +1 @@
+test

tests/fixtures/config/change-queues/main.yaml

@@ -8,3 +8,4 @@
           - org/project
           - org/project2
           - org/project3
+          - org/project4

tests/fixtures/config/single-tenant/git/common-config/zuul.yaml

@@ -137,6 +137,7 @@
 
 - project:
     name: org/project1
+    queue: integrated
     check:
       jobs:
         - project-merge
@@ -147,7 +148,8 @@
         - project1-project2-integration:
             dependencies: project-merge
     gate:
-      queue: integrated
+      # This will be overridden on project level
+      queue: integrated-overridden
       jobs:
         - project-merge
         - project-test1:

tests/fixtures/layouts/template-project-queue.yaml

@@ -0,0 +1,27 @@
+- pipeline:
+    name: gate
+    manager: dependent
+    trigger: {}
+
+- job:
+    name: base
+    parent: null
+    run: playbooks/base.yaml
+
+- project-template:
+    name: integrated-jobs
+    gate:
+      jobs:
+        - base
+
+- project:
+    name: org/project1
+    queue: integrated
+    templates:
+      - integrated-jobs
+
+- project:
+    name: org/project2
+    queue: integrated
+    templates:
+      - integrated-jobs

tests/unit/test_scheduler.py

@@ -3203,6 +3203,27 @@ class TestScheduler(ZuulTestCase):
         self.assertEqual(q1.name, 'integrated')
         self.assertEqual(q2.name, 'integrated')
 
+    @simple_layout("layouts/template-project-queue.yaml")
+    def test_template_project_queue(self):
+        "Test a shared queue can be constructed from a project-template"
+        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
+        (trusted, project1) = tenant.getProject('org/project1')
+        (trusted, project2) = tenant.getProject('org/project2')
+
+        # Change queues are created lazy by the dependent pipeline manager
+        # so retrieve the queue first without having to really enqueue a
+        # change first.
+        gate = tenant.layout.pipelines['gate']
+        FakeChange = namedtuple('FakeChange', ['project', 'branch'])
+        fake_a = FakeChange(project1, 'master')
+        fake_b = FakeChange(project2, 'master')
+        gate.manager.getChangeQueue(fake_a, None)
+        gate.manager.getChangeQueue(fake_b, None)
+        q1 = gate.getQueue(project1, None)
+        q2 = gate.getQueue(project2, None)
+        self.assertEqual(q1.name, 'integrated')
+        self.assertEqual(q2.name, 'integrated')
+
     @simple_layout("layouts/regex-template-queue.yaml")
     def test_regex_template_queue(self):
         "Test a shared queue can be constructed from a regex project-template"
@@ -6461,6 +6482,15 @@ class TestChangeQueues(ZuulTestCase):
             'org/project3', queue_name='integrated-untrusted',
             queue_repo='org/project3')
 
+    def test_dependent_queues_per_branch_project_queue(self):
+        """
+        Test that change queues can be different for different branches.
+
+        In this case we create changes for two branches in a repo that
+        references the queue on project level instead of pipeline level.
+        """
+        self._test_dependent_queues_per_branch('org/project4')
+
 
 class TestJobUpdateBrokenConfig(ZuulTestCase):
     tenant_config_file = 'config/job-update-broken/main.yaml'

tests/unit/test_web.py

@@ -708,6 +708,7 @@ class TestWeb(BaseTestWeb):
                 'configs': [{
                     'templates': [],
                     'default_branch': 'master',
+                    'queue_name': 'integrated',
                     'merge_mode': 'merge-resolve',
                     'pipelines': [{
                         'name': 'check',
@@ -715,7 +716,7 @@ class TestWeb(BaseTestWeb):
                         'jobs': jobs,
                     }, {
                         'name': 'gate',
-                        'queue_name': 'integrated',
+                        'queue_name': 'integrated-overridden',
                         'jobs': jobs,
                     }, {'name': 'post',
                         'queue_name': None,

zuul/configloader.py

@@ -972,13 +972,14 @@ class ProjectTemplateParser(object):
         self.schema = self.getSchema()
         self.not_pipelines = ['name', 'description', 'templates',
                               'merge-mode', 'default-branch', 'vars',
-                              '_source_context', '_start_mark']
+                              'queue', '_source_context', '_start_mark']
 
     def getSchema(self):
         job = {str: vs.Any(str, JobParser.job_attributes)}
         job_list = [vs.Any(str, job)]
 
         pipeline_contents = {
+            # TODO(tobiash): Remove pipeline specific queue after deprecation
             'queue': str,
             'debug': bool,
             'fail-fast': bool,
@@ -988,6 +989,7 @@ class ProjectTemplateParser(object):
         project = {
             'name': str,
             'description': str,
+            'queue': str,
             'vars': ansible_vars_dict,
             str: pipeline_contents,
             '_source_context': model.SourceContext,
@@ -1004,11 +1006,13 @@ class ProjectTemplateParser(object):
         project_template = model.ProjectConfig(conf.get('name'))
         project_template.source_context = conf['_source_context']
         project_template.start_mark = conf['_start_mark']
+        project_template.queue_name = conf.get('queue')
         for pipeline_name, conf_pipeline in conf.items():
             if pipeline_name in self.not_pipelines:
                 continue
             project_pipeline = model.ProjectPipelineConfig()
             project_template.pipelines[pipeline_name] = project_pipeline
+            # TODO(tobiash): Remove pipeline specific queue after deprecation
             project_pipeline.queue_name = conf_pipeline.get('queue')
             project_pipeline.debug = conf_pipeline.get('debug')
             project_pipeline.fail_fast = conf_pipeline.get(
@@ -1063,6 +1067,7 @@ class ProjectParser(object):
         job_list = [vs.Any(str, job)]
 
         pipeline_contents = {
+            # TODO(tobiash): Remove pipeline specific queue after deprecation
             'queue': str,
             'debug': bool,
             'fail-fast': bool,
@@ -1077,6 +1082,7 @@ class ProjectParser(object):
             'merge-mode': vs.Any('merge', 'merge-resolve',
                                  'cherry-pick', 'squash-merge'),
             'default-branch': str,
+            'queue': str,
             str: pipeline_contents,
             '_source_context': model.SourceContext,
             '_start_mark': ZuulMark,
@@ -1142,6 +1148,8 @@ class ProjectParser(object):
         default_branch = conf.get('default-branch', 'master')
         project_config.default_branch = default_branch
 
+        project_config.queue_name = conf.get('queue', None)
+
         variables = conf.get('vars', {})
         if variables:
             if 'zuul' in variables or 'nodepool' in variables:

zuul/manager/__init__.py

@@ -72,7 +72,8 @@ class PipelineManager(metaclass=ABCMeta):
 
         for project_name, project_configs in layout_project_configs.items():
             (trusted, project) = tenant.getProject(project_name)
-            queue_name = None
+            project_queue_name = None
+            pipeline_queue_name = None
             project_in_pipeline = False
             for project_config in layout.getAllProjectConfigs(project_name):
                 project_pipeline_config = project_config.pipelines.get(
@@ -80,11 +81,17 @@ class PipelineManager(metaclass=ABCMeta):
                 if project_pipeline_config is None:
                     continue
                 project_in_pipeline = True
-                queue_name = project_pipeline_config.queue_name
-                if queue_name:
-                    break
+                if not pipeline_queue_name:
+                    pipeline_queue_name = project_pipeline_config.queue_name
+                if not project_queue_name:
+                    project_queue_name = project_config.queue_name
             if not project_in_pipeline:
                 continue
+
+            # Note: we currently support queue name per pipeline and per
+            # project while project has precedence.
+            queue_name = project_queue_name or pipeline_queue_name
+
             if not queue_name:
                 continue
             if queue_name in change_queues:

zuul/manager/shared.py

@@ -71,20 +71,27 @@ class SharedQueuePipelineManager(PipelineManager, metaclass=ABCMeta):
 
         for project_name, project_configs in layout_project_configs.items():
             (trusted, project) = tenant.getProject(project_name)
-            queue_name = None
+            project_queue_name = None
+            pipeline_queue_name = None
             project_in_pipeline = False
             for project_config in layout.getAllProjectConfigs(project_name):
                 project_pipeline_config = project_config.pipelines.get(
                     self.pipeline.name)
+                if not project_queue_name:
+                    project_queue_name = project_config.queue_name
                 if project_pipeline_config is None:
                     continue
                 project_in_pipeline = True
-                queue_name = project_pipeline_config.queue_name
-                if queue_name:
-                    break
+                # TODO(tobiash): Remove pipeline_queue_name after deprecation
+                if not pipeline_queue_name:
+                    pipeline_queue_name = project_pipeline_config.queue_name
             if not project_in_pipeline:
                 continue
 
+            # Note: we currently support queue name per pipeline and per
+            # project while project has precedence.
+            queue_name = project_queue_name or pipeline_queue_name
+
             # Check if the queue is global or per branch
             queue = layout.queues.get(queue_name)
             per_branch = queue and queue.per_branch

zuul/model.py

@@ -3522,6 +3522,7 @@ class ProjectConfig(ConfigObject):
         # stanzas.
         self.merge_mode = None
         self.default_branch = None
+        self.queue_name = None
 
     def __repr__(self):
         return '<ProjectConfig %s source: %s %s>' % (
@@ -3537,6 +3538,7 @@ class ProjectConfig(ConfigObject):
         r.variables = self.variables
         r.merge_mode = self.merge_mode
         r.default_branch = self.default_branch
+        r.queue_name = self.queue_name
         return r
 
     def setImpliedBranchMatchers(self, branches):
@@ -3564,6 +3566,7 @@ class ProjectConfig(ConfigObject):
         else:
             d['merge_mode'] = None
         d['templates'] = self.templates
+        d['queue_name'] = self.queue_name
         return d