Add queue-name parameter to job

Allow an optional queue-name parameter to be set for a job.
As projects with that job are combined with others into shared
change queues, give the queue that name.  This allows us to, say,
set the queue name of the tempest gate job to 'integrated' and
end up with the shared change queue of all the OpenStack integrated
projects named 'integrated'.

With that, we can do things like emit stats for the 'integrated'
queue.

Change-Id: Iafd218d7cd519312ccbf97de7c070e8d3b82038c

doc/source/zuul.rst

@@ -563,6 +563,18 @@ each job as it builds a list from the project specification.
   The name of the job.  This field is treated as a regular expression
   and will be applied to each job that matches.
 
+**queue-name (optional)**
+  Zuul will automatically combine projects that share a job into
+  shared change queues for dependent pipeline managers.  In order to
+  report statistics about these queues, it is convenient for them to
+  have names.  Zuul can automatically name change queues, however
+  these can grow quite long and are prone to changing as projects in
+  the queue change.  If you assign a queue-name to a job, Zuul will
+  use that as the name for the shared change queue that contains that
+  job instead of the automatically generated one.  It is an error for
+  a shared change queue to have more than one job with a queue-name if
+  they are not the same.
+
 **failure-message (optional)**
   The message that should be reported to Gerrit if the job fails.
 

tests/fixtures/layout-bad-queue.yaml

@@ -0,0 +1,74 @@
+pipelines:
+  - name: check
+    manager: IndependentPipelineManager
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        verified: 1
+    failure:
+      gerrit:
+        verified: -1
+
+  - name: post
+    manager: IndependentPipelineManager
+    trigger:
+      gerrit:
+        - event: ref-updated
+          ref: ^(?!refs/).*$
+
+  - name: gate
+    manager: DependentPipelineManager
+    failure-message: Build failed.  For information on how to proceed, see http://wiki.example.org/Test_Failures
+    trigger:
+      gerrit:
+        - event: comment-added
+          approval:
+            - approved: 1
+    success:
+      gerrit:
+        verified: 2
+        submit: true
+    failure:
+      gerrit:
+        verified: -2
+    start:
+      gerrit:
+        verified: 0
+    precedence: high
+
+jobs:
+  - name: project1-project2-integration
+    queue-name: integration
+  - name: project1-test1
+    queue-name: not_integration
+
+projects:
+  - name: org/project1
+    check:
+      - project1-merge:
+        - project1-test1
+        - project1-test2
+        - project1-project2-integration
+    gate:
+      - project1-merge:
+        - project1-test1
+        - project1-test2
+        - project1-project2-integration
+    post:
+      - project1-post
+
+  - name: org/project2
+    check:
+      - project2-merge:
+        - project2-test1
+        - project2-test2
+        - project1-project2-integration
+    gate:
+      - project2-merge:
+        - project2-test1
+        - project2-test2
+        - project1-project2-integration
+    post:
+      - project2-post

tests/fixtures/layout.yaml

@@ -104,6 +104,8 @@ jobs:
       - '.*-requires'
   - name: node-project-test1
     parameter-function: select_debian_node
+  - name: project1-project2-integration
+    queue-name: integration
 
 project-templates:
   - name: test-one-and-two

tests/test_scheduler.py

@@ -2893,6 +2893,21 @@ class TestScheduler(testtools.TestCase):
         self.assertTrue(re.search("project-test2.*SUCCESS", desc))
         self.assertTrue(re.search("Reported result.*SUCCESS", desc))
 
+    def test_queue_names(self):
+        "Test shared change queue names"
+        project1 = self.sched.layout.projects['org/project1']
+        project2 = self.sched.layout.projects['org/project2']
+        q1 = self.sched.layout.pipelines['gate'].getQueue(project1)
+        q2 = self.sched.layout.pipelines['gate'].getQueue(project2)
+        self.assertEqual(q1.name, 'integration')
+        self.assertEqual(q2.name, 'integration')
+
+        self.config.set('zuul', 'layout_config',
+                        'tests/fixtures/layout-bad-queue.yaml')
+        with testtools.ExpectedException(
+            Exception, "More than one name assigned to change queue"):
+            self.sched.reconfigure(self.config)
+
     def test_queue_precedence(self):
         "Test that queue precedence works"
 

zuul/layoutvalidator.py

@@ -97,6 +97,7 @@ class LayoutSchema(object):
     project_templates = [project_template]
 
     job = {v.Required('name'): str,
+           'queue-name': str,
            'failure-message': str,
            'success-message': str,
            'failure-pattern': str,

zuul/model.py

@@ -403,6 +403,8 @@ class ChangeQueue(object):
                  window_decrease_type='exponential', window_decrease_factor=2):
         self.pipeline = pipeline
         self.name = ''
+        self.assigned_name = None
+        self.generated_name = None
         self.projects = []
         self._jobs = set()
         self.queue = []
@@ -423,10 +425,21 @@ class ChangeQueue(object):
     def addProject(self, project):
         if project not in self.projects:
             self.projects.append(project)
+            self._jobs |= set(self.pipeline.getJobTree(project).getJobs())
+
             names = [x.name for x in self.projects]
             names.sort()
-            self.name = ', '.join(names)
-            self._jobs |= set(self.pipeline.getJobTree(project).getJobs())
+            self.generated_name = ', '.join(names)
+
+            for job in self._jobs:
+                if job.queue_name:
+                    if (self.assigned_name and
+                        job.queue_name != self.assigned_name):
+                        raise Exception("More than one name assigned to "
+                                        "change queue: %s != %s" %
+                                        (self.assigned_name, job.queue_name))
+                    self.assigned_name = job.queue_name
+            self.name = self.assigned_name or self.generated_name
 
     def enqueueChange(self, change):
         item = QueueItem(self.pipeline, change)
@@ -520,6 +533,7 @@ class Job(object):
     def __init__(self, name):
         # If you add attributes here, be sure to add them to the copy method.
         self.name = name
+        self.queue_name = None
         self.failure_message = None
         self.success_message = None
         self.failure_pattern = None

zuul/scheduler.py

@@ -304,6 +304,9 @@ class Scheduler(threading.Thread):
             job = layout.getJob(config_job['name'])
             # Be careful to only set attributes explicitly present on
             # this job, to avoid squashing attributes set by a meta-job.
+            m = config_job.get('queue-name', None)
+            if m:
+                job.queue_name = m
             m = config_job.get('failure-message', None)
             if m:
                 job.failure_message = m
@@ -1628,7 +1631,8 @@ class DependentPipelineManager(BasePipelineManager):
         self.log.info("  Shared change queues:")
         for queue in new_change_queues:
             self.pipeline.addQueue(queue)
-            self.log.info("    %s" % queue)
+            self.log.info("    %s containing %s" % (
+                queue, queue.generated_name))
 
     def combineChangeQueues(self, change_queues):
         self.log.debug("Combining shared queues")