From c8a1e05849f6ff2f05560bcd2c14d0e2be28c861 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Tue, 25 Feb 2014 09:29:26 -0800 Subject: [PATCH] Add queue-name parameter to job Allow an optional queue-name parameter to be set for a job. As projects with that job are combined with others into shared change queues, give the queue that name. This allows us to, say, set the queue name of the tempest gate job to 'integrated' and end up with the shared change queue of all the OpenStack integrated projects named 'integrated'. With that, we can do things like emit stats for the 'integrated' queue. Change-Id: Iafd218d7cd519312ccbf97de7c070e8d3b82038c --- doc/source/zuul.rst | 12 +++++ tests/fixtures/layout-bad-queue.yaml | 74 ++++++++++++++++++++++++++++ tests/fixtures/layout.yaml | 2 + tests/test_scheduler.py | 15 ++++++ zuul/layoutvalidator.py | 1 + zuul/model.py | 18 ++++++- zuul/scheduler.py | 6 ++- 7 files changed, 125 insertions(+), 3 deletions(-) create mode 100644 tests/fixtures/layout-bad-queue.yaml diff --git a/doc/source/zuul.rst b/doc/source/zuul.rst index 1a6a23de9b..826fb9a702 100644 --- a/doc/source/zuul.rst +++ b/doc/source/zuul.rst @@ -563,6 +563,18 @@ each job as it builds a list from the project specification. The name of the job. This field is treated as a regular expression and will be applied to each job that matches. +**queue-name (optional)** + Zuul will automatically combine projects that share a job into + shared change queues for dependent pipeline managers. In order to + report statistics about these queues, it is convenient for them to + have names. Zuul can automatically name change queues, however + these can grow quite long and are prone to changing as projects in + the queue change. If you assign a queue-name to a job, Zuul will + use that as the name for the shared change queue that contains that + job instead of the automatically generated one. It is an error for + a shared change queue to have more than one job with a queue-name if + they are not the same. + **failure-message (optional)** The message that should be reported to Gerrit if the job fails. diff --git a/tests/fixtures/layout-bad-queue.yaml b/tests/fixtures/layout-bad-queue.yaml new file mode 100644 index 0000000000..3eb2051927 --- /dev/null +++ b/tests/fixtures/layout-bad-queue.yaml @@ -0,0 +1,74 @@ +pipelines: + - name: check + manager: IndependentPipelineManager + trigger: + gerrit: + - event: patchset-created + success: + gerrit: + verified: 1 + failure: + gerrit: + verified: -1 + + - name: post + manager: IndependentPipelineManager + trigger: + gerrit: + - event: ref-updated + ref: ^(?!refs/).*$ + + - name: gate + manager: DependentPipelineManager + failure-message: Build failed. For information on how to proceed, see http://wiki.example.org/Test_Failures + trigger: + gerrit: + - event: comment-added + approval: + - approved: 1 + success: + gerrit: + verified: 2 + submit: true + failure: + gerrit: + verified: -2 + start: + gerrit: + verified: 0 + precedence: high + +jobs: + - name: project1-project2-integration + queue-name: integration + - name: project1-test1 + queue-name: not_integration + +projects: + - name: org/project1 + check: + - project1-merge: + - project1-test1 + - project1-test2 + - project1-project2-integration + gate: + - project1-merge: + - project1-test1 + - project1-test2 + - project1-project2-integration + post: + - project1-post + + - name: org/project2 + check: + - project2-merge: + - project2-test1 + - project2-test2 + - project1-project2-integration + gate: + - project2-merge: + - project2-test1 + - project2-test2 + - project1-project2-integration + post: + - project2-post diff --git a/tests/fixtures/layout.yaml b/tests/fixtures/layout.yaml index b1c94de0ce..70e664faf6 100644 --- a/tests/fixtures/layout.yaml +++ b/tests/fixtures/layout.yaml @@ -104,6 +104,8 @@ jobs: - '.*-requires' - name: node-project-test1 parameter-function: select_debian_node + - name: project1-project2-integration + queue-name: integration project-templates: - name: test-one-and-two diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 95764404e6..47581554dd 100755 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -2893,6 +2893,21 @@ class TestScheduler(testtools.TestCase): self.assertTrue(re.search("project-test2.*SUCCESS", desc)) self.assertTrue(re.search("Reported result.*SUCCESS", desc)) + def test_queue_names(self): + "Test shared change queue names" + project1 = self.sched.layout.projects['org/project1'] + project2 = self.sched.layout.projects['org/project2'] + q1 = self.sched.layout.pipelines['gate'].getQueue(project1) + q2 = self.sched.layout.pipelines['gate'].getQueue(project2) + self.assertEqual(q1.name, 'integration') + self.assertEqual(q2.name, 'integration') + + self.config.set('zuul', 'layout_config', + 'tests/fixtures/layout-bad-queue.yaml') + with testtools.ExpectedException( + Exception, "More than one name assigned to change queue"): + self.sched.reconfigure(self.config) + def test_queue_precedence(self): "Test that queue precedence works" diff --git a/zuul/layoutvalidator.py b/zuul/layoutvalidator.py index 48aab03490..6eef594ef1 100644 --- a/zuul/layoutvalidator.py +++ b/zuul/layoutvalidator.py @@ -97,6 +97,7 @@ class LayoutSchema(object): project_templates = [project_template] job = {v.Required('name'): str, + 'queue-name': str, 'failure-message': str, 'success-message': str, 'failure-pattern': str, diff --git a/zuul/model.py b/zuul/model.py index 22475e6895..8188f3737c 100644 --- a/zuul/model.py +++ b/zuul/model.py @@ -403,6 +403,8 @@ class ChangeQueue(object): window_decrease_type='exponential', window_decrease_factor=2): self.pipeline = pipeline self.name = '' + self.assigned_name = None + self.generated_name = None self.projects = [] self._jobs = set() self.queue = [] @@ -423,10 +425,21 @@ class ChangeQueue(object): def addProject(self, project): if project not in self.projects: self.projects.append(project) + self._jobs |= set(self.pipeline.getJobTree(project).getJobs()) + names = [x.name for x in self.projects] names.sort() - self.name = ', '.join(names) - self._jobs |= set(self.pipeline.getJobTree(project).getJobs()) + self.generated_name = ', '.join(names) + + for job in self._jobs: + if job.queue_name: + if (self.assigned_name and + job.queue_name != self.assigned_name): + raise Exception("More than one name assigned to " + "change queue: %s != %s" % + (self.assigned_name, job.queue_name)) + self.assigned_name = job.queue_name + self.name = self.assigned_name or self.generated_name def enqueueChange(self, change): item = QueueItem(self.pipeline, change) @@ -520,6 +533,7 @@ class Job(object): def __init__(self, name): # If you add attributes here, be sure to add them to the copy method. self.name = name + self.queue_name = None self.failure_message = None self.success_message = None self.failure_pattern = None diff --git a/zuul/scheduler.py b/zuul/scheduler.py index 815da8c776..fe86b93e1e 100644 --- a/zuul/scheduler.py +++ b/zuul/scheduler.py @@ -304,6 +304,9 @@ class Scheduler(threading.Thread): job = layout.getJob(config_job['name']) # Be careful to only set attributes explicitly present on # this job, to avoid squashing attributes set by a meta-job. + m = config_job.get('queue-name', None) + if m: + job.queue_name = m m = config_job.get('failure-message', None) if m: job.failure_message = m @@ -1628,7 +1631,8 @@ class DependentPipelineManager(BasePipelineManager): self.log.info(" Shared change queues:") for queue in new_change_queues: self.pipeline.addQueue(queue) - self.log.info(" %s" % queue) + self.log.info(" %s containing %s" % ( + queue, queue.generated_name)) def combineChangeQueues(self, change_queues): self.log.debug("Combining shared queues")