Add ignore-dependencies option

With the previous CRD changes in effect, the merge-check pipeline
became rather large since it needlessly stacked dependent (both
git-dependent and commit-dependent) changes for each change tested.
Since commit-dependencies should not affect whether a change has
a merge conflict, and git-dependencies are already taken care of,
add an option to allow independent pipelines to ignore dependencies.

We will only set this for the merge-check pipeline in OpenStack.

Change-Id: I553446374ac12aa3e3f2e4bbea7ca8fafba42294

doc/source/zuul.rst

@@ -569,6 +569,15 @@ explanation of each of the parameters::
   well.  To suppress this behavior (and allow jobs to continue
   running), set this to ``false``.  Default: ``true``.
 
+**ignore-dependencies**
+  In any kind of pipeline (dependent or independent), Zuul will
+  attempt to enqueue all dependencies ahead of the current change so
+  that they are tested together (independent pipelines report the
+  results of each change regardless of the results of changes ahead).
+  To ignore dependencies completely in an independent pipeline, set
+  this to ``true``.  This option is ignored by dependent pipelines.
+  The default is: ``false``.
+
 **success**
   Describes where Zuul should report to if all the jobs complete
   successfully.

tests/fixtures/layout-ignore-dependencies.yaml

@@ -0,0 +1,28 @@
+pipelines:
+  - name: check
+    manager: IndependentPipelineManager
+    ignore-dependencies: true
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        verified: 1
+    failure:
+      gerrit:
+        verified: -1
+
+projects:
+  - name: org/project1
+    check:
+      - project1-merge:
+        - project1-test1
+        - project1-test2
+        - project1-project2-integration
+
+  - name: org/project2
+    check:
+      - project2-merge:
+        - project2-test1
+        - project2-test2
+        - project1-project2-integration

tests/fixtures/layout-zuultrigger-merged.yaml

@@ -36,6 +36,7 @@ pipelines:
   - name: merge-check
     manager: IndependentPipelineManager
     source: gerrit
+    ignore-dependencies: true
     trigger:
       zuul:
         - event: project-change-merged

tests/fixtures/layouts/good_layout.yaml

@@ -43,6 +43,17 @@ pipelines:
         verified: -2
         workinprogress: true
 
+  - name: merge-check
+    manager: IndependentPipelineManager
+    source: gerrit
+    ignore-dependencies: true
+    trigger:
+      zuul:
+        - event: project-change-merged
+    merge-failure:
+      gerrit:
+        verified: -1
+
 jobs:
   - name: ^.*-merge$
     failure-message: Unable to merge change

tests/test_scheduler.py

@@ -3390,3 +3390,48 @@ For CI problems and help debugging, contact ci@example.org"""
 
         self.assertEqual(self.history[0].changes, '2,1 1,1')
         self.assertEqual(len(self.sched.layout.pipelines['check'].queues), 0)
+
+    def test_crd_check_ignore_dependencies(self):
+        "Test cross-repo dependencies can be ignored"
+        self.config.set('zuul', 'layout_config',
+                        'tests/fixtures/layout-ignore-dependencies.yaml')
+        self.sched.reconfigure(self.config)
+        self.registerJobs()
+
+        self.gearman_server.hold_jobs_in_queue = True
+        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+        B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
+        C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
+
+        # A Depends-On: B
+        A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+            A.subject, B.data['id'])
+        # C git-depends on B
+        C.setDependsOn(B, 1)
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+        self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        # Make sure none of the items share a change queue, and all
+        # are live.
+        check_pipeline = self.sched.layout.pipelines['check']
+        self.assertEqual(len(check_pipeline.queues), 3)
+        self.assertEqual(len(check_pipeline.getAllItems()), 3)
+        for item in check_pipeline.getAllItems():
+            self.assertTrue(item.live)
+
+        self.gearman_server.hold_jobs_in_queue = False
+        self.gearman_server.release()
+        self.waitUntilSettled()
+
+        self.assertEqual(A.data['status'], 'NEW')
+        self.assertEqual(B.data['status'], 'NEW')
+        self.assertEqual(C.data['status'], 'NEW')
+        self.assertEqual(A.reported, 1)
+        self.assertEqual(B.reported, 1)
+        self.assertEqual(C.reported, 1)
+
+        # Each job should have tested exactly one change
+        for job in self.history:
+            self.assertEqual(len(job.changes.split()), 1)

zuul/layoutvalidator.py

@@ -106,6 +106,7 @@ class LayoutSchema(object):
                 'merge-failure-message': str,
                 'footer-message': str,
                 'dequeue-on-new-patchset': bool,
+                'ignore-dependencies': bool,
                 'trigger': trigger,
                 'success': report_actions,
                 'failure': report_actions,

zuul/model.py

@@ -73,6 +73,7 @@ class Pipeline(object):
         self.success_message = None
         self.footer_message = None
         self.dequeue_on_new_patchset = True
+        self.ignore_dependencies = False
         self.job_trees = {}  # project -> JobTree
         self.manager = None
         self.queues = []

zuul/scheduler.py

@@ -251,6 +251,8 @@ class Scheduler(threading.Thread):
             pipeline.footer_message = conf_pipeline.get('footer-message', "")
             pipeline.dequeue_on_new_patchset = conf_pipeline.get(
                 'dequeue-on-new-patchset', True)
+            pipeline.ignore_dependencies = conf_pipeline.get(
+                'ignore-dependencies', False)
 
             action_reporters = {}
             for action in ['start', 'success', 'failure', 'merge-failure']:
@@ -1767,6 +1769,8 @@ class IndependentPipelineManager(BasePipelineManager):
         return True
 
     def checkForChangesNeededBy(self, change, change_queue):
+        if self.pipeline.ignore_dependencies:
+            return True
         self.log.debug("Checking for changes needed by %s:" % change)
         # Return true if okay to proceed enqueing this change,
         # false if the change should not be enqueued.