Merge "Add a Zuul trigger"

doc/source/triggers.rst

@@ -4,8 +4,7 @@ Triggers
 ========
 
 The process of merging a change starts with proposing a change to be
-merged.  Primarily, Zuul supports Gerrit as a triggering system, as
-well as a facility for triggering jobs based on a timer.
+merged.  Primarily, Zuul supports Gerrit as a triggering system.
 Zuul's design is modular, so alternate triggering and reporting
 systems can be supported.
 
@@ -40,3 +39,8 @@ Timer
 
 A simple timer trigger is available as well.  It supports triggering
 jobs in a pipeline based on cron-style time instructions.
+
+Zuul
+----
+
+The Zuul trigger generates events based on internal actions in Zuul.

doc/source/zuul.rst

@@ -389,7 +389,7 @@ explanation of each of the parameters::
     DependentPipelineManager, see: :doc:`gating`.
 
 **trigger**
-  Exactly one trigger source must be supplied for each pipeline.
+  At least one trigger source must be supplied for each pipeline.
   Triggers are not exclusive -- matching events may be placed in
   multiple pipelines, and they will behave independently in each of
   the pipelines they match.  You may select from the following:
@@ -475,6 +475,31 @@ explanation of each of the parameters::
     supported, not the symbolic names.  Example: ``0 0 * * *`` runs
     at midnight.
 
+  **zuul**
+    This trigger supplies events generated internally by Zuul.
+    Multiple events may be listed.
+
+    *event*
+    The event name.  Currently supported:
+
+      *project-change-merged* when Zuul merges a change to a project,
+      it generates this event for every open change in the project.
+
+      *parent-change-enqueued* when Zuul enqueues a change into any
+      pipeline, it generates this event for every child of that
+      change.
+
+    *pipeline*
+    Only available for ``parent-change-enqueued`` events.  This is the
+    name of the pipeline in which the parent change was enqueued.
+
+    *require-approval*
+    This may be used for any event.  It requires that a certain kind
+    of approval be present for the current patchset of the change (the
+    approval could be added by the event in question).  It follows the
+    same syntax as the "approval" pipeline requirement below.
+
+
 **require**
   If this section is present, it established pre-requisites for any
   kind of item entering the Pipeline.  Regardless of how the item is

tests/base.py

@@ -52,6 +52,7 @@ import zuul.reporter.gerrit
 import zuul.reporter.smtp
 import zuul.trigger.gerrit
 import zuul.trigger.timer
+import zuul.trigger.zuultrigger
 
 FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
                            'fixtures')
@@ -401,6 +402,11 @@ class FakeGerrit(object):
             return change.query()
         return {}
 
+    def simpleQuery(self, query):
+        # This is currently only used to return all open changes for a
+        # project
+        return [change.query() for change in self.changes.values()]
+
     def startWatching(self, *args, **kw):
         pass
 
@@ -906,6 +912,8 @@ class ZuulTestCase(testtools.TestCase):
         self.sched.registerTrigger(self.gerrit)
         self.timer = zuul.trigger.timer.Timer(self.config, self.sched)
         self.sched.registerTrigger(self.timer)
+        self.zuultrigger = zuul.trigger.zuultrigger.ZuulTrigger(self.config, self.sched)
+        self.sched.registerTrigger(self.zuultrigger)
 
         self.sched.registerReporter(
             zuul.reporter.gerrit.Reporter(self.gerrit))

tests/fixtures/layout-zuultrigger-enqueued.yaml

@@ -0,0 +1,53 @@
+pipelines:
+  - name: check
+    manager: IndependentPipelineManager
+    source: gerrit
+    require:
+      approval:
+        - verified: -1
+    trigger:
+      gerrit:
+        - event: patchset-created
+      zuul:
+        - event: parent-change-enqueued
+          pipeline: gate
+    success:
+      gerrit:
+        verified: 1
+    failure:
+      gerrit:
+        verified: -1
+
+  - name: gate
+    manager: DependentPipelineManager
+    failure-message: Build failed.  For information on how to proceed, see http://wiki.example.org/Test_Failures
+    source: gerrit
+    require:
+      approval:
+        - verified: 1
+    trigger:
+      gerrit:
+        - event: comment-added
+          approval:
+            - approved: 1
+      zuul:
+        - event: parent-change-enqueued
+          pipeline: gate
+    success:
+      gerrit:
+        verified: 2
+        submit: true
+    failure:
+      gerrit:
+        verified: -2
+    start:
+      gerrit:
+        verified: 0
+    precedence: high
+
+projects:
+  - name: org/project
+    check:
+      - project-check
+    gate:
+      - project-gate

tests/fixtures/layout-zuultrigger-merged.yaml

@@ -0,0 +1,53 @@
+pipelines:
+  - name: check
+    manager: IndependentPipelineManager
+    source: gerrit
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        verified: 1
+    failure:
+      gerrit:
+        verified: -1
+
+  - name: gate
+    manager: DependentPipelineManager
+    failure-message: Build failed.  For information on how to proceed, see http://wiki.example.org/Test_Failures
+    source: gerrit
+    trigger:
+      gerrit:
+        - event: comment-added
+          approval:
+            - approved: 1
+    success:
+      gerrit:
+        verified: 2
+        submit: true
+    failure:
+      gerrit:
+        verified: -2
+    start:
+      gerrit:
+        verified: 0
+    precedence: high
+
+  - name: merge-check
+    manager: IndependentPipelineManager
+    source: gerrit
+    trigger:
+      zuul:
+        - event: project-change-merged
+    merge-failure:
+      gerrit:
+        verified: -1
+
+projects:
+  - name: org/project
+    check:
+      - project-check
+    gate:
+      - project-gate
+    merge-check:
+      - noop

tests/test_scheduler.py

@@ -1742,6 +1742,7 @@ class TestScheduler(ZuulTestCase):
         sched = zuul.scheduler.Scheduler()
         sched.registerTrigger(None, 'gerrit')
         sched.registerTrigger(None, 'timer')
+        sched.registerTrigger(None, 'zuul')
         sched.testConfig(self.config.get('zuul', 'layout_config'))
 
     def test_build_description(self):

tests/test_zuultrigger.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python
+
+# Copyright 2014 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+import time
+
+from tests.base import ZuulTestCase
+
+logging.basicConfig(level=logging.DEBUG,
+                    format='%(asctime)s %(name)-32s '
+                    '%(levelname)-8s %(message)s')
+
+
+class TestZuulTrigger(ZuulTestCase):
+    """Test Zuul Trigger"""
+
+    def test_zuul_trigger_parent_change_enqueued(self):
+        "Test Zuul trigger event: parent-change-enqueued"
+        self.config.set('zuul', 'layout_config',
+                        'tests/fixtures/layout-zuultrigger-enqueued.yaml')
+        self.sched.reconfigure(self.config)
+        self.registerJobs()
+
+        # This test has the following three changes:
+        # B1 -> A; B2 -> A
+        # When A is enqueued in the gate, B1 and B2 should both attempt
+        # to be enqueued in both pipelines.  B1 should end up in check
+        # and B2 in gate because of differing pipeline requirements.
+        self.worker.hold_jobs_in_build = True
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+        B1 = self.fake_gerrit.addFakeChange('org/project', 'master', 'B1')
+        B2 = self.fake_gerrit.addFakeChange('org/project', 'master', 'B2')
+        A.addApproval('CRVW', 2)
+        B1.addApproval('CRVW', 2)
+        B2.addApproval('CRVW', 2)
+        A.addApproval('VRFY', 1)  # required by gate
+        B1.addApproval('VRFY', -1) # should go to check
+        B2.addApproval('VRFY', 1)  # should go to gate
+        B1.addApproval('APRV', 1)
+        B2.addApproval('APRV', 1)
+        B1.setDependsOn(A, 1)
+        B2.setDependsOn(A, 1)
+        self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
+        # Jobs are being held in build to make sure that 3,1 has time
+        # to enqueue behind 1,1 so that the test is more
+        # deterministic.
+        self.waitUntilSettled()
+        self.worker.hold_jobs_in_build = False
+        self.worker.release()
+        self.waitUntilSettled()
+
+        self.assertEqual(len(self.history), 3)
+        for job in self.history:
+            if job.changes == '1,1':
+                self.assertEqual(job.name, 'project-gate')
+            elif job.changes == '2,1':
+                self.assertEqual(job.name, 'project-check')
+            elif job.changes == '1,1 3,1':
+                self.assertEqual(job.name, 'project-gate')
+            else:
+                raise Exception("Unknown job")
+
+    def test_zuul_trigger_project_change_merged(self):
+        "Test Zuul trigger event: project-change-merged"
+        self.config.set('zuul', 'layout_config',
+                        'tests/fixtures/layout-zuultrigger-merged.yaml')
+        self.sched.reconfigure(self.config)
+        self.registerJobs()
+
+        # This test has the following three changes:
+        # A, B, C;  B conflicts with A, but C does not.
+        # When A is merged, B and C should be checked for conflicts,
+        # and B should receive a -1.
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
+        C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
+        A.addPatchset(['conflict'])
+        B.addPatchset(['conflict'])
+        A.addApproval('CRVW', 2)
+        self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
+        self.waitUntilSettled()
+
+        self.assertEqual(len(self.history), 1)
+        self.assertEqual(self.history[0].name, 'project-gate')
+        self.assertEqual(A.reported, 2)
+        self.assertEqual(B.reported, 1)
+        self.assertEqual(C.reported, 0)
+        self.assertEqual(B.messages[0],
+            "Merge Failed.\n\nThis change was unable to be automatically "
+            "merged with the current state of the repository. Please rebase "
+            "your change and upload a new patchset.")

zuul/cmd/server.py

@@ -87,6 +87,7 @@ class Server(zuul.cmd.ZuulApp):
         self.sched.registerReporter(None, 'smtp')
         self.sched.registerTrigger(None, 'gerrit')
         self.sched.registerTrigger(None, 'timer')
+        self.sched.registerTrigger(None, 'zuul')
         layout = self.sched.testConfig(self.config.get('zuul',
                                                        'layout_config'))
         if not job_list_path:
@@ -145,6 +146,7 @@ class Server(zuul.cmd.ZuulApp):
         import zuul.reporter.smtp
         import zuul.trigger.gerrit
         import zuul.trigger.timer
+        import zuul.trigger.zuultrigger
         import zuul.webapp
         import zuul.rpclistener
 
@@ -163,6 +165,7 @@ class Server(zuul.cmd.ZuulApp):
         merger = zuul.merger.client.MergeClient(self.config, self.sched)
         gerrit = zuul.trigger.gerrit.Gerrit(self.config, self.sched)
         timer = zuul.trigger.timer.Timer(self.config, self.sched)
+        zuultrigger = zuul.trigger.zuultrigger.ZuulTrigger(self.config, self.sched)
         if self.config.has_option('zuul', 'status_expiry'):
             cache_expiry = self.config.getint('zuul', 'status_expiry')
         else:
@@ -185,6 +188,7 @@ class Server(zuul.cmd.ZuulApp):
         self.sched.setMerger(merger)
         self.sched.registerTrigger(gerrit)
         self.sched.registerTrigger(timer)
+        self.sched.registerTrigger(zuultrigger)
         self.sched.registerReporter(gerrit_reporter)
         self.sched.registerReporter(smtp_reporter)
 

zuul/layoutvalidator.py

@@ -65,8 +65,16 @@ class LayoutSchema(object):
 
     timer_trigger = {v.Required('time'): str}
 
-    trigger = v.Required(v.Any({'gerrit': toList(gerrit_trigger)},
-                               {'timer': toList(timer_trigger)}))
+    zuul_trigger = {v.Required('event'):
+                    toList(v.Any('parent-change-enqueued',
+                                 'project-change-merged')),
+                    'pipeline': toList(str),
+                    'require-approval': toList(require_approval),
+                    }
+
+    trigger = v.Required({'gerrit': toList(gerrit_trigger),
+                          'timer': toList(timer_trigger),
+                          'zuul': toList(zuul_trigger)})
 
     report_actions = {'gerrit': variable_dict,
                       'smtp': {'to': str,

zuul/lib/gerrit.py

@@ -144,6 +144,23 @@ class Gerrit(object):
                        (pprint.pformat(data)))
         return data
 
+    def simpleQuery(self, query):
+        args = '--current-patch-set'
+        cmd = 'gerrit query --format json %s %s' % (
+            args, query)
+        out, err = self._ssh(cmd)
+        if not out:
+            return False
+        lines = out.split('\n')
+        if not lines:
+            return False
+        data = [json.loads(line) for line in lines[:-1]]
+        if not data:
+            return False
+        self.log.debug("Received data from Gerrit query: \n%s" %
+                       (pprint.pformat(data)))
+        return data
+
     def _open(self):
         client = paramiko.SSHClient()
         client.load_system_host_keys()

zuul/model.py

@@ -947,6 +947,8 @@ class TriggerEvent(object):
         self.newrev = None
         # timer
         self.timespec = None
+        # zuultrigger
+        self.pipeline_name = None
         # For events that arrive with a destination pipeline (eg, from
         # an admin command, etc):
         self.forced_pipeline = None
@@ -1026,7 +1028,7 @@ class BaseFilter(object):
 class EventFilter(BaseFilter):
     def __init__(self, trigger, types=[], branches=[], refs=[],
                  event_approvals={}, comments=[], emails=[], usernames=[],
-                 timespecs=[], required_approvals=[]):
+                 timespecs=[], required_approvals=[], pipelines=[]):
         super(EventFilter, self).__init__(
             required_approvals=required_approvals)
         self.trigger = trigger
@@ -1036,12 +1038,14 @@ class EventFilter(BaseFilter):
         self._comments = comments
         self._emails = emails
         self._usernames = usernames
+        self._pipelines = pipelines
         self.types = [re.compile(x) for x in types]
         self.branches = [re.compile(x) for x in branches]
         self.refs = [re.compile(x) for x in refs]
         self.comments = [re.compile(x) for x in comments]
         self.emails = [re.compile(x) for x in emails]
         self.usernames = [re.compile(x) for x in usernames]
+        self.pipelines = [re.compile(x) for x in pipelines]
         self.event_approvals = event_approvals
         self.timespecs = timespecs
 
@@ -1050,6 +1054,8 @@ class EventFilter(BaseFilter):
 
         if self._types:
             ret += ' types: %s' % ', '.join(self._types)
+        if self._pipelines:
+            ret += ' pipelines: %s' % ', '.join(self._pipelines)
         if self._branches:
             ret += ' branches: %s' % ', '.join(self._branches)
         if self._refs:
@@ -1081,6 +1087,14 @@ class EventFilter(BaseFilter):
         if self.types and not matches_type:
             return False
 
+        # pipelines are ORed
+        matches_pipeline = False
+        for epipe in self.pipelines:
+            if epipe.match(event.pipeline_name):
+                matches_pipeline = True
+        if self.pipelines and not matches_pipeline:
+            return False
+
         # branches are ORed
         matches_branch = False
         for branch in self.branches:

zuul/scheduler.py

@@ -1,4 +1,4 @@
-# Copyright 2012 Hewlett-Packard Development Company, L.P.
+# Copyright 2012-2014 Hewlett-Packard Development Company, L.P.
 # Copyright 2013 OpenStack Foundation
 # Copyright 2013 Antoine "hashar" Musso
 # Copyright 2013 Wikimedia Foundation Inc.
@@ -326,12 +326,20 @@ class Scheduler(threading.Thread):
                                     required_approvals=
                                     toList(trigger.get('require-approval')))
                     manager.event_filters.append(f)
-            elif 'timer' in conf_pipeline['trigger']:
+            if 'timer' in conf_pipeline['trigger']:
                 for trigger in toList(conf_pipeline['trigger']['timer']):
                     f = EventFilter(trigger=self.triggers['timer'],
                                     types=['timer'],
                                     timespecs=toList(trigger['time']))
                     manager.event_filters.append(f)
+            if 'zuul' in conf_pipeline['trigger']:
+                for trigger in toList(conf_pipeline['trigger']['zuul']):
+                    f = EventFilter(trigger=self.triggers['zuul'],
+                                    types=toList(trigger['event']),
+                                    pipelines=toList(trigger.get('pipeline')),
+                                    required_approvals=
+                                    toList(trigger.get('require-approval')))
+                    manager.event_filters.append(f)
 
         for project_template in data.get('project-templates', []):
             # Make sure the template only contains valid pipelines
@@ -1153,6 +1161,7 @@ class BasePipelineManager(object):
                 item.enqueue_time = enqueue_time
             self.reportStats(item)
             self.enqueueChangesBehind(change, quiet, ignore_requirements)
+            self.sched.triggers['zuul'].onChangeEnqueued(item.change, self.pipeline)
         else:
             self.log.error("Unable to find change queue for project %s" %
                            change.project)
@@ -1427,6 +1436,7 @@ class BasePipelineManager(object):
                 change_queue.increaseWindowSize()
                 self.log.debug("%s window size increased to %s" %
                                (change_queue, change_queue.window))
+                self.sched.triggers['zuul'].onChangeMerged(item.change)
 
     def _reportItem(self, item):
         self.log.debug("Reporting change %s" % item.change)

zuul/trigger/gerrit.py

@@ -323,6 +323,14 @@ class Gerrit(object):
             raise
         return change
 
+    def getProjectOpenChanges(self, project):
+        data = self.gerrit.simpleQuery("project:%s status:open" % project.name)
+        changes = []
+        for record in data:
+            changes.append(self._getChange(record['number'],
+                                           record['currentPatchSet']['number']))
+        return changes
+
     def updateChange(self, change):
         self.log.info("Updating information for %s,%s" %
                       (change.number, change.patchset))

zuul/trigger/zuultrigger.py

@@ -0,0 +1,117 @@
+# Copyright 2012-2014 Hewlett-Packard Development Company, L.P.
+# Copyright 2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+from zuul.model import TriggerEvent
+
+
+class ZuulTrigger(object):
+    name = 'zuul'
+    log = logging.getLogger("zuul.ZuulTrigger")
+
+    def __init__(self, config, sched):
+        self.sched = sched
+        self.config = config
+        self._handle_parent_change_enqueued_events = False
+        self._handle_project_change_merged_events = False
+
+    def stop(self):
+        pass
+
+    def isMerged(self, change, head=None):
+        raise Exception("Zuul trigger does not support checking if "
+                        "a change is merged.")
+
+    def canMerge(self, change, allow_needs):
+        raise Exception("Zuul trigger does not support checking if "
+                        "a change can merge.")
+
+    def maintainCache(self, relevant):
+        return
+
+    def onChangeMerged(self, change):
+        # Called each time zuul merges a change
+        if self._handle_project_change_merged_events:
+            try:
+                self._createProjectChangeMergedEvents(change)
+            except Exception:
+                self.log.exception("Unable to create project-change-merged events for %s" % (change,))
+
+    def onChangeEnqueued(self, change, pipeline):
+        # Called each time a change is enqueued in a pipeline
+        if self._handle_parent_change_enqueued_events:
+            try:
+                self._createParentChangeEnqueuedEvents(change, pipeline)
+            except Exception:
+                self.log.exception("Unable to create parent-change-enqueued events for %s in %s" % (change, pipeline))
+
+    def _createProjectChangeMergedEvents(self, change):
+        changes = self.sched.triggers['gerrit'].getProjectOpenChanges(change.project)
+        for change in changes:
+            self._createProjectChangeMergedEvent(change)
+
+    def _createProjectChangeMergedEvent(self, change):
+        event = TriggerEvent()
+        event.type = 'project-change-merged'
+        event.trigger_name = self.name
+        event.project_name = change.project.name
+        event.change_number = change.number
+        event.branch = change.branch
+        event.change_url = change.url
+        event.patch_number = change.patchset
+        event.refspec = change.refspec
+        self.sched.addEvent(event)
+
+    def _createParentChangeEnqueuedEvents(self, change, pipeline):
+        self.log.debug("Checking for changes needing %s:" % change)
+        if not hasattr(change, 'needed_by_changes'):
+            self.log.debug("  Changeish does not support dependencies")
+            return
+        for needs in change.needed_by_changes:
+            self._createParentChangeEnqueuedEvent(needs, pipeline)
+
+    def _createParentChangeEnqueuedEvent(self, change, pipeline):
+        event = TriggerEvent()
+        event.type = 'parent-change-enqueued'
+        event.trigger_name = self.name
+        event.pipeline_name = pipeline.name
+        event.project_name = change.project.name
+        event.change_number = change.number
+        event.branch = change.branch
+        event.change_url = change.url
+        event.patch_number = change.patchset
+        event.refspec = change.refspec
+        self.sched.addEvent(event)
+
+    def postConfig(self):
+        self._handle_parent_change_enqueued_events = False
+        self._handle_project_change_merged_events = False
+        for pipeline in self.sched.layout.pipelines.values():
+            for ef in pipeline.manager.event_filters:
+                if ef.trigger != self:
+                    continue
+                if 'parent-change-enqueued' in ef._types:
+                    self._handle_parent_change_enqueued_events = True
+                elif 'project-change-merged' in ef._types:
+                    self._handle_project_change_merged_events = True
+
+    def getChange(self, number, patchset, refresh=False):
+        raise Exception("Zuul trigger does not support changes.")
+
+    def getGitUrl(self, project):
+        raise Exception("Zuul trigger does not support changes.")
+
+    def getGitwebUrl(self, project, sha=None):
+        raise Exception("Zuul trigger does not support changes.")