Git driver

This patch improves the existing git driver by adding
a refs watcher thread. This refs watcher looks at
refs added, deleted, updated and trigger a ref-updated
event.

When a refs is updated and that the related commits
from oldrev to newrev include a change on .zuul.yaml/zuul.yaml
or zuul.d/*.yaml then tenants including that ref is reconfigured.

Furthermore the patch includes a triggering model. Events are
sent to the scheduler so jobs can be attached to a pipeline for
running jobs.

Change-Id: I529660cb20d011f36814abe64f837945dd3f1f33

doc/source/admin/connections.rst

@@ -55,6 +55,7 @@ Zuul includes the following drivers:
 
    drivers/gerrit
    drivers/github
+   drivers/git
    drivers/smtp
    drivers/sql
    drivers/timer

doc/source/admin/drivers/git.rst

@@ -0,0 +1,59 @@
+:title: Git Driver
+
+Git
+===
+
+This driver can be used to load Zuul configuration from public Git repositories,
+for instance from ``openstack-infra/zuul-jobs`` that is suitable for use by
+any Zuul system. It can also be used to trigger jobs from ``ref-updated`` events
+in a pipeline.
+
+Connection Configuration
+------------------------
+
+The supported options in ``zuul.conf`` connections are:
+
+.. attr:: <git connection>
+
+   .. attr:: driver
+      :required:
+
+      .. value:: git
+
+         The connection must set ``driver=git`` for Git connections.
+
+   .. attr:: baseurl
+
+      Path to the base Git URL. Git repos name will be appended to it.
+
+   .. attr:: poll_delay
+      :default: 7200
+
+      The delay in seconds of the Git repositories polling loop.
+
+Trigger Configuration
+---------------------
+
+.. attr:: pipeline.trigger.<git source>
+
+   The dictionary passed to the Git pipeline ``trigger`` attribute
+   supports the following attributes:
+
+   .. attr:: event
+      :required:
+
+      Only ``ref-updated`` is supported.
+
+   .. attr:: ref
+
+      On ref-updated events, a ref such as ``refs/heads/master`` or
+      ``^refs/tags/.*$``. This field is treated as a regular expression,
+      and multiple refs may be listed.
+
+   .. attr:: ignore-deletes
+      :default: true
+
+      When a ref is deleted, a ref-updated event is emitted with a
+      newrev of all zeros specified. The ``ignore-deletes`` field is a
+      boolean value that describes whether or not these newrevs
+      trigger ref-updated events.

tests/base.py

@@ -2833,6 +2833,16 @@ class ZuulTestCase(BaseTestCase):
                         os.path.join(FIXTURE_DIR, f.name))
         self.setupAllProjectKeys()
 
+    def addTagToRepo(self, project, name, sha):
+        path = os.path.join(self.upstream_root, project)
+        repo = git.Repo(path)
+        repo.git.tag(name, sha)
+
+    def delTagFromRepo(self, project, name):
+        path = os.path.join(self.upstream_root, project)
+        repo = git.Repo(path)
+        repo.git.tag('-d', name)
+
     def addCommitToRepo(self, project, message, files,
                         branch='master', tag=None):
         path = os.path.join(self.upstream_root, project)

tests/fixtures/config/git-driver/git/common-config/playbooks/project-test2.yaml

@@ -0,0 +1,2 @@
+- hosts: all
+  tasks: []

tests/fixtures/config/git-driver/git/common-config/zuul.yaml

@@ -19,6 +19,10 @@
     name: project-test1
     run: playbooks/project-test1.yaml
 
+- job:
+    name: project-test2
+    run: playbooks/project-test2.yaml
+
 - project:
     name: org/project
     check:

tests/fixtures/layouts/basic-git.yaml

@@ -0,0 +1,37 @@
+- pipeline:
+    name: post
+    manager: independent
+    trigger:
+      git:
+        - event: ref-updated
+          ref: ^refs/heads/.*$
+
+- pipeline:
+    name: tag
+    manager: independent
+    trigger:
+      git:
+        - event: ref-updated
+          ref: ^refs/tags/.*$
+
+- job:
+    name: base
+    parent: null
+    run: playbooks/base.yaml
+
+- job:
+    name: post-job
+    run: playbooks/post-job.yaml
+
+- job:
+    name: tag-job
+    run: playbooks/post-job.yaml
+
+- project:
+    name: org/project
+    post:
+      jobs:
+        - post-job
+    tag:
+      jobs:
+        - tag-job

tests/fixtures/zuul-git-driver.conf

@@ -21,6 +21,7 @@ sshkey=none
 [connection git]
 driver=git
 baseurl=""
+poll_delay=0.1
 
 [connection outgoing_smtp]
 driver=smtp

tests/unit/test_git_driver.py

@@ -12,7 +12,12 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-from tests.base import ZuulTestCase
+
+import os
+import time
+import yaml
+
+from tests.base import ZuulTestCase, simple_layout
 
 
 class TestGitDriver(ZuulTestCase):
@@ -23,7 +28,7 @@ class TestGitDriver(ZuulTestCase):
         super(TestGitDriver, self).setup_config()
         self.config.set('connection git', 'baseurl', self.upstream_root)
 
-    def test_git_driver(self):
+    def test_basic(self):
         tenant = self.sched.abide.tenants.get('tenant-one')
         # Check that we have the git source for common-config and the
         # gerrit source for the project.
@@ -40,3 +45,127 @@ class TestGitDriver(ZuulTestCase):
         self.waitUntilSettled()
         self.assertEqual(len(self.history), 1)
         self.assertEqual(A.reported, 1)
+
+    def test_config_refreshed(self):
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.assertEqual(len(self.history), 1)
+        self.assertEqual(A.reported, 1)
+        self.assertEqual(self.history[0].name, 'project-test1')
+
+        # Update zuul.yaml to force a tenant reconfiguration
+        path = os.path.join(self.upstream_root, 'common-config', 'zuul.yaml')
+        config = yaml.load(open(path, 'r').read())
+        change = {
+            'name': 'org/project',
+            'check': {
+                'jobs': [
+                    'project-test2'
+                ]
+            }
+        }
+        config[4]['project'] = change
+        files = {'zuul.yaml': yaml.dump(config)}
+        self.addCommitToRepo(
+            'common-config', 'Change zuul.yaml configuration', files)
+
+        # Let some time for the tenant reconfiguration to happen
+        time.sleep(2)
+        self.waitUntilSettled()
+
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.assertEqual(len(self.history), 2)
+        self.assertEqual(A.reported, 1)
+        # We make sure the new job has run
+        self.assertEqual(self.history[1].name, 'project-test2')
+
+        # Let's stop the git Watcher to let us merge some changes commits
+        # We want to verify that config changes are detected for commits
+        # on the range oldrev..newrev
+        self.sched.connections.getSource('git').connection.w_pause = True
+        # Add a config change
+        change = {
+            'name': 'org/project',
+            'check': {
+                'jobs': [
+                    'project-test1'
+                ]
+            }
+        }
+        config[4]['project'] = change
+        files = {'zuul.yaml': yaml.dump(config)}
+        self.addCommitToRepo(
+            'common-config', 'Change zuul.yaml configuration', files)
+        # Add two other changes
+        self.addCommitToRepo(
+            'common-config', 'Adding f1',
+            {'f1': "Content"})
+        self.addCommitToRepo(
+            'common-config', 'Adding f2',
+            {'f2': "Content"})
+        # Restart the git watcher
+        self.sched.connections.getSource('git').connection.w_pause = False
+
+        # Let some time for the tenant reconfiguration to happen
+        time.sleep(2)
+        self.waitUntilSettled()
+
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.assertEqual(len(self.history), 3)
+        self.assertEqual(A.reported, 1)
+        # We make sure the new job has run
+        self.assertEqual(self.history[2].name, 'project-test1')
+
+    def ensure_watcher_has_context(self):
+        # Make sure watcher have read initial refs shas
+        cnx = self.sched.connections.getSource('git').connection
+        delay = 0.1
+        max_delay = 1
+        while not cnx.projects_refs:
+            time.sleep(delay)
+            max_delay -= delay
+            if max_delay <= 0:
+                raise Exception("Timeout waiting for initial read")
+
+    @simple_layout('layouts/basic-git.yaml', driver='git')
+    def test_ref_updated_event(self):
+        self.ensure_watcher_has_context()
+        # Add a commit to trigger a ref-updated event
+        self.addCommitToRepo(
+            'org/project', 'A change for ref-updated', {'f1': 'Content'})
+        # Let some time for the git watcher to detect the ref-update event
+        time.sleep(0.2)
+        self.waitUntilSettled()
+        self.assertEqual(len(self.history), 1)
+        self.assertEqual('SUCCESS',
+                         self.getJobFromHistory('post-job').result)
+
+    @simple_layout('layouts/basic-git.yaml', driver='git')
+    def test_ref_created(self):
+        self.ensure_watcher_has_context()
+        # Tag HEAD to trigger a ref-updated event
+        self.addTagToRepo(
+            'org/project', 'atag', 'HEAD')
+        # Let some time for the git watcher to detect the ref-update event
+        time.sleep(0.2)
+        self.waitUntilSettled()
+        self.assertEqual(len(self.history), 1)
+        self.assertEqual('SUCCESS',
+                         self.getJobFromHistory('tag-job').result)
+
+    @simple_layout('layouts/basic-git.yaml', driver='git')
+    def test_ref_deleted(self):
+        self.ensure_watcher_has_context()
+        # Delete default tag init to trigger a ref-updated event
+        self.delTagFromRepo(
+            'org/project', 'init')
+        # Let some time for the git watcher to detect the ref-update event
+        time.sleep(0.2)
+        self.waitUntilSettled()
+        # Make sure no job as run as ignore-delete is True by default
+        self.assertEqual(len(self.history), 0)

zuul/driver/git/__init__.py

@@ -15,6 +15,7 @@
 from zuul.driver import Driver, ConnectionInterface, SourceInterface
 from zuul.driver.git import gitconnection
 from zuul.driver.git import gitsource
+from zuul.driver.git import gittrigger
 
 
 class GitDriver(Driver, ConnectionInterface, SourceInterface):
@@ -23,9 +24,15 @@ class GitDriver(Driver, ConnectionInterface, SourceInterface):
     def getConnection(self, name, config):
         return gitconnection.GitConnection(self, name, config)
 
+    def getTrigger(self, connection, config=None):
+        return gittrigger.GitTrigger(self, connection, config)
+
     def getSource(self, connection):
         return gitsource.GitSource(self, connection)
 
+    def getTriggerSchema(self):
+        return gittrigger.getSchema()
+
     def getRequireSchema(self):
         return {}
 

zuul/driver/git/gitconnection.py

@@ -13,12 +13,119 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import os
+import git
+import time
 import logging
 import urllib
+import threading
 
 import voluptuous as v
 
 from zuul.connection import BaseConnection
+from zuul.driver.git.gitmodel import GitTriggerEvent, EMPTY_GIT_REF
+from zuul.model import Ref, Branch
+
+
+class GitWatcher(threading.Thread):
+    log = logging.getLogger("connection.git.GitWatcher")
+
+    def __init__(self, git_connection, baseurl, poll_delay):
+        threading.Thread.__init__(self)
+        self.daemon = True
+        self.git_connection = git_connection
+        self.baseurl = baseurl
+        self.poll_delay = poll_delay
+        self._stopped = False
+        self.projects_refs = self.git_connection.projects_refs
+
+    def compareRefs(self, project, refs):
+        partial_events = []
+        # Fetch previous refs state
+        base_refs = self.projects_refs.get(project)
+        # Create list of created refs
+        rcreateds = set(refs.keys()) - set(base_refs.keys())
+        # Create list of deleted refs
+        rdeleteds = set(base_refs.keys()) - set(refs.keys())
+        # Create the list of updated refs
+        updateds = {}
+        for ref, sha in refs.items():
+            if ref in base_refs and base_refs[ref] != sha:
+                updateds[ref] = sha
+        for ref in rcreateds:
+            event = {
+                'ref': ref,
+                'branch_created': True,
+                'oldrev': EMPTY_GIT_REF,
+                'newrev': refs[ref]
+            }
+            partial_events.append(event)
+        for ref in rdeleteds:
+            event = {
+                'ref': ref,
+                'branch_deleted': True,
+                'oldrev': base_refs[ref],
+                'newrev': EMPTY_GIT_REF
+            }
+            partial_events.append(event)
+        for ref, sha in updateds.items():
+            event = {
+                'ref': ref,
+                'branch_updated': True,
+                'oldrev': base_refs[ref],
+                'newrev': sha
+            }
+            partial_events.append(event)
+        events = []
+        for pevent in partial_events:
+            event = GitTriggerEvent()
+            event.type = 'ref-updated'
+            event.project_hostname = self.git_connection.canonical_hostname
+            event.project_name = project
+            for attr in ('ref', 'oldrev', 'newrev', 'branch_created',
+                         'branch_deleted', 'branch_updated'):
+                if attr in pevent:
+                    setattr(event, attr, pevent[attr])
+            events.append(event)
+        return events
+
+    def _run(self):
+        self.log.debug("Walk through projects refs for connection: %s" %
+                       self.git_connection.connection_name)
+        try:
+            for project in self.git_connection.projects:
+                refs = self.git_connection.lsRemote(project)
+                self.log.debug("Read refs %s for project %s" % (refs, project))
+                if not self.projects_refs.get(project):
+                    # State for this project does not exist yet so add it.
+                    # No event will be triggered in this loop as
+                    # projects_refs['project'] and refs are equal
+                    self.projects_refs[project] = refs
+                events = self.compareRefs(project, refs)
+                self.projects_refs[project] = refs
+                # Send events to the scheduler
+                for event in events:
+                    self.log.debug("Handling event: %s" % event)
+                    # Force changes cache update before passing
+                    # the event to the scheduler
+                    self.git_connection.getChange(event)
+                    self.git_connection.logEvent(event)
+                    # Pass the event to the scheduler
+                    self.git_connection.sched.addEvent(event)
+        except Exception as e:
+            self.log.debug("Unexpected issue in _run loop: %s" % str(e))
+
+    def run(self):
+        while not self._stopped:
+            if not self.git_connection.w_pause:
+                self._run()
+                # Polling wait delay
+            else:
+                self.log.debug("Watcher is on pause")
+            time.sleep(self.poll_delay)
+
+    def stop(self):
+        self._stopped = True
 
 
 class GitConnection(BaseConnection):
@@ -32,6 +139,8 @@ class GitConnection(BaseConnection):
             raise Exception('baseurl is required for git connections in '
                             '%s' % self.connection_name)
         self.baseurl = self.connection_config.get('baseurl')
+        self.poll_timeout = float(
+            self.connection_config.get('poll_delay', 3600 * 2))
         self.canonical_hostname = self.connection_config.get(
             'canonical_hostname')
         if not self.canonical_hostname:
@@ -40,7 +149,10 @@ class GitConnection(BaseConnection):
                 self.canonical_hostname = r.hostname
             else:
                 self.canonical_hostname = 'localhost'
+        self.w_pause = False
         self.projects = {}
+        self.projects_refs = {}
+        self._change_cache = {}
 
     def getProject(self, name):
         return self.projects.get(name)
@@ -48,15 +160,97 @@ class GitConnection(BaseConnection):
     def addProject(self, project):
         self.projects[project.name] = project
 
+    def getChangeFilesUpdated(self, project_name, branch, tosha):
+        job = self.sched.merger.getFilesChanges(
+            self.connection_name, project_name, branch, tosha)
+        self.log.debug("Waiting for fileschanges job %s" % job)
+        job.wait()
+        if not job.updated:
+            raise Exception("Fileschanges job %s failed" % job)
+        self.log.debug("Fileschanges job %s got changes on files %s" %
+                       (job, job.files))
+        return job.files
+
+    def lsRemote(self, project):
+        refs = {}
+        client = git.cmd.Git()
+        output = client.ls_remote(
+            os.path.join(self.baseurl, project))
+        for line in output.splitlines():
+            sha, ref = line.split('\t')
+            if ref.startswith('refs/'):
+                refs[ref] = sha
+        return refs
+
+    def maintainCache(self, relevant):
+        remove = {}
+        for branch, refschange in self._change_cache.items():
+            for ref, change in refschange.items():
+                if change not in relevant:
+                    remove.setdefault(branch, []).append(ref)
+        for branch, refs in remove.items():
+            for ref in refs:
+                del self._change_cache[branch][ref]
+            if not self._change_cache[branch]:
+                del self._change_cache[branch]
+
+    def getChange(self, event, refresh=False):
+        if event.ref and event.ref.startswith('refs/heads/'):
+            branch = event.ref[len('refs/heads/'):]
+            change = self._change_cache.get(branch, {}).get(event.newrev)
+            if change:
+                return change
+            project = self.getProject(event.project_name)
+            change = Branch(project)
+            change.branch = branch
+            for attr in ('ref', 'oldrev', 'newrev'):
+                setattr(change, attr, getattr(event, attr))
+            change.url = ""
+            change.files = self.getChangeFilesUpdated(
+                event.project_name, change.branch, event.oldrev)
+            self._change_cache.setdefault(branch, {})[event.newrev] = change
+        elif event.ref:
+            # catch-all ref (ie, not a branch or head)
+            project = self.getProject(event.project_name)
+            change = Ref(project)
+            for attr in ('ref', 'oldrev', 'newrev'):
+                setattr(change, attr, getattr(event, attr))
+            change.url = ""
+        else:
+            self.log.warning("Unable to get change for %s" % (event,))
+            change = None
+        return change
+
     def getProjectBranches(self, project, tenant):
-        # TODO(jeblair): implement; this will need to handle local or
-        # remote git urls.
-        return ['master']
+        refs = self.lsRemote(project.name)
+        branches = [ref[len('refs/heads/'):] for ref in
+                    refs if ref.startswith('refs/heads/')]
+        return branches
 
     def getGitUrl(self, project):
         url = '%s/%s' % (self.baseurl, project.name)
         return url
 
+    def onLoad(self):
+        self.log.debug("Starting Git Watcher")
+        self._start_watcher_thread()
+
+    def onStop(self):
+        self.log.debug("Stopping Git Watcher")
+        self._stop_watcher_thread()
+
+    def _stop_watcher_thread(self):
+        if self.watcher_thread:
+            self.watcher_thread.stop()
+            self.watcher_thread.join()
+
+    def _start_watcher_thread(self):
+        self.watcher_thread = GitWatcher(
+            self,
+            self.baseurl,
+            self.poll_timeout)
+        self.watcher_thread.start()
+
 
 def getSchema():
     git_connection = v.Any(str, v.Schema(dict))

zuul/driver/git/gitmodel.py

@@ -0,0 +1,86 @@
+# Copyright 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import re
+
+from zuul.model import TriggerEvent
+from zuul.model import EventFilter
+
+
+EMPTY_GIT_REF = '0' * 40  # git sha of all zeros, used during creates/deletes
+
+
+class GitTriggerEvent(TriggerEvent):
+    """Incoming event from an external system."""
+
+    def __repr__(self):
+        ret = '<GitTriggerEvent %s %s' % (self.type,
+                                          self.project_name)
+
+        if self.branch:
+            ret += " %s" % self.branch
+        ret += " oldrev:%s" % self.oldrev
+        ret += " newrev:%s" % self.newrev
+        ret += '>'
+
+        return ret
+
+
+class GitEventFilter(EventFilter):
+    def __init__(self, trigger, types=[], refs=[],
+                 ignore_deletes=True):
+
+        super().__init__(trigger)
+
+        self._refs = refs
+        self.types = types
+        self.refs = [re.compile(x) for x in refs]
+        self.ignore_deletes = ignore_deletes
+
+    def __repr__(self):
+        ret = '<GitEventFilter'
+
+        if self.types:
+            ret += ' types: %s' % ', '.join(self.types)
+        if self._refs:
+            ret += ' refs: %s' % ', '.join(self._refs)
+        if self.ignore_deletes:
+            ret += ' ignore_deletes: %s' % self.ignore_deletes
+        ret += '>'
+
+        return ret
+
+    def matches(self, event, change):
+        # event types are ORed
+        matches_type = False
+        for etype in self.types:
+            if etype == event.type:
+                matches_type = True
+        if self.types and not matches_type:
+            return False
+
+        # refs are ORed
+        matches_ref = False
+        if event.ref is not None:
+            for ref in self.refs:
+                if ref.match(event.ref):
+                    matches_ref = True
+        if self.refs and not matches_ref:
+            return False
+        if self.ignore_deletes and event.newrev == EMPTY_GIT_REF:
+            # If the updated ref has an empty git sha (all 0s),
+            # then the ref is being deleted
+            return False
+
+        return True

zuul/driver/git/gitsource.py

@@ -36,7 +36,7 @@ class GitSource(BaseSource):
         raise NotImplemented()
 
     def getChange(self, event, refresh=False):
-        raise NotImplemented()
+        return self.connection.getChange(event, refresh)
 
     def getProject(self, name):
         p = self.connection.getProject(name)

zuul/driver/git/gittrigger.py

@@ -0,0 +1,49 @@
+# Copyright 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+import voluptuous as v
+from zuul.trigger import BaseTrigger
+from zuul.driver.git.gitmodel import GitEventFilter
+from zuul.driver.util import scalar_or_list, to_list
+
+
+class GitTrigger(BaseTrigger):
+    name = 'git'
+    log = logging.getLogger("zuul.GitTrigger")
+
+    def getEventFilters(self, trigger_conf):
+        efilters = []
+        for trigger in to_list(trigger_conf):
+            f = GitEventFilter(
+                trigger=self,
+                types=to_list(trigger['event']),
+                refs=to_list(trigger.get('ref')),
+                ignore_deletes=trigger.get(
+                    'ignore-deletes', True)
+            )
+            efilters.append(f)
+
+        return efilters
+
+
+def getSchema():
+    git_trigger = {
+        v.Required('event'):
+            scalar_or_list(v.Any('ref-updated')),
+        'ref': scalar_or_list(str),
+        'ignore-deletes': bool,
+    }
+
+    return git_trigger

zuul/executor/server.py

@@ -1706,6 +1706,7 @@ class ExecutorServer(object):
         self.merger_worker.registerFunction("merger:merge")
         self.merger_worker.registerFunction("merger:cat")
         self.merger_worker.registerFunction("merger:refstate")
+        self.merger_worker.registerFunction("merger:fileschanges")
 
     def register_work(self):
         if self._running:
@@ -1859,6 +1860,9 @@ class ExecutorServer(object):
             elif job.name == 'merger:refstate':
                 self.log.debug("Got refstate job: %s" % job.unique)
                 self.refstate(job)
+            elif job.name == 'merger:fileschanges':
+                self.log.debug("Got fileschanges job: %s" % job.unique)
+                self.fileschanges(job)
             else:
                 self.log.error("Unable to handle job %s" % job.name)
                 job.sendWorkFail()
@@ -1970,6 +1974,19 @@ class ExecutorServer(object):
                       files=files)
         job.sendWorkComplete(json.dumps(result))
 
+    def fileschanges(self, job):
+        args = json.loads(job.arguments)
+        task = self.update(args['connection'], args['project'])
+        task.wait()
+        with self.merger_lock:
+            files = self.merger.getFilesChanges(
+                args['connection'], args['project'],
+                args['branch'],
+                args['tosha'])
+        result = dict(updated=True,
+                      files=files)
+        job.sendWorkComplete(json.dumps(result))
+
     def refstate(self, job):
         args = json.loads(job.arguments)
         with self.merger_lock:

zuul/merger/client.py

@@ -131,6 +131,15 @@ class MergeClient(object):
         job = self.submitJob('merger:cat', data, None, precedence)
         return job
 
+    def getFilesChanges(self, connection_name, project_name, branch,
+                        tosha=None, precedence=zuul.model.PRECEDENCE_HIGH):
+        data = dict(connection=connection_name,
+                    project=project_name,
+                    branch=branch,
+                    tosha=tosha)
+        job = self.submitJob('merger:fileschanges', data, None, precedence)
+        return job
+
     def onBuildCompleted(self, job):
         data = getJobData(job)
         merged = data.get('merged', False)

zuul/merger/merger.py

@@ -314,6 +314,18 @@ class Repo(object):
                             'utf-8')
         return ret
 
+    def getFilesChanges(self, branch, tosha=None):
+        repo = self.createRepoObject()
+        files = set()
+        head = repo.heads[branch].commit
+        files.update(set(head.stats.files.keys()))
+        if tosha:
+            for cmt in head.iter_parents():
+                if cmt.hexsha == tosha:
+                    break
+                files.update(set(cmt.stats.files.keys()))
+        return list(files)
+
     def deleteRemote(self, remote):
         repo = self.createRepoObject()
         repo.delete_remote(repo.remotes[remote])
@@ -581,3 +593,8 @@ class Merger(object):
     def getFiles(self, connection_name, project_name, branch, files, dirs=[]):
         repo = self.getRepo(connection_name, project_name)
         return repo.getFiles(files, dirs, branch=branch)
+
+    def getFilesChanges(self, connection_name, project_name, branch,
+                        tosha=None):
+        repo = self.getRepo(connection_name, project_name)
+        return repo.getFilesChanges(branch, tosha)

zuul/merger/server.py

@@ -81,6 +81,7 @@ class MergeServer(object):
         self.worker.registerFunction("merger:merge")
         self.worker.registerFunction("merger:cat")
         self.worker.registerFunction("merger:refstate")
+        self.worker.registerFunction("merger:fileschanges")
 
     def stop(self):
         self.log.debug("Stopping")
@@ -117,6 +118,9 @@ class MergeServer(object):
                     elif job.name == 'merger:refstate':
                         self.log.debug("Got refstate job: %s" % job.unique)
                         self.refstate(job)
+                    elif job.name == 'merger:fileschanges':
+                        self.log.debug("Got fileschanges job: %s" % job.unique)
+                        self.fileschanges(job)
                     else:
                         self.log.error("Unable to handle job %s" % job.name)
                         job.sendWorkFail()
@@ -158,3 +162,12 @@ class MergeServer(object):
         result = dict(updated=True,
                       files=files)
         job.sendWorkComplete(json.dumps(result))
+
+    def fileschanges(self, job):
+        args = json.loads(job.arguments)
+        self.merger.updateRepo(args['connection'], args['project'])
+        files = self.merger.getFilesChanges(
+            args['connection'], args['project'], args['branch'], args['tosha'])
+        result = dict(updated=True,
+                      files=files)
+        job.sendWorkComplete(json.dumps(result))