Git driver

This patch improves the existing git driver by adding
a refs watcher thread. This refs watcher looks at
refs added, deleted, updated and trigger a ref-updated
event.

When a refs is updated and that the related commits
from oldrev to newrev include a change on .zuul.yaml/zuul.yaml
or zuul.d/*.yaml then tenants including that ref is reconfigured.

Furthermore the patch includes a triggering model. Events are
sent to the scheduler so jobs can be attached to a pipeline for
running jobs.

Change-Id: I529660cb20d011f36814abe64f837945dd3f1f33
This commit is contained in:
Fabien Boucher 2017-12-02 18:17:58 +01:00
parent 9dc1b9f000
commit 194a2bf237
17 changed files with 641 additions and 6 deletions

View File

@ -55,6 +55,7 @@ Zuul includes the following drivers:
drivers/gerrit drivers/gerrit
drivers/github drivers/github
drivers/git
drivers/smtp drivers/smtp
drivers/sql drivers/sql
drivers/timer drivers/timer

View File

@ -0,0 +1,59 @@
:title: Git Driver
Git
===
This driver can be used to load Zuul configuration from public Git repositories,
for instance from ``openstack-infra/zuul-jobs`` that is suitable for use by
any Zuul system. It can also be used to trigger jobs from ``ref-updated`` events
in a pipeline.
Connection Configuration
------------------------
The supported options in ``zuul.conf`` connections are:
.. attr:: <git connection>
.. attr:: driver
:required:
.. value:: git
The connection must set ``driver=git`` for Git connections.
.. attr:: baseurl
Path to the base Git URL. Git repos name will be appended to it.
.. attr:: poll_delay
:default: 7200
The delay in seconds of the Git repositories polling loop.
Trigger Configuration
---------------------
.. attr:: pipeline.trigger.<git source>
The dictionary passed to the Git pipeline ``trigger`` attribute
supports the following attributes:
.. attr:: event
:required:
Only ``ref-updated`` is supported.
.. attr:: ref
On ref-updated events, a ref such as ``refs/heads/master`` or
``^refs/tags/.*$``. This field is treated as a regular expression,
and multiple refs may be listed.
.. attr:: ignore-deletes
:default: true
When a ref is deleted, a ref-updated event is emitted with a
newrev of all zeros specified. The ``ignore-deletes`` field is a
boolean value that describes whether or not these newrevs
trigger ref-updated events.

View File

@ -2833,6 +2833,16 @@ class ZuulTestCase(BaseTestCase):
os.path.join(FIXTURE_DIR, f.name)) os.path.join(FIXTURE_DIR, f.name))
self.setupAllProjectKeys() self.setupAllProjectKeys()
def addTagToRepo(self, project, name, sha):
path = os.path.join(self.upstream_root, project)
repo = git.Repo(path)
repo.git.tag(name, sha)
def delTagFromRepo(self, project, name):
path = os.path.join(self.upstream_root, project)
repo = git.Repo(path)
repo.git.tag('-d', name)
def addCommitToRepo(self, project, message, files, def addCommitToRepo(self, project, message, files,
branch='master', tag=None): branch='master', tag=None):
path = os.path.join(self.upstream_root, project) path = os.path.join(self.upstream_root, project)

View File

@ -0,0 +1,2 @@
- hosts: all
tasks: []

View File

@ -19,6 +19,10 @@
name: project-test1 name: project-test1
run: playbooks/project-test1.yaml run: playbooks/project-test1.yaml
- job:
name: project-test2
run: playbooks/project-test2.yaml
- project: - project:
name: org/project name: org/project
check: check:

37
tests/fixtures/layouts/basic-git.yaml vendored Normal file
View File

@ -0,0 +1,37 @@
- pipeline:
name: post
manager: independent
trigger:
git:
- event: ref-updated
ref: ^refs/heads/.*$
- pipeline:
name: tag
manager: independent
trigger:
git:
- event: ref-updated
ref: ^refs/tags/.*$
- job:
name: base
parent: null
run: playbooks/base.yaml
- job:
name: post-job
run: playbooks/post-job.yaml
- job:
name: tag-job
run: playbooks/post-job.yaml
- project:
name: org/project
post:
jobs:
- post-job
tag:
jobs:
- tag-job

View File

@ -21,6 +21,7 @@ sshkey=none
[connection git] [connection git]
driver=git driver=git
baseurl="" baseurl=""
poll_delay=0.1
[connection outgoing_smtp] [connection outgoing_smtp]
driver=smtp driver=smtp

View File

@ -12,7 +12,12 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
from tests.base import ZuulTestCase
import os
import time
import yaml
from tests.base import ZuulTestCase, simple_layout
class TestGitDriver(ZuulTestCase): class TestGitDriver(ZuulTestCase):
@ -23,7 +28,7 @@ class TestGitDriver(ZuulTestCase):
super(TestGitDriver, self).setup_config() super(TestGitDriver, self).setup_config()
self.config.set('connection git', 'baseurl', self.upstream_root) self.config.set('connection git', 'baseurl', self.upstream_root)
def test_git_driver(self): def test_basic(self):
tenant = self.sched.abide.tenants.get('tenant-one') tenant = self.sched.abide.tenants.get('tenant-one')
# Check that we have the git source for common-config and the # Check that we have the git source for common-config and the
# gerrit source for the project. # gerrit source for the project.
@ -40,3 +45,127 @@ class TestGitDriver(ZuulTestCase):
self.waitUntilSettled() self.waitUntilSettled()
self.assertEqual(len(self.history), 1) self.assertEqual(len(self.history), 1)
self.assertEqual(A.reported, 1) self.assertEqual(A.reported, 1)
def test_config_refreshed(self):
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(len(self.history), 1)
self.assertEqual(A.reported, 1)
self.assertEqual(self.history[0].name, 'project-test1')
# Update zuul.yaml to force a tenant reconfiguration
path = os.path.join(self.upstream_root, 'common-config', 'zuul.yaml')
config = yaml.load(open(path, 'r').read())
change = {
'name': 'org/project',
'check': {
'jobs': [
'project-test2'
]
}
}
config[4]['project'] = change
files = {'zuul.yaml': yaml.dump(config)}
self.addCommitToRepo(
'common-config', 'Change zuul.yaml configuration', files)
# Let some time for the tenant reconfiguration to happen
time.sleep(2)
self.waitUntilSettled()
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(len(self.history), 2)
self.assertEqual(A.reported, 1)
# We make sure the new job has run
self.assertEqual(self.history[1].name, 'project-test2')
# Let's stop the git Watcher to let us merge some changes commits
# We want to verify that config changes are detected for commits
# on the range oldrev..newrev
self.sched.connections.getSource('git').connection.w_pause = True
# Add a config change
change = {
'name': 'org/project',
'check': {
'jobs': [
'project-test1'
]
}
}
config[4]['project'] = change
files = {'zuul.yaml': yaml.dump(config)}
self.addCommitToRepo(
'common-config', 'Change zuul.yaml configuration', files)
# Add two other changes
self.addCommitToRepo(
'common-config', 'Adding f1',
{'f1': "Content"})
self.addCommitToRepo(
'common-config', 'Adding f2',
{'f2': "Content"})
# Restart the git watcher
self.sched.connections.getSource('git').connection.w_pause = False
# Let some time for the tenant reconfiguration to happen
time.sleep(2)
self.waitUntilSettled()
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(len(self.history), 3)
self.assertEqual(A.reported, 1)
# We make sure the new job has run
self.assertEqual(self.history[2].name, 'project-test1')
def ensure_watcher_has_context(self):
# Make sure watcher have read initial refs shas
cnx = self.sched.connections.getSource('git').connection
delay = 0.1
max_delay = 1
while not cnx.projects_refs:
time.sleep(delay)
max_delay -= delay
if max_delay <= 0:
raise Exception("Timeout waiting for initial read")
@simple_layout('layouts/basic-git.yaml', driver='git')
def test_ref_updated_event(self):
self.ensure_watcher_has_context()
# Add a commit to trigger a ref-updated event
self.addCommitToRepo(
'org/project', 'A change for ref-updated', {'f1': 'Content'})
# Let some time for the git watcher to detect the ref-update event
time.sleep(0.2)
self.waitUntilSettled()
self.assertEqual(len(self.history), 1)
self.assertEqual('SUCCESS',
self.getJobFromHistory('post-job').result)
@simple_layout('layouts/basic-git.yaml', driver='git')
def test_ref_created(self):
self.ensure_watcher_has_context()
# Tag HEAD to trigger a ref-updated event
self.addTagToRepo(
'org/project', 'atag', 'HEAD')
# Let some time for the git watcher to detect the ref-update event
time.sleep(0.2)
self.waitUntilSettled()
self.assertEqual(len(self.history), 1)
self.assertEqual('SUCCESS',
self.getJobFromHistory('tag-job').result)
@simple_layout('layouts/basic-git.yaml', driver='git')
def test_ref_deleted(self):
self.ensure_watcher_has_context()
# Delete default tag init to trigger a ref-updated event
self.delTagFromRepo(
'org/project', 'init')
# Let some time for the git watcher to detect the ref-update event
time.sleep(0.2)
self.waitUntilSettled()
# Make sure no job as run as ignore-delete is True by default
self.assertEqual(len(self.history), 0)

View File

@ -15,6 +15,7 @@
from zuul.driver import Driver, ConnectionInterface, SourceInterface from zuul.driver import Driver, ConnectionInterface, SourceInterface
from zuul.driver.git import gitconnection from zuul.driver.git import gitconnection
from zuul.driver.git import gitsource from zuul.driver.git import gitsource
from zuul.driver.git import gittrigger
class GitDriver(Driver, ConnectionInterface, SourceInterface): class GitDriver(Driver, ConnectionInterface, SourceInterface):
@ -23,9 +24,15 @@ class GitDriver(Driver, ConnectionInterface, SourceInterface):
def getConnection(self, name, config): def getConnection(self, name, config):
return gitconnection.GitConnection(self, name, config) return gitconnection.GitConnection(self, name, config)
def getTrigger(self, connection, config=None):
return gittrigger.GitTrigger(self, connection, config)
def getSource(self, connection): def getSource(self, connection):
return gitsource.GitSource(self, connection) return gitsource.GitSource(self, connection)
def getTriggerSchema(self):
return gittrigger.getSchema()
def getRequireSchema(self): def getRequireSchema(self):
return {} return {}

View File

@ -13,12 +13,119 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
import os
import git
import time
import logging import logging
import urllib import urllib
import threading
import voluptuous as v import voluptuous as v
from zuul.connection import BaseConnection from zuul.connection import BaseConnection
from zuul.driver.git.gitmodel import GitTriggerEvent, EMPTY_GIT_REF
from zuul.model import Ref, Branch
class GitWatcher(threading.Thread):
log = logging.getLogger("connection.git.GitWatcher")
def __init__(self, git_connection, baseurl, poll_delay):
threading.Thread.__init__(self)
self.daemon = True
self.git_connection = git_connection
self.baseurl = baseurl
self.poll_delay = poll_delay
self._stopped = False
self.projects_refs = self.git_connection.projects_refs
def compareRefs(self, project, refs):
partial_events = []
# Fetch previous refs state
base_refs = self.projects_refs.get(project)
# Create list of created refs
rcreateds = set(refs.keys()) - set(base_refs.keys())
# Create list of deleted refs
rdeleteds = set(base_refs.keys()) - set(refs.keys())
# Create the list of updated refs
updateds = {}
for ref, sha in refs.items():
if ref in base_refs and base_refs[ref] != sha:
updateds[ref] = sha
for ref in rcreateds:
event = {
'ref': ref,
'branch_created': True,
'oldrev': EMPTY_GIT_REF,
'newrev': refs[ref]
}
partial_events.append(event)
for ref in rdeleteds:
event = {
'ref': ref,
'branch_deleted': True,
'oldrev': base_refs[ref],
'newrev': EMPTY_GIT_REF
}
partial_events.append(event)
for ref, sha in updateds.items():
event = {
'ref': ref,
'branch_updated': True,
'oldrev': base_refs[ref],
'newrev': sha
}
partial_events.append(event)
events = []
for pevent in partial_events:
event = GitTriggerEvent()
event.type = 'ref-updated'
event.project_hostname = self.git_connection.canonical_hostname
event.project_name = project
for attr in ('ref', 'oldrev', 'newrev', 'branch_created',
'branch_deleted', 'branch_updated'):
if attr in pevent:
setattr(event, attr, pevent[attr])
events.append(event)
return events
def _run(self):
self.log.debug("Walk through projects refs for connection: %s" %
self.git_connection.connection_name)
try:
for project in self.git_connection.projects:
refs = self.git_connection.lsRemote(project)
self.log.debug("Read refs %s for project %s" % (refs, project))
if not self.projects_refs.get(project):
# State for this project does not exist yet so add it.
# No event will be triggered in this loop as
# projects_refs['project'] and refs are equal
self.projects_refs[project] = refs
events = self.compareRefs(project, refs)
self.projects_refs[project] = refs
# Send events to the scheduler
for event in events:
self.log.debug("Handling event: %s" % event)
# Force changes cache update before passing
# the event to the scheduler
self.git_connection.getChange(event)
self.git_connection.logEvent(event)
# Pass the event to the scheduler
self.git_connection.sched.addEvent(event)
except Exception as e:
self.log.debug("Unexpected issue in _run loop: %s" % str(e))
def run(self):
while not self._stopped:
if not self.git_connection.w_pause:
self._run()
# Polling wait delay
else:
self.log.debug("Watcher is on pause")
time.sleep(self.poll_delay)
def stop(self):
self._stopped = True
class GitConnection(BaseConnection): class GitConnection(BaseConnection):
@ -32,6 +139,8 @@ class GitConnection(BaseConnection):
raise Exception('baseurl is required for git connections in ' raise Exception('baseurl is required for git connections in '
'%s' % self.connection_name) '%s' % self.connection_name)
self.baseurl = self.connection_config.get('baseurl') self.baseurl = self.connection_config.get('baseurl')
self.poll_timeout = float(
self.connection_config.get('poll_delay', 3600 * 2))
self.canonical_hostname = self.connection_config.get( self.canonical_hostname = self.connection_config.get(
'canonical_hostname') 'canonical_hostname')
if not self.canonical_hostname: if not self.canonical_hostname:
@ -40,7 +149,10 @@ class GitConnection(BaseConnection):
self.canonical_hostname = r.hostname self.canonical_hostname = r.hostname
else: else:
self.canonical_hostname = 'localhost' self.canonical_hostname = 'localhost'
self.w_pause = False
self.projects = {} self.projects = {}
self.projects_refs = {}
self._change_cache = {}
def getProject(self, name): def getProject(self, name):
return self.projects.get(name) return self.projects.get(name)
@ -48,15 +160,97 @@ class GitConnection(BaseConnection):
def addProject(self, project): def addProject(self, project):
self.projects[project.name] = project self.projects[project.name] = project
def getChangeFilesUpdated(self, project_name, branch, tosha):
job = self.sched.merger.getFilesChanges(
self.connection_name, project_name, branch, tosha)
self.log.debug("Waiting for fileschanges job %s" % job)
job.wait()
if not job.updated:
raise Exception("Fileschanges job %s failed" % job)
self.log.debug("Fileschanges job %s got changes on files %s" %
(job, job.files))
return job.files
def lsRemote(self, project):
refs = {}
client = git.cmd.Git()
output = client.ls_remote(
os.path.join(self.baseurl, project))
for line in output.splitlines():
sha, ref = line.split('\t')
if ref.startswith('refs/'):
refs[ref] = sha
return refs
def maintainCache(self, relevant):
remove = {}
for branch, refschange in self._change_cache.items():
for ref, change in refschange.items():
if change not in relevant:
remove.setdefault(branch, []).append(ref)
for branch, refs in remove.items():
for ref in refs:
del self._change_cache[branch][ref]
if not self._change_cache[branch]:
del self._change_cache[branch]
def getChange(self, event, refresh=False):
if event.ref and event.ref.startswith('refs/heads/'):
branch = event.ref[len('refs/heads/'):]
change = self._change_cache.get(branch, {}).get(event.newrev)
if change:
return change
project = self.getProject(event.project_name)
change = Branch(project)
change.branch = branch
for attr in ('ref', 'oldrev', 'newrev'):
setattr(change, attr, getattr(event, attr))
change.url = ""
change.files = self.getChangeFilesUpdated(
event.project_name, change.branch, event.oldrev)
self._change_cache.setdefault(branch, {})[event.newrev] = change
elif event.ref:
# catch-all ref (ie, not a branch or head)
project = self.getProject(event.project_name)
change = Ref(project)
for attr in ('ref', 'oldrev', 'newrev'):
setattr(change, attr, getattr(event, attr))
change.url = ""
else:
self.log.warning("Unable to get change for %s" % (event,))
change = None
return change
def getProjectBranches(self, project, tenant): def getProjectBranches(self, project, tenant):
# TODO(jeblair): implement; this will need to handle local or refs = self.lsRemote(project.name)
# remote git urls. branches = [ref[len('refs/heads/'):] for ref in
return ['master'] refs if ref.startswith('refs/heads/')]
return branches
def getGitUrl(self, project): def getGitUrl(self, project):
url = '%s/%s' % (self.baseurl, project.name) url = '%s/%s' % (self.baseurl, project.name)
return url return url
def onLoad(self):
self.log.debug("Starting Git Watcher")
self._start_watcher_thread()
def onStop(self):
self.log.debug("Stopping Git Watcher")
self._stop_watcher_thread()
def _stop_watcher_thread(self):
if self.watcher_thread:
self.watcher_thread.stop()
self.watcher_thread.join()
def _start_watcher_thread(self):
self.watcher_thread = GitWatcher(
self,
self.baseurl,
self.poll_timeout)
self.watcher_thread.start()
def getSchema(): def getSchema():
git_connection = v.Any(str, v.Schema(dict)) git_connection = v.Any(str, v.Schema(dict))

View File

@ -0,0 +1,86 @@
# Copyright 2017 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import re
from zuul.model import TriggerEvent
from zuul.model import EventFilter
EMPTY_GIT_REF = '0' * 40 # git sha of all zeros, used during creates/deletes
class GitTriggerEvent(TriggerEvent):
"""Incoming event from an external system."""
def __repr__(self):
ret = '<GitTriggerEvent %s %s' % (self.type,
self.project_name)
if self.branch:
ret += " %s" % self.branch
ret += " oldrev:%s" % self.oldrev
ret += " newrev:%s" % self.newrev
ret += '>'
return ret
class GitEventFilter(EventFilter):
def __init__(self, trigger, types=[], refs=[],
ignore_deletes=True):
super().__init__(trigger)
self._refs = refs
self.types = types
self.refs = [re.compile(x) for x in refs]
self.ignore_deletes = ignore_deletes
def __repr__(self):
ret = '<GitEventFilter'
if self.types:
ret += ' types: %s' % ', '.join(self.types)
if self._refs:
ret += ' refs: %s' % ', '.join(self._refs)
if self.ignore_deletes:
ret += ' ignore_deletes: %s' % self.ignore_deletes
ret += '>'
return ret
def matches(self, event, change):
# event types are ORed
matches_type = False
for etype in self.types:
if etype == event.type:
matches_type = True
if self.types and not matches_type:
return False
# refs are ORed
matches_ref = False
if event.ref is not None:
for ref in self.refs:
if ref.match(event.ref):
matches_ref = True
if self.refs and not matches_ref:
return False
if self.ignore_deletes and event.newrev == EMPTY_GIT_REF:
# If the updated ref has an empty git sha (all 0s),
# then the ref is being deleted
return False
return True

View File

@ -36,7 +36,7 @@ class GitSource(BaseSource):
raise NotImplemented() raise NotImplemented()
def getChange(self, event, refresh=False): def getChange(self, event, refresh=False):
raise NotImplemented() return self.connection.getChange(event, refresh)
def getProject(self, name): def getProject(self, name):
p = self.connection.getProject(name) p = self.connection.getProject(name)

View File

@ -0,0 +1,49 @@
# Copyright 2017 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import logging
import voluptuous as v
from zuul.trigger import BaseTrigger
from zuul.driver.git.gitmodel import GitEventFilter
from zuul.driver.util import scalar_or_list, to_list
class GitTrigger(BaseTrigger):
name = 'git'
log = logging.getLogger("zuul.GitTrigger")
def getEventFilters(self, trigger_conf):
efilters = []
for trigger in to_list(trigger_conf):
f = GitEventFilter(
trigger=self,
types=to_list(trigger['event']),
refs=to_list(trigger.get('ref')),
ignore_deletes=trigger.get(
'ignore-deletes', True)
)
efilters.append(f)
return efilters
def getSchema():
git_trigger = {
v.Required('event'):
scalar_or_list(v.Any('ref-updated')),
'ref': scalar_or_list(str),
'ignore-deletes': bool,
}
return git_trigger

View File

@ -1706,6 +1706,7 @@ class ExecutorServer(object):
self.merger_worker.registerFunction("merger:merge") self.merger_worker.registerFunction("merger:merge")
self.merger_worker.registerFunction("merger:cat") self.merger_worker.registerFunction("merger:cat")
self.merger_worker.registerFunction("merger:refstate") self.merger_worker.registerFunction("merger:refstate")
self.merger_worker.registerFunction("merger:fileschanges")
def register_work(self): def register_work(self):
if self._running: if self._running:
@ -1859,6 +1860,9 @@ class ExecutorServer(object):
elif job.name == 'merger:refstate': elif job.name == 'merger:refstate':
self.log.debug("Got refstate job: %s" % job.unique) self.log.debug("Got refstate job: %s" % job.unique)
self.refstate(job) self.refstate(job)
elif job.name == 'merger:fileschanges':
self.log.debug("Got fileschanges job: %s" % job.unique)
self.fileschanges(job)
else: else:
self.log.error("Unable to handle job %s" % job.name) self.log.error("Unable to handle job %s" % job.name)
job.sendWorkFail() job.sendWorkFail()
@ -1970,6 +1974,19 @@ class ExecutorServer(object):
files=files) files=files)
job.sendWorkComplete(json.dumps(result)) job.sendWorkComplete(json.dumps(result))
def fileschanges(self, job):
args = json.loads(job.arguments)
task = self.update(args['connection'], args['project'])
task.wait()
with self.merger_lock:
files = self.merger.getFilesChanges(
args['connection'], args['project'],
args['branch'],
args['tosha'])
result = dict(updated=True,
files=files)
job.sendWorkComplete(json.dumps(result))
def refstate(self, job): def refstate(self, job):
args = json.loads(job.arguments) args = json.loads(job.arguments)
with self.merger_lock: with self.merger_lock:

View File

@ -131,6 +131,15 @@ class MergeClient(object):
job = self.submitJob('merger:cat', data, None, precedence) job = self.submitJob('merger:cat', data, None, precedence)
return job return job
def getFilesChanges(self, connection_name, project_name, branch,
tosha=None, precedence=zuul.model.PRECEDENCE_HIGH):
data = dict(connection=connection_name,
project=project_name,
branch=branch,
tosha=tosha)
job = self.submitJob('merger:fileschanges', data, None, precedence)
return job
def onBuildCompleted(self, job): def onBuildCompleted(self, job):
data = getJobData(job) data = getJobData(job)
merged = data.get('merged', False) merged = data.get('merged', False)

View File

@ -314,6 +314,18 @@ class Repo(object):
'utf-8') 'utf-8')
return ret return ret
def getFilesChanges(self, branch, tosha=None):
repo = self.createRepoObject()
files = set()
head = repo.heads[branch].commit
files.update(set(head.stats.files.keys()))
if tosha:
for cmt in head.iter_parents():
if cmt.hexsha == tosha:
break
files.update(set(cmt.stats.files.keys()))
return list(files)
def deleteRemote(self, remote): def deleteRemote(self, remote):
repo = self.createRepoObject() repo = self.createRepoObject()
repo.delete_remote(repo.remotes[remote]) repo.delete_remote(repo.remotes[remote])
@ -581,3 +593,8 @@ class Merger(object):
def getFiles(self, connection_name, project_name, branch, files, dirs=[]): def getFiles(self, connection_name, project_name, branch, files, dirs=[]):
repo = self.getRepo(connection_name, project_name) repo = self.getRepo(connection_name, project_name)
return repo.getFiles(files, dirs, branch=branch) return repo.getFiles(files, dirs, branch=branch)
def getFilesChanges(self, connection_name, project_name, branch,
tosha=None):
repo = self.getRepo(connection_name, project_name)
return repo.getFilesChanges(branch, tosha)

View File

@ -81,6 +81,7 @@ class MergeServer(object):
self.worker.registerFunction("merger:merge") self.worker.registerFunction("merger:merge")
self.worker.registerFunction("merger:cat") self.worker.registerFunction("merger:cat")
self.worker.registerFunction("merger:refstate") self.worker.registerFunction("merger:refstate")
self.worker.registerFunction("merger:fileschanges")
def stop(self): def stop(self):
self.log.debug("Stopping") self.log.debug("Stopping")
@ -117,6 +118,9 @@ class MergeServer(object):
elif job.name == 'merger:refstate': elif job.name == 'merger:refstate':
self.log.debug("Got refstate job: %s" % job.unique) self.log.debug("Got refstate job: %s" % job.unique)
self.refstate(job) self.refstate(job)
elif job.name == 'merger:fileschanges':
self.log.debug("Got fileschanges job: %s" % job.unique)
self.fileschanges(job)
else: else:
self.log.error("Unable to handle job %s" % job.name) self.log.error("Unable to handle job %s" % job.name)
job.sendWorkFail() job.sendWorkFail()
@ -158,3 +162,12 @@ class MergeServer(object):
result = dict(updated=True, result = dict(updated=True,
files=files) files=files)
job.sendWorkComplete(json.dumps(result)) job.sendWorkComplete(json.dumps(result))
def fileschanges(self, job):
args = json.loads(job.arguments)
self.merger.updateRepo(args['connection'], args['project'])
files = self.merger.getFilesChanges(
args['connection'], args['project'], args['branch'], args['tosha'])
result = dict(updated=True,
files=files)
job.sendWorkComplete(json.dumps(result))