Merge "Git driver" into feature/zuulv3
This commit is contained in:
commit
17a4ae8c78
|
@ -55,6 +55,7 @@ Zuul includes the following drivers:
|
|||
|
||||
drivers/gerrit
|
||||
drivers/github
|
||||
drivers/git
|
||||
drivers/smtp
|
||||
drivers/sql
|
||||
drivers/timer
|
||||
|
|
|
@ -0,0 +1,59 @@
|
|||
:title: Git Driver
|
||||
|
||||
Git
|
||||
===
|
||||
|
||||
This driver can be used to load Zuul configuration from public Git repositories,
|
||||
for instance from ``openstack-infra/zuul-jobs`` that is suitable for use by
|
||||
any Zuul system. It can also be used to trigger jobs from ``ref-updated`` events
|
||||
in a pipeline.
|
||||
|
||||
Connection Configuration
|
||||
------------------------
|
||||
|
||||
The supported options in ``zuul.conf`` connections are:
|
||||
|
||||
.. attr:: <git connection>
|
||||
|
||||
.. attr:: driver
|
||||
:required:
|
||||
|
||||
.. value:: git
|
||||
|
||||
The connection must set ``driver=git`` for Git connections.
|
||||
|
||||
.. attr:: baseurl
|
||||
|
||||
Path to the base Git URL. Git repos name will be appended to it.
|
||||
|
||||
.. attr:: poll_delay
|
||||
:default: 7200
|
||||
|
||||
The delay in seconds of the Git repositories polling loop.
|
||||
|
||||
Trigger Configuration
|
||||
---------------------
|
||||
|
||||
.. attr:: pipeline.trigger.<git source>
|
||||
|
||||
The dictionary passed to the Git pipeline ``trigger`` attribute
|
||||
supports the following attributes:
|
||||
|
||||
.. attr:: event
|
||||
:required:
|
||||
|
||||
Only ``ref-updated`` is supported.
|
||||
|
||||
.. attr:: ref
|
||||
|
||||
On ref-updated events, a ref such as ``refs/heads/master`` or
|
||||
``^refs/tags/.*$``. This field is treated as a regular expression,
|
||||
and multiple refs may be listed.
|
||||
|
||||
.. attr:: ignore-deletes
|
||||
:default: true
|
||||
|
||||
When a ref is deleted, a ref-updated event is emitted with a
|
||||
newrev of all zeros specified. The ``ignore-deletes`` field is a
|
||||
boolean value that describes whether or not these newrevs
|
||||
trigger ref-updated events.
|
|
@ -2833,6 +2833,16 @@ class ZuulTestCase(BaseTestCase):
|
|||
os.path.join(FIXTURE_DIR, f.name))
|
||||
self.setupAllProjectKeys()
|
||||
|
||||
def addTagToRepo(self, project, name, sha):
|
||||
path = os.path.join(self.upstream_root, project)
|
||||
repo = git.Repo(path)
|
||||
repo.git.tag(name, sha)
|
||||
|
||||
def delTagFromRepo(self, project, name):
|
||||
path = os.path.join(self.upstream_root, project)
|
||||
repo = git.Repo(path)
|
||||
repo.git.tag('-d', name)
|
||||
|
||||
def addCommitToRepo(self, project, message, files,
|
||||
branch='master', tag=None):
|
||||
path = os.path.join(self.upstream_root, project)
|
||||
|
|
2
tests/fixtures/config/git-driver/git/common-config/playbooks/project-test2.yaml
vendored
Normal file
2
tests/fixtures/config/git-driver/git/common-config/playbooks/project-test2.yaml
vendored
Normal file
|
@ -0,0 +1,2 @@
|
|||
- hosts: all
|
||||
tasks: []
|
|
@ -19,6 +19,10 @@
|
|||
name: project-test1
|
||||
run: playbooks/project-test1.yaml
|
||||
|
||||
- job:
|
||||
name: project-test2
|
||||
run: playbooks/project-test2.yaml
|
||||
|
||||
- project:
|
||||
name: org/project
|
||||
check:
|
||||
|
|
|
@ -0,0 +1,37 @@
|
|||
- pipeline:
|
||||
name: post
|
||||
manager: independent
|
||||
trigger:
|
||||
git:
|
||||
- event: ref-updated
|
||||
ref: ^refs/heads/.*$
|
||||
|
||||
- pipeline:
|
||||
name: tag
|
||||
manager: independent
|
||||
trigger:
|
||||
git:
|
||||
- event: ref-updated
|
||||
ref: ^refs/tags/.*$
|
||||
|
||||
- job:
|
||||
name: base
|
||||
parent: null
|
||||
run: playbooks/base.yaml
|
||||
|
||||
- job:
|
||||
name: post-job
|
||||
run: playbooks/post-job.yaml
|
||||
|
||||
- job:
|
||||
name: tag-job
|
||||
run: playbooks/post-job.yaml
|
||||
|
||||
- project:
|
||||
name: org/project
|
||||
post:
|
||||
jobs:
|
||||
- post-job
|
||||
tag:
|
||||
jobs:
|
||||
- tag-job
|
|
@ -21,6 +21,7 @@ sshkey=none
|
|||
[connection git]
|
||||
driver=git
|
||||
baseurl=""
|
||||
poll_delay=0.1
|
||||
|
||||
[connection outgoing_smtp]
|
||||
driver=smtp
|
||||
|
|
|
@ -12,7 +12,12 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from tests.base import ZuulTestCase
|
||||
|
||||
import os
|
||||
import time
|
||||
import yaml
|
||||
|
||||
from tests.base import ZuulTestCase, simple_layout
|
||||
|
||||
|
||||
class TestGitDriver(ZuulTestCase):
|
||||
|
@ -23,7 +28,7 @@ class TestGitDriver(ZuulTestCase):
|
|||
super(TestGitDriver, self).setup_config()
|
||||
self.config.set('connection git', 'baseurl', self.upstream_root)
|
||||
|
||||
def test_git_driver(self):
|
||||
def test_basic(self):
|
||||
tenant = self.sched.abide.tenants.get('tenant-one')
|
||||
# Check that we have the git source for common-config and the
|
||||
# gerrit source for the project.
|
||||
|
@ -40,3 +45,127 @@ class TestGitDriver(ZuulTestCase):
|
|||
self.waitUntilSettled()
|
||||
self.assertEqual(len(self.history), 1)
|
||||
self.assertEqual(A.reported, 1)
|
||||
|
||||
def test_config_refreshed(self):
|
||||
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
|
||||
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
|
||||
self.waitUntilSettled()
|
||||
self.assertEqual(len(self.history), 1)
|
||||
self.assertEqual(A.reported, 1)
|
||||
self.assertEqual(self.history[0].name, 'project-test1')
|
||||
|
||||
# Update zuul.yaml to force a tenant reconfiguration
|
||||
path = os.path.join(self.upstream_root, 'common-config', 'zuul.yaml')
|
||||
config = yaml.load(open(path, 'r').read())
|
||||
change = {
|
||||
'name': 'org/project',
|
||||
'check': {
|
||||
'jobs': [
|
||||
'project-test2'
|
||||
]
|
||||
}
|
||||
}
|
||||
config[4]['project'] = change
|
||||
files = {'zuul.yaml': yaml.dump(config)}
|
||||
self.addCommitToRepo(
|
||||
'common-config', 'Change zuul.yaml configuration', files)
|
||||
|
||||
# Let some time for the tenant reconfiguration to happen
|
||||
time.sleep(2)
|
||||
self.waitUntilSettled()
|
||||
|
||||
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
|
||||
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
|
||||
self.waitUntilSettled()
|
||||
self.assertEqual(len(self.history), 2)
|
||||
self.assertEqual(A.reported, 1)
|
||||
# We make sure the new job has run
|
||||
self.assertEqual(self.history[1].name, 'project-test2')
|
||||
|
||||
# Let's stop the git Watcher to let us merge some changes commits
|
||||
# We want to verify that config changes are detected for commits
|
||||
# on the range oldrev..newrev
|
||||
self.sched.connections.getSource('git').connection.w_pause = True
|
||||
# Add a config change
|
||||
change = {
|
||||
'name': 'org/project',
|
||||
'check': {
|
||||
'jobs': [
|
||||
'project-test1'
|
||||
]
|
||||
}
|
||||
}
|
||||
config[4]['project'] = change
|
||||
files = {'zuul.yaml': yaml.dump(config)}
|
||||
self.addCommitToRepo(
|
||||
'common-config', 'Change zuul.yaml configuration', files)
|
||||
# Add two other changes
|
||||
self.addCommitToRepo(
|
||||
'common-config', 'Adding f1',
|
||||
{'f1': "Content"})
|
||||
self.addCommitToRepo(
|
||||
'common-config', 'Adding f2',
|
||||
{'f2': "Content"})
|
||||
# Restart the git watcher
|
||||
self.sched.connections.getSource('git').connection.w_pause = False
|
||||
|
||||
# Let some time for the tenant reconfiguration to happen
|
||||
time.sleep(2)
|
||||
self.waitUntilSettled()
|
||||
|
||||
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
|
||||
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
|
||||
self.waitUntilSettled()
|
||||
self.assertEqual(len(self.history), 3)
|
||||
self.assertEqual(A.reported, 1)
|
||||
# We make sure the new job has run
|
||||
self.assertEqual(self.history[2].name, 'project-test1')
|
||||
|
||||
def ensure_watcher_has_context(self):
|
||||
# Make sure watcher have read initial refs shas
|
||||
cnx = self.sched.connections.getSource('git').connection
|
||||
delay = 0.1
|
||||
max_delay = 1
|
||||
while not cnx.projects_refs:
|
||||
time.sleep(delay)
|
||||
max_delay -= delay
|
||||
if max_delay <= 0:
|
||||
raise Exception("Timeout waiting for initial read")
|
||||
|
||||
@simple_layout('layouts/basic-git.yaml', driver='git')
|
||||
def test_ref_updated_event(self):
|
||||
self.ensure_watcher_has_context()
|
||||
# Add a commit to trigger a ref-updated event
|
||||
self.addCommitToRepo(
|
||||
'org/project', 'A change for ref-updated', {'f1': 'Content'})
|
||||
# Let some time for the git watcher to detect the ref-update event
|
||||
time.sleep(0.2)
|
||||
self.waitUntilSettled()
|
||||
self.assertEqual(len(self.history), 1)
|
||||
self.assertEqual('SUCCESS',
|
||||
self.getJobFromHistory('post-job').result)
|
||||
|
||||
@simple_layout('layouts/basic-git.yaml', driver='git')
|
||||
def test_ref_created(self):
|
||||
self.ensure_watcher_has_context()
|
||||
# Tag HEAD to trigger a ref-updated event
|
||||
self.addTagToRepo(
|
||||
'org/project', 'atag', 'HEAD')
|
||||
# Let some time for the git watcher to detect the ref-update event
|
||||
time.sleep(0.2)
|
||||
self.waitUntilSettled()
|
||||
self.assertEqual(len(self.history), 1)
|
||||
self.assertEqual('SUCCESS',
|
||||
self.getJobFromHistory('tag-job').result)
|
||||
|
||||
@simple_layout('layouts/basic-git.yaml', driver='git')
|
||||
def test_ref_deleted(self):
|
||||
self.ensure_watcher_has_context()
|
||||
# Delete default tag init to trigger a ref-updated event
|
||||
self.delTagFromRepo(
|
||||
'org/project', 'init')
|
||||
# Let some time for the git watcher to detect the ref-update event
|
||||
time.sleep(0.2)
|
||||
self.waitUntilSettled()
|
||||
# Make sure no job as run as ignore-delete is True by default
|
||||
self.assertEqual(len(self.history), 0)
|
||||
|
|
|
@ -15,6 +15,7 @@
|
|||
from zuul.driver import Driver, ConnectionInterface, SourceInterface
|
||||
from zuul.driver.git import gitconnection
|
||||
from zuul.driver.git import gitsource
|
||||
from zuul.driver.git import gittrigger
|
||||
|
||||
|
||||
class GitDriver(Driver, ConnectionInterface, SourceInterface):
|
||||
|
@ -23,9 +24,15 @@ class GitDriver(Driver, ConnectionInterface, SourceInterface):
|
|||
def getConnection(self, name, config):
|
||||
return gitconnection.GitConnection(self, name, config)
|
||||
|
||||
def getTrigger(self, connection, config=None):
|
||||
return gittrigger.GitTrigger(self, connection, config)
|
||||
|
||||
def getSource(self, connection):
|
||||
return gitsource.GitSource(self, connection)
|
||||
|
||||
def getTriggerSchema(self):
|
||||
return gittrigger.getSchema()
|
||||
|
||||
def getRequireSchema(self):
|
||||
return {}
|
||||
|
||||
|
|
|
@ -13,12 +13,119 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import os
|
||||
import git
|
||||
import time
|
||||
import logging
|
||||
import urllib
|
||||
import threading
|
||||
|
||||
import voluptuous as v
|
||||
|
||||
from zuul.connection import BaseConnection
|
||||
from zuul.driver.git.gitmodel import GitTriggerEvent, EMPTY_GIT_REF
|
||||
from zuul.model import Ref, Branch
|
||||
|
||||
|
||||
class GitWatcher(threading.Thread):
|
||||
log = logging.getLogger("connection.git.GitWatcher")
|
||||
|
||||
def __init__(self, git_connection, baseurl, poll_delay):
|
||||
threading.Thread.__init__(self)
|
||||
self.daemon = True
|
||||
self.git_connection = git_connection
|
||||
self.baseurl = baseurl
|
||||
self.poll_delay = poll_delay
|
||||
self._stopped = False
|
||||
self.projects_refs = self.git_connection.projects_refs
|
||||
|
||||
def compareRefs(self, project, refs):
|
||||
partial_events = []
|
||||
# Fetch previous refs state
|
||||
base_refs = self.projects_refs.get(project)
|
||||
# Create list of created refs
|
||||
rcreateds = set(refs.keys()) - set(base_refs.keys())
|
||||
# Create list of deleted refs
|
||||
rdeleteds = set(base_refs.keys()) - set(refs.keys())
|
||||
# Create the list of updated refs
|
||||
updateds = {}
|
||||
for ref, sha in refs.items():
|
||||
if ref in base_refs and base_refs[ref] != sha:
|
||||
updateds[ref] = sha
|
||||
for ref in rcreateds:
|
||||
event = {
|
||||
'ref': ref,
|
||||
'branch_created': True,
|
||||
'oldrev': EMPTY_GIT_REF,
|
||||
'newrev': refs[ref]
|
||||
}
|
||||
partial_events.append(event)
|
||||
for ref in rdeleteds:
|
||||
event = {
|
||||
'ref': ref,
|
||||
'branch_deleted': True,
|
||||
'oldrev': base_refs[ref],
|
||||
'newrev': EMPTY_GIT_REF
|
||||
}
|
||||
partial_events.append(event)
|
||||
for ref, sha in updateds.items():
|
||||
event = {
|
||||
'ref': ref,
|
||||
'branch_updated': True,
|
||||
'oldrev': base_refs[ref],
|
||||
'newrev': sha
|
||||
}
|
||||
partial_events.append(event)
|
||||
events = []
|
||||
for pevent in partial_events:
|
||||
event = GitTriggerEvent()
|
||||
event.type = 'ref-updated'
|
||||
event.project_hostname = self.git_connection.canonical_hostname
|
||||
event.project_name = project
|
||||
for attr in ('ref', 'oldrev', 'newrev', 'branch_created',
|
||||
'branch_deleted', 'branch_updated'):
|
||||
if attr in pevent:
|
||||
setattr(event, attr, pevent[attr])
|
||||
events.append(event)
|
||||
return events
|
||||
|
||||
def _run(self):
|
||||
self.log.debug("Walk through projects refs for connection: %s" %
|
||||
self.git_connection.connection_name)
|
||||
try:
|
||||
for project in self.git_connection.projects:
|
||||
refs = self.git_connection.lsRemote(project)
|
||||
self.log.debug("Read refs %s for project %s" % (refs, project))
|
||||
if not self.projects_refs.get(project):
|
||||
# State for this project does not exist yet so add it.
|
||||
# No event will be triggered in this loop as
|
||||
# projects_refs['project'] and refs are equal
|
||||
self.projects_refs[project] = refs
|
||||
events = self.compareRefs(project, refs)
|
||||
self.projects_refs[project] = refs
|
||||
# Send events to the scheduler
|
||||
for event in events:
|
||||
self.log.debug("Handling event: %s" % event)
|
||||
# Force changes cache update before passing
|
||||
# the event to the scheduler
|
||||
self.git_connection.getChange(event)
|
||||
self.git_connection.logEvent(event)
|
||||
# Pass the event to the scheduler
|
||||
self.git_connection.sched.addEvent(event)
|
||||
except Exception as e:
|
||||
self.log.debug("Unexpected issue in _run loop: %s" % str(e))
|
||||
|
||||
def run(self):
|
||||
while not self._stopped:
|
||||
if not self.git_connection.w_pause:
|
||||
self._run()
|
||||
# Polling wait delay
|
||||
else:
|
||||
self.log.debug("Watcher is on pause")
|
||||
time.sleep(self.poll_delay)
|
||||
|
||||
def stop(self):
|
||||
self._stopped = True
|
||||
|
||||
|
||||
class GitConnection(BaseConnection):
|
||||
|
@ -32,6 +139,8 @@ class GitConnection(BaseConnection):
|
|||
raise Exception('baseurl is required for git connections in '
|
||||
'%s' % self.connection_name)
|
||||
self.baseurl = self.connection_config.get('baseurl')
|
||||
self.poll_timeout = float(
|
||||
self.connection_config.get('poll_delay', 3600 * 2))
|
||||
self.canonical_hostname = self.connection_config.get(
|
||||
'canonical_hostname')
|
||||
if not self.canonical_hostname:
|
||||
|
@ -40,7 +149,10 @@ class GitConnection(BaseConnection):
|
|||
self.canonical_hostname = r.hostname
|
||||
else:
|
||||
self.canonical_hostname = 'localhost'
|
||||
self.w_pause = False
|
||||
self.projects = {}
|
||||
self.projects_refs = {}
|
||||
self._change_cache = {}
|
||||
|
||||
def getProject(self, name):
|
||||
return self.projects.get(name)
|
||||
|
@ -48,15 +160,97 @@ class GitConnection(BaseConnection):
|
|||
def addProject(self, project):
|
||||
self.projects[project.name] = project
|
||||
|
||||
def getChangeFilesUpdated(self, project_name, branch, tosha):
|
||||
job = self.sched.merger.getFilesChanges(
|
||||
self.connection_name, project_name, branch, tosha)
|
||||
self.log.debug("Waiting for fileschanges job %s" % job)
|
||||
job.wait()
|
||||
if not job.updated:
|
||||
raise Exception("Fileschanges job %s failed" % job)
|
||||
self.log.debug("Fileschanges job %s got changes on files %s" %
|
||||
(job, job.files))
|
||||
return job.files
|
||||
|
||||
def lsRemote(self, project):
|
||||
refs = {}
|
||||
client = git.cmd.Git()
|
||||
output = client.ls_remote(
|
||||
os.path.join(self.baseurl, project))
|
||||
for line in output.splitlines():
|
||||
sha, ref = line.split('\t')
|
||||
if ref.startswith('refs/'):
|
||||
refs[ref] = sha
|
||||
return refs
|
||||
|
||||
def maintainCache(self, relevant):
|
||||
remove = {}
|
||||
for branch, refschange in self._change_cache.items():
|
||||
for ref, change in refschange.items():
|
||||
if change not in relevant:
|
||||
remove.setdefault(branch, []).append(ref)
|
||||
for branch, refs in remove.items():
|
||||
for ref in refs:
|
||||
del self._change_cache[branch][ref]
|
||||
if not self._change_cache[branch]:
|
||||
del self._change_cache[branch]
|
||||
|
||||
def getChange(self, event, refresh=False):
|
||||
if event.ref and event.ref.startswith('refs/heads/'):
|
||||
branch = event.ref[len('refs/heads/'):]
|
||||
change = self._change_cache.get(branch, {}).get(event.newrev)
|
||||
if change:
|
||||
return change
|
||||
project = self.getProject(event.project_name)
|
||||
change = Branch(project)
|
||||
change.branch = branch
|
||||
for attr in ('ref', 'oldrev', 'newrev'):
|
||||
setattr(change, attr, getattr(event, attr))
|
||||
change.url = ""
|
||||
change.files = self.getChangeFilesUpdated(
|
||||
event.project_name, change.branch, event.oldrev)
|
||||
self._change_cache.setdefault(branch, {})[event.newrev] = change
|
||||
elif event.ref:
|
||||
# catch-all ref (ie, not a branch or head)
|
||||
project = self.getProject(event.project_name)
|
||||
change = Ref(project)
|
||||
for attr in ('ref', 'oldrev', 'newrev'):
|
||||
setattr(change, attr, getattr(event, attr))
|
||||
change.url = ""
|
||||
else:
|
||||
self.log.warning("Unable to get change for %s" % (event,))
|
||||
change = None
|
||||
return change
|
||||
|
||||
def getProjectBranches(self, project, tenant):
|
||||
# TODO(jeblair): implement; this will need to handle local or
|
||||
# remote git urls.
|
||||
return ['master']
|
||||
refs = self.lsRemote(project.name)
|
||||
branches = [ref[len('refs/heads/'):] for ref in
|
||||
refs if ref.startswith('refs/heads/')]
|
||||
return branches
|
||||
|
||||
def getGitUrl(self, project):
|
||||
url = '%s/%s' % (self.baseurl, project.name)
|
||||
return url
|
||||
|
||||
def onLoad(self):
|
||||
self.log.debug("Starting Git Watcher")
|
||||
self._start_watcher_thread()
|
||||
|
||||
def onStop(self):
|
||||
self.log.debug("Stopping Git Watcher")
|
||||
self._stop_watcher_thread()
|
||||
|
||||
def _stop_watcher_thread(self):
|
||||
if self.watcher_thread:
|
||||
self.watcher_thread.stop()
|
||||
self.watcher_thread.join()
|
||||
|
||||
def _start_watcher_thread(self):
|
||||
self.watcher_thread = GitWatcher(
|
||||
self,
|
||||
self.baseurl,
|
||||
self.poll_timeout)
|
||||
self.watcher_thread.start()
|
||||
|
||||
|
||||
def getSchema():
|
||||
git_connection = v.Any(str, v.Schema(dict))
|
||||
|
|
|
@ -0,0 +1,86 @@
|
|||
# Copyright 2017 Red Hat, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import re
|
||||
|
||||
from zuul.model import TriggerEvent
|
||||
from zuul.model import EventFilter
|
||||
|
||||
|
||||
EMPTY_GIT_REF = '0' * 40 # git sha of all zeros, used during creates/deletes
|
||||
|
||||
|
||||
class GitTriggerEvent(TriggerEvent):
|
||||
"""Incoming event from an external system."""
|
||||
|
||||
def __repr__(self):
|
||||
ret = '<GitTriggerEvent %s %s' % (self.type,
|
||||
self.project_name)
|
||||
|
||||
if self.branch:
|
||||
ret += " %s" % self.branch
|
||||
ret += " oldrev:%s" % self.oldrev
|
||||
ret += " newrev:%s" % self.newrev
|
||||
ret += '>'
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
class GitEventFilter(EventFilter):
|
||||
def __init__(self, trigger, types=[], refs=[],
|
||||
ignore_deletes=True):
|
||||
|
||||
super().__init__(trigger)
|
||||
|
||||
self._refs = refs
|
||||
self.types = types
|
||||
self.refs = [re.compile(x) for x in refs]
|
||||
self.ignore_deletes = ignore_deletes
|
||||
|
||||
def __repr__(self):
|
||||
ret = '<GitEventFilter'
|
||||
|
||||
if self.types:
|
||||
ret += ' types: %s' % ', '.join(self.types)
|
||||
if self._refs:
|
||||
ret += ' refs: %s' % ', '.join(self._refs)
|
||||
if self.ignore_deletes:
|
||||
ret += ' ignore_deletes: %s' % self.ignore_deletes
|
||||
ret += '>'
|
||||
|
||||
return ret
|
||||
|
||||
def matches(self, event, change):
|
||||
# event types are ORed
|
||||
matches_type = False
|
||||
for etype in self.types:
|
||||
if etype == event.type:
|
||||
matches_type = True
|
||||
if self.types and not matches_type:
|
||||
return False
|
||||
|
||||
# refs are ORed
|
||||
matches_ref = False
|
||||
if event.ref is not None:
|
||||
for ref in self.refs:
|
||||
if ref.match(event.ref):
|
||||
matches_ref = True
|
||||
if self.refs and not matches_ref:
|
||||
return False
|
||||
if self.ignore_deletes and event.newrev == EMPTY_GIT_REF:
|
||||
# If the updated ref has an empty git sha (all 0s),
|
||||
# then the ref is being deleted
|
||||
return False
|
||||
|
||||
return True
|
|
@ -36,7 +36,7 @@ class GitSource(BaseSource):
|
|||
raise NotImplemented()
|
||||
|
||||
def getChange(self, event, refresh=False):
|
||||
raise NotImplemented()
|
||||
return self.connection.getChange(event, refresh)
|
||||
|
||||
def getProject(self, name):
|
||||
p = self.connection.getProject(name)
|
||||
|
|
|
@ -0,0 +1,49 @@
|
|||
# Copyright 2017 Red Hat, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import logging
|
||||
import voluptuous as v
|
||||
from zuul.trigger import BaseTrigger
|
||||
from zuul.driver.git.gitmodel import GitEventFilter
|
||||
from zuul.driver.util import scalar_or_list, to_list
|
||||
|
||||
|
||||
class GitTrigger(BaseTrigger):
|
||||
name = 'git'
|
||||
log = logging.getLogger("zuul.GitTrigger")
|
||||
|
||||
def getEventFilters(self, trigger_conf):
|
||||
efilters = []
|
||||
for trigger in to_list(trigger_conf):
|
||||
f = GitEventFilter(
|
||||
trigger=self,
|
||||
types=to_list(trigger['event']),
|
||||
refs=to_list(trigger.get('ref')),
|
||||
ignore_deletes=trigger.get(
|
||||
'ignore-deletes', True)
|
||||
)
|
||||
efilters.append(f)
|
||||
|
||||
return efilters
|
||||
|
||||
|
||||
def getSchema():
|
||||
git_trigger = {
|
||||
v.Required('event'):
|
||||
scalar_or_list(v.Any('ref-updated')),
|
||||
'ref': scalar_or_list(str),
|
||||
'ignore-deletes': bool,
|
||||
}
|
||||
|
||||
return git_trigger
|
|
@ -1706,6 +1706,7 @@ class ExecutorServer(object):
|
|||
self.merger_worker.registerFunction("merger:merge")
|
||||
self.merger_worker.registerFunction("merger:cat")
|
||||
self.merger_worker.registerFunction("merger:refstate")
|
||||
self.merger_worker.registerFunction("merger:fileschanges")
|
||||
|
||||
def register_work(self):
|
||||
if self._running:
|
||||
|
@ -1859,6 +1860,9 @@ class ExecutorServer(object):
|
|||
elif job.name == 'merger:refstate':
|
||||
self.log.debug("Got refstate job: %s" % job.unique)
|
||||
self.refstate(job)
|
||||
elif job.name == 'merger:fileschanges':
|
||||
self.log.debug("Got fileschanges job: %s" % job.unique)
|
||||
self.fileschanges(job)
|
||||
else:
|
||||
self.log.error("Unable to handle job %s" % job.name)
|
||||
job.sendWorkFail()
|
||||
|
@ -1970,6 +1974,19 @@ class ExecutorServer(object):
|
|||
files=files)
|
||||
job.sendWorkComplete(json.dumps(result))
|
||||
|
||||
def fileschanges(self, job):
|
||||
args = json.loads(job.arguments)
|
||||
task = self.update(args['connection'], args['project'])
|
||||
task.wait()
|
||||
with self.merger_lock:
|
||||
files = self.merger.getFilesChanges(
|
||||
args['connection'], args['project'],
|
||||
args['branch'],
|
||||
args['tosha'])
|
||||
result = dict(updated=True,
|
||||
files=files)
|
||||
job.sendWorkComplete(json.dumps(result))
|
||||
|
||||
def refstate(self, job):
|
||||
args = json.loads(job.arguments)
|
||||
with self.merger_lock:
|
||||
|
|
|
@ -131,6 +131,15 @@ class MergeClient(object):
|
|||
job = self.submitJob('merger:cat', data, None, precedence)
|
||||
return job
|
||||
|
||||
def getFilesChanges(self, connection_name, project_name, branch,
|
||||
tosha=None, precedence=zuul.model.PRECEDENCE_HIGH):
|
||||
data = dict(connection=connection_name,
|
||||
project=project_name,
|
||||
branch=branch,
|
||||
tosha=tosha)
|
||||
job = self.submitJob('merger:fileschanges', data, None, precedence)
|
||||
return job
|
||||
|
||||
def onBuildCompleted(self, job):
|
||||
data = getJobData(job)
|
||||
merged = data.get('merged', False)
|
||||
|
|
|
@ -314,6 +314,18 @@ class Repo(object):
|
|||
'utf-8')
|
||||
return ret
|
||||
|
||||
def getFilesChanges(self, branch, tosha=None):
|
||||
repo = self.createRepoObject()
|
||||
files = set()
|
||||
head = repo.heads[branch].commit
|
||||
files.update(set(head.stats.files.keys()))
|
||||
if tosha:
|
||||
for cmt in head.iter_parents():
|
||||
if cmt.hexsha == tosha:
|
||||
break
|
||||
files.update(set(cmt.stats.files.keys()))
|
||||
return list(files)
|
||||
|
||||
def deleteRemote(self, remote):
|
||||
repo = self.createRepoObject()
|
||||
repo.delete_remote(repo.remotes[remote])
|
||||
|
@ -581,3 +593,8 @@ class Merger(object):
|
|||
def getFiles(self, connection_name, project_name, branch, files, dirs=[]):
|
||||
repo = self.getRepo(connection_name, project_name)
|
||||
return repo.getFiles(files, dirs, branch=branch)
|
||||
|
||||
def getFilesChanges(self, connection_name, project_name, branch,
|
||||
tosha=None):
|
||||
repo = self.getRepo(connection_name, project_name)
|
||||
return repo.getFilesChanges(branch, tosha)
|
||||
|
|
|
@ -81,6 +81,7 @@ class MergeServer(object):
|
|||
self.worker.registerFunction("merger:merge")
|
||||
self.worker.registerFunction("merger:cat")
|
||||
self.worker.registerFunction("merger:refstate")
|
||||
self.worker.registerFunction("merger:fileschanges")
|
||||
|
||||
def stop(self):
|
||||
self.log.debug("Stopping")
|
||||
|
@ -117,6 +118,9 @@ class MergeServer(object):
|
|||
elif job.name == 'merger:refstate':
|
||||
self.log.debug("Got refstate job: %s" % job.unique)
|
||||
self.refstate(job)
|
||||
elif job.name == 'merger:fileschanges':
|
||||
self.log.debug("Got fileschanges job: %s" % job.unique)
|
||||
self.fileschanges(job)
|
||||
else:
|
||||
self.log.error("Unable to handle job %s" % job.name)
|
||||
job.sendWorkFail()
|
||||
|
@ -158,3 +162,12 @@ class MergeServer(object):
|
|||
result = dict(updated=True,
|
||||
files=files)
|
||||
job.sendWorkComplete(json.dumps(result))
|
||||
|
||||
def fileschanges(self, job):
|
||||
args = json.loads(job.arguments)
|
||||
self.merger.updateRepo(args['connection'], args['project'])
|
||||
files = self.merger.getFilesChanges(
|
||||
args['connection'], args['project'], args['branch'], args['tosha'])
|
||||
result = dict(updated=True,
|
||||
files=files)
|
||||
job.sendWorkComplete(json.dumps(result))
|
||||
|
|
Loading…
Reference in New Issue