Git driver
This patch improves the existing git driver by adding a refs watcher thread. This refs watcher looks at refs added, deleted, updated and trigger a ref-updated event. When a refs is updated and that the related commits from oldrev to newrev include a change on .zuul.yaml/zuul.yaml or zuul.d/*.yaml then tenants including that ref is reconfigured. Furthermore the patch includes a triggering model. Events are sent to the scheduler so jobs can be attached to a pipeline for running jobs. Change-Id: I529660cb20d011f36814abe64f837945dd3f1f33
This commit is contained in:
parent
9dc1b9f000
commit
194a2bf237
|
@ -55,6 +55,7 @@ Zuul includes the following drivers:
|
||||||
|
|
||||||
drivers/gerrit
|
drivers/gerrit
|
||||||
drivers/github
|
drivers/github
|
||||||
|
drivers/git
|
||||||
drivers/smtp
|
drivers/smtp
|
||||||
drivers/sql
|
drivers/sql
|
||||||
drivers/timer
|
drivers/timer
|
||||||
|
|
|
@ -0,0 +1,59 @@
|
||||||
|
:title: Git Driver
|
||||||
|
|
||||||
|
Git
|
||||||
|
===
|
||||||
|
|
||||||
|
This driver can be used to load Zuul configuration from public Git repositories,
|
||||||
|
for instance from ``openstack-infra/zuul-jobs`` that is suitable for use by
|
||||||
|
any Zuul system. It can also be used to trigger jobs from ``ref-updated`` events
|
||||||
|
in a pipeline.
|
||||||
|
|
||||||
|
Connection Configuration
|
||||||
|
------------------------
|
||||||
|
|
||||||
|
The supported options in ``zuul.conf`` connections are:
|
||||||
|
|
||||||
|
.. attr:: <git connection>
|
||||||
|
|
||||||
|
.. attr:: driver
|
||||||
|
:required:
|
||||||
|
|
||||||
|
.. value:: git
|
||||||
|
|
||||||
|
The connection must set ``driver=git`` for Git connections.
|
||||||
|
|
||||||
|
.. attr:: baseurl
|
||||||
|
|
||||||
|
Path to the base Git URL. Git repos name will be appended to it.
|
||||||
|
|
||||||
|
.. attr:: poll_delay
|
||||||
|
:default: 7200
|
||||||
|
|
||||||
|
The delay in seconds of the Git repositories polling loop.
|
||||||
|
|
||||||
|
Trigger Configuration
|
||||||
|
---------------------
|
||||||
|
|
||||||
|
.. attr:: pipeline.trigger.<git source>
|
||||||
|
|
||||||
|
The dictionary passed to the Git pipeline ``trigger`` attribute
|
||||||
|
supports the following attributes:
|
||||||
|
|
||||||
|
.. attr:: event
|
||||||
|
:required:
|
||||||
|
|
||||||
|
Only ``ref-updated`` is supported.
|
||||||
|
|
||||||
|
.. attr:: ref
|
||||||
|
|
||||||
|
On ref-updated events, a ref such as ``refs/heads/master`` or
|
||||||
|
``^refs/tags/.*$``. This field is treated as a regular expression,
|
||||||
|
and multiple refs may be listed.
|
||||||
|
|
||||||
|
.. attr:: ignore-deletes
|
||||||
|
:default: true
|
||||||
|
|
||||||
|
When a ref is deleted, a ref-updated event is emitted with a
|
||||||
|
newrev of all zeros specified. The ``ignore-deletes`` field is a
|
||||||
|
boolean value that describes whether or not these newrevs
|
||||||
|
trigger ref-updated events.
|
|
@ -2833,6 +2833,16 @@ class ZuulTestCase(BaseTestCase):
|
||||||
os.path.join(FIXTURE_DIR, f.name))
|
os.path.join(FIXTURE_DIR, f.name))
|
||||||
self.setupAllProjectKeys()
|
self.setupAllProjectKeys()
|
||||||
|
|
||||||
|
def addTagToRepo(self, project, name, sha):
|
||||||
|
path = os.path.join(self.upstream_root, project)
|
||||||
|
repo = git.Repo(path)
|
||||||
|
repo.git.tag(name, sha)
|
||||||
|
|
||||||
|
def delTagFromRepo(self, project, name):
|
||||||
|
path = os.path.join(self.upstream_root, project)
|
||||||
|
repo = git.Repo(path)
|
||||||
|
repo.git.tag('-d', name)
|
||||||
|
|
||||||
def addCommitToRepo(self, project, message, files,
|
def addCommitToRepo(self, project, message, files,
|
||||||
branch='master', tag=None):
|
branch='master', tag=None):
|
||||||
path = os.path.join(self.upstream_root, project)
|
path = os.path.join(self.upstream_root, project)
|
||||||
|
|
2
tests/fixtures/config/git-driver/git/common-config/playbooks/project-test2.yaml
vendored
Normal file
2
tests/fixtures/config/git-driver/git/common-config/playbooks/project-test2.yaml
vendored
Normal file
|
@ -0,0 +1,2 @@
|
||||||
|
- hosts: all
|
||||||
|
tasks: []
|
|
@ -19,6 +19,10 @@
|
||||||
name: project-test1
|
name: project-test1
|
||||||
run: playbooks/project-test1.yaml
|
run: playbooks/project-test1.yaml
|
||||||
|
|
||||||
|
- job:
|
||||||
|
name: project-test2
|
||||||
|
run: playbooks/project-test2.yaml
|
||||||
|
|
||||||
- project:
|
- project:
|
||||||
name: org/project
|
name: org/project
|
||||||
check:
|
check:
|
||||||
|
|
|
@ -0,0 +1,37 @@
|
||||||
|
- pipeline:
|
||||||
|
name: post
|
||||||
|
manager: independent
|
||||||
|
trigger:
|
||||||
|
git:
|
||||||
|
- event: ref-updated
|
||||||
|
ref: ^refs/heads/.*$
|
||||||
|
|
||||||
|
- pipeline:
|
||||||
|
name: tag
|
||||||
|
manager: independent
|
||||||
|
trigger:
|
||||||
|
git:
|
||||||
|
- event: ref-updated
|
||||||
|
ref: ^refs/tags/.*$
|
||||||
|
|
||||||
|
- job:
|
||||||
|
name: base
|
||||||
|
parent: null
|
||||||
|
run: playbooks/base.yaml
|
||||||
|
|
||||||
|
- job:
|
||||||
|
name: post-job
|
||||||
|
run: playbooks/post-job.yaml
|
||||||
|
|
||||||
|
- job:
|
||||||
|
name: tag-job
|
||||||
|
run: playbooks/post-job.yaml
|
||||||
|
|
||||||
|
- project:
|
||||||
|
name: org/project
|
||||||
|
post:
|
||||||
|
jobs:
|
||||||
|
- post-job
|
||||||
|
tag:
|
||||||
|
jobs:
|
||||||
|
- tag-job
|
|
@ -21,6 +21,7 @@ sshkey=none
|
||||||
[connection git]
|
[connection git]
|
||||||
driver=git
|
driver=git
|
||||||
baseurl=""
|
baseurl=""
|
||||||
|
poll_delay=0.1
|
||||||
|
|
||||||
[connection outgoing_smtp]
|
[connection outgoing_smtp]
|
||||||
driver=smtp
|
driver=smtp
|
||||||
|
|
|
@ -12,7 +12,12 @@
|
||||||
# License for the specific language governing permissions and limitations
|
# License for the specific language governing permissions and limitations
|
||||||
# under the License.
|
# under the License.
|
||||||
|
|
||||||
from tests.base import ZuulTestCase
|
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
from tests.base import ZuulTestCase, simple_layout
|
||||||
|
|
||||||
|
|
||||||
class TestGitDriver(ZuulTestCase):
|
class TestGitDriver(ZuulTestCase):
|
||||||
|
@ -23,7 +28,7 @@ class TestGitDriver(ZuulTestCase):
|
||||||
super(TestGitDriver, self).setup_config()
|
super(TestGitDriver, self).setup_config()
|
||||||
self.config.set('connection git', 'baseurl', self.upstream_root)
|
self.config.set('connection git', 'baseurl', self.upstream_root)
|
||||||
|
|
||||||
def test_git_driver(self):
|
def test_basic(self):
|
||||||
tenant = self.sched.abide.tenants.get('tenant-one')
|
tenant = self.sched.abide.tenants.get('tenant-one')
|
||||||
# Check that we have the git source for common-config and the
|
# Check that we have the git source for common-config and the
|
||||||
# gerrit source for the project.
|
# gerrit source for the project.
|
||||||
|
@ -40,3 +45,127 @@ class TestGitDriver(ZuulTestCase):
|
||||||
self.waitUntilSettled()
|
self.waitUntilSettled()
|
||||||
self.assertEqual(len(self.history), 1)
|
self.assertEqual(len(self.history), 1)
|
||||||
self.assertEqual(A.reported, 1)
|
self.assertEqual(A.reported, 1)
|
||||||
|
|
||||||
|
def test_config_refreshed(self):
|
||||||
|
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
|
||||||
|
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
|
||||||
|
self.waitUntilSettled()
|
||||||
|
self.assertEqual(len(self.history), 1)
|
||||||
|
self.assertEqual(A.reported, 1)
|
||||||
|
self.assertEqual(self.history[0].name, 'project-test1')
|
||||||
|
|
||||||
|
# Update zuul.yaml to force a tenant reconfiguration
|
||||||
|
path = os.path.join(self.upstream_root, 'common-config', 'zuul.yaml')
|
||||||
|
config = yaml.load(open(path, 'r').read())
|
||||||
|
change = {
|
||||||
|
'name': 'org/project',
|
||||||
|
'check': {
|
||||||
|
'jobs': [
|
||||||
|
'project-test2'
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
config[4]['project'] = change
|
||||||
|
files = {'zuul.yaml': yaml.dump(config)}
|
||||||
|
self.addCommitToRepo(
|
||||||
|
'common-config', 'Change zuul.yaml configuration', files)
|
||||||
|
|
||||||
|
# Let some time for the tenant reconfiguration to happen
|
||||||
|
time.sleep(2)
|
||||||
|
self.waitUntilSettled()
|
||||||
|
|
||||||
|
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
|
||||||
|
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
|
||||||
|
self.waitUntilSettled()
|
||||||
|
self.assertEqual(len(self.history), 2)
|
||||||
|
self.assertEqual(A.reported, 1)
|
||||||
|
# We make sure the new job has run
|
||||||
|
self.assertEqual(self.history[1].name, 'project-test2')
|
||||||
|
|
||||||
|
# Let's stop the git Watcher to let us merge some changes commits
|
||||||
|
# We want to verify that config changes are detected for commits
|
||||||
|
# on the range oldrev..newrev
|
||||||
|
self.sched.connections.getSource('git').connection.w_pause = True
|
||||||
|
# Add a config change
|
||||||
|
change = {
|
||||||
|
'name': 'org/project',
|
||||||
|
'check': {
|
||||||
|
'jobs': [
|
||||||
|
'project-test1'
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
config[4]['project'] = change
|
||||||
|
files = {'zuul.yaml': yaml.dump(config)}
|
||||||
|
self.addCommitToRepo(
|
||||||
|
'common-config', 'Change zuul.yaml configuration', files)
|
||||||
|
# Add two other changes
|
||||||
|
self.addCommitToRepo(
|
||||||
|
'common-config', 'Adding f1',
|
||||||
|
{'f1': "Content"})
|
||||||
|
self.addCommitToRepo(
|
||||||
|
'common-config', 'Adding f2',
|
||||||
|
{'f2': "Content"})
|
||||||
|
# Restart the git watcher
|
||||||
|
self.sched.connections.getSource('git').connection.w_pause = False
|
||||||
|
|
||||||
|
# Let some time for the tenant reconfiguration to happen
|
||||||
|
time.sleep(2)
|
||||||
|
self.waitUntilSettled()
|
||||||
|
|
||||||
|
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
|
||||||
|
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
|
||||||
|
self.waitUntilSettled()
|
||||||
|
self.assertEqual(len(self.history), 3)
|
||||||
|
self.assertEqual(A.reported, 1)
|
||||||
|
# We make sure the new job has run
|
||||||
|
self.assertEqual(self.history[2].name, 'project-test1')
|
||||||
|
|
||||||
|
def ensure_watcher_has_context(self):
|
||||||
|
# Make sure watcher have read initial refs shas
|
||||||
|
cnx = self.sched.connections.getSource('git').connection
|
||||||
|
delay = 0.1
|
||||||
|
max_delay = 1
|
||||||
|
while not cnx.projects_refs:
|
||||||
|
time.sleep(delay)
|
||||||
|
max_delay -= delay
|
||||||
|
if max_delay <= 0:
|
||||||
|
raise Exception("Timeout waiting for initial read")
|
||||||
|
|
||||||
|
@simple_layout('layouts/basic-git.yaml', driver='git')
|
||||||
|
def test_ref_updated_event(self):
|
||||||
|
self.ensure_watcher_has_context()
|
||||||
|
# Add a commit to trigger a ref-updated event
|
||||||
|
self.addCommitToRepo(
|
||||||
|
'org/project', 'A change for ref-updated', {'f1': 'Content'})
|
||||||
|
# Let some time for the git watcher to detect the ref-update event
|
||||||
|
time.sleep(0.2)
|
||||||
|
self.waitUntilSettled()
|
||||||
|
self.assertEqual(len(self.history), 1)
|
||||||
|
self.assertEqual('SUCCESS',
|
||||||
|
self.getJobFromHistory('post-job').result)
|
||||||
|
|
||||||
|
@simple_layout('layouts/basic-git.yaml', driver='git')
|
||||||
|
def test_ref_created(self):
|
||||||
|
self.ensure_watcher_has_context()
|
||||||
|
# Tag HEAD to trigger a ref-updated event
|
||||||
|
self.addTagToRepo(
|
||||||
|
'org/project', 'atag', 'HEAD')
|
||||||
|
# Let some time for the git watcher to detect the ref-update event
|
||||||
|
time.sleep(0.2)
|
||||||
|
self.waitUntilSettled()
|
||||||
|
self.assertEqual(len(self.history), 1)
|
||||||
|
self.assertEqual('SUCCESS',
|
||||||
|
self.getJobFromHistory('tag-job').result)
|
||||||
|
|
||||||
|
@simple_layout('layouts/basic-git.yaml', driver='git')
|
||||||
|
def test_ref_deleted(self):
|
||||||
|
self.ensure_watcher_has_context()
|
||||||
|
# Delete default tag init to trigger a ref-updated event
|
||||||
|
self.delTagFromRepo(
|
||||||
|
'org/project', 'init')
|
||||||
|
# Let some time for the git watcher to detect the ref-update event
|
||||||
|
time.sleep(0.2)
|
||||||
|
self.waitUntilSettled()
|
||||||
|
# Make sure no job as run as ignore-delete is True by default
|
||||||
|
self.assertEqual(len(self.history), 0)
|
||||||
|
|
|
@ -15,6 +15,7 @@
|
||||||
from zuul.driver import Driver, ConnectionInterface, SourceInterface
|
from zuul.driver import Driver, ConnectionInterface, SourceInterface
|
||||||
from zuul.driver.git import gitconnection
|
from zuul.driver.git import gitconnection
|
||||||
from zuul.driver.git import gitsource
|
from zuul.driver.git import gitsource
|
||||||
|
from zuul.driver.git import gittrigger
|
||||||
|
|
||||||
|
|
||||||
class GitDriver(Driver, ConnectionInterface, SourceInterface):
|
class GitDriver(Driver, ConnectionInterface, SourceInterface):
|
||||||
|
@ -23,9 +24,15 @@ class GitDriver(Driver, ConnectionInterface, SourceInterface):
|
||||||
def getConnection(self, name, config):
|
def getConnection(self, name, config):
|
||||||
return gitconnection.GitConnection(self, name, config)
|
return gitconnection.GitConnection(self, name, config)
|
||||||
|
|
||||||
|
def getTrigger(self, connection, config=None):
|
||||||
|
return gittrigger.GitTrigger(self, connection, config)
|
||||||
|
|
||||||
def getSource(self, connection):
|
def getSource(self, connection):
|
||||||
return gitsource.GitSource(self, connection)
|
return gitsource.GitSource(self, connection)
|
||||||
|
|
||||||
|
def getTriggerSchema(self):
|
||||||
|
return gittrigger.getSchema()
|
||||||
|
|
||||||
def getRequireSchema(self):
|
def getRequireSchema(self):
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
|
|
|
@ -13,12 +13,119 @@
|
||||||
# License for the specific language governing permissions and limitations
|
# License for the specific language governing permissions and limitations
|
||||||
# under the License.
|
# under the License.
|
||||||
|
|
||||||
|
import os
|
||||||
|
import git
|
||||||
|
import time
|
||||||
import logging
|
import logging
|
||||||
import urllib
|
import urllib
|
||||||
|
import threading
|
||||||
|
|
||||||
import voluptuous as v
|
import voluptuous as v
|
||||||
|
|
||||||
from zuul.connection import BaseConnection
|
from zuul.connection import BaseConnection
|
||||||
|
from zuul.driver.git.gitmodel import GitTriggerEvent, EMPTY_GIT_REF
|
||||||
|
from zuul.model import Ref, Branch
|
||||||
|
|
||||||
|
|
||||||
|
class GitWatcher(threading.Thread):
|
||||||
|
log = logging.getLogger("connection.git.GitWatcher")
|
||||||
|
|
||||||
|
def __init__(self, git_connection, baseurl, poll_delay):
|
||||||
|
threading.Thread.__init__(self)
|
||||||
|
self.daemon = True
|
||||||
|
self.git_connection = git_connection
|
||||||
|
self.baseurl = baseurl
|
||||||
|
self.poll_delay = poll_delay
|
||||||
|
self._stopped = False
|
||||||
|
self.projects_refs = self.git_connection.projects_refs
|
||||||
|
|
||||||
|
def compareRefs(self, project, refs):
|
||||||
|
partial_events = []
|
||||||
|
# Fetch previous refs state
|
||||||
|
base_refs = self.projects_refs.get(project)
|
||||||
|
# Create list of created refs
|
||||||
|
rcreateds = set(refs.keys()) - set(base_refs.keys())
|
||||||
|
# Create list of deleted refs
|
||||||
|
rdeleteds = set(base_refs.keys()) - set(refs.keys())
|
||||||
|
# Create the list of updated refs
|
||||||
|
updateds = {}
|
||||||
|
for ref, sha in refs.items():
|
||||||
|
if ref in base_refs and base_refs[ref] != sha:
|
||||||
|
updateds[ref] = sha
|
||||||
|
for ref in rcreateds:
|
||||||
|
event = {
|
||||||
|
'ref': ref,
|
||||||
|
'branch_created': True,
|
||||||
|
'oldrev': EMPTY_GIT_REF,
|
||||||
|
'newrev': refs[ref]
|
||||||
|
}
|
||||||
|
partial_events.append(event)
|
||||||
|
for ref in rdeleteds:
|
||||||
|
event = {
|
||||||
|
'ref': ref,
|
||||||
|
'branch_deleted': True,
|
||||||
|
'oldrev': base_refs[ref],
|
||||||
|
'newrev': EMPTY_GIT_REF
|
||||||
|
}
|
||||||
|
partial_events.append(event)
|
||||||
|
for ref, sha in updateds.items():
|
||||||
|
event = {
|
||||||
|
'ref': ref,
|
||||||
|
'branch_updated': True,
|
||||||
|
'oldrev': base_refs[ref],
|
||||||
|
'newrev': sha
|
||||||
|
}
|
||||||
|
partial_events.append(event)
|
||||||
|
events = []
|
||||||
|
for pevent in partial_events:
|
||||||
|
event = GitTriggerEvent()
|
||||||
|
event.type = 'ref-updated'
|
||||||
|
event.project_hostname = self.git_connection.canonical_hostname
|
||||||
|
event.project_name = project
|
||||||
|
for attr in ('ref', 'oldrev', 'newrev', 'branch_created',
|
||||||
|
'branch_deleted', 'branch_updated'):
|
||||||
|
if attr in pevent:
|
||||||
|
setattr(event, attr, pevent[attr])
|
||||||
|
events.append(event)
|
||||||
|
return events
|
||||||
|
|
||||||
|
def _run(self):
|
||||||
|
self.log.debug("Walk through projects refs for connection: %s" %
|
||||||
|
self.git_connection.connection_name)
|
||||||
|
try:
|
||||||
|
for project in self.git_connection.projects:
|
||||||
|
refs = self.git_connection.lsRemote(project)
|
||||||
|
self.log.debug("Read refs %s for project %s" % (refs, project))
|
||||||
|
if not self.projects_refs.get(project):
|
||||||
|
# State for this project does not exist yet so add it.
|
||||||
|
# No event will be triggered in this loop as
|
||||||
|
# projects_refs['project'] and refs are equal
|
||||||
|
self.projects_refs[project] = refs
|
||||||
|
events = self.compareRefs(project, refs)
|
||||||
|
self.projects_refs[project] = refs
|
||||||
|
# Send events to the scheduler
|
||||||
|
for event in events:
|
||||||
|
self.log.debug("Handling event: %s" % event)
|
||||||
|
# Force changes cache update before passing
|
||||||
|
# the event to the scheduler
|
||||||
|
self.git_connection.getChange(event)
|
||||||
|
self.git_connection.logEvent(event)
|
||||||
|
# Pass the event to the scheduler
|
||||||
|
self.git_connection.sched.addEvent(event)
|
||||||
|
except Exception as e:
|
||||||
|
self.log.debug("Unexpected issue in _run loop: %s" % str(e))
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
while not self._stopped:
|
||||||
|
if not self.git_connection.w_pause:
|
||||||
|
self._run()
|
||||||
|
# Polling wait delay
|
||||||
|
else:
|
||||||
|
self.log.debug("Watcher is on pause")
|
||||||
|
time.sleep(self.poll_delay)
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
self._stopped = True
|
||||||
|
|
||||||
|
|
||||||
class GitConnection(BaseConnection):
|
class GitConnection(BaseConnection):
|
||||||
|
@ -32,6 +139,8 @@ class GitConnection(BaseConnection):
|
||||||
raise Exception('baseurl is required for git connections in '
|
raise Exception('baseurl is required for git connections in '
|
||||||
'%s' % self.connection_name)
|
'%s' % self.connection_name)
|
||||||
self.baseurl = self.connection_config.get('baseurl')
|
self.baseurl = self.connection_config.get('baseurl')
|
||||||
|
self.poll_timeout = float(
|
||||||
|
self.connection_config.get('poll_delay', 3600 * 2))
|
||||||
self.canonical_hostname = self.connection_config.get(
|
self.canonical_hostname = self.connection_config.get(
|
||||||
'canonical_hostname')
|
'canonical_hostname')
|
||||||
if not self.canonical_hostname:
|
if not self.canonical_hostname:
|
||||||
|
@ -40,7 +149,10 @@ class GitConnection(BaseConnection):
|
||||||
self.canonical_hostname = r.hostname
|
self.canonical_hostname = r.hostname
|
||||||
else:
|
else:
|
||||||
self.canonical_hostname = 'localhost'
|
self.canonical_hostname = 'localhost'
|
||||||
|
self.w_pause = False
|
||||||
self.projects = {}
|
self.projects = {}
|
||||||
|
self.projects_refs = {}
|
||||||
|
self._change_cache = {}
|
||||||
|
|
||||||
def getProject(self, name):
|
def getProject(self, name):
|
||||||
return self.projects.get(name)
|
return self.projects.get(name)
|
||||||
|
@ -48,15 +160,97 @@ class GitConnection(BaseConnection):
|
||||||
def addProject(self, project):
|
def addProject(self, project):
|
||||||
self.projects[project.name] = project
|
self.projects[project.name] = project
|
||||||
|
|
||||||
|
def getChangeFilesUpdated(self, project_name, branch, tosha):
|
||||||
|
job = self.sched.merger.getFilesChanges(
|
||||||
|
self.connection_name, project_name, branch, tosha)
|
||||||
|
self.log.debug("Waiting for fileschanges job %s" % job)
|
||||||
|
job.wait()
|
||||||
|
if not job.updated:
|
||||||
|
raise Exception("Fileschanges job %s failed" % job)
|
||||||
|
self.log.debug("Fileschanges job %s got changes on files %s" %
|
||||||
|
(job, job.files))
|
||||||
|
return job.files
|
||||||
|
|
||||||
|
def lsRemote(self, project):
|
||||||
|
refs = {}
|
||||||
|
client = git.cmd.Git()
|
||||||
|
output = client.ls_remote(
|
||||||
|
os.path.join(self.baseurl, project))
|
||||||
|
for line in output.splitlines():
|
||||||
|
sha, ref = line.split('\t')
|
||||||
|
if ref.startswith('refs/'):
|
||||||
|
refs[ref] = sha
|
||||||
|
return refs
|
||||||
|
|
||||||
|
def maintainCache(self, relevant):
|
||||||
|
remove = {}
|
||||||
|
for branch, refschange in self._change_cache.items():
|
||||||
|
for ref, change in refschange.items():
|
||||||
|
if change not in relevant:
|
||||||
|
remove.setdefault(branch, []).append(ref)
|
||||||
|
for branch, refs in remove.items():
|
||||||
|
for ref in refs:
|
||||||
|
del self._change_cache[branch][ref]
|
||||||
|
if not self._change_cache[branch]:
|
||||||
|
del self._change_cache[branch]
|
||||||
|
|
||||||
|
def getChange(self, event, refresh=False):
|
||||||
|
if event.ref and event.ref.startswith('refs/heads/'):
|
||||||
|
branch = event.ref[len('refs/heads/'):]
|
||||||
|
change = self._change_cache.get(branch, {}).get(event.newrev)
|
||||||
|
if change:
|
||||||
|
return change
|
||||||
|
project = self.getProject(event.project_name)
|
||||||
|
change = Branch(project)
|
||||||
|
change.branch = branch
|
||||||
|
for attr in ('ref', 'oldrev', 'newrev'):
|
||||||
|
setattr(change, attr, getattr(event, attr))
|
||||||
|
change.url = ""
|
||||||
|
change.files = self.getChangeFilesUpdated(
|
||||||
|
event.project_name, change.branch, event.oldrev)
|
||||||
|
self._change_cache.setdefault(branch, {})[event.newrev] = change
|
||||||
|
elif event.ref:
|
||||||
|
# catch-all ref (ie, not a branch or head)
|
||||||
|
project = self.getProject(event.project_name)
|
||||||
|
change = Ref(project)
|
||||||
|
for attr in ('ref', 'oldrev', 'newrev'):
|
||||||
|
setattr(change, attr, getattr(event, attr))
|
||||||
|
change.url = ""
|
||||||
|
else:
|
||||||
|
self.log.warning("Unable to get change for %s" % (event,))
|
||||||
|
change = None
|
||||||
|
return change
|
||||||
|
|
||||||
def getProjectBranches(self, project, tenant):
|
def getProjectBranches(self, project, tenant):
|
||||||
# TODO(jeblair): implement; this will need to handle local or
|
refs = self.lsRemote(project.name)
|
||||||
# remote git urls.
|
branches = [ref[len('refs/heads/'):] for ref in
|
||||||
return ['master']
|
refs if ref.startswith('refs/heads/')]
|
||||||
|
return branches
|
||||||
|
|
||||||
def getGitUrl(self, project):
|
def getGitUrl(self, project):
|
||||||
url = '%s/%s' % (self.baseurl, project.name)
|
url = '%s/%s' % (self.baseurl, project.name)
|
||||||
return url
|
return url
|
||||||
|
|
||||||
|
def onLoad(self):
|
||||||
|
self.log.debug("Starting Git Watcher")
|
||||||
|
self._start_watcher_thread()
|
||||||
|
|
||||||
|
def onStop(self):
|
||||||
|
self.log.debug("Stopping Git Watcher")
|
||||||
|
self._stop_watcher_thread()
|
||||||
|
|
||||||
|
def _stop_watcher_thread(self):
|
||||||
|
if self.watcher_thread:
|
||||||
|
self.watcher_thread.stop()
|
||||||
|
self.watcher_thread.join()
|
||||||
|
|
||||||
|
def _start_watcher_thread(self):
|
||||||
|
self.watcher_thread = GitWatcher(
|
||||||
|
self,
|
||||||
|
self.baseurl,
|
||||||
|
self.poll_timeout)
|
||||||
|
self.watcher_thread.start()
|
||||||
|
|
||||||
|
|
||||||
def getSchema():
|
def getSchema():
|
||||||
git_connection = v.Any(str, v.Schema(dict))
|
git_connection = v.Any(str, v.Schema(dict))
|
||||||
|
|
|
@ -0,0 +1,86 @@
|
||||||
|
# Copyright 2017 Red Hat, Inc.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
|
from zuul.model import TriggerEvent
|
||||||
|
from zuul.model import EventFilter
|
||||||
|
|
||||||
|
|
||||||
|
EMPTY_GIT_REF = '0' * 40 # git sha of all zeros, used during creates/deletes
|
||||||
|
|
||||||
|
|
||||||
|
class GitTriggerEvent(TriggerEvent):
|
||||||
|
"""Incoming event from an external system."""
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
ret = '<GitTriggerEvent %s %s' % (self.type,
|
||||||
|
self.project_name)
|
||||||
|
|
||||||
|
if self.branch:
|
||||||
|
ret += " %s" % self.branch
|
||||||
|
ret += " oldrev:%s" % self.oldrev
|
||||||
|
ret += " newrev:%s" % self.newrev
|
||||||
|
ret += '>'
|
||||||
|
|
||||||
|
return ret
|
||||||
|
|
||||||
|
|
||||||
|
class GitEventFilter(EventFilter):
|
||||||
|
def __init__(self, trigger, types=[], refs=[],
|
||||||
|
ignore_deletes=True):
|
||||||
|
|
||||||
|
super().__init__(trigger)
|
||||||
|
|
||||||
|
self._refs = refs
|
||||||
|
self.types = types
|
||||||
|
self.refs = [re.compile(x) for x in refs]
|
||||||
|
self.ignore_deletes = ignore_deletes
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
ret = '<GitEventFilter'
|
||||||
|
|
||||||
|
if self.types:
|
||||||
|
ret += ' types: %s' % ', '.join(self.types)
|
||||||
|
if self._refs:
|
||||||
|
ret += ' refs: %s' % ', '.join(self._refs)
|
||||||
|
if self.ignore_deletes:
|
||||||
|
ret += ' ignore_deletes: %s' % self.ignore_deletes
|
||||||
|
ret += '>'
|
||||||
|
|
||||||
|
return ret
|
||||||
|
|
||||||
|
def matches(self, event, change):
|
||||||
|
# event types are ORed
|
||||||
|
matches_type = False
|
||||||
|
for etype in self.types:
|
||||||
|
if etype == event.type:
|
||||||
|
matches_type = True
|
||||||
|
if self.types and not matches_type:
|
||||||
|
return False
|
||||||
|
|
||||||
|
# refs are ORed
|
||||||
|
matches_ref = False
|
||||||
|
if event.ref is not None:
|
||||||
|
for ref in self.refs:
|
||||||
|
if ref.match(event.ref):
|
||||||
|
matches_ref = True
|
||||||
|
if self.refs and not matches_ref:
|
||||||
|
return False
|
||||||
|
if self.ignore_deletes and event.newrev == EMPTY_GIT_REF:
|
||||||
|
# If the updated ref has an empty git sha (all 0s),
|
||||||
|
# then the ref is being deleted
|
||||||
|
return False
|
||||||
|
|
||||||
|
return True
|
|
@ -36,7 +36,7 @@ class GitSource(BaseSource):
|
||||||
raise NotImplemented()
|
raise NotImplemented()
|
||||||
|
|
||||||
def getChange(self, event, refresh=False):
|
def getChange(self, event, refresh=False):
|
||||||
raise NotImplemented()
|
return self.connection.getChange(event, refresh)
|
||||||
|
|
||||||
def getProject(self, name):
|
def getProject(self, name):
|
||||||
p = self.connection.getProject(name)
|
p = self.connection.getProject(name)
|
||||||
|
|
|
@ -0,0 +1,49 @@
|
||||||
|
# Copyright 2017 Red Hat, Inc.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import voluptuous as v
|
||||||
|
from zuul.trigger import BaseTrigger
|
||||||
|
from zuul.driver.git.gitmodel import GitEventFilter
|
||||||
|
from zuul.driver.util import scalar_or_list, to_list
|
||||||
|
|
||||||
|
|
||||||
|
class GitTrigger(BaseTrigger):
|
||||||
|
name = 'git'
|
||||||
|
log = logging.getLogger("zuul.GitTrigger")
|
||||||
|
|
||||||
|
def getEventFilters(self, trigger_conf):
|
||||||
|
efilters = []
|
||||||
|
for trigger in to_list(trigger_conf):
|
||||||
|
f = GitEventFilter(
|
||||||
|
trigger=self,
|
||||||
|
types=to_list(trigger['event']),
|
||||||
|
refs=to_list(trigger.get('ref')),
|
||||||
|
ignore_deletes=trigger.get(
|
||||||
|
'ignore-deletes', True)
|
||||||
|
)
|
||||||
|
efilters.append(f)
|
||||||
|
|
||||||
|
return efilters
|
||||||
|
|
||||||
|
|
||||||
|
def getSchema():
|
||||||
|
git_trigger = {
|
||||||
|
v.Required('event'):
|
||||||
|
scalar_or_list(v.Any('ref-updated')),
|
||||||
|
'ref': scalar_or_list(str),
|
||||||
|
'ignore-deletes': bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
return git_trigger
|
|
@ -1706,6 +1706,7 @@ class ExecutorServer(object):
|
||||||
self.merger_worker.registerFunction("merger:merge")
|
self.merger_worker.registerFunction("merger:merge")
|
||||||
self.merger_worker.registerFunction("merger:cat")
|
self.merger_worker.registerFunction("merger:cat")
|
||||||
self.merger_worker.registerFunction("merger:refstate")
|
self.merger_worker.registerFunction("merger:refstate")
|
||||||
|
self.merger_worker.registerFunction("merger:fileschanges")
|
||||||
|
|
||||||
def register_work(self):
|
def register_work(self):
|
||||||
if self._running:
|
if self._running:
|
||||||
|
@ -1859,6 +1860,9 @@ class ExecutorServer(object):
|
||||||
elif job.name == 'merger:refstate':
|
elif job.name == 'merger:refstate':
|
||||||
self.log.debug("Got refstate job: %s" % job.unique)
|
self.log.debug("Got refstate job: %s" % job.unique)
|
||||||
self.refstate(job)
|
self.refstate(job)
|
||||||
|
elif job.name == 'merger:fileschanges':
|
||||||
|
self.log.debug("Got fileschanges job: %s" % job.unique)
|
||||||
|
self.fileschanges(job)
|
||||||
else:
|
else:
|
||||||
self.log.error("Unable to handle job %s" % job.name)
|
self.log.error("Unable to handle job %s" % job.name)
|
||||||
job.sendWorkFail()
|
job.sendWorkFail()
|
||||||
|
@ -1970,6 +1974,19 @@ class ExecutorServer(object):
|
||||||
files=files)
|
files=files)
|
||||||
job.sendWorkComplete(json.dumps(result))
|
job.sendWorkComplete(json.dumps(result))
|
||||||
|
|
||||||
|
def fileschanges(self, job):
|
||||||
|
args = json.loads(job.arguments)
|
||||||
|
task = self.update(args['connection'], args['project'])
|
||||||
|
task.wait()
|
||||||
|
with self.merger_lock:
|
||||||
|
files = self.merger.getFilesChanges(
|
||||||
|
args['connection'], args['project'],
|
||||||
|
args['branch'],
|
||||||
|
args['tosha'])
|
||||||
|
result = dict(updated=True,
|
||||||
|
files=files)
|
||||||
|
job.sendWorkComplete(json.dumps(result))
|
||||||
|
|
||||||
def refstate(self, job):
|
def refstate(self, job):
|
||||||
args = json.loads(job.arguments)
|
args = json.loads(job.arguments)
|
||||||
with self.merger_lock:
|
with self.merger_lock:
|
||||||
|
|
|
@ -131,6 +131,15 @@ class MergeClient(object):
|
||||||
job = self.submitJob('merger:cat', data, None, precedence)
|
job = self.submitJob('merger:cat', data, None, precedence)
|
||||||
return job
|
return job
|
||||||
|
|
||||||
|
def getFilesChanges(self, connection_name, project_name, branch,
|
||||||
|
tosha=None, precedence=zuul.model.PRECEDENCE_HIGH):
|
||||||
|
data = dict(connection=connection_name,
|
||||||
|
project=project_name,
|
||||||
|
branch=branch,
|
||||||
|
tosha=tosha)
|
||||||
|
job = self.submitJob('merger:fileschanges', data, None, precedence)
|
||||||
|
return job
|
||||||
|
|
||||||
def onBuildCompleted(self, job):
|
def onBuildCompleted(self, job):
|
||||||
data = getJobData(job)
|
data = getJobData(job)
|
||||||
merged = data.get('merged', False)
|
merged = data.get('merged', False)
|
||||||
|
|
|
@ -314,6 +314,18 @@ class Repo(object):
|
||||||
'utf-8')
|
'utf-8')
|
||||||
return ret
|
return ret
|
||||||
|
|
||||||
|
def getFilesChanges(self, branch, tosha=None):
|
||||||
|
repo = self.createRepoObject()
|
||||||
|
files = set()
|
||||||
|
head = repo.heads[branch].commit
|
||||||
|
files.update(set(head.stats.files.keys()))
|
||||||
|
if tosha:
|
||||||
|
for cmt in head.iter_parents():
|
||||||
|
if cmt.hexsha == tosha:
|
||||||
|
break
|
||||||
|
files.update(set(cmt.stats.files.keys()))
|
||||||
|
return list(files)
|
||||||
|
|
||||||
def deleteRemote(self, remote):
|
def deleteRemote(self, remote):
|
||||||
repo = self.createRepoObject()
|
repo = self.createRepoObject()
|
||||||
repo.delete_remote(repo.remotes[remote])
|
repo.delete_remote(repo.remotes[remote])
|
||||||
|
@ -581,3 +593,8 @@ class Merger(object):
|
||||||
def getFiles(self, connection_name, project_name, branch, files, dirs=[]):
|
def getFiles(self, connection_name, project_name, branch, files, dirs=[]):
|
||||||
repo = self.getRepo(connection_name, project_name)
|
repo = self.getRepo(connection_name, project_name)
|
||||||
return repo.getFiles(files, dirs, branch=branch)
|
return repo.getFiles(files, dirs, branch=branch)
|
||||||
|
|
||||||
|
def getFilesChanges(self, connection_name, project_name, branch,
|
||||||
|
tosha=None):
|
||||||
|
repo = self.getRepo(connection_name, project_name)
|
||||||
|
return repo.getFilesChanges(branch, tosha)
|
||||||
|
|
|
@ -81,6 +81,7 @@ class MergeServer(object):
|
||||||
self.worker.registerFunction("merger:merge")
|
self.worker.registerFunction("merger:merge")
|
||||||
self.worker.registerFunction("merger:cat")
|
self.worker.registerFunction("merger:cat")
|
||||||
self.worker.registerFunction("merger:refstate")
|
self.worker.registerFunction("merger:refstate")
|
||||||
|
self.worker.registerFunction("merger:fileschanges")
|
||||||
|
|
||||||
def stop(self):
|
def stop(self):
|
||||||
self.log.debug("Stopping")
|
self.log.debug("Stopping")
|
||||||
|
@ -117,6 +118,9 @@ class MergeServer(object):
|
||||||
elif job.name == 'merger:refstate':
|
elif job.name == 'merger:refstate':
|
||||||
self.log.debug("Got refstate job: %s" % job.unique)
|
self.log.debug("Got refstate job: %s" % job.unique)
|
||||||
self.refstate(job)
|
self.refstate(job)
|
||||||
|
elif job.name == 'merger:fileschanges':
|
||||||
|
self.log.debug("Got fileschanges job: %s" % job.unique)
|
||||||
|
self.fileschanges(job)
|
||||||
else:
|
else:
|
||||||
self.log.error("Unable to handle job %s" % job.name)
|
self.log.error("Unable to handle job %s" % job.name)
|
||||||
job.sendWorkFail()
|
job.sendWorkFail()
|
||||||
|
@ -158,3 +162,12 @@ class MergeServer(object):
|
||||||
result = dict(updated=True,
|
result = dict(updated=True,
|
||||||
files=files)
|
files=files)
|
||||||
job.sendWorkComplete(json.dumps(result))
|
job.sendWorkComplete(json.dumps(result))
|
||||||
|
|
||||||
|
def fileschanges(self, job):
|
||||||
|
args = json.loads(job.arguments)
|
||||||
|
self.merger.updateRepo(args['connection'], args['project'])
|
||||||
|
files = self.merger.getFilesChanges(
|
||||||
|
args['connection'], args['project'], args['branch'], args['tosha'])
|
||||||
|
result = dict(updated=True,
|
||||||
|
files=files)
|
||||||
|
job.sendWorkComplete(json.dumps(result))
|
||||||
|
|
Loading…
Reference in New Issue