From 194a2bf2372ab1c71c6734da16ecc5ad23a3d548 Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Sat, 2 Dec 2017 18:17:58 +0100 Subject: [PATCH] Git driver This patch improves the existing git driver by adding a refs watcher thread. This refs watcher looks at refs added, deleted, updated and trigger a ref-updated event. When a refs is updated and that the related commits from oldrev to newrev include a change on .zuul.yaml/zuul.yaml or zuul.d/*.yaml then tenants including that ref is reconfigured. Furthermore the patch includes a triggering model. Events are sent to the scheduler so jobs can be attached to a pipeline for running jobs. Change-Id: I529660cb20d011f36814abe64f837945dd3f1f33 --- doc/source/admin/connections.rst | 1 + doc/source/admin/drivers/git.rst | 59 ++++++ tests/base.py | 10 + .../playbooks/project-test2.yaml | 2 + .../git-driver/git/common-config/zuul.yaml | 4 + tests/fixtures/layouts/basic-git.yaml | 37 ++++ tests/fixtures/zuul-git-driver.conf | 1 + tests/unit/test_git_driver.py | 133 +++++++++++- zuul/driver/git/__init__.py | 7 + zuul/driver/git/gitconnection.py | 200 +++++++++++++++++- zuul/driver/git/gitmodel.py | 86 ++++++++ zuul/driver/git/gitsource.py | 2 +- zuul/driver/git/gittrigger.py | 49 +++++ zuul/executor/server.py | 17 ++ zuul/merger/client.py | 9 + zuul/merger/merger.py | 17 ++ zuul/merger/server.py | 13 ++ 17 files changed, 641 insertions(+), 6 deletions(-) create mode 100644 doc/source/admin/drivers/git.rst create mode 100644 tests/fixtures/config/git-driver/git/common-config/playbooks/project-test2.yaml create mode 100644 tests/fixtures/layouts/basic-git.yaml create mode 100644 zuul/driver/git/gitmodel.py create mode 100644 zuul/driver/git/gittrigger.py diff --git a/doc/source/admin/connections.rst b/doc/source/admin/connections.rst index 29ca3be7c0..55ac629c18 100644 --- a/doc/source/admin/connections.rst +++ b/doc/source/admin/connections.rst @@ -55,6 +55,7 @@ Zuul includes the following drivers: drivers/gerrit drivers/github + drivers/git drivers/smtp drivers/sql drivers/timer diff --git a/doc/source/admin/drivers/git.rst b/doc/source/admin/drivers/git.rst new file mode 100644 index 0000000000..e0acec1168 --- /dev/null +++ b/doc/source/admin/drivers/git.rst @@ -0,0 +1,59 @@ +:title: Git Driver + +Git +=== + +This driver can be used to load Zuul configuration from public Git repositories, +for instance from ``openstack-infra/zuul-jobs`` that is suitable for use by +any Zuul system. It can also be used to trigger jobs from ``ref-updated`` events +in a pipeline. + +Connection Configuration +------------------------ + +The supported options in ``zuul.conf`` connections are: + +.. attr:: + + .. attr:: driver + :required: + + .. value:: git + + The connection must set ``driver=git`` for Git connections. + + .. attr:: baseurl + + Path to the base Git URL. Git repos name will be appended to it. + + .. attr:: poll_delay + :default: 7200 + + The delay in seconds of the Git repositories polling loop. + +Trigger Configuration +--------------------- + +.. attr:: pipeline.trigger. + + The dictionary passed to the Git pipeline ``trigger`` attribute + supports the following attributes: + + .. attr:: event + :required: + + Only ``ref-updated`` is supported. + + .. attr:: ref + + On ref-updated events, a ref such as ``refs/heads/master`` or + ``^refs/tags/.*$``. This field is treated as a regular expression, + and multiple refs may be listed. + + .. attr:: ignore-deletes + :default: true + + When a ref is deleted, a ref-updated event is emitted with a + newrev of all zeros specified. The ``ignore-deletes`` field is a + boolean value that describes whether or not these newrevs + trigger ref-updated events. diff --git a/tests/base.py b/tests/base.py index 69d9f55227..a51eeddcdc 100755 --- a/tests/base.py +++ b/tests/base.py @@ -2833,6 +2833,16 @@ class ZuulTestCase(BaseTestCase): os.path.join(FIXTURE_DIR, f.name)) self.setupAllProjectKeys() + def addTagToRepo(self, project, name, sha): + path = os.path.join(self.upstream_root, project) + repo = git.Repo(path) + repo.git.tag(name, sha) + + def delTagFromRepo(self, project, name): + path = os.path.join(self.upstream_root, project) + repo = git.Repo(path) + repo.git.tag('-d', name) + def addCommitToRepo(self, project, message, files, branch='master', tag=None): path = os.path.join(self.upstream_root, project) diff --git a/tests/fixtures/config/git-driver/git/common-config/playbooks/project-test2.yaml b/tests/fixtures/config/git-driver/git/common-config/playbooks/project-test2.yaml new file mode 100644 index 0000000000..f679dceaef --- /dev/null +++ b/tests/fixtures/config/git-driver/git/common-config/playbooks/project-test2.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/git-driver/git/common-config/zuul.yaml b/tests/fixtures/config/git-driver/git/common-config/zuul.yaml index 784b5f2b62..53fc210734 100644 --- a/tests/fixtures/config/git-driver/git/common-config/zuul.yaml +++ b/tests/fixtures/config/git-driver/git/common-config/zuul.yaml @@ -19,6 +19,10 @@ name: project-test1 run: playbooks/project-test1.yaml +- job: + name: project-test2 + run: playbooks/project-test2.yaml + - project: name: org/project check: diff --git a/tests/fixtures/layouts/basic-git.yaml b/tests/fixtures/layouts/basic-git.yaml new file mode 100644 index 0000000000..068d0a0ea2 --- /dev/null +++ b/tests/fixtures/layouts/basic-git.yaml @@ -0,0 +1,37 @@ +- pipeline: + name: post + manager: independent + trigger: + git: + - event: ref-updated + ref: ^refs/heads/.*$ + +- pipeline: + name: tag + manager: independent + trigger: + git: + - event: ref-updated + ref: ^refs/tags/.*$ + +- job: + name: base + parent: null + run: playbooks/base.yaml + +- job: + name: post-job + run: playbooks/post-job.yaml + +- job: + name: tag-job + run: playbooks/post-job.yaml + +- project: + name: org/project + post: + jobs: + - post-job + tag: + jobs: + - tag-job diff --git a/tests/fixtures/zuul-git-driver.conf b/tests/fixtures/zuul-git-driver.conf index b24b0a1b44..23a2a622c0 100644 --- a/tests/fixtures/zuul-git-driver.conf +++ b/tests/fixtures/zuul-git-driver.conf @@ -21,6 +21,7 @@ sshkey=none [connection git] driver=git baseurl="" +poll_delay=0.1 [connection outgoing_smtp] driver=smtp diff --git a/tests/unit/test_git_driver.py b/tests/unit/test_git_driver.py index 1cfadf4703..b9e6c6e921 100644 --- a/tests/unit/test_git_driver.py +++ b/tests/unit/test_git_driver.py @@ -12,7 +12,12 @@ # License for the specific language governing permissions and limitations # under the License. -from tests.base import ZuulTestCase + +import os +import time +import yaml + +from tests.base import ZuulTestCase, simple_layout class TestGitDriver(ZuulTestCase): @@ -23,7 +28,7 @@ class TestGitDriver(ZuulTestCase): super(TestGitDriver, self).setup_config() self.config.set('connection git', 'baseurl', self.upstream_root) - def test_git_driver(self): + def test_basic(self): tenant = self.sched.abide.tenants.get('tenant-one') # Check that we have the git source for common-config and the # gerrit source for the project. @@ -40,3 +45,127 @@ class TestGitDriver(ZuulTestCase): self.waitUntilSettled() self.assertEqual(len(self.history), 1) self.assertEqual(A.reported, 1) + + def test_config_refreshed(self): + A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + self.assertEqual(len(self.history), 1) + self.assertEqual(A.reported, 1) + self.assertEqual(self.history[0].name, 'project-test1') + + # Update zuul.yaml to force a tenant reconfiguration + path = os.path.join(self.upstream_root, 'common-config', 'zuul.yaml') + config = yaml.load(open(path, 'r').read()) + change = { + 'name': 'org/project', + 'check': { + 'jobs': [ + 'project-test2' + ] + } + } + config[4]['project'] = change + files = {'zuul.yaml': yaml.dump(config)} + self.addCommitToRepo( + 'common-config', 'Change zuul.yaml configuration', files) + + # Let some time for the tenant reconfiguration to happen + time.sleep(2) + self.waitUntilSettled() + + A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + self.assertEqual(len(self.history), 2) + self.assertEqual(A.reported, 1) + # We make sure the new job has run + self.assertEqual(self.history[1].name, 'project-test2') + + # Let's stop the git Watcher to let us merge some changes commits + # We want to verify that config changes are detected for commits + # on the range oldrev..newrev + self.sched.connections.getSource('git').connection.w_pause = True + # Add a config change + change = { + 'name': 'org/project', + 'check': { + 'jobs': [ + 'project-test1' + ] + } + } + config[4]['project'] = change + files = {'zuul.yaml': yaml.dump(config)} + self.addCommitToRepo( + 'common-config', 'Change zuul.yaml configuration', files) + # Add two other changes + self.addCommitToRepo( + 'common-config', 'Adding f1', + {'f1': "Content"}) + self.addCommitToRepo( + 'common-config', 'Adding f2', + {'f2': "Content"}) + # Restart the git watcher + self.sched.connections.getSource('git').connection.w_pause = False + + # Let some time for the tenant reconfiguration to happen + time.sleep(2) + self.waitUntilSettled() + + A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + self.assertEqual(len(self.history), 3) + self.assertEqual(A.reported, 1) + # We make sure the new job has run + self.assertEqual(self.history[2].name, 'project-test1') + + def ensure_watcher_has_context(self): + # Make sure watcher have read initial refs shas + cnx = self.sched.connections.getSource('git').connection + delay = 0.1 + max_delay = 1 + while not cnx.projects_refs: + time.sleep(delay) + max_delay -= delay + if max_delay <= 0: + raise Exception("Timeout waiting for initial read") + + @simple_layout('layouts/basic-git.yaml', driver='git') + def test_ref_updated_event(self): + self.ensure_watcher_has_context() + # Add a commit to trigger a ref-updated event + self.addCommitToRepo( + 'org/project', 'A change for ref-updated', {'f1': 'Content'}) + # Let some time for the git watcher to detect the ref-update event + time.sleep(0.2) + self.waitUntilSettled() + self.assertEqual(len(self.history), 1) + self.assertEqual('SUCCESS', + self.getJobFromHistory('post-job').result) + + @simple_layout('layouts/basic-git.yaml', driver='git') + def test_ref_created(self): + self.ensure_watcher_has_context() + # Tag HEAD to trigger a ref-updated event + self.addTagToRepo( + 'org/project', 'atag', 'HEAD') + # Let some time for the git watcher to detect the ref-update event + time.sleep(0.2) + self.waitUntilSettled() + self.assertEqual(len(self.history), 1) + self.assertEqual('SUCCESS', + self.getJobFromHistory('tag-job').result) + + @simple_layout('layouts/basic-git.yaml', driver='git') + def test_ref_deleted(self): + self.ensure_watcher_has_context() + # Delete default tag init to trigger a ref-updated event + self.delTagFromRepo( + 'org/project', 'init') + # Let some time for the git watcher to detect the ref-update event + time.sleep(0.2) + self.waitUntilSettled() + # Make sure no job as run as ignore-delete is True by default + self.assertEqual(len(self.history), 0) diff --git a/zuul/driver/git/__init__.py b/zuul/driver/git/__init__.py index 0faa0365aa..1fe43f6439 100644 --- a/zuul/driver/git/__init__.py +++ b/zuul/driver/git/__init__.py @@ -15,6 +15,7 @@ from zuul.driver import Driver, ConnectionInterface, SourceInterface from zuul.driver.git import gitconnection from zuul.driver.git import gitsource +from zuul.driver.git import gittrigger class GitDriver(Driver, ConnectionInterface, SourceInterface): @@ -23,9 +24,15 @@ class GitDriver(Driver, ConnectionInterface, SourceInterface): def getConnection(self, name, config): return gitconnection.GitConnection(self, name, config) + def getTrigger(self, connection, config=None): + return gittrigger.GitTrigger(self, connection, config) + def getSource(self, connection): return gitsource.GitSource(self, connection) + def getTriggerSchema(self): + return gittrigger.getSchema() + def getRequireSchema(self): return {} diff --git a/zuul/driver/git/gitconnection.py b/zuul/driver/git/gitconnection.py index f93824d2f8..03b24cadcd 100644 --- a/zuul/driver/git/gitconnection.py +++ b/zuul/driver/git/gitconnection.py @@ -13,12 +13,119 @@ # License for the specific language governing permissions and limitations # under the License. +import os +import git +import time import logging import urllib +import threading import voluptuous as v from zuul.connection import BaseConnection +from zuul.driver.git.gitmodel import GitTriggerEvent, EMPTY_GIT_REF +from zuul.model import Ref, Branch + + +class GitWatcher(threading.Thread): + log = logging.getLogger("connection.git.GitWatcher") + + def __init__(self, git_connection, baseurl, poll_delay): + threading.Thread.__init__(self) + self.daemon = True + self.git_connection = git_connection + self.baseurl = baseurl + self.poll_delay = poll_delay + self._stopped = False + self.projects_refs = self.git_connection.projects_refs + + def compareRefs(self, project, refs): + partial_events = [] + # Fetch previous refs state + base_refs = self.projects_refs.get(project) + # Create list of created refs + rcreateds = set(refs.keys()) - set(base_refs.keys()) + # Create list of deleted refs + rdeleteds = set(base_refs.keys()) - set(refs.keys()) + # Create the list of updated refs + updateds = {} + for ref, sha in refs.items(): + if ref in base_refs and base_refs[ref] != sha: + updateds[ref] = sha + for ref in rcreateds: + event = { + 'ref': ref, + 'branch_created': True, + 'oldrev': EMPTY_GIT_REF, + 'newrev': refs[ref] + } + partial_events.append(event) + for ref in rdeleteds: + event = { + 'ref': ref, + 'branch_deleted': True, + 'oldrev': base_refs[ref], + 'newrev': EMPTY_GIT_REF + } + partial_events.append(event) + for ref, sha in updateds.items(): + event = { + 'ref': ref, + 'branch_updated': True, + 'oldrev': base_refs[ref], + 'newrev': sha + } + partial_events.append(event) + events = [] + for pevent in partial_events: + event = GitTriggerEvent() + event.type = 'ref-updated' + event.project_hostname = self.git_connection.canonical_hostname + event.project_name = project + for attr in ('ref', 'oldrev', 'newrev', 'branch_created', + 'branch_deleted', 'branch_updated'): + if attr in pevent: + setattr(event, attr, pevent[attr]) + events.append(event) + return events + + def _run(self): + self.log.debug("Walk through projects refs for connection: %s" % + self.git_connection.connection_name) + try: + for project in self.git_connection.projects: + refs = self.git_connection.lsRemote(project) + self.log.debug("Read refs %s for project %s" % (refs, project)) + if not self.projects_refs.get(project): + # State for this project does not exist yet so add it. + # No event will be triggered in this loop as + # projects_refs['project'] and refs are equal + self.projects_refs[project] = refs + events = self.compareRefs(project, refs) + self.projects_refs[project] = refs + # Send events to the scheduler + for event in events: + self.log.debug("Handling event: %s" % event) + # Force changes cache update before passing + # the event to the scheduler + self.git_connection.getChange(event) + self.git_connection.logEvent(event) + # Pass the event to the scheduler + self.git_connection.sched.addEvent(event) + except Exception as e: + self.log.debug("Unexpected issue in _run loop: %s" % str(e)) + + def run(self): + while not self._stopped: + if not self.git_connection.w_pause: + self._run() + # Polling wait delay + else: + self.log.debug("Watcher is on pause") + time.sleep(self.poll_delay) + + def stop(self): + self._stopped = True class GitConnection(BaseConnection): @@ -32,6 +139,8 @@ class GitConnection(BaseConnection): raise Exception('baseurl is required for git connections in ' '%s' % self.connection_name) self.baseurl = self.connection_config.get('baseurl') + self.poll_timeout = float( + self.connection_config.get('poll_delay', 3600 * 2)) self.canonical_hostname = self.connection_config.get( 'canonical_hostname') if not self.canonical_hostname: @@ -40,7 +149,10 @@ class GitConnection(BaseConnection): self.canonical_hostname = r.hostname else: self.canonical_hostname = 'localhost' + self.w_pause = False self.projects = {} + self.projects_refs = {} + self._change_cache = {} def getProject(self, name): return self.projects.get(name) @@ -48,15 +160,97 @@ class GitConnection(BaseConnection): def addProject(self, project): self.projects[project.name] = project + def getChangeFilesUpdated(self, project_name, branch, tosha): + job = self.sched.merger.getFilesChanges( + self.connection_name, project_name, branch, tosha) + self.log.debug("Waiting for fileschanges job %s" % job) + job.wait() + if not job.updated: + raise Exception("Fileschanges job %s failed" % job) + self.log.debug("Fileschanges job %s got changes on files %s" % + (job, job.files)) + return job.files + + def lsRemote(self, project): + refs = {} + client = git.cmd.Git() + output = client.ls_remote( + os.path.join(self.baseurl, project)) + for line in output.splitlines(): + sha, ref = line.split('\t') + if ref.startswith('refs/'): + refs[ref] = sha + return refs + + def maintainCache(self, relevant): + remove = {} + for branch, refschange in self._change_cache.items(): + for ref, change in refschange.items(): + if change not in relevant: + remove.setdefault(branch, []).append(ref) + for branch, refs in remove.items(): + for ref in refs: + del self._change_cache[branch][ref] + if not self._change_cache[branch]: + del self._change_cache[branch] + + def getChange(self, event, refresh=False): + if event.ref and event.ref.startswith('refs/heads/'): + branch = event.ref[len('refs/heads/'):] + change = self._change_cache.get(branch, {}).get(event.newrev) + if change: + return change + project = self.getProject(event.project_name) + change = Branch(project) + change.branch = branch + for attr in ('ref', 'oldrev', 'newrev'): + setattr(change, attr, getattr(event, attr)) + change.url = "" + change.files = self.getChangeFilesUpdated( + event.project_name, change.branch, event.oldrev) + self._change_cache.setdefault(branch, {})[event.newrev] = change + elif event.ref: + # catch-all ref (ie, not a branch or head) + project = self.getProject(event.project_name) + change = Ref(project) + for attr in ('ref', 'oldrev', 'newrev'): + setattr(change, attr, getattr(event, attr)) + change.url = "" + else: + self.log.warning("Unable to get change for %s" % (event,)) + change = None + return change + def getProjectBranches(self, project, tenant): - # TODO(jeblair): implement; this will need to handle local or - # remote git urls. - return ['master'] + refs = self.lsRemote(project.name) + branches = [ref[len('refs/heads/'):] for ref in + refs if ref.startswith('refs/heads/')] + return branches def getGitUrl(self, project): url = '%s/%s' % (self.baseurl, project.name) return url + def onLoad(self): + self.log.debug("Starting Git Watcher") + self._start_watcher_thread() + + def onStop(self): + self.log.debug("Stopping Git Watcher") + self._stop_watcher_thread() + + def _stop_watcher_thread(self): + if self.watcher_thread: + self.watcher_thread.stop() + self.watcher_thread.join() + + def _start_watcher_thread(self): + self.watcher_thread = GitWatcher( + self, + self.baseurl, + self.poll_timeout) + self.watcher_thread.start() + def getSchema(): git_connection = v.Any(str, v.Schema(dict)) diff --git a/zuul/driver/git/gitmodel.py b/zuul/driver/git/gitmodel.py new file mode 100644 index 0000000000..5d12b36da1 --- /dev/null +++ b/zuul/driver/git/gitmodel.py @@ -0,0 +1,86 @@ +# Copyright 2017 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import re + +from zuul.model import TriggerEvent +from zuul.model import EventFilter + + +EMPTY_GIT_REF = '0' * 40 # git sha of all zeros, used during creates/deletes + + +class GitTriggerEvent(TriggerEvent): + """Incoming event from an external system.""" + + def __repr__(self): + ret = '