Gitlab - Basic handling of merge_requests event

This patch also adds the basics for the unittests in order
to be able to validate an incoming event.

Also it brings a minimal trigger definition to support
the merge_requests opened event.

Change-Id: I37978a2cf976f1f3a25fd70ccc5d48d77afd8b83
This commit is contained in:
Fabien Boucher 2019-10-01 17:11:08 +02:00 committed by Tristan Cacqueray
parent 6821db44dd
commit 7264e7553d
7 changed files with 536 additions and 18 deletions

View File

@ -1502,11 +1502,191 @@ class FakeGitlabConnection(gitlabconnection.GitlabConnection):
changes_db=None, upstream_root=None):
super(FakeGitlabConnection, self).__init__(driver, connection_name,
connection_config)
self.merge_requests = changes_db
self.gl_client = FakeGitlabAPIClient(
self.baseurl, self.api_token, merge_requests_db=changes_db)
self.rpcclient = rpcclient
self.upstream_root = upstream_root
self.mr_number = 0
def getGitUrl(self, project):
return 'file://' + os.path.join(self.upstream_root, project.name)
def openFakeMergeRequest(self, project,
branch, title, description='', files=[]):
self.mr_number += 1
merge_request = FakeGitlabMergeRequest(
self, self.mr_number, project, branch, title, self.upstream_root,
files=files, description=description)
self.merge_requests.setdefault(
project, {})[str(self.mr_number)] = merge_request
return merge_request
def emitEvent(self, event, use_zuulweb=False, project=None):
name, payload = event
if use_zuulweb:
payload = json.dumps(payload).encode('utf-8')
headers = {'x-gitlab-token': self.webhook_token}
return requests.post(
'http://127.0.0.1:%s/api/connection/%s/payload'
% (self.zuul_web_port, self.connection_name),
data=payload, headers=headers)
else:
job = self.rpcclient.submitJob(
'gitlab:%s:payload' % self.connection_name,
{'payload': payload})
return json.loads(job.data[0])
def setZuulWebPort(self, port):
self.zuul_web_port = port
class FakeGitlabAPIClient(gitlabconnection.GitlabAPIClient):
log = logging.getLogger("zuul.test.FakeGitlabAPIClient")
def __init__(self, baseurl, api_token, merge_requests_db={}):
super(FakeGitlabAPIClient, self).__init__(baseurl, api_token)
self.merge_requests = merge_requests_db
def gen_error(self, verb):
return {
'message': 'some error',
}, 503, "", verb
def _get_mr(self, match):
project, number = match.groups()
project = urllib.parse.unquote(project)
mr = self.merge_requests.get(project, {}).get(number)
if not mr:
return self.gen_error("GET")
return mr
def get(self, url):
self.log.debug("Getting resource %s ..." % url)
match = re.match(r'.+/projects/(.+)/merge_requests/(\d+)$', url)
if match:
mr = self._get_mr(match)
return {
'target_branch': mr.branch,
'title': mr.title,
'state': mr.state,
'description': mr.description,
'updated_at': mr.updated_at.strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
'sha': mr.patch_number,
'labels': mr.labels,
'merged_at': mr.merged_at,
'merge_status': mr.merge_status,
}, 200, "", "GET"
match = re.match('.+/projects/(.+)/repository/branches$', url)
if match:
return [{'name': 'master'}], 200, "", "GET"
class GitlabChangeReference(git.Reference):
_common_path_default = "refs/merge-requests"
_points_to_commits_only = True
class FakeGitlabMergeRequest(object):
log = logging.getLogger("zuul.test.FakeGitlabMergeRequest")
def __init__(self, gitlab, number, project, branch,
title, upstream_root, files=[], description=''):
self.gitlab = gitlab
self.source = gitlab
self.number = number
self.project = project
self.branch = branch
self.title = title
self.description = description
self.upstream_root = upstream_root
self.number_of_commits = 0
self.created_at = datetime.datetime.now()
self.updated_at = self.created_at
self.merged_at = None
self.patch_number = None
self.state = 'opened'
self.merge_status = 'can_be_merged'
self.uuid = uuid.uuid4().hex
self.labels = []
self.upstream_root = upstream_root
self.url = "https://%s/%s/merge_requests/%s" % (
self.gitlab.server, urllib.parse.quote_plus(
self.project), self.number)
self.is_merged = False
self.mr_ref = self._createMRRef()
self._addCommitInMR(files=files)
def _getRepo(self):
repo_path = os.path.join(self.upstream_root, self.project)
return git.Repo(repo_path)
def _createMRRef(self):
repo = self._getRepo()
return GitlabChangeReference.create(
repo, self.getMRReference(), 'refs/tags/init')
def getMRReference(self):
return '%s/head' % self.number
def _addCommitInMR(self, files=[], reset=False):
repo = self._getRepo()
ref = repo.references[self.getMRReference()]
if reset:
self.number_of_commits = 0
ref.set_object('refs/tags/init')
self.number_of_commits += 1
repo.head.reference = ref
repo.git.clean('-x', '-f', '-d')
if files:
self.files = files
else:
fn = '%s-%s' % (self.branch.replace('/', '_'), self.number)
self.files = {fn: "test %s %s\n" % (self.branch, self.number)}
msg = self.title + '-' + str(self.number_of_commits)
for fn, content in self.files.items():
fn = os.path.join(repo.working_dir, fn)
with open(fn, 'w') as f:
f.write(content)
repo.index.add([fn])
self.patch_number = repo.index.commit(msg).hexsha
repo.create_head(self.getMRReference(), self.patch_number, force=True)
self.mr_ref.set_commit(self.patch_number)
repo.head.reference = 'master'
repo.git.clean('-x', '-f', '-d')
repo.heads['master'].checkout()
def _updateTimeStamp(self):
self.updated_at = datetime.datetime.now()
def getMergeRequestEvent(self):
name = 'gl_merge_request'
data = {
'object_kind': 'merge_request',
'project': {
'path_with_namespace': self.project
},
'object_attributes': {
'title': self.title,
'created_at': self.created_at.strftime(
'%Y-%m-%d %H:%M:%S UTC'),
'updated_at': self.updated_at.strftime(
'%Y-%m-%d %H:%M:%S UTC'),
'iid': self.number,
'target_branch': self.branch,
'last_commit': {
'id': self.patch_number,
}
},
}
return (name, data)
class GithubChangeReference(git.Reference):
_common_path_default = "refs/pull"
_points_to_commits_only = True
@ -2902,6 +3082,7 @@ class ZuulWebFixture(fixtures.Fixture):
config,
include_drivers=[zuul.driver.sql.SQLDriver,
zuul.driver.github.GithubDriver,
zuul.driver.gitlab.GitlabDriver,
zuul.driver.pagure.PagureDriver])
self.authenticators = zuul.lib.auth.AuthenticatorRegistry()
self.authenticators.configure(config)

View File

@ -1,4 +1,23 @@
- pipeline:
name: check
manager: independent
trigger: {}
trigger:
gitlab:
- event: gl_merge_request
action:
- opened
- job:
name: base
parent: null
run: playbooks/base.yaml
- job:
name: project-test1
run: playbooks/project-test1.yaml
- project:
name: org/project
check:
jobs:
- project-test1

View File

@ -46,4 +46,24 @@ class TestGitlabWebhook(ZuulTestCase):
@simple_layout('layouts/basic-gitlab.yaml', driver='gitlab')
def test_webhook(self):
pass
A = self.fake_gitlab.openFakeMergeRequest(
'org/project', 'master', 'A')
self.fake_gitlab.emitEvent(A.getMergeRequestEvent(),
use_zuulweb=False,
project='org/project')
self.waitUntilSettled()
self.assertEqual('SUCCESS',
self.getJobFromHistory('project-test1').result)
@simple_layout('layouts/basic-gitlab.yaml', driver='gitlab')
def test_webhook_via_zuulweb(self):
A = self.fake_gitlab.openFakeMergeRequest(
'org/project', 'master', 'A')
self.fake_gitlab.emitEvent(A.getMergeRequestEvent(),
use_zuulweb=True,
project='org/project')
self.waitUntilSettled()
self.assertEqual('SUCCESS',
self.getJobFromHistory('project-test1').result)

View File

@ -19,11 +19,16 @@ import queue
import cherrypy
import voluptuous as v
import time
import requests
from urllib.parse import quote_plus
from datetime import datetime
from zuul.connection import BaseConnection
from zuul.web.handler import BaseWebController
from zuul.lib.gearworker import ZuulGearWorker
from zuul.driver.gitlab.gitlabmodel import GitlabTriggerEvent, MergeRequest
class GitlabGearmanWorker(object):
"""A thread that answers gearman requests"""
@ -48,8 +53,7 @@ class GitlabGearmanWorker(object):
payload = args["payload"]
self.log.info(
"Gitlab Webhook Received (event kind: %(object_kind)s ",
"event name: %(event_name)s)" % payload)
"Gitlab Webhook Received event kind: %(object_kind)s" % payload)
try:
self.__dispatch_event(payload)
@ -61,7 +65,8 @@ class GitlabGearmanWorker(object):
job.sendWorkComplete(json.dumps(output))
def __dispatch_event(self, payload):
event = payload['event_name']
self.log.info(payload)
event = payload['object_kind']
try:
self.log.info("Dispatching event %s" % event)
self.connection.addEvent(payload, event)
@ -87,12 +92,36 @@ class GitlabEventConnector(threading.Thread):
self.daemon = True
self.connection = connection
self._stopped = False
self.event_handler_mapping = {}
self.event_handler_mapping = {
'merge_request': self._event_merge_request,
}
def stop(self):
self._stopped = True
self.connection.addEvent(None)
def _event_merge_request(self, body):
event = GitlabTriggerEvent()
attrs = body['object_attributes']
event.title = attrs['title']
event.updated_at = int(datetime.strptime(
attrs['updated_at'], '%Y-%m-%d %H:%M:%S %Z').strftime('%s'))
event.created_at = int(datetime.strptime(
attrs['created_at'], '%Y-%m-%d %H:%M:%S %Z').strftime('%s'))
event.project_name = body['project']['path_with_namespace']
event.change_number = attrs['iid']
event.branch = attrs['target_branch']
event.change_url = self.connection.getPullUrl(event.project_name,
event.change_number)
event.ref = "refs/merge-requests/%s/head" % event.change_number
event.patch_number = attrs['last_commit']['id']
if event.created_at == event.updated_at:
event.action = 'opened'
else:
event.action = 'changed'
event.type = 'gl_merge_request'
return event
def _handleEvent(self):
ts, json_body, event_type = self.connection.getEvent()
if self._stopped:
@ -105,6 +134,30 @@ class GitlabEventConnector(threading.Thread):
self.log.info(message)
return
if event_type in self.event_handler_mapping:
self.log.debug("Handling event: %s" % event_type)
try:
event = self.event_handler_mapping[event_type](json_body)
except Exception:
self.log.exception(
'Exception when handling event: %s' % event_type)
event = None
if event:
event.timestamp = ts
if event.change_number:
project = self.connection.source.getProject(event.project_name)
self.connection._getChange(project,
event.change_number,
event.patch_number,
refresh=True,
url=event.change_url,
event=event)
event.project_hostname = self.connection.canonical_hostname
self.connection.logEvent(event)
self.connection.sched.addEvent(event)
def run(self):
while True:
if self._stopped:
@ -117,6 +170,53 @@ class GitlabEventConnector(threading.Thread):
self.connection.eventDone()
class GitlabAPIClientException(Exception):
pass
class GitlabAPIClient():
log = logging.getLogger("zuul.GitlabAPIClient")
def __init__(self, baseurl, api_token):
self.session = requests.Session()
self.baseurl = '%s/api/v4/' % baseurl
self.api_token = api_token
self.headers = {'Authorization': 'Authorization: Bearer %s' % (
self.api_token)}
def _manage_error(self, data, code, url, verb):
if code < 400:
return
else:
raise GitlabAPIClientException(
"Unable to %s on %s (code: %s) due to: %s" % (
verb, url, code, data
))
def get(self, url):
self.log.debug("Getting resource %s ..." % url)
ret = self.session.get(url, headers=self.headers)
self.log.debug("GET returned (code: %s): %s" % (
ret.status_code, ret.text))
return ret.json(), ret.status_code, ret.url, 'GET'
# https://docs.gitlab.com/ee/api/merge_requests.html#get-single-mr
def get_mr(self, project_name, number):
path = "/projects/%s/merge_requests/%s" % (
quote_plus(project_name), number)
resp = self.get(self.baseurl + path)
self._manage_error(*resp)
return resp[0]
# https://docs.gitlab.com/ee/api/branches.html#list-repository-branches
def get_project_branches(self, project_name):
path = "/projects/%s/repository/branches" % (
quote_plus(project_name))
resp = self.get(self.baseurl + path)
self._manage_error(*resp)
return [branch['name'] for branch in resp[0]]
class GitlabConnection(BaseConnection):
driver_name = 'gitlab'
log = logging.getLogger("zuul.GitlabConnection")
@ -126,13 +226,23 @@ class GitlabConnection(BaseConnection):
super(GitlabConnection, self).__init__(
driver, connection_name, connection_config)
self.projects = {}
self.project_branch_cache = {}
self._change_cache = {}
self.server = self.connection_config.get('server', 'gitlab.com')
self.baseurl = self.connection_config.get(
'baseurl', 'https://%s' % self.server).rstrip('/')
self.cloneurl = self.connection_config.get(
'cloneurl', self.baseurl).rstrip('/')
self.canonical_hostname = self.connection_config.get(
'canonical_hostname', self.server)
self.webhook_token = self.connection_config.get(
'webhook_token', '')
self.api_token = self.connection_config.get(
'api_token', '')
self.gl_client = GitlabAPIClient(self.baseurl, self.api_token)
self.sched = None
self.event_queue = queue.Queue()
self.source = driver.getSource(self)
def _start_event_connector(self):
self.gitlab_event_connector = GitlabEventConnector(self)
@ -168,15 +278,113 @@ class GitlabConnection(BaseConnection):
def getWebController(self, zuul_web):
return GitlabWebController(zuul_web, self)
def getChange(self, event):
return None
def getProject(self, name):
return self.projects.get(name)
def addProject(self, project):
self.projects[project.name] = project
def getProjectBranches(self, project, tenant):
branches = self.project_branch_cache.get(project.name)
if branches is not None:
return branches
branches = self.gl_client.get_project_branches(project.name)
self.project_branch_cache[project.name] = branches
self.log.info("Got branches for %s" % project.name)
return branches
def clearBranchCache(self):
self.project_branch_cache = {}
def getGitwebUrl(self, project, sha=None):
url = '%s/%s' % (self.baseurl, project)
if sha is not None:
url += '/tree/%s' % sha
return url
def getPullUrl(self, project, number):
return '%s/%s/merge_requests/%s' % (self.baseurl, project, number)
def getGitUrl(self, project):
return '%s/%s.git' % (self.cloneurl, project.name)
def getChange(self, event, refresh=False):
project = self.source.getProject(event.project_name)
if event.change_number:
self.log.info("Getting change for %s#%s" % (
project, event.change_number))
change = self._getChange(
project, event.change_number, event.patch_number,
refresh=refresh, event=event)
change.source_event = event
change.is_current_patchset = (change.patchset ==
event.patch_number)
else:
self.log.info("Getting change for %s ref:%s" % (
project, event.ref))
raise NotImplementedError
return change
def _getChange(self, project, number, patchset=None,
refresh=False, url=None, event=None):
key = (project.name, number, patchset)
change = self._change_cache.get(key)
if change and not refresh:
self.log.debug("Getting change from cache %s" % str(key))
return change
if not change:
change = MergeRequest(project.name)
change.project = project
change.number = number
# patchset is the tips commit of the PR
change.patchset = patchset
change.url = url
change.uris = list(url)
self._change_cache[key] = change
try:
self.log.debug("Getting change mr#%s from project %s" % (
number, project.name))
self._updateChange(change, event)
except Exception:
if key in self._change_cache:
del self._change_cache[key]
raise
return change
def _updateChange(self, change, event):
self.log.info("Updating change from Gitlab %s" % change)
change.mr = self.getPull(change.project.name, change.number)
change.ref = "refs/merge-requests/%s/head" % change.number
change.branch = change.mr['target_branch']
change.patchset = change.mr['sha']
# Files changes are not part of the Merge Request data
# See api/merge_requests.html#get-single-mr-changes
# this endpoint includes file changes information
change.files = None
change.title = change.mr['title']
change.open = change.mr['state'] == 'opened'
change.is_merged = change.mr['merged_at'] is not None
# Can be "can_be_merged"
change.merge_status = change.mr['merge_status']
change.message = change.mr['description']
change.labels = change.mr['labels']
change.updated_at = int(datetime.strptime(
change.mr['updated_at'], '%Y-%m-%dT%H:%M:%S.%fZ').strftime('%s'))
self.log.info("Updated change from Gitlab %s" % change)
if self.sched:
self.sched.onChangeUpdated(change, event)
return change
def getPull(self, project_name, number):
mr = self.gl_client.get_mr(project_name, number)
self.log.info('Got MR %s#%s', project_name, number)
return mr
class GitlabWebController(BaseWebController):

View File

@ -12,22 +12,98 @@
# License for the specific language governing permissions and limitations
# under the License.
import re
from zuul.model import Change, TriggerEvent, EventFilter, RefFilter
class PullRequest(Change):
class MergeRequest(Change):
def __init__(self, project):
super(PullRequest, self).__init__(project)
super(MergeRequest, self).__init__(project)
self.updated_at = None
def __repr__(self):
r = ['<Change 0x%x' % id(self)]
if self.project:
r.append('project: %s' % self.project)
if self.number:
r.append('number: %s' % self.number)
if self.patchset:
r.append('patchset: %s' % self.patchset)
if self.updated_at:
r.append('updated: %s' % self.updated_at)
if self.is_merged:
r.append('state: merged')
if self.open:
r.append('state: open')
return ' '.join(r) + '>'
def isUpdateOf(self, other):
if (self.project == other.project and
hasattr(other, 'number') and self.number == other.number and
hasattr(other, 'updated_at') and
self.updated_at > other.updated_at):
return True
return False
class GitlabTriggerEvent(TriggerEvent):
def __init__(self):
super(GitlabTriggerEvent, self).__init__()
self.trigger_name = 'gitlab'
self.title = None
self.action = None
self.change_number = None
def _repr(self):
r = [super(GitlabTriggerEvent, self)._repr()]
if self.action:
r.append("action:%s" % self.action)
r.append("project:%s" % self.canonical_project_name)
if self.change_number:
r.append("mr:%s" % self.change_number)
return ' '.join(r)
def isPatchsetCreated(self):
if self.type == 'gl_pull_request':
return self.action in ['opened', 'changed']
return False
class GitlabEventFilter(EventFilter):
def __init__(self, trigger):
super(GitlabEventFilter, self).__init__()
def __init__(self, trigger, types=[], actions=[]):
super(GitlabEventFilter, self).__init__(self)
self._types = types
self.types = [re.compile(x) for x in types]
self.actions = actions
def __repr__(self):
ret = '<GitlabEventFilter'
if self._types:
ret += ' types: %s' % ', '.join(self._types)
if self.actions:
ret += ' actions: %s' % ', '.join(self.actions)
ret += '>'
return ret
def matches(self, event, change):
matches_type = False
for etype in self.types:
if etype.match(event.type):
matches_type = True
if self.types and not matches_type:
return False
matches_action = False
for action in self.actions:
if (event.action == action):
matches_action = True
if self.actions and not matches_action:
return False
return True
# The RefFilter should be understood as RequireFilter (it maps to

View File

@ -50,7 +50,7 @@ class GitlabSource(BaseSource):
raise NotImplementedError()
def getChange(self, event, refresh=False):
raise NotImplementedError()
return self.connection.getChange(event, refresh)
def getChangeByURL(self, url):
raise NotImplementedError()
@ -59,7 +59,7 @@ class GitlabSource(BaseSource):
raise NotImplementedError()
def getCachedChanges(self):
raise NotImplementedError()
return self.connection._change_cache.values()
def getProject(self, name):
p = self.connection.getProject(name)
@ -69,7 +69,7 @@ class GitlabSource(BaseSource):
return p
def getProjectBranches(self, project, tenant):
raise NotImplementedError()
return self.connection.getProjectBranches(project, tenant)
def getProjectOpenChanges(self, project):
"""Get the open changes for a project."""
@ -81,7 +81,7 @@ class GitlabSource(BaseSource):
def getGitUrl(self, project):
"""Get the git url for a project."""
raise NotImplementedError()
return self.connection.getGitUrl(project)
def getGitwebUrl(self, project, sha=None):
"""Get the git-web url for a project."""

View File

@ -13,7 +13,10 @@
# under the License.
import logging
import voluptuous as v
from zuul.driver.gitlab.gitlabmodel import GitlabEventFilter
from zuul.trigger import BaseTrigger
from zuul.driver.util import scalar_or_list, to_list
class GitlabTrigger(BaseTrigger):
@ -22,6 +25,13 @@ class GitlabTrigger(BaseTrigger):
def getEventFilters(self, trigger_config):
efilters = []
for trigger in to_list(trigger_config):
f = GitlabEventFilter(
trigger=self,
types=to_list(trigger['event']),
actions=to_list(trigger.get('action')),
)
efilters.append(f)
return efilters
def onPullRequest(self, payload):
@ -29,5 +39,9 @@ class GitlabTrigger(BaseTrigger):
def getSchema():
gitlab_trigger = {}
gitlab_trigger = {
v.Required('event'):
scalar_or_list(v.Any('gl_merge_request')),
'action': scalar_or_list(str),
}
return gitlab_trigger