gitlab: implement git push support

Change-Id: I61f3ad44c4e81b580063be53ada8323be9be4e02

doc/source/examples/pipelines/gitlab-reference-pipelines.yaml

@@ -25,3 +25,16 @@
       gitlab.com:
         comment: true
       sqlreporter:
+
+- pipeline:
+    name: post
+    post-review: true
+    manager: independent
+    trigger:
+      gitlab.com:
+        - event: gl_push
+          ref: ^refs/heads/.*$
+    success:
+      sqlreporter:
+    failure:
+      sqlreporter:

doc/source/reference/drivers/gitlab.rst

@@ -29,6 +29,7 @@ Each project to be integrated with Zuul needs in "Settings/Webhooks":
 - "URL" set to
   ``http://<zuul-web>/zuul/api/connection/<conn-name>/payload``
 - "Merge request events" set to "on"
+- "Push events" set to "on"
 - "Comments" set to "on"
 - Define a "Secret Token"
 
@@ -102,6 +103,8 @@ the following options.
 
       .. value:: gl_merge_request
 
+      .. value:: gl_push
+
    .. attr:: action
 
       A :value:`pipeline.trigger.<gitlab source>.event.gl_merge_request`
@@ -129,6 +132,13 @@ the following options.
       match when comments containing 'retrigger' somewhere in the
       comment text are added to a merge request.
 
+   .. attr:: ref
+
+      This is only used for ``gl_push`` events. This field is treated as
+      a regular expression and multiple refs may be listed. GitLab
+      always sends full ref name, eg. ``refs/heads/bar`` and this
+      string is matched against the regular expression.
+
 Reporter Configuration
 ----------------------
 Zuul reports back to GitLab via the API. Available reports include a Merge Request

tests/base.py

@@ -1594,6 +1594,25 @@ class FakeGitlabConnection(gitlabconnection.GitlabConnection):
     def setZuulWebPort(self, port):
         self.zuul_web_port = port
 
+    def getPushEvent(
+            self, project, before=None, after=None,
+            branch='refs/heads/master'):
+        name = 'gl_push'
+        if not after:
+            repo_path = os.path.join(self.upstream_root, project)
+            repo = git.Repo(repo_path)
+            after = repo.head.commit.hexsha
+        data = {
+            'object_kind': 'push',
+            'before': before or '1' * 40,
+            'after': after,
+            'ref': branch,
+            'project': {
+                'path_with_namespace': project
+            },
+        }
+        return (name, data)
+
 
 class FakeGitlabAPIClient(gitlabconnection.GitlabAPIClient):
     log = logging.getLogger("zuul.test.FakeGitlabAPIClient")

tests/fixtures/layouts/basic-gitlab.yaml

@@ -19,6 +19,15 @@
       gitlab:
         comment: True
 
+- pipeline:
+    name: post
+    post-review: true
+    manager: independent
+    trigger:
+      gitlab:
+        - event: gl_push
+          ref: ^refs/heads/.*$
+
 - job:
     name: base
     parent: null
@@ -32,9 +41,16 @@
     name: project-test2
     run: playbooks/project-test2.yaml
 
+- job:
+    name: project-post-job
+    run: playbooks/project-post.yaml
+
 - project:
     name: org/project
     check:
       jobs:
         - project-test1
         - project-test2
+    post:
+      jobs:
+        - project-post-job

tests/unit/test_gitlab_driver.py

@@ -13,6 +13,8 @@
 # under the License.
 
 import re
+import os
+import git
 import socket
 
 from tests.base import ZuulTestCase, simple_layout
@@ -127,3 +129,67 @@ class TestGitlabDriver(ZuulTestCase):
             A.getMergeRequestCommentedEvent('recheck'))
         self.waitUntilSettled()
         self.assertEqual(4, len(self.history))
+
+    @simple_layout('layouts/basic-gitlab.yaml', driver='gitlab')
+    def test_ref_updated(self):
+
+        event = self.fake_gitlab.getPushEvent('org/project')
+        expected_newrev = event[1]['after']
+        expected_oldrev = event[1]['before']
+        self.fake_gitlab.emitEvent(event)
+        self.waitUntilSettled()
+        self.assertEqual(1, len(self.history))
+        self.assertEqual(
+            'SUCCESS',
+            self.getJobFromHistory('project-post-job').result)
+
+        job = self.getJobFromHistory('project-post-job')
+        zuulvars = job.parameters['zuul']
+        self.assertEqual('refs/heads/master', zuulvars['ref'])
+        self.assertEqual('post', zuulvars['pipeline'])
+        self.assertEqual('project-post-job', zuulvars['job'])
+        self.assertEqual('master', zuulvars['branch'])
+        self.assertEqual(
+            'https://gitlab/org/project/tree/%s' % zuulvars['newrev'],
+            zuulvars['change_url'])
+        self.assertEqual(expected_newrev, zuulvars['newrev'])
+        self.assertEqual(expected_oldrev, zuulvars['oldrev'])
+
+    @simple_layout('layouts/basic-gitlab.yaml', driver='gitlab')
+    def test_ref_created(self):
+
+        self.create_branch('org/project', 'stable-1.0')
+        path = os.path.join(self.upstream_root, 'org/project')
+        repo = git.Repo(path)
+        newrev = repo.commit('refs/heads/stable-1.0').hexsha
+        event = self.fake_gitlab.getPushEvent(
+            'org/project', branch='refs/heads/stable-1.0',
+            before='0' * 40, after=newrev)
+        old = self.scheds.first.sched.tenant_last_reconfigured.get(
+            'tenant-one', 0)
+        self.fake_gitlab.emitEvent(event)
+        self.waitUntilSettled()
+        new = self.scheds.first.sched.tenant_last_reconfigured.get(
+            'tenant-one', 0)
+        # New timestamp should be greater than the old timestamp
+        self.assertLess(old, new)
+        self.assertEqual(1, len(self.history))
+        self.assertEqual(
+            'SUCCESS',
+            self.getJobFromHistory('project-post-job').result)
+        job = self.getJobFromHistory('project-post-job')
+        zuulvars = job.parameters['zuul']
+        self.assertEqual('refs/heads/stable-1.0', zuulvars['ref'])
+        self.assertEqual('post', zuulvars['pipeline'])
+        self.assertEqual('project-post-job', zuulvars['job'])
+        self.assertEqual('stable-1.0', zuulvars['branch'])
+        self.assertEqual(newrev, zuulvars['newrev'])
+
+    @simple_layout('layouts/basic-gitlab.yaml', driver='gitlab')
+    def test_ref_deleted(self):
+
+        event = self.fake_gitlab.getPushEvent(
+            'org/project', 'stable-1.0', after='0' * 40)
+        self.fake_gitlab.emitEvent(event)
+        self.waitUntilSettled()
+        self.assertEqual(0, len(self.history))

zuul/driver/gitlab/gitlabconnection.py

@@ -28,6 +28,7 @@ from zuul.connection import BaseConnection
 from zuul.web.handler import BaseWebController
 from zuul.lib.gearworker import ZuulGearWorker
 from zuul.lib.logutil import get_annotated_logger
+from zuul.model import Ref, Branch
 
 from zuul.driver.gitlab.gitlabmodel import GitlabTriggerEvent, MergeRequest
 
@@ -97,6 +98,7 @@ class GitlabEventConnector(threading.Thread):
         self.event_handler_mapping = {
             'merge_request': self._event_merge_request,
             'note': self._event_note,
+            'push': self._event_push,
         }
 
     def stop(self):
@@ -105,13 +107,13 @@ class GitlabEventConnector(threading.Thread):
 
     def _event_base(self, body):
         event = GitlabTriggerEvent()
-        attrs = body['object_attributes']
-        event.updated_at = int(datetime.strptime(
-            attrs['updated_at'], '%Y-%m-%d %H:%M:%S %Z').strftime('%s'))
-        event.created_at = int(datetime.strptime(
-            attrs['created_at'], '%Y-%m-%d %H:%M:%S %Z').strftime('%s'))
+        attrs = body.get('object_attributes')
+        if attrs:
+            event.updated_at = int(datetime.strptime(
+                attrs['updated_at'], '%Y-%m-%d %H:%M:%S %Z').strftime('%s'))
+            event.created_at = int(datetime.strptime(
+                attrs['created_at'], '%Y-%m-%d %H:%M:%S %Z').strftime('%s'))
         event.project_name = body['project']['path_with_namespace']
-        event.ref = "refs/merge-requests/%s/head" % event.change_number
         return event
 
     # https://docs.gitlab.com/ee/user/project/integrations/webhooks.html#merge-request-events
@@ -120,6 +122,7 @@ class GitlabEventConnector(threading.Thread):
         attrs = body['object_attributes']
         event.title = attrs['title']
         event.change_number = attrs['iid']
+        event.ref = "refs/merge-requests/%s/head" % event.change_number
         event.branch = attrs['target_branch']
         event.patch_number = attrs['last_commit']['id']
         event.change_url = self.connection.getPullUrl(event.project_name,
@@ -138,6 +141,7 @@ class GitlabEventConnector(threading.Thread):
         mr = body['merge_request']
         event.title = mr['title']
         event.change_number = mr['iid']
+        event.ref = "refs/merge-requests/%s/head" % event.change_number
         event.branch = mr['target_branch']
         event.patch_number = mr['last_commit']['id']
         event.change_url = self.connection.getPullUrl(event.project_name,
@@ -146,6 +150,22 @@ class GitlabEventConnector(threading.Thread):
         event.type = 'gl_merge_request'
         return event
 
+    # https://docs.gitlab.com/ee/user/project/integrations/webhooks.html#push-events
+    def _event_push(self, body):
+        event = self._event_base(body)
+        event.branch = body['ref'].replace('refs/heads/', '')
+        event.ref = body['ref']
+        event.newrev = body['after']
+        event.oldrev = body['before']
+        if event.newrev == '0' * 40:
+            event.branch_deleted = True
+        elif event.oldrev == '0' * 40:
+            event.branch_created = True
+        else:
+            event.branch_updated = True
+        event.type = 'gl_push'
+        return event
+
     def _handleEvent(self):
         ts, json_body, event_type = self.connection.getEvent()
         if self._stopped:
@@ -371,7 +391,22 @@ class GitlabConnection(BaseConnection):
         else:
             self.log.info("Getting change for %s ref:%s" % (
                 project, event.ref))
-            raise NotImplementedError
+            if event.ref and event.ref.startswith('refs/tags/'):
+                raise NotImplementedError
+            elif event.ref and event.ref.startswith('refs/heads/'):
+                change = Branch(project)
+                change.branch = event.branch
+            else:
+                change = Ref(project)
+                change.branch = None
+            change.ref = event.ref
+            change.oldrev = event.oldrev
+            change.newrev = event.newrev
+            change.url = self.getGitwebUrl(project, sha=event.newrev)
+
+            change.files = None
+
+            change.source_event = event
         return change
 
     def _getChange(self, project, number, patchset=None,

zuul/driver/gitlab/gitlabmodel.py

@@ -16,6 +16,8 @@ import re
 
 from zuul.model import Change, TriggerEvent, EventFilter, RefFilter
 
+EMPTY_GIT_REF = '0' * 40  # git sha of all zeros, used during creates/deletes
+
 
 class MergeRequest(Change):
     def __init__(self, project):
@@ -71,13 +73,18 @@ class GitlabTriggerEvent(TriggerEvent):
 
 
 class GitlabEventFilter(EventFilter):
-    def __init__(self, trigger, types=[], actions=[], comments=[]):
+    def __init__(
+            self, trigger, types=[], actions=[],
+            comments=[], refs=[], ignore_deletes=True):
         super(GitlabEventFilter, self).__init__(self)
         self._types = types
         self.types = [re.compile(x) for x in types]
         self.actions = actions
         self._comments = comments
         self.comments = [re.compile(x) for x in comments]
+        self._refs = refs
+        self.refs = [re.compile(x) for x in refs]
+        self.ignore_deletes = ignore_deletes
 
     def __repr__(self):
         ret = '<GitlabEventFilter'
@@ -88,6 +95,10 @@ class GitlabEventFilter(EventFilter):
             ret += ' actions: %s' % ', '.join(self.actions)
         if self._comments:
             ret += ' comments: %s' % ', '.join(self._comments)
+        if self._refs:
+            ret += ' refs: %s' % ', '.join(self._refs)
+        if self.ignore_deletes:
+            ret += ' ignore_deletes: %s' % self.ignore_deletes
         ret += '>'
 
         return ret
@@ -100,6 +111,18 @@ class GitlabEventFilter(EventFilter):
         if self.types and not matches_type:
             return False
 
+        matches_ref = False
+        if event.ref is not None:
+            for ref in self.refs:
+                if ref.match(event.ref):
+                    matches_ref = True
+        if self.refs and not matches_ref:
+            return False
+        if self.ignore_deletes and event.newrev == EMPTY_GIT_REF:
+            # If the updated ref has an empty git sha (all 0s),
+            # then the ref is being deleted
+            return False
+
         matches_action = False
         for action in self.actions:
             if (event.action == action):

zuul/driver/gitlab/gitlabtrigger.py

@@ -31,6 +31,7 @@ class GitlabTrigger(BaseTrigger):
                 types=to_list(trigger['event']),
                 actions=to_list(trigger.get('action')),
                 comments=to_list(trigger.get('comment')),
+                refs=to_list(trigger.get('ref')),
             )
             efilters.append(f)
         return efilters
@@ -42,8 +43,13 @@ class GitlabTrigger(BaseTrigger):
 def getSchema():
     gitlab_trigger = {
         v.Required('event'):
-            scalar_or_list(v.Any('gl_merge_request')),
+            scalar_or_list(
+                v.Any(
+                    'gl_merge_request',
+                    'gl_push',
+                )),
         'action': scalar_or_list(str),
         'comment': scalar_or_list(str),
+        'ref': scalar_or_list(str),
     }
     return gitlab_trigger