Merge "gitlab: Add access token name, Update docs, Fix webhook"

doc/source/reference/drivers/gitlab.rst

@@ -17,20 +17,35 @@ Zuul needs to interact with projects by:
 - receiving events via web-hooks
 - performing actions via the API
 
-The Zuul user's API token configured in zuul.conf must have the
+web-hooks
-following ACL rights: "api". The API token must be created in user Settings,
+^^^^^^^^^
-Access tokens.
 
-Each project to be integrated with Zuul needs in "Settings/Webhooks":
+Projects to be integrated with Zuul needs to send events using webhooks.
+This can be enabled at Group level or Project level in "Settings/Webhooks"
 
 - "URL" set to
-  ``http://<zuul-web>/zuul/api/connection/<conn-name>/payload``
+  ``http://<zuul-web>/api/connection/<conn-name>/payload``
 - "Merge request events" set to "on"
 - "Push events" set to "on"
 - "Tag push events" set to "on"
 - "Comments" set to "on"
 - Define a "Secret Token"
 
+
+API
+^^^
+
+| Even though bot users exist: https://docs.gitlab.com/ce/user/project/settings/project_access_tokens.html#project-bot-users
+| They are only available at project level.
+
+In order to manage multiple projects using a single connection, Zuul needs a
+global access to projects, which can only be achieved by creating a specific
+Zuul user. This user counts as a licensed seat.
+
+The API token must be created in user Settings, Access tokens. The Zuul user's
+API token configured in zuul.conf must have the following ACL rights: "api".
+
+
 Connection Configuration
 ------------------------
 
@@ -45,13 +60,18 @@ The supported options in ``zuul.conf`` connections are:
 
          The connection must set ``driver=gitlab`` for GitLab connections.
 
+   .. attr:: api_token_name
+
+      The user's personal access token name (Used if **cloneurl** is http(s))
+      Set this parameter if authentication to clone projects is required
+
    .. attr:: api_token
 
-      The user's API token.
+      The user's personal access token
 
    .. attr:: webhook_token
 
-      The project's webhook secret token.
+      The webhook secret token.
 
    .. attr:: server
       :default: gitlab.com
@@ -75,10 +95,19 @@ The supported options in ``zuul.conf`` connections are:
 
       Path to the GitLab web and API interface.
 
+   .. attr:: sshkey
+
+      Path to SSH key to use (Used if **cloneurl** is ssh)
+
    .. attr:: cloneurl
       :default: {baseurl}
-
+      
-      Path to the GitLab Git repositories. Used to clone.
+      Omit to clone using http(s) or set to ``ssh://git@{server}``.
+      If **api_token_name** is set and **cloneurl** is either omitted or is
+      set without credentials, **cloneurl** will be modified to use credentials
+      as this: ``http(s)://<api_token_name>:<api_token>@<server>``.
+      If **cloneurl** is defined with credentials, it will be used as is,
+      without modification from the driver.
 
 
 Trigger Configuration

tests/base.py

@@ -1836,6 +1836,9 @@ class FakeGitlabConnection(gitlabconnection.GitlabConnection):
     def getGitUrl(self, project):
         return 'file://' + os.path.join(self.upstream_root, project.name)
 
+    def real_getGitUrl(self, project):
+        return super(FakeGitlabConnection, self).getGitUrl(project)
+
     def openFakeMergeRequest(self, project,
                              branch, title, description='', files=[]):
         self.mr_number += 1

tests/fixtures/zuul-gitlab-driver.conf

@@ -20,3 +20,28 @@ api_token=0000000000000000000000000000000000000000
 [database]
 dburi=$MYSQL_FIXTURE_DBURI$
 
+[connection gitlab2]
+driver=gitlab
+server=gitlabtwo
+api_token=2222
+cloneurl=http://myusername:2222@gitlab
+
+[connection gitlab3]
+driver=gitlab
+server=gitlabthree
+api_token_name=tokenname3
+api_token=3333
+cloneurl=http://myusername:2222@gitlabthree
+
+[connection gitlab4]
+driver=gitlab
+server=gitlabfour
+api_token_name=tokenname4
+api_token=444
+
+[connection gitlab5]
+driver=gitlab
+server=gitlabfive
+api_token_name=tokenname5
+api_token=555
+cloneurl=http://gitlabfivvve

tests/unit/test_gitlab_driver.py

@@ -703,6 +703,61 @@ class TestGitlabDriver(ZuulTestCase):
         self.assertTrue(A.is_merged)
         self.assertTrue(B.is_merged)
 
+    @simple_layout('layouts/crd-gitlab.yaml', driver='gitlab')
+    def test_api_token(self):
+        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
+        _, project = tenant.getProject('org/project1')
+
+        project_git_url = self.fake_gitlab.real_getGitUrl(project)
+        # cloneurl created from config 'server' should be used
+        # without credentials
+        self.assertEqual("https://gitlab/org/project1.git", project_git_url)
+
+    @simple_layout('layouts/crd-gitlab.yaml', driver='gitlab2')
+    def test_api_token_cloneurl(self):
+        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
+        _, project = tenant.getProject('org/project1')
+
+        project_git_url = self.fake_gitlab2.real_getGitUrl(project)
+        # cloneurl from config file should be used as it defines token name and
+        # secret
+        self.assertEqual("http://myusername:2222@gitlab/org/project1.git",
+                         project_git_url)
+
+    @simple_layout('layouts/crd-gitlab.yaml', driver='gitlab3')
+    def test_api_token_name_cloneurl(self):
+        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
+        _, project = tenant.getProject('org/project1')
+
+        project_git_url = self.fake_gitlab3.real_getGitUrl(project)
+        # cloneurl from config file should be used as it defines token name and
+        # secret, even if token name and token secret are defined
+        self.assertEqual("http://myusername:2222@gitlabthree/org/project1.git",
+                         project_git_url)
+
+    @simple_layout('layouts/crd-gitlab.yaml', driver='gitlab4')
+    def test_api_token_name(self):
+        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
+        _, project = tenant.getProject('org/project1')
+
+        project_git_url = self.fake_gitlab4.real_getGitUrl(project)
+        # cloneurl is not set, generate one from token name, token secret and
+        # server
+        self.assertEqual("https://tokenname4:444@gitlabfour/org/project1.git",
+                         project_git_url)
+
+    @simple_layout('layouts/crd-gitlab.yaml', driver='gitlab5')
+    def test_api_token_name_cloneurl_server(self):
+        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
+        _, project = tenant.getProject('org/project1')
+
+        project_git_url = self.fake_gitlab5.real_getGitUrl(project)
+        # cloneurl defines a url, without credentials. As token name is
+        # set, include token name and secret in cloneurl, 'server' is
+        # overwritten
+        self.assertEqual("http://tokenname5:555@gitlabfivvve/org/project1.git",
+                         project_git_url)
+
 
 class TestGitlabUnprotectedBranches(ZuulTestCase):
     config_file = 'zuul-gitlab-driver.conf'

zuul/driver/gitlab/gitlabconnection.py

@@ -19,6 +19,7 @@ import cherrypy
 import voluptuous as v
 import time
 import uuid
+import re
 import requests
 
 import dateutil.parser
@@ -396,6 +397,8 @@ class GitlabConnection(CachedBranchConnection):
             'canonical_hostname', self.server)
         self.webhook_token = self.connection_config.get(
             'webhook_token', '')
+        self.api_token_name = self.connection_config.get(
+            'api_token_name', '')
         self.api_token = self.connection_config.get(
             'api_token', '')
         self.gl_client = GitlabAPIClient(self.baseurl, self.api_token)
@@ -458,7 +461,19 @@ class GitlabConnection(CachedBranchConnection):
         return '%s/%s/merge_requests/%s' % (self.baseurl, project, number)
 
     def getGitUrl(self, project):
-        return '%s/%s.git' % (self.cloneurl, project.name)
+        cloneurl = '%s/%s.git' % (self.cloneurl, project.name)
+        # https://gitlab.com/gitlab-org/gitlab/-/issues/212953
+        # any login name can be used, but it's likely going to be reduce to
+        # username/token-name
+        if (cloneurl.startswith('http') and self.api_token_name != '' and
+            not re.match("http?://.+:.+@.+", cloneurl)):
+            cloneurl = '%s://%s:%s@%s/%s.git' % (
+                self.cloneurl.split('://')[0],
+                self.api_token_name,
+                self.api_token,
+                self.cloneurl.split('://')[1],
+                project.name)
+        return cloneurl
 
     def getChange(self, event, refresh=False):
         project = self.source.getProject(event.project_name)