From c4d120551a869ecd55d2e78dd780d22c3188af18 Mon Sep 17 00:00:00 2001 From: Guillaume Chauvel Date: Sat, 16 Jan 2021 16:09:07 +0100 Subject: [PATCH] gitlab: Add access token name, Update docs, Fix webhook Highlight that Access Token: - can be created at project level and user level - can be used to authenticate when cloning repositories To clone internal or private projects, authentication is required. It can be achieved using the Access Token. Currently, any names can be used but [1] shows that it's likely going to be restricted to username and token-name. As token-name is directly related to an Access Token, use this value. webhook can be created foreach project but also at group level, so at the topmost group. Therefor only one webhook at the upper group can be used to send events to Zuul: remove "project" from webhook_token description [1] https://gitlab.com/gitlab-org/gitlab/-/issues/212953 Change-Id: I57adb5e03658d31bd3900d2d9037101cf491466e --- doc/source/reference/drivers/gitlab.rst | 47 +++++++++++++++++---- tests/base.py | 3 ++ tests/fixtures/zuul-gitlab-driver.conf | 25 +++++++++++ tests/unit/test_gitlab_driver.py | 55 +++++++++++++++++++++++++ zuul/driver/gitlab/gitlabconnection.py | 17 +++++++- 5 files changed, 137 insertions(+), 10 deletions(-) diff --git a/doc/source/reference/drivers/gitlab.rst b/doc/source/reference/drivers/gitlab.rst index 374e255785..4bbd21a495 100644 --- a/doc/source/reference/drivers/gitlab.rst +++ b/doc/source/reference/drivers/gitlab.rst @@ -17,20 +17,35 @@ Zuul needs to interact with projects by: - receiving events via web-hooks - performing actions via the API -The Zuul user's API token configured in zuul.conf must have the -following ACL rights: "api". The API token must be created in user Settings, -Access tokens. +web-hooks +^^^^^^^^^ -Each project to be integrated with Zuul needs in "Settings/Webhooks": +Projects to be integrated with Zuul needs to send events using webhooks. +This can be enabled at Group level or Project level in "Settings/Webhooks" - "URL" set to - ``http:///zuul/api/connection//payload`` + ``http:///api/connection//payload`` - "Merge request events" set to "on" - "Push events" set to "on" - "Tag push events" set to "on" - "Comments" set to "on" - Define a "Secret Token" + +API +^^^ + +| Even though bot users exist: https://docs.gitlab.com/ce/user/project/settings/project_access_tokens.html#project-bot-users +| They are only available at project level. + +In order to manage multiple projects using a single connection, Zuul needs a +global access to projects, which can only be achieved by creating a specific +Zuul user. This user counts as a licensed seat. + +The API token must be created in user Settings, Access tokens. The Zuul user's +API token configured in zuul.conf must have the following ACL rights: "api". + + Connection Configuration ------------------------ @@ -45,13 +60,18 @@ The supported options in ``zuul.conf`` connections are: The connection must set ``driver=gitlab`` for GitLab connections. + .. attr:: api_token_name + + The user's personal access token name (Used if **cloneurl** is http(s)) + Set this parameter if authentication to clone projects is required + .. attr:: api_token - The user's API token. + The user's personal access token .. attr:: webhook_token - The project's webhook secret token. + The webhook secret token. .. attr:: server :default: gitlab.com @@ -75,10 +95,19 @@ The supported options in ``zuul.conf`` connections are: Path to the GitLab web and API interface. + .. attr:: sshkey + + Path to SSH key to use (Used if **cloneurl** is ssh) + .. attr:: cloneurl :default: {baseurl} - - Path to the GitLab Git repositories. Used to clone. + + Omit to clone using http(s) or set to ``ssh://git@{server}``. + If **api_token_name** is set and **cloneurl** is either omitted or is + set without credentials, **cloneurl** will be modified to use credentials + as this: ``http(s)://:@``. + If **cloneurl** is defined with credentials, it will be used as is, + without modification from the driver. Trigger Configuration diff --git a/tests/base.py b/tests/base.py index 3d87593acb..3a9537c7eb 100644 --- a/tests/base.py +++ b/tests/base.py @@ -1817,6 +1817,9 @@ class FakeGitlabConnection(gitlabconnection.GitlabConnection): def getGitUrl(self, project): return 'file://' + os.path.join(self.upstream_root, project.name) + def real_getGitUrl(self, project): + return super(FakeGitlabConnection, self).getGitUrl(project) + def openFakeMergeRequest(self, project, branch, title, description='', files=[]): self.mr_number += 1 diff --git a/tests/fixtures/zuul-gitlab-driver.conf b/tests/fixtures/zuul-gitlab-driver.conf index 9a54060451..d1ab4f8b80 100644 --- a/tests/fixtures/zuul-gitlab-driver.conf +++ b/tests/fixtures/zuul-gitlab-driver.conf @@ -20,3 +20,28 @@ api_token=0000000000000000000000000000000000000000 [database] dburi=$MYSQL_FIXTURE_DBURI$ +[connection gitlab2] +driver=gitlab +server=gitlabtwo +api_token=2222 +cloneurl=http://myusername:2222@gitlab + +[connection gitlab3] +driver=gitlab +server=gitlabthree +api_token_name=tokenname3 +api_token=3333 +cloneurl=http://myusername:2222@gitlabthree + +[connection gitlab4] +driver=gitlab +server=gitlabfour +api_token_name=tokenname4 +api_token=444 + +[connection gitlab5] +driver=gitlab +server=gitlabfive +api_token_name=tokenname5 +api_token=555 +cloneurl=http://gitlabfivvve diff --git a/tests/unit/test_gitlab_driver.py b/tests/unit/test_gitlab_driver.py index 4600809c2a..5401d8cfca 100644 --- a/tests/unit/test_gitlab_driver.py +++ b/tests/unit/test_gitlab_driver.py @@ -699,6 +699,61 @@ class TestGitlabDriver(ZuulTestCase): self.assertTrue(A.is_merged) self.assertTrue(B.is_merged) + @simple_layout('layouts/crd-gitlab.yaml', driver='gitlab') + def test_api_token(self): + tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') + _, project = tenant.getProject('org/project1') + + project_git_url = self.fake_gitlab.real_getGitUrl(project) + # cloneurl created from config 'server' should be used + # without credentials + self.assertEqual("https://gitlab/org/project1.git", project_git_url) + + @simple_layout('layouts/crd-gitlab.yaml', driver='gitlab2') + def test_api_token_cloneurl(self): + tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') + _, project = tenant.getProject('org/project1') + + project_git_url = self.fake_gitlab2.real_getGitUrl(project) + # cloneurl from config file should be used as it defines token name and + # secret + self.assertEqual("http://myusername:2222@gitlab/org/project1.git", + project_git_url) + + @simple_layout('layouts/crd-gitlab.yaml', driver='gitlab3') + def test_api_token_name_cloneurl(self): + tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') + _, project = tenant.getProject('org/project1') + + project_git_url = self.fake_gitlab3.real_getGitUrl(project) + # cloneurl from config file should be used as it defines token name and + # secret, even if token name and token secret are defined + self.assertEqual("http://myusername:2222@gitlabthree/org/project1.git", + project_git_url) + + @simple_layout('layouts/crd-gitlab.yaml', driver='gitlab4') + def test_api_token_name(self): + tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') + _, project = tenant.getProject('org/project1') + + project_git_url = self.fake_gitlab4.real_getGitUrl(project) + # cloneurl is not set, generate one from token name, token secret and + # server + self.assertEqual("https://tokenname4:444@gitlabfour/org/project1.git", + project_git_url) + + @simple_layout('layouts/crd-gitlab.yaml', driver='gitlab5') + def test_api_token_name_cloneurl_server(self): + tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') + _, project = tenant.getProject('org/project1') + + project_git_url = self.fake_gitlab5.real_getGitUrl(project) + # cloneurl defines a url, without credentials. As token name is + # set, include token name and secret in cloneurl, 'server' is + # overwritten + self.assertEqual("http://tokenname5:555@gitlabfivvve/org/project1.git", + project_git_url) + class TestGitlabUnprotectedBranches(ZuulTestCase): config_file = 'zuul-gitlab-driver.conf' diff --git a/zuul/driver/gitlab/gitlabconnection.py b/zuul/driver/gitlab/gitlabconnection.py index 9e36f71b6d..3f8b889e4e 100644 --- a/zuul/driver/gitlab/gitlabconnection.py +++ b/zuul/driver/gitlab/gitlabconnection.py @@ -20,6 +20,7 @@ import cherrypy import voluptuous as v import time import uuid +import re import requests import dateutil.parser @@ -410,6 +411,8 @@ class GitlabConnection(CachedBranchConnection): 'canonical_hostname', self.server) self.webhook_token = self.connection_config.get( 'webhook_token', '') + self.api_token_name = self.connection_config.get( + 'api_token_name', '') self.api_token = self.connection_config.get( 'api_token', '') self.gl_client = GitlabAPIClient(self.baseurl, self.api_token) @@ -479,7 +482,19 @@ class GitlabConnection(CachedBranchConnection): return '%s/%s/merge_requests/%s' % (self.baseurl, project, number) def getGitUrl(self, project): - return '%s/%s.git' % (self.cloneurl, project.name) + cloneurl = '%s/%s.git' % (self.cloneurl, project.name) + # https://gitlab.com/gitlab-org/gitlab/-/issues/212953 + # any login name can be used, but it's likely going to be reduce to + # username/token-name + if (cloneurl.startswith('http') and self.api_token_name != '' and + not re.match("http?://.+:.+@.+", cloneurl)): + cloneurl = '%s://%s:%s@%s/%s.git' % ( + self.cloneurl.split('://')[0], + self.api_token_name, + self.api_token, + self.cloneurl.split('://')[1], + project.name) + return cloneurl def getChange(self, event, refresh=False): project = self.source.getProject(event.project_name)