Perform actual HTTP requests in gitlab tests

In order to exercise our use of the requests library and the
GitlabApiClient in Zuul, this removes the FakeGitlabApiClient in
tests, instead running a real web server that we perform actual
requests against.  The existing work done by the fake client has
been ported over to the web server.

Change-Id: I553449d0e6d986378a38bf006347fd11a46876dc
This commit is contained in:
James E. Blair 2021-10-08 13:26:45 -07:00
parent 96c61208a0
commit 6ddac4cb0c
3 changed files with 288 additions and 172 deletions

View File

@ -62,7 +62,6 @@ import testtools
import testtools.content
import testtools.content_type
from git.exc import NoSuchPathError
from git.util import IterableList
import yaml
import paramiko
import prometheus_client.exposition
@ -126,6 +125,7 @@ from zuul.lib.config import get_default
from zuul.lib.logutil import get_annotated_logger
import tests.fakegithub
import tests.fakegitlab
FIXTURE_DIR = os.path.join(os.path.dirname(__file__), 'fixtures')
@ -325,7 +325,7 @@ class GitlabDriverMock(GitlabDriver):
changes_db=db,
upstream_root=self.upstream_root)
setattr(self.registry, 'fake_' + name, connection)
registerProjects(connection.source.name, connection.gl_client,
registerProjects(connection.source.name, connection,
self.config)
return connection
@ -1918,33 +1918,51 @@ class FakePagureConnection(pagureconnection.PagureConnection):
self.zuul_web_port = port
FakeGitlabBranch = namedtuple('Branch', ('name', 'protected'))
class FakeGitlabConnection(gitlabconnection.GitlabConnection):
log = logging.getLogger("zuul.test.FakeGitlabConnection")
def __init__(self, driver, connection_name, connection_config, rpcclient,
changes_db=None, upstream_root=None):
super(FakeGitlabConnection, self).__init__(driver, connection_name,
connection_config)
self.merge_requests = changes_db
self.gl_client = FakeGitlabAPIClient(
self.baseurl, self.api_token, 60, merge_requests_db=changes_db)
self.rpcclient = rpcclient
self.upstream_root = upstream_root
self.mr_number = 0
self._test_web_server = tests.fakegitlab.GitlabWebServer(changes_db)
self._test_web_server.start()
self._test_baseurl = 'http://localhost:%s' % self._test_web_server.port
connection_config['baseurl'] = self._test_baseurl
super(FakeGitlabConnection, self).__init__(driver, connection_name,
connection_config)
def onStop(self):
super().onStop()
self._test_web_server.stop()
def addProject(self, project):
super(FakeGitlabConnection, self).addProject(project)
self.gl_client.addProject(project)
self.addProjectByName(project.name)
def addProjectByName(self, project_name):
owner, proj = project_name.split('/')
repo = self._test_web_server.fake_repos[(owner, proj)]
branch = FakeGitlabBranch('master', False)
if 'master' not in repo:
repo.append(branch)
def protectBranch(self, owner, project, branch, protected=True):
if branch in self.gl_client.fake_repos[(owner, project)]:
del self.gl_client.fake_repos[(owner, project)][branch]
fake_branch = FakeBranch(branch, protected=protected)
self.gl_client.fake_repos[(owner, project)].append(fake_branch)
if branch in self._test_web_server.fake_repos[(owner, project)]:
del self._test_web_server.fake_repos[(owner, project)][branch]
fake_branch = FakeGitlabBranch(branch, protected=protected)
self._test_web_server.fake_repos[(owner, project)].append(fake_branch)
def deleteBranch(self, owner, project, branch):
if branch in self.gl_client.fake_repos[(owner, project)]:
del self.gl_client.fake_repos[(owner, project)][branch]
if branch in self._test_web_server.fake_repos[(owner, project)]:
del self._test_web_server.fake_repos[(owner, project)][branch]
def getGitUrl(self, project):
return 'file://' + os.path.join(self.upstream_root, project.name)
@ -2013,162 +2031,9 @@ class FakeGitlabConnection(gitlabconnection.GitlabConnection):
@contextmanager
def enable_community_edition(self):
self.gl_client.community_edition = True
self._test_web_server.options['community_edition'] = True
yield
self.gl_client.community_edition = False
FakeBranch = namedtuple('Branch', ('name', 'protected'))
class FakeGitlabAPIClient(gitlabconnection.GitlabAPIClient):
log = logging.getLogger("zuul.test.FakeGitlabAPIClient")
def __init__(self, baseurl, api_token, keepalive,
merge_requests_db={}):
super(FakeGitlabAPIClient, self).__init__(
baseurl, api_token, keepalive)
self.merge_requests = merge_requests_db
self.fake_repos = defaultdict(lambda: IterableList('name'))
self.community_edition = False
def gen_error(self, verb):
return {
'message': 'some error',
}, 503, "", verb
def _get_mr(self, match):
project, number = match.groups()
project = urllib.parse.unquote(project)
mr = self.merge_requests.get(project, {}).get(number)
if not mr:
return self.gen_error("GET")
return mr
def get(self, url, zuul_event_id=None):
log = get_annotated_logger(self.log, zuul_event_id)
log.debug("Getting resource %s ..." % url)
match = re.match(r'.+/projects/(.+)/merge_requests/(\d+)$', url)
if match:
mr = self._get_mr(match)
return {
'target_branch': mr.branch,
'title': mr.subject,
'state': mr.state,
'description': mr.description,
'author': {
'name': 'Administrator',
'username': 'admin'
},
'updated_at': mr.updated_at.strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
'sha': mr.sha,
'labels': mr.labels,
'merged_at': mr.merged_at,
'diff_refs': {
'base_sha': 'c380d3acebd181f13629a25d2e2acca46ffe1e00',
'head_sha': '2be7ddb704c7b6b83732fdd5b9f09d5a397b5f8f',
'start_sha': 'c380d3acebd181f13629a25d2e2acca46ffe1e00'
},
'merge_status': mr.merge_status,
}, 200, "", "GET"
match = re.match('.+/projects/(.+)/'
'repository/branches\\?.*$', url)
if match:
project = urllib.parse.unquote(match.group(1)).split('/')
req = urllib.parse.urlparse(url)
query = urllib.parse.parse_qs(req.query)
per_page = int(query["per_page"][0])
page = int(query["page"][0])
repo = self.fake_repos[tuple(project)]
first_entry = (page - 1) * per_page
last_entry = min(len(repo), (page) * per_page)
if first_entry >= len(repo):
branches = []
else:
branches = [{'name': repo[i].name,
'protected': repo[i].protected}
for i in range(first_entry, last_entry)]
return branches, 200, "", "GET"
match = re.match(
r'.+/projects/(.+)/merge_requests/(\d+)/approvals$', url)
if match:
mr = self._get_mr(match)
if not self.community_edition:
return {
'approvals_left': 0 if mr.approved else 1,
}, 200, "", "GET"
else:
return {
'approved': mr.approved,
}, 200, "", "GET"
match = re.match(r'.+/projects/(.+)/repository/branches/(.+)$', url)
if match:
project, branch = match.groups()
project = urllib.parse.unquote(project)
branch = urllib.parse.unquote(branch)
owner, name = project.split('/')
if branch in self.fake_repos[(owner, name)]:
protected = self.fake_repos[(owner, name)][branch].protected
return {'protected': protected}, 200, "", "GET"
else:
return {}, 404, "", "GET"
def post(self, url, params=None, zuul_event_id=None):
self.log.info(
"Posting on resource %s, params (%s) ..." % (url, params))
match = re.match(r'.+/projects/(.+)/merge_requests/(\d+)/notes$', url)
if match:
mr = self._get_mr(match)
mr.addNote(params['body'])
match = re.match(
r'.+/projects/(.+)/merge_requests/(\d+)/approve$', url)
if match:
assert 'sha' in params
mr = self._get_mr(match)
if params['sha'] != mr.sha:
return {'message': 'SHA does not match HEAD of source '
'branch: <new_sha>'}, 409, "", "POST"
mr.approved = True
match = re.match(
r'.+/projects/(.+)/merge_requests/(\d+)/unapprove$', url)
if match:
mr = self._get_mr(match)
mr.approved = False
return {}, 200, "", "POST"
def put(self, url, params=None, zuul_event_id=None):
self.log.info(
"Put on resource %s, params (%s) ..." % (url, params))
match = re.match(r'.+/projects/(.+)/merge_requests/(\d+)/merge$', url)
if match:
mr = self._get_mr(match)
mr.mergeMergeRequest()
return {'state': 'merged'}, 200, "", "PUT"
def addProject(self, project):
self.addProjectByName(project.name)
def addProjectByName(self, project_name):
owner, proj = project_name.split('/')
repo = self.fake_repos[(owner, proj)]
branch = FakeBranch('master', False)
if 'master' not in repo:
repo.append(branch)
self._test_web_server.options['community_edition'] = False
class GitlabChangeReference(git.Reference):

246
tests/fakegitlab.py Normal file
View File

@ -0,0 +1,246 @@
# Copyright 2016 Red Hat, Inc.
# Copyright 2021 Acme Gating, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from collections import defaultdict
import http.server
import json
import logging
import re
import socketserver
import threading
import urllib.parse
from git.util import IterableList
class GitlabWebServer(object):
def __init__(self, merge_requests):
super(GitlabWebServer, self).__init__()
self.merge_requests = merge_requests
self.fake_repos = defaultdict(lambda: IterableList('name'))
# A dictionary so we can mutate it
self.options = dict(community_edition=False)
def start(self):
merge_requests = self.merge_requests
fake_repos = self.fake_repos
options = self.options
class Server(http.server.SimpleHTTPRequestHandler):
log = logging.getLogger("zuul.test.GitlabWebServer")
branches_re = re.compile(r'.+/projects/(?P<project>.+)/'
r'repository/branches\\?.*$')
branch_re = re.compile(r'.+/projects/(?P<project>.+)/'
r'repository/branches/(?P<branch>.+)$')
mr_re = re.compile(r'.+/projects/(?P<project>.+)/'
r'merge_requests/(?P<mr>\d+)$')
mr_approvals_re = re.compile(
r'.+/projects/(?P<project>.+)/'
r'merge_requests/(?P<mr>\d+)/approvals$')
mr_notes_re = re.compile(
r'.+/projects/(?P<project>.+)/'
r'merge_requests/(?P<mr>\d+)/notes$')
mr_approve_re = re.compile(
r'.+/projects/(?P<project>.+)/'
r'merge_requests/(?P<mr>\d+)/approve$')
mr_unapprove_re = re.compile(
r'.+/projects/(?P<project>.+)/'
r'merge_requests/(?P<mr>\d+)/unapprove$')
mr_merge_re = re.compile(r'.+/projects/(?P<project>.+)/'
r'merge_requests/(?P<mr>\d+)/merge$')
def _get_mr(self, project, number):
project = urllib.parse.unquote(project)
mr = merge_requests.get(project, {}).get(number)
if not mr:
# Find out what gitlab does in this case
raise NotImplementedError()
return mr
def do_GET(self):
path = self.path
self.log.debug("Got GET %s", path)
m = self.mr_re.match(path)
if m:
return self.get_mr(**m.groupdict())
m = self.mr_approvals_re.match(path)
if m:
return self.get_mr_approvals(**m.groupdict())
m = self.branch_re.match(path)
if m:
return self.get_branch(**m.groupdict())
m = self.branches_re.match(path)
if m:
return self.get_branches(path, **m.groupdict())
self.send_response(500)
self.end_headers()
def do_POST(self):
path = self.path
self.log.debug("Got POST %s", path)
data = self.rfile.read(int(self.headers['Content-Length']))
if (self.headers['Content-Type'] ==
'application/x-www-form-urlencoded'):
data = urllib.parse.parse_qs(data.decode('utf-8'))
self.log.debug("Got data %s", data)
m = self.mr_notes_re.match(path)
if m:
return self.post_mr_notes(data, **m.groupdict())
m = self.mr_approve_re.match(path)
if m:
return self.post_mr_approve(data, **m.groupdict())
m = self.mr_unapprove_re.match(path)
if m:
return self.post_mr_unapprove(data, **m.groupdict())
self.send_response(500)
self.end_headers()
def do_PUT(self):
path = self.path
self.log.debug("Got PUT %s", path)
data = self.rfile.read(int(self.headers['Content-Length']))
if (self.headers['Content-Type'] ==
'application/x-www-form-urlencoded'):
data = urllib.parse.parse_qs(data.decode('utf-8'))
self.log.debug("Got data %s", data)
m = self.mr_merge_re.match(path)
if m:
return self.put_mr_merge(data, **m.groupdict())
self.send_response(500)
self.end_headers()
def send_data(self, data, code=200):
data = json.dumps(data).encode('utf-8')
self.send_response(code)
self.send_header('Content-Type', 'application/json')
self.send_header('Content-Length', len(data))
self.end_headers()
self.wfile.write(data)
def get_mr(self, project, mr):
mr = self._get_mr(project, mr)
data = {
'target_branch': mr.branch,
'title': mr.subject,
'state': mr.state,
'description': mr.description,
'author': {
'name': 'Administrator',
'username': 'admin'
},
'updated_at':
mr.updated_at.strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
'sha': mr.sha,
'labels': mr.labels,
'merged_at': mr.merged_at.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
if mr.merged_at else mr.merged_at,
'diff_refs': {
'base_sha': 'c380d3acebd181f13629a25d2e2acca46ffe1e00',
'head_sha': '2be7ddb704c7b6b83732fdd5b9f09d5a397b5f8f',
'start_sha': 'c380d3acebd181f13629a25d2e2acca46ffe1e00'
},
'merge_status': mr.merge_status,
}
self.send_data(data)
def get_mr_approvals(self, project, mr):
mr = self._get_mr(project, mr)
if not options['community_edition']:
self.send_data({
'approvals_left': 0 if mr.approved else 1,
})
else:
self.send_data({
'approved': mr.approved,
})
def get_branch(self, project, branch):
project = urllib.parse.unquote(project)
branch = urllib.parse.unquote(branch)
owner, name = project.split('/')
if branch in fake_repos[(owner, name)]:
protected = fake_repos[(owner, name)][branch].protected
self.send_data({'protected': protected})
else:
return self.send_data({}, code=404)
def get_branches(self, url, project):
project = urllib.parse.unquote(project).split('/')
req = urllib.parse.urlparse(url)
query = urllib.parse.parse_qs(req.query)
per_page = int(query["per_page"][0])
page = int(query["page"][0])
repo = fake_repos[tuple(project)]
first_entry = (page - 1) * per_page
last_entry = min(len(repo), (page) * per_page)
if first_entry >= len(repo):
branches = []
else:
branches = [{'name': repo[i].name,
'protected': repo[i].protected}
for i in range(first_entry, last_entry)]
self.send_data(branches)
def post_mr_notes(self, data, project, mr):
mr = self._get_mr(project, mr)
mr.addNote(data['body'][0])
self.send_data({})
def post_mr_approve(self, data, project, mr):
assert 'sha' in data
mr = self._get_mr(project, mr)
if data['sha'][0] != mr.sha:
return self.send_data(
{'message': 'SHA does not match HEAD of source '
'branch: <new_sha>'}, code=409)
mr.approved = True
self.send_data({})
def post_mr_unapprove(self, data, project, mr):
mr = self._get_mr(project, mr)
mr.approved = False
self.send_data({})
def put_mr_merge(self, data, project, mr):
mr = self._get_mr(project, mr)
mr.mergeMergeRequest()
self.send_data({'state': 'merged'})
def log_message(self, fmt, *args):
self.log.debug(fmt, *args)
self.httpd = socketserver.ThreadingTCPServer(('', 0), Server)
self.port = self.httpd.socket.getsockname()[1]
self.thread = threading.Thread(name='GitlabWebServer',
target=self.httpd.serve_forever)
self.thread.daemon = True
self.thread.start()
def stop(self):
self.httpd.shutdown()
self.thread.join()

View File

@ -108,7 +108,8 @@ class TestGitlabDriver(ZuulTestCase):
self.assertEqual(str(A.number), zuulvars['change'])
self.assertEqual(str(A.sha), zuulvars['patchset'])
self.assertEqual('master', zuulvars['branch'])
self.assertEquals('https://gitlab/org/project/merge_requests/1',
self.assertEquals(f'{self.fake_gitlab._test_baseurl}/'
'org/project/merge_requests/1',
zuulvars['items'][0]['change_url'])
self.assertEqual(zuulvars["message"], strings.b64encode(description))
self.assertEqual(2, len(self.history))
@ -282,7 +283,8 @@ class TestGitlabDriver(ZuulTestCase):
self.assertEqual('project-post-job', zuulvars['job'])
self.assertEqual('master', zuulvars['branch'])
self.assertEqual(
'https://gitlab/org/project/tree/%s' % zuulvars['newrev'],
f'{self.fake_gitlab._test_baseurl}/org/project/tree/'
f'{zuulvars["newrev"]}',
zuulvars['change_url'])
self.assertEqual(expected_newrev, zuulvars['newrev'])
self.assertEqual(expected_oldrev, zuulvars['oldrev'])
@ -711,7 +713,8 @@ class TestGitlabDriver(ZuulTestCase):
project_git_url = self.fake_gitlab.real_getGitUrl(project)
# cloneurl created from config 'server' should be used
# without credentials
self.assertEqual("https://gitlab/org/project1.git", project_git_url)
self.assertEqual(f"{self.fake_gitlab._test_baseurl}/org/project1.git",
project_git_url)
@simple_layout('layouts/crd-gitlab.yaml', driver='gitlab2')
def test_api_token_cloneurl(self):
@ -743,7 +746,9 @@ class TestGitlabDriver(ZuulTestCase):
project_git_url = self.fake_gitlab4.real_getGitUrl(project)
# cloneurl is not set, generate one from token name, token secret and
# server
self.assertEqual("https://tokenname4:444@gitlabfour/org/project1.git",
self.assertEqual("http://tokenname4:444@localhost:"
f"{self.fake_gitlab4._test_web_server.port}"
"/org/project1.git",
project_git_url)
@simple_layout('layouts/crd-gitlab.yaml', driver='gitlab5')