From 7dc1edb4cdd365b292540e6f130922db0ac7b32c Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Fri, 21 Sep 2018 16:13:21 +0200 Subject: [PATCH] Pagure driver - https://pagure.io/pagure/ This change adds a Pagure driver for Zuul. Pagure is a Github like forge https://pagure.io/pagure/. This driver is usable starting with Pagure 5.3. For history: first Pagure PR gated by Zuul is https://pagure.io/test-zuul/pull-request/22 Change-Id: I1711653355ae26a3fff3bb6de3c6fca7113cdd01 --- doc/source/admin/connections.rst | 1 + doc/source/admin/drivers/pagure.rst | 305 ++++++ tests/base.py | 349 ++++++- .../config/cross-source-pagure/gerrit.yaml | 11 + .../playbooks/project-merge.yaml | 2 + .../playbooks/project-test1.yaml | 2 + .../playbooks/project-test2.yaml | 2 + .../project1-project2-integration.yaml | 2 + .../git/common-config-gerrit/zuul.yaml | 137 +++ .../git/gerrit_project1/README | 1 + .../playbooks/project-merge.yaml | 2 + .../playbooks/project-test1.yaml | 2 + .../playbooks/project-test2.yaml | 2 + .../project1-project2-integration.yaml | 2 + .../git/github_common-config/zuul.yaml | 135 +++ .../git/github_project1/README | 1 + .../git/pagure_project2/README | 1 + .../config/cross-source-pagure/github.yaml | 11 + tests/fixtures/layouts/basic-pagure.yaml | 60 ++ tests/fixtures/layouts/crd-pagure.yaml | 65 ++ tests/fixtures/layouts/merging-pagure.yaml | 46 + .../fixtures/layouts/requirements-pagure.yaml | 66 ++ tests/fixtures/zuul-crd-pagure.conf | 39 + tests/fixtures/zuul-pagure-driver.conf | 18 + tests/unit/test_pagure_driver.py | 872 +++++++++++++++++ zuul/cmd/web.py | 3 +- zuul/driver/pagure/__init__.py | 50 + zuul/driver/pagure/pagureconnection.py | 886 ++++++++++++++++++ zuul/driver/pagure/paguremodel.py | 206 ++++ zuul/driver/pagure/pagurereporter.py | 141 +++ zuul/driver/pagure/paguresource.py | 151 +++ zuul/driver/pagure/paguretrigger.py | 61 ++ zuul/lib/connections.py | 2 + 33 files changed, 3630 insertions(+), 4 deletions(-) create mode 100644 doc/source/admin/drivers/pagure.rst create mode 100644 tests/fixtures/config/cross-source-pagure/gerrit.yaml create mode 100644 tests/fixtures/config/cross-source-pagure/git/common-config-gerrit/playbooks/project-merge.yaml create mode 100644 tests/fixtures/config/cross-source-pagure/git/common-config-gerrit/playbooks/project-test1.yaml create mode 100644 tests/fixtures/config/cross-source-pagure/git/common-config-gerrit/playbooks/project-test2.yaml create mode 100644 tests/fixtures/config/cross-source-pagure/git/common-config-gerrit/playbooks/project1-project2-integration.yaml create mode 100644 tests/fixtures/config/cross-source-pagure/git/common-config-gerrit/zuul.yaml create mode 100644 tests/fixtures/config/cross-source-pagure/git/gerrit_project1/README create mode 100644 tests/fixtures/config/cross-source-pagure/git/github_common-config/playbooks/project-merge.yaml create mode 100644 tests/fixtures/config/cross-source-pagure/git/github_common-config/playbooks/project-test1.yaml create mode 100644 tests/fixtures/config/cross-source-pagure/git/github_common-config/playbooks/project-test2.yaml create mode 100644 tests/fixtures/config/cross-source-pagure/git/github_common-config/playbooks/project1-project2-integration.yaml create mode 100644 tests/fixtures/config/cross-source-pagure/git/github_common-config/zuul.yaml create mode 100644 tests/fixtures/config/cross-source-pagure/git/github_project1/README create mode 100644 tests/fixtures/config/cross-source-pagure/git/pagure_project2/README create mode 100644 tests/fixtures/config/cross-source-pagure/github.yaml create mode 100644 tests/fixtures/layouts/basic-pagure.yaml create mode 100644 tests/fixtures/layouts/crd-pagure.yaml create mode 100644 tests/fixtures/layouts/merging-pagure.yaml create mode 100644 tests/fixtures/layouts/requirements-pagure.yaml create mode 100644 tests/fixtures/zuul-crd-pagure.conf create mode 100644 tests/fixtures/zuul-pagure-driver.conf create mode 100644 tests/unit/test_pagure_driver.py create mode 100644 zuul/driver/pagure/__init__.py create mode 100644 zuul/driver/pagure/pagureconnection.py create mode 100644 zuul/driver/pagure/paguremodel.py create mode 100644 zuul/driver/pagure/pagurereporter.py create mode 100644 zuul/driver/pagure/paguresource.py create mode 100644 zuul/driver/pagure/paguretrigger.py diff --git a/doc/source/admin/connections.rst b/doc/source/admin/connections.rst index 420c551e7e..a551c853e7 100644 --- a/doc/source/admin/connections.rst +++ b/doc/source/admin/connections.rst @@ -64,6 +64,7 @@ Zuul includes the following drivers: drivers/gerrit drivers/github + drivers/pagure drivers/git drivers/mqtt drivers/smtp diff --git a/doc/source/admin/drivers/pagure.rst b/doc/source/admin/drivers/pagure.rst new file mode 100644 index 0000000000..115a9b9829 --- /dev/null +++ b/doc/source/admin/drivers/pagure.rst @@ -0,0 +1,305 @@ +:title: Pagure Driver + +.. _pagure_driver: + +Pagure +====== + +The Pagure driver supports sources, triggers, and reporters. It can +interact with the public Pagure.io service as well as site-local +installations of Pagure. + +Configure Pagure +---------------- + +Pagure's project owner must give project Admin access to the Pagure's user +that own the API key defined in the Zuul configuration. The API key +must at least have the ``Modify an existing project`` access. + +Furthermore Project owner must set the web hook target url in project settings +such as: ``http:///zuul/api/connection//payload`` + +Connection Configuration +------------------------ + +The supported options in ``zuul.conf`` connections are: + +.. attr:: + + .. attr:: driver + :required: + + .. value:: pagure + + The connection must set ``driver=pagure`` for Pagure connections. + + .. attr:: api_token + + The user's API token with the ``Modify an existing project`` capability. + + .. attr:: server + :default: pagure.io + + Hostname of the Pagure server. + + .. attr:: canonical_hostname + + The canonical hostname associated with the git repos on the + Pagure server. Defaults to the value of :attr:`.server`. This is used to identify projects from + this connection by name and in preparing repos on the filesystem + for use by jobs. Note that Zuul will still only communicate + with the Pagure server identified by **server**; this option is + useful if users customarily use a different hostname to clone or + pull git repos so that when Zuul places them in the job's + working directory, they appear under this directory name. + + .. attr:: baseurl + :default: https://{server} + + Path to the Pagure web and API interface. + + .. attr:: cloneurl + :default: https://{baseurl} + + Path to the Pagure Git repositories. Used to clone. + +Trigger Configuration +--------------------- +Pagure webhook events can be configured as triggers. + +A connection name with the Pagure driver can take multiple events with +the following options. + +.. attr:: pipeline.trigger. + + The dictionary passed to the Pagure pipeline ``trigger`` attribute + supports the following attributes: + + .. attr:: event + :required: + + The event from Pagure. Supported events are: + + .. value:: pg_pull_request + + .. value:: pg_pull_request_review + + .. value:: pg_push + + .. attr:: action + + A :value:`pipeline.trigger..event.pg_pull_request` + event will have associated action(s) to trigger from. The + supported actions are: + + .. value:: opened + + Pull request opened. + + .. value:: changed + + Pull request synchronized. + + .. value:: comment + + Comment added to pull request. + + .. value:: status + + Status set on pull request. + + A :value:`pipeline.trigger..event.pg_pull_request_review` event will have associated + action(s) to trigger from. The supported actions are: + + .. value:: thumbsup + + Positive pull request review added. + + .. value:: thumbsdown + + Negative pull request review added. + + .. attr:: comment + + This is only used for ``pg_pull_request`` ``comment`` actions. It + accepts a list of regexes that are searched for in the comment + string. If any of these regexes matches a portion of the comment + string the trigger is matched. ``comment: retrigger`` will + match when comments containing 'retrigger' somewhere in the + comment text are added to a pull request. + + .. attr:: status + + This is used for ``pg_pull_request`` and ``status`` actions. It + accepts a list of strings each of which matches the user setting + the status, the status context, and the status itself in the + format of ``status``. For example, ``success`` or ``failure``. + + .. attr:: ref + + This is only used for ``pg_push`` events. This field is treated as + a regular expression and multiple refs may be listed. Pagure + always sends full ref name, eg. ``refs/tags/bar`` and this + string is matched against the regular expression. + +Reporter Configuration +---------------------- +Zuul reports back to Pagure via Pagure API. Available reports include a PR +comment containing the build results, a commit status on start, success and +failure, and a merge of the PR itself. Status name, description, and context +is taken from the pipeline. + +.. attr:: pipeline.. + + To report to Pagure, the dictionaries passed to any of the pipeline + :ref:`reporter` attributes support the following + attributes: + + .. attr:: status + + String value (``pending``, ``success``, ``failure``) that the + reporter should set as the commit status on Pagure. + + .. attr:: status-url + :default: web.status_url or the empty string + + String value for a link url to set in the Pagure status. Defaults to the + zuul server status_url, or the empty string if that is unset. + + .. attr:: comment + :default: true + + Boolean value that determines if the reporter should add a + comment to the pipeline status to the Pagure Pull Request. Only + used for Pull Request based items. + + .. attr:: merge + :default: false + + Boolean value that determines if the reporter should merge the + pull Request. Only used for Pull Request based items. + + +Requirements Configuration +-------------------------- + +As described in :attr:`pipeline.require` pipelines may specify that items meet +certain conditions in order to be enqueued into the pipeline. These conditions +vary according to the source of the project in question. To supply +requirements for changes from a Pagure source named ``pagure``, create a +configuration such as the following:: + + pipeline: + require: + pagure: + score: 1 + merged: false + status: success + +This indicates that changes originating from the Pagure connection +must have a score of *1*, a CI status *success* and not being already merged. + +.. attr:: pipeline.require. + + The dictionary passed to the Pagure pipeline `require` attribute + supports the following attributes: + + .. attr:: score + + If present, the minimal score a Pull Request must reached. + + .. attr:: status + + If present, the CI status a Pull Request must have. + + .. attr:: merged + + A boolean value (``true`` or ``false``) that indicates whether + the Pull Request must be merged or not in order to be enqueued. + + .. attr:: open + + A boolean value (``true`` or ``false``) that indicates whether + the Pull Request must be open or closed in order to be enqueued. + +Reference pipelines configuration +--------------------------------- + +Here is an example of standard pipelines you may want to define:: + + - pipeline: + name: check + manager: independent + require: + pagure.io: + merged: False + trigger: + pagure.io: + - event: pg_pull_request + action: comment + comment: (?i)^\s*recheck\s*$ + - event: pg_pull_request + action: + - opened + - changed + start: + pagure.io: + status: 'pending' + comment: false + sqlreporter: + success: + pagure.io: + status: 'success' + sqlreporter: + failure: + pagure.io: + status: 'failure' + sqlreporter: + + - pipeline: + name: gate + manager: dependent + precedence: high + require: + pagure.io: + score: 1 + merged: False + status: success + sqlreporter: + trigger: + pagure.io: + - event: pg_pull_request + action: status + status: success + - event: pg_pull_request_review + action: thumbsup + start: + pagure.io: + status: 'pending' + comment: false + sqlreporter: + success: + pagure.io: + status: 'success' + merge: true + comment: true + sqlreporter: + failure: + pagure.io: + status: 'failure' + comment: true + sqlreporter: + + - pipeline: + name: post + post-review: true + manager: independent + precedence: low + trigger: + pagure.io: + - event: pg_push + ref: ^refs/heads/.*$ + success: + sqlreporter: diff --git a/tests/base.py b/tests/base.py index bc79987f53..9ee32cfbbb 100644 --- a/tests/base.py +++ b/tests/base.py @@ -61,6 +61,7 @@ import tests.fakegithub import zuul.driver.gerrit.gerritsource as gerritsource import zuul.driver.gerrit.gerritconnection as gerritconnection import zuul.driver.github.githubconnection as githubconnection +import zuul.driver.pagure.pagureconnection as pagureconnection import zuul.driver.github import zuul.driver.sql import zuul.scheduler @@ -800,6 +801,331 @@ class FakeGerritConnection(gerritconnection.GerritConnection): return 'file://' + os.path.join(self.upstream_root, project.name) +class PagureChangeReference(git.Reference): + _common_path_default = "refs/pull" + _points_to_commits_only = True + + +class FakePagurePullRequest(object): + log = logging.getLogger("zuul.test.FakePagurePullRequest") + + def __init__(self, pagure, number, project, branch, + subject, upstream_root, files=[], number_of_commits=1, + initial_comment=None): + self.pagure = pagure + self.source = pagure + self.number = number + self.project = project + self.branch = branch + self.subject = subject + self.upstream_root = upstream_root + self.number_of_commits = 0 + self.status = 'Open' + self.initial_comment = initial_comment + self.uuid = uuid.uuid4().hex + self.comments = [] + self.flags = [] + self.files = {} + self.cached_merge_status = '' + self.threshold_reached = False + self.commit_stop = None + self.commit_start = None + self.threshold_reached = False + self.upstream_root = upstream_root + self.cached_merge_status = 'MERGE' + self.url = "https://%s/%s/pull-request/%s" % ( + self.pagure.server, self.project, self.number) + self.is_merged = False + self.pr_ref = self._createPRRef() + self._addCommitInPR(files=files) + self._updateTimeStamp() + + def _getPullRequestEvent(self, action): + name = 'pg_pull_request' + data = { + 'msg': { + 'pullrequest': { + 'branch': self.branch, + 'comments': self.comments, + 'commit_start': self.commit_start, + 'commit_stop': self.commit_stop, + 'date_created': '0', + 'id': self.number, + 'project': { + 'fullname': self.project, + }, + 'status': self.status, + 'subject': self.subject, + 'uid': self.uuid, + } + }, + 'msg_id': str(uuid.uuid4()), + 'timestamp': 1427459070, + 'topic': action + } + if action == 'pull-request.flag.added': + data['msg']['flag'] = self.flags[0] + return (name, data) + + def getPullRequestOpenedEvent(self): + return self._getPullRequestEvent('pull-request.new') + + def getPullRequestUpdatedEvent(self): + self._addCommitInPR() + self.addComment( + "**1 new commit added**\n\n * ``Bump``\n", + True) + return self._getPullRequestEvent('pull-request.comment.added') + + def getPullRequestCommentedEvent(self, message): + self.addComment(message) + return self._getPullRequestEvent('pull-request.comment.added') + + def getPullRequestStatusSetEvent(self, status): + self.addFlag( + status, "https://url", "Build %s" % status) + return self._getPullRequestEvent('pull-request.flag.added') + + def addFlag(self, status, url, comment, username="Pingou"): + flag = { + "username": username, + "comment": comment, + "status": status, + "url": url + } + self.flags.insert(0, flag) + self._updateTimeStamp() + + def editInitialComment(self, initial_comment): + self.initial_comment = initial_comment + self._updateTimeStamp() + + def addComment(self, message, notification=False, fullname=None): + self.comments.append({ + 'comment': message, + 'notification': notification, + 'date_created': str(int(time.time())), + 'user': { + 'fullname': fullname or 'Pingou' + }} + ) + self._updateTimeStamp() + + def getPRReference(self): + return '%s/head' % self.number + + def _getRepo(self): + repo_path = os.path.join(self.upstream_root, self.project) + return git.Repo(repo_path) + + def _createPRRef(self): + repo = self._getRepo() + return PagureChangeReference.create( + repo, self.getPRReference(), 'refs/tags/init') + + def addCommit(self, files=[]): + """Adds a commit on top of the actual PR head.""" + self._addCommitInPR(files=files) + self._updateTimeStamp() + + def forcePush(self, files=[]): + """Clears actual commits and add a commit on top of the base.""" + self._addCommitInPR(files=files, reset=True) + self._updateTimeStamp() + + def _addCommitInPR(self, files=[], reset=False): + repo = self._getRepo() + ref = repo.references[self.getPRReference()] + if reset: + self.number_of_commits = 0 + ref.set_object('refs/tags/init') + self.number_of_commits += 1 + repo.head.reference = ref + repo.git.clean('-x', '-f', '-d') + + if files: + self.files = files + else: + fn = '%s-%s' % (self.branch.replace('/', '_'), self.number) + self.files = {fn: "test %s %s\n" % (self.branch, self.number)} + msg = self.subject + '-' + str(self.number_of_commits) + for fn, content in self.files.items(): + fn = os.path.join(repo.working_dir, fn) + with open(fn, 'w') as f: + f.write(content) + repo.index.add([fn]) + + self.commit_stop = repo.index.commit(msg).hexsha + if not self.commit_start: + self.commit_start = self.commit_stop + + repo.create_head(self.getPRReference(), self.commit_stop, force=True) + self.pr_ref.set_commit(self.commit_stop) + repo.head.reference = 'master' + repo.git.clean('-x', '-f', '-d') + repo.heads['master'].checkout() + + def _updateTimeStamp(self): + self.last_updated = str(int(time.time())) + + +class FakePagureAPIClient(pagureconnection.PagureAPIClient): + log = logging.getLogger("zuul.test.FakePagureAPIClient") + + def __init__(self, baseurl, api_token, project, + token_exp_date=None, pull_requests_db={}): + super(FakePagureAPIClient, self).__init__( + baseurl, api_token, project, token_exp_date) + self.session = None + self.pull_requests = pull_requests_db + + def gen_error(self): + return { + 'error': 'some error', + 'error_code': 'some error code' + } + + def _get_pr(self, match): + project, number = match.groups() + pr = self.pull_requests.get(project, {}).get(number) + if not pr: + return self.gen_error() + return pr + + def get(self, url): + self.log.debug("Getting resource %s ..." % url) + + match = re.match(r'.+/api/0/(.+)/pull-request/(\d+)$', url) + if match: + pr = self._get_pr(match) + return { + 'branch': pr.branch, + 'subject': pr.subject, + 'status': pr.status, + 'initial_comment': pr.initial_comment, + 'last_updated': pr.last_updated, + 'comments': pr.comments, + 'commit_stop': pr.commit_stop, + 'threshold_reached': pr.threshold_reached, + 'cached_merge_status': pr.cached_merge_status + } + + match = re.match(r'.+/api/0/(.+)/pull-request/(\d+)/flag$', url) + if match: + pr = self._get_pr(match) + return {'flags': pr.flags} + + match = re.match('.+/api/0/(.+)/git/branches$', url) + if match: + # project = match.groups()[0] + return {'branches': ['master']} + + match = re.match(r'.+/api/0/(.+)/pull-request/(\d+)/diffstats$', url) + if match: + pr = self._get_pr(match) + return pr.files + + def post(self, url, params=None): + + self.log.info( + "Posting on resource %s, params (%s) ..." % (url, params)) + + match = re.match(r'.+/api/0/(.+)/pull-request/(\d+)/merge$', url) + if match: + pr = self._get_pr(match) + pr.status = 'Merged' + pr.is_merged = True + + if not params: + return self.gen_error() + + match = re.match(r'.+/api/0/(.+)/pull-request/(\d+)/flag$', url) + if match: + pr = self._get_pr(match) + pr.flags.insert(0, params) + + match = re.match(r'.+/api/0/(.+)/pull-request/(\d+)/comment$', url) + if match: + pr = self._get_pr(match) + pr.addComment(params['comment']) + + +class FakePagureConnection(pagureconnection.PagureConnection): + log = logging.getLogger("zuul.test.FakePagureConnection") + + def __init__(self, driver, connection_name, connection_config, rpcclient, + changes_db=None, upstream_root=None): + super(FakePagureConnection, self).__init__(driver, connection_name, + connection_config) + self.connection_name = connection_name + self.pr_number = 0 + self.pull_requests = changes_db + self.statuses = {} + self.upstream_root = upstream_root + self.reports = [] + self.rpcclient = rpcclient + self.cloneurl = self.upstream_root + + def _refresh_project_connectors(self, project): + connector = self.connectors.setdefault( + project, {'api_client': None, 'webhook_token': None}) + api_token_exp_date = int(time.time()) + 60 * 24 * 3600 + connector['api_client'] = FakePagureAPIClient( + self.baseurl, "fake_api_token-%s" % project, project, + token_exp_date=api_token_exp_date, + pull_requests_db=self.pull_requests) + connector['webhook_token'] = "fake_webhook_token-%s" % project + return connector + + def emitEvent(self, event, use_zuulweb=False, project=None): + name, payload = event + secret = 'fake_webhook_token-%s' % project + if use_zuulweb: + payload = json.dumps(payload).encode('utf-8') + signature, _ = pagureconnection._sign_request(payload, secret) + headers = {'x-pagure-signature': signature, + 'x-pagure-project': project} + return requests.post( + 'http://127.0.0.1:%s/api/connection/%s/payload' + % (self.zuul_web_port, self.connection_name), + data=payload, headers=headers) + else: + job = self.rpcclient.submitJob( + 'pagure:%s:payload' % self.connection_name, + {'payload': payload}) + return json.loads(job.data[0]) + + def openFakePullRequest(self, project, branch, subject, files=[], + initial_comment=None): + self.pr_number += 1 + pull_request = FakePagurePullRequest( + self, self.pr_number, project, branch, subject, self.upstream_root, + files=files, initial_comment=initial_comment) + self.pull_requests.setdefault( + project, {})[str(self.pr_number)] = pull_request + return pull_request + + def getGitReceiveEvent(self, project): + name = 'pg_push' + repo_path = os.path.join(self.upstream_root, project) + repo = git.Repo(repo_path) + headsha = repo.head.commit.hexsha + data = { + 'msg': { + 'project_fullname': project, + 'branch': 'master', + 'stop_commit': headsha, + }, + 'msg_id': str(uuid.uuid4()), + 'timestamp': 1427459070, + 'topic': 'git.receive', + } + return (name, data) + + def setZuulWebPort(self, port): + self.zuul_web_port = port + + class GithubChangeReference(git.Reference): _common_path_default = "refs/pull" _points_to_commits_only = True @@ -1378,7 +1704,7 @@ class FakeBuild(object): self.changes = None items = self.parameters['zuul']['items'] self.changes = ' '.join(['%s,%s' % (x['change'], x['patchset']) - for x in items if 'change' in x]) + for x in items if 'change' in x]) if 'change' in items[-1]: self.change = ' '.join((items[-1]['change'], items[-1]['patchset'])) @@ -2184,7 +2510,8 @@ class ZuulWebFixture(fixtures.Fixture): self.connections.configure( config, include_drivers=[zuul.driver.sql.SQLDriver, - zuul.driver.github.GithubDriver]) + zuul.driver.github.GithubDriver, + zuul.driver.pagure.PagureDriver]) if info is None: self.info = zuul.model.WebInfo() else: @@ -2635,6 +2962,7 @@ class ZuulTestCase(BaseTestCase): # a virtual canonical database given by the configured hostname self.gerrit_changes_dbs = {} self.github_changes_dbs = {} + self.pagure_changes_dbs = {} def getGerritConnection(driver, name, config): db = self.gerrit_changes_dbs.setdefault(config['server'], {}) @@ -2692,6 +3020,22 @@ class ZuulTestCase(BaseTestCase): 'zuul.driver.github.GithubDriver.getConnection', getGithubConnection)) + def getPagureConnection(driver, name, config): + server = config.get('server', 'pagure.io') + db = self.pagure_changes_dbs.setdefault(server, {}) + con = FakePagureConnection( + driver, name, config, + self.rpcclient, + changes_db=db, + upstream_root=self.upstream_root) + self.event_queues.append(con.event_queue) + setattr(self, 'fake_' + name, con) + return con + + self.useFixture(fixtures.MonkeyPatch( + 'zuul.driver.pagure.PagureDriver.getConnection', + getPagureConnection)) + # Set up smtp related fakes # TODO(jhesketh): This should come from lib.connections for better # coverage @@ -3623,7 +3967,6 @@ class ZuulTestCase(BaseTestCase): self.addCleanup(_restoreTenantConfig) def addEvent(self, connection, event): - """Inject a Fake (Gerrit) event. This method accepts a JSON-encoded event and simulates Zuul diff --git a/tests/fixtures/config/cross-source-pagure/gerrit.yaml b/tests/fixtures/config/cross-source-pagure/gerrit.yaml new file mode 100644 index 0000000000..c5d9d24db8 --- /dev/null +++ b/tests/fixtures/config/cross-source-pagure/gerrit.yaml @@ -0,0 +1,11 @@ +- tenant: + name: tenant-one + source: + gerrit: + config-projects: + - common-config-gerrit + untrusted-projects: + - gerrit/project1 + pagure: + untrusted-projects: + - pagure/project2 diff --git a/tests/fixtures/config/cross-source-pagure/git/common-config-gerrit/playbooks/project-merge.yaml b/tests/fixtures/config/cross-source-pagure/git/common-config-gerrit/playbooks/project-merge.yaml new file mode 100644 index 0000000000..f679dceaef --- /dev/null +++ b/tests/fixtures/config/cross-source-pagure/git/common-config-gerrit/playbooks/project-merge.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/cross-source-pagure/git/common-config-gerrit/playbooks/project-test1.yaml b/tests/fixtures/config/cross-source-pagure/git/common-config-gerrit/playbooks/project-test1.yaml new file mode 100644 index 0000000000..f679dceaef --- /dev/null +++ b/tests/fixtures/config/cross-source-pagure/git/common-config-gerrit/playbooks/project-test1.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/cross-source-pagure/git/common-config-gerrit/playbooks/project-test2.yaml b/tests/fixtures/config/cross-source-pagure/git/common-config-gerrit/playbooks/project-test2.yaml new file mode 100644 index 0000000000..f679dceaef --- /dev/null +++ b/tests/fixtures/config/cross-source-pagure/git/common-config-gerrit/playbooks/project-test2.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/cross-source-pagure/git/common-config-gerrit/playbooks/project1-project2-integration.yaml b/tests/fixtures/config/cross-source-pagure/git/common-config-gerrit/playbooks/project1-project2-integration.yaml new file mode 100644 index 0000000000..f679dceaef --- /dev/null +++ b/tests/fixtures/config/cross-source-pagure/git/common-config-gerrit/playbooks/project1-project2-integration.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/cross-source-pagure/git/common-config-gerrit/zuul.yaml b/tests/fixtures/config/cross-source-pagure/git/common-config-gerrit/zuul.yaml new file mode 100644 index 0000000000..57566f260d --- /dev/null +++ b/tests/fixtures/config/cross-source-pagure/git/common-config-gerrit/zuul.yaml @@ -0,0 +1,137 @@ +- pipeline: + name: check + manager: independent + trigger: + gerrit: + - event: patchset-created + pagure: + - event: pg_pull_request + action: + - changed + success: + gerrit: + Verified: 1 + pagure: {} + failure: + gerrit: + Verified: -1 + pagure: {} + +- pipeline: + name: gate + manager: dependent + success-message: Build succeeded (gate). + require: + pagure: + score: 1 + merged: False + status: success + gerrit: + approval: + - Approved: 1 + trigger: + gerrit: + - event: comment-added + approval: + - Approved: 1 + pagure: + - event: pg_pull_request + action: status + status: success + - event: pg_pull_request_review + action: thumbsup + success: + gerrit: + Verified: 2 + submit: true + pagure: + merge: true + failure: + gerrit: + Verified: -2 + pagure: {} + start: + gerrit: + Verified: 0 + pagure: {} + precedence: high + +- job: + name: base + parent: null + +- job: + name: project-merge + nodeset: + nodes: + - name: controller + label: label1 + run: playbooks/project-merge.yaml + +- job: + name: project-test1 + nodeset: + nodes: + - name: controller + label: label1 + run: playbooks/project-test1.yaml + +- job: + name: project-test2 + nodeset: + nodes: + - name: controller + label: label1 + run: playbooks/project-test2.yaml + +- job: + name: project1-project2-integration + nodeset: + nodes: + - name: controller + label: label1 + run: playbooks/project1-project2-integration.yaml + +- project: + name: gerrit/project1 + check: + jobs: + - project-merge + - project-test1: + dependencies: project-merge + - project-test2: + dependencies: project-merge + - project1-project2-integration: + dependencies: project-merge + gate: + queue: integrated + jobs: + - project-merge + - project-test1: + dependencies: project-merge + - project-test2: + dependencies: project-merge + - project1-project2-integration: + dependencies: project-merge + +- project: + name: pagure/project2 + check: + jobs: + - project-merge + - project-test1: + dependencies: project-merge + - project-test2: + dependencies: project-merge + - project1-project2-integration: + dependencies: project-merge + gate: + queue: integrated + jobs: + - project-merge + - project-test1: + dependencies: project-merge + - project-test2: + dependencies: project-merge + - project1-project2-integration: + dependencies: project-merge diff --git a/tests/fixtures/config/cross-source-pagure/git/gerrit_project1/README b/tests/fixtures/config/cross-source-pagure/git/gerrit_project1/README new file mode 100644 index 0000000000..9daeafb986 --- /dev/null +++ b/tests/fixtures/config/cross-source-pagure/git/gerrit_project1/README @@ -0,0 +1 @@ +test diff --git a/tests/fixtures/config/cross-source-pagure/git/github_common-config/playbooks/project-merge.yaml b/tests/fixtures/config/cross-source-pagure/git/github_common-config/playbooks/project-merge.yaml new file mode 100644 index 0000000000..f679dceaef --- /dev/null +++ b/tests/fixtures/config/cross-source-pagure/git/github_common-config/playbooks/project-merge.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/cross-source-pagure/git/github_common-config/playbooks/project-test1.yaml b/tests/fixtures/config/cross-source-pagure/git/github_common-config/playbooks/project-test1.yaml new file mode 100644 index 0000000000..f679dceaef --- /dev/null +++ b/tests/fixtures/config/cross-source-pagure/git/github_common-config/playbooks/project-test1.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/cross-source-pagure/git/github_common-config/playbooks/project-test2.yaml b/tests/fixtures/config/cross-source-pagure/git/github_common-config/playbooks/project-test2.yaml new file mode 100644 index 0000000000..f679dceaef --- /dev/null +++ b/tests/fixtures/config/cross-source-pagure/git/github_common-config/playbooks/project-test2.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/cross-source-pagure/git/github_common-config/playbooks/project1-project2-integration.yaml b/tests/fixtures/config/cross-source-pagure/git/github_common-config/playbooks/project1-project2-integration.yaml new file mode 100644 index 0000000000..f679dceaef --- /dev/null +++ b/tests/fixtures/config/cross-source-pagure/git/github_common-config/playbooks/project1-project2-integration.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/cross-source-pagure/git/github_common-config/zuul.yaml b/tests/fixtures/config/cross-source-pagure/git/github_common-config/zuul.yaml new file mode 100644 index 0000000000..f7bbd47843 --- /dev/null +++ b/tests/fixtures/config/cross-source-pagure/git/github_common-config/zuul.yaml @@ -0,0 +1,135 @@ +- pipeline: + name: check + manager: independent + trigger: + github: + - event: pull_request + action: + - edited + pagure: + - event: pg_pull_request + action: + - changed + success: + github: {} + pagure: {} + failure: + github: {} + pagure: {} + +- pipeline: + name: gate + manager: dependent + success-message: Build succeeded (gate). + require: + pagure: + score: 1 + merged: False + status: success + github: + label: approved + trigger: + github: + - event: pull_request + action: edited + - event: pull_request + action: labeled + label: approved + pagure: + - event: pg_pull_request + action: status + status: success + - event: pg_pull_request_review + action: thumbsup + success: + github: + merge: true + pagure: + merge: true + failure: + github: {} + pagure: {} + start: + github: {} + pagure: {} + precedence: high + +- job: + name: base + parent: null + +- job: + name: project-merge + nodeset: + nodes: + - name: controller + label: label1 + run: playbooks/project-merge.yaml + +- job: + name: project-test1 + nodeset: + nodes: + - name: controller + label: label1 + run: playbooks/project-test1.yaml + +- job: + name: project-test2 + nodeset: + nodes: + - name: controller + label: label1 + run: playbooks/project-test2.yaml + +- job: + name: project1-project2-integration + nodeset: + nodes: + - name: controller + label: label1 + run: playbooks/project1-project2-integration.yaml + +- project: + name: github/project1 + check: + jobs: + - project-merge + - project-test1: + dependencies: project-merge + - project-test2: + dependencies: project-merge + - project1-project2-integration: + dependencies: project-merge + gate: + queue: integrated + jobs: + - project-merge + - project-test1: + dependencies: project-merge + - project-test2: + dependencies: project-merge + - project1-project2-integration: + dependencies: project-merge + +- project: + name: pagure/project2 + check: + jobs: + - project-merge + - project-test1: + dependencies: project-merge + - project-test2: + dependencies: project-merge + - project1-project2-integration: + dependencies: project-merge + gate: + queue: integrated + jobs: + - project-merge + - project-test1: + dependencies: project-merge + - project-test2: + dependencies: project-merge + - project1-project2-integration: + dependencies: project-merge diff --git a/tests/fixtures/config/cross-source-pagure/git/github_project1/README b/tests/fixtures/config/cross-source-pagure/git/github_project1/README new file mode 100644 index 0000000000..9daeafb986 --- /dev/null +++ b/tests/fixtures/config/cross-source-pagure/git/github_project1/README @@ -0,0 +1 @@ +test diff --git a/tests/fixtures/config/cross-source-pagure/git/pagure_project2/README b/tests/fixtures/config/cross-source-pagure/git/pagure_project2/README new file mode 100644 index 0000000000..9daeafb986 --- /dev/null +++ b/tests/fixtures/config/cross-source-pagure/git/pagure_project2/README @@ -0,0 +1 @@ +test diff --git a/tests/fixtures/config/cross-source-pagure/github.yaml b/tests/fixtures/config/cross-source-pagure/github.yaml new file mode 100644 index 0000000000..b5bc5f8863 --- /dev/null +++ b/tests/fixtures/config/cross-source-pagure/github.yaml @@ -0,0 +1,11 @@ +- tenant: + name: tenant-one + source: + github: + config-projects: + - github/common-config + untrusted-projects: + - github/project1 + pagure: + untrusted-projects: + - pagure/project2 diff --git a/tests/fixtures/layouts/basic-pagure.yaml b/tests/fixtures/layouts/basic-pagure.yaml new file mode 100644 index 0000000000..b7aa5b87a4 --- /dev/null +++ b/tests/fixtures/layouts/basic-pagure.yaml @@ -0,0 +1,60 @@ +- pipeline: + name: check + manager: independent + trigger: + pagure: + - event: pg_pull_request + action: comment + comment: (?i)^\s*recheck\s*$ + - event: pg_pull_request + action: + - opened + - changed + start: + pagure: + status: 'pending' + comment: True + success: + pagure: + status: 'success' + comment: True + failure: + pagure: + status: 'failure' + comment: True + +- pipeline: + name: post + post-review: true + manager: independent + trigger: + pagure: + - event: pg_push + ref: ^refs/heads/.*$ + +- job: + name: base + parent: null + run: playbooks/base.yaml + +- job: + name: project-test1 + run: playbooks/project-test1.yaml + +- job: + name: project-test2 + run: playbooks/project-test2.yaml + +- job: + name: project-post-job + run: playbooks/project-post.yaml + +- project: + name: org/project + check: + jobs: + - project-test1 + - project-test2 + post: + jobs: + - project-post-job diff --git a/tests/fixtures/layouts/crd-pagure.yaml b/tests/fixtures/layouts/crd-pagure.yaml new file mode 100644 index 0000000000..e0828aeeb1 --- /dev/null +++ b/tests/fixtures/layouts/crd-pagure.yaml @@ -0,0 +1,65 @@ +- pipeline: + name: check + manager: independent + trigger: + pagure: + - event: pg_pull_request + action: comment + +- pipeline: + name: gate + manager: dependent + trigger: + pagure: + - event: pg_pull_request + action: comment + success: + pagure: + merge: true + +- job: + name: base + parent: null + run: playbooks/base.yaml + +- job: + name: project1-test + run: playbooks/project1-test.yaml + +- job: + name: project2-test + run: playbooks/project2-test.yaml + +- job: + name: project3-test + run: playbooks/project3-test.yaml + +- job: + name: project4-test + run: playbooks/project4-test.yaml + +- project: + name: org/project1 + check: + jobs: + - project1-test + +- project: + name: org/project2 + check: + jobs: + - project2-test + +- project: + name: org/project3 + gate: + queue: cogated + jobs: + - project3-test + +- project: + name: org/project4 + gate: + queue: cogated + jobs: + - project4-test diff --git a/tests/fixtures/layouts/merging-pagure.yaml b/tests/fixtures/layouts/merging-pagure.yaml new file mode 100644 index 0000000000..d7cdd30015 --- /dev/null +++ b/tests/fixtures/layouts/merging-pagure.yaml @@ -0,0 +1,46 @@ +- pipeline: + name: check-merge + manager: independent + trigger: + pagure: + - event: pg_pull_request + action: + - opened + success: + pagure: + status: 'success' + merge: true + +- pipeline: + name: gate-merge + manager: dependent + trigger: + pagure: + - event: pg_pull_request + action: + - opened + success: + pagure: + status: 'success' + merge: true + +- job: + name: base + parent: null + run: playbooks/base.yaml + +- job: + name: project-test + run: playbooks/project-test.yaml + +- project: + name: org/project1 + check-merge: + jobs: + - project-test + +- project: + name: org/project2 + gate-merge: + jobs: + - project-test diff --git a/tests/fixtures/layouts/requirements-pagure.yaml b/tests/fixtures/layouts/requirements-pagure.yaml new file mode 100644 index 0000000000..47ca6c1355 --- /dev/null +++ b/tests/fixtures/layouts/requirements-pagure.yaml @@ -0,0 +1,66 @@ +- pipeline: + name: req-score-1 + manager: independent + require: + pagure: + score: 1 + trigger: + pagure: + - event: pg_pull_request_review + action: thumbsup + success: + pagure: + status: 'success' + +- pipeline: + name: req-score-2 + manager: independent + require: + pagure: + score: 2 + trigger: + pagure: + - event: pg_pull_request_review + action: thumbsup + success: + pagure: + status: 'success' + +- pipeline: + name: trigger-flag + manager: independent + trigger: + pagure: + - event: pg_pull_request + action: status + status: success + success: + pagure: + status: 'success' + +- job: + name: base + parent: null + run: playbooks/base.yaml + +- job: + name: project-test + run: playbooks/project-test.yaml + +- project: + name: org/project1 + req-score-1: + jobs: + - project-test + +- project: + name: org/project2 + req-score-2: + jobs: + - project-test + +- project: + name: org/project3 + trigger-flag: + jobs: + - project-test diff --git a/tests/fixtures/zuul-crd-pagure.conf b/tests/fixtures/zuul-crd-pagure.conf new file mode 100644 index 0000000000..f0b870e655 --- /dev/null +++ b/tests/fixtures/zuul-crd-pagure.conf @@ -0,0 +1,39 @@ +[gearman] +server=127.0.0.1 + +[statsd] +# note, use 127.0.0.1 rather than localhost to avoid getting ipv6 +# see: https://github.com/jsocol/pystatsd/issues/61 +server=127.0.0.1 + +[scheduler] +tenant_config=main.yaml + +[merger] +git_dir=/tmp/zuul-test/merger-git +git_user_email=zuul@example.com +git_user_name=zuul + +[executor] +git_dir=/tmp/zuul-test/executor-git + +[connection gerrit] +driver=gerrit +server=review.example.com +user=jenkins +sshkey=fake_id_rsa_path + +[connection github] +driver=github +webhook_token=0000000000000000000000000000000000000000 + +[connection pagure] +driver=pagure +api_token=0000000000000000000000000000000000000000 + +[connection smtp] +driver=smtp +server=localhost +port=25 +default_from=zuul@example.com +default_to=you@example.com diff --git a/tests/fixtures/zuul-pagure-driver.conf b/tests/fixtures/zuul-pagure-driver.conf new file mode 100644 index 0000000000..8e87a4bd13 --- /dev/null +++ b/tests/fixtures/zuul-pagure-driver.conf @@ -0,0 +1,18 @@ +[gearman] +server=127.0.0.1 + +[web] +status_url=http://zuul.example.com/status/#{change.number},{change.patchset} + +[merger] +git_dir=/tmp/zuul-test/git +git_user_email=zuul@example.com +git_user_name=zuul + +[executor] +git_dir=/tmp/zuul-test/executor-git + +[connection pagure] +driver=pagure +server=pagure +api_token=0000000000000000000000000000000000000000 diff --git a/tests/unit/test_pagure_driver.py b/tests/unit/test_pagure_driver.py new file mode 100644 index 0000000000..ffc3b6f8e6 --- /dev/null +++ b/tests/unit/test_pagure_driver.py @@ -0,0 +1,872 @@ +# Copyright 2019 Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import re +import yaml +import time +import socket + +from testtools.matchers import MatchesRegex + +import zuul.rpcclient + +from tests.base import ZuulTestCase, simple_layout +from tests.base import ZuulWebFixture + + +class TestPagureDriver(ZuulTestCase): + config_file = 'zuul-pagure-driver.conf' + + @simple_layout('layouts/basic-pagure.yaml', driver='pagure') + def test_pull_request_opened(self): + + initial_comment = "This is the\nPR initial_comment." + A = self.fake_pagure.openFakePullRequest( + 'org/project', 'master', 'A', initial_comment=initial_comment) + self.fake_pagure.emitEvent(A.getPullRequestOpenedEvent()) + self.waitUntilSettled() + + self.assertEqual('SUCCESS', + self.getJobFromHistory('project-test1').result) + self.assertEqual('SUCCESS', + self.getJobFromHistory('project-test2').result) + + job = self.getJobFromHistory('project-test2') + zuulvars = job.parameters['zuul'] + self.assertEqual(str(A.number), zuulvars['change']) + self.assertEqual(str(A.commit_stop), zuulvars['patchset']) + self.assertEqual('master', zuulvars['branch']) + self.assertEquals('https://pagure/org/project/pull-request/1', + zuulvars['items'][0]['change_url']) + self.assertEqual(zuulvars["message"], initial_comment) + self.assertEqual(2, len(self.history)) + self.assertEqual(2, len(A.comments)) + self.assertEqual( + A.comments[0]['comment'], "Starting check jobs.") + self.assertThat( + A.comments[1]['comment'], + MatchesRegex(r'.*\[project-test1 \]\(.*\).*', re.DOTALL)) + self.assertThat( + A.comments[1]['comment'], + MatchesRegex(r'.*\[project-test2 \]\(.*\).*', re.DOTALL)) + self.assertEqual(2, len(A.flags)) + self.assertEqual('success', A.flags[0]['status']) + self.assertEqual('pending', A.flags[1]['status']) + + @simple_layout('layouts/basic-pagure.yaml', driver='pagure') + def test_pull_request_updated(self): + + A = self.fake_pagure.openFakePullRequest('org/project', 'master', 'A') + pr_tip1 = A.commit_stop + self.fake_pagure.emitEvent(A.getPullRequestOpenedEvent()) + self.waitUntilSettled() + self.assertEqual(2, len(self.history)) + self.assertHistory( + [ + {'name': 'project-test1', 'changes': '1,%s' % pr_tip1}, + {'name': 'project-test2', 'changes': '1,%s' % pr_tip1}, + ], ordered=False + ) + + self.fake_pagure.emitEvent(A.getPullRequestUpdatedEvent()) + pr_tip2 = A.commit_stop + self.waitUntilSettled() + self.assertEqual(4, len(self.history)) + self.assertHistory( + [ + {'name': 'project-test1', 'changes': '1,%s' % pr_tip1}, + {'name': 'project-test2', 'changes': '1,%s' % pr_tip1}, + {'name': 'project-test1', 'changes': '1,%s' % pr_tip2}, + {'name': 'project-test2', 'changes': '1,%s' % pr_tip2} + ], ordered=False + ) + + @simple_layout('layouts/basic-pagure.yaml', driver='pagure') + def test_pull_request_updated_builds_aborted(self): + + A = self.fake_pagure.openFakePullRequest('org/project', 'master', 'A') + pr_tip1 = A.commit_stop + + self.executor_server.hold_jobs_in_build = True + + self.fake_pagure.emitEvent(A.getPullRequestOpenedEvent()) + self.waitUntilSettled() + + self.fake_pagure.emitEvent(A.getPullRequestUpdatedEvent()) + pr_tip2 = A.commit_stop + self.waitUntilSettled() + + self.executor_server.hold_jobs_in_build = False + self.executor_server.release() + self.waitUntilSettled() + + self.assertHistory( + [ + {'name': 'project-test1', 'result': 'ABORTED', + 'changes': '1,%s' % pr_tip1}, + {'name': 'project-test2', 'result': 'ABORTED', + 'changes': '1,%s' % pr_tip1}, + {'name': 'project-test1', 'changes': '1,%s' % pr_tip2}, + {'name': 'project-test2', 'changes': '1,%s' % pr_tip2} + ], ordered=False + ) + + @simple_layout('layouts/basic-pagure.yaml', driver='pagure') + def test_pull_request_commented(self): + + A = self.fake_pagure.openFakePullRequest('org/project', 'master', 'A') + self.fake_pagure.emitEvent(A.getPullRequestOpenedEvent()) + self.waitUntilSettled() + self.assertEqual(2, len(self.history)) + + self.fake_pagure.emitEvent( + A.getPullRequestCommentedEvent('I like that change')) + self.waitUntilSettled() + self.assertEqual(2, len(self.history)) + + self.fake_pagure.emitEvent( + A.getPullRequestCommentedEvent('recheck')) + self.waitUntilSettled() + self.assertEqual(4, len(self.history)) + + @simple_layout('layouts/basic-pagure.yaml', driver='pagure') + def test_pull_request_with_dyn_reconf(self): + + zuul_yaml = [ + {'job': { + 'name': 'project-test3', + 'run': 'job.yaml' + }}, + {'project': { + 'check': { + 'jobs': [ + 'project-test3' + ] + } + }} + ] + playbook = "- hosts: all\n tasks: []" + + A = self.fake_pagure.openFakePullRequest( + 'org/project', 'master', 'A') + A.addCommit( + {'.zuul.yaml': yaml.dump(zuul_yaml), + 'job.yaml': playbook} + ) + self.fake_pagure.emitEvent(A.getPullRequestOpenedEvent()) + self.waitUntilSettled() + + self.assertEqual('SUCCESS', + self.getJobFromHistory('project-test1').result) + self.assertEqual('SUCCESS', + self.getJobFromHistory('project-test2').result) + self.assertEqual('SUCCESS', + self.getJobFromHistory('project-test3').result) + + @simple_layout('layouts/basic-pagure.yaml', driver='pagure') + def test_ref_updated(self): + + event = self.fake_pagure.getGitReceiveEvent('org/project') + expected_newrev = event[1]['msg']['stop_commit'] + self.fake_pagure.emitEvent(event) + self.waitUntilSettled() + self.assertEqual(1, len(self.history)) + self.assertEqual( + 'SUCCESS', + self.getJobFromHistory('project-post-job').result) + + job = self.getJobFromHistory('project-post-job') + zuulvars = job.parameters['zuul'] + self.assertEqual('refs/heads/master', zuulvars['ref']) + self.assertEqual('post', zuulvars['pipeline']) + self.assertEqual('project-post-job', zuulvars['job']) + self.assertEqual('master', zuulvars['branch']) + self.assertEqual( + 'https://pagure/org/project/commit/%s' % zuulvars['newrev'], + zuulvars['change_url']) + self.assertEqual(expected_newrev, zuulvars['newrev']) + + @simple_layout('layouts/basic-pagure.yaml', driver='pagure') + def test_ref_updated_and_tenant_reconfigure(self): + + self.waitUntilSettled() + old = self.sched.tenant_last_reconfigured.get('tenant-one', 0) + time.sleep(1) + + zuul_yaml = [ + {'job': { + 'name': 'project-post-job2', + 'run': 'job.yaml' + }}, + {'project': { + 'post': { + 'jobs': [ + 'project-post-job2' + ] + } + }} + ] + playbook = "- hosts: all\n tasks: []" + self.create_commit( + 'org/project', + {'.zuul.yaml': yaml.dump(zuul_yaml), + 'job.yaml': playbook}, + message='Add InRepo configuration' + ) + event = self.fake_pagure.getGitReceiveEvent('org/project') + self.fake_pagure.emitEvent(event) + self.waitUntilSettled() + + new = self.sched.tenant_last_reconfigured.get('tenant-one', 0) + # New timestamp should be greater than the old timestamp + self.assertLess(old, new) + + self.assertHistory( + [{'name': 'project-post-job'}, + {'name': 'project-post-job2'}, + ], ordered=False + ) + + @simple_layout('layouts/basic-pagure.yaml', driver='pagure') + def test_client_dequeue_change_pagure(self): + + client = zuul.rpcclient.RPCClient('127.0.0.1', + self.gearman_server.port) + self.addCleanup(client.shutdown) + + self.executor_server.hold_jobs_in_build = True + A = self.fake_pagure.openFakePullRequest('org/project', 'master', 'A') + + self.fake_pagure.emitEvent(A.getPullRequestOpenedEvent()) + self.waitUntilSettled() + + client.dequeue( + tenant='tenant-one', + pipeline='check', + project='org/project', + change='%s,%s' % (A.number, A.commit_stop), + ref=None) + + self.waitUntilSettled() + + tenant = self.sched.abide.tenants.get('tenant-one') + check_pipeline = tenant.layout.pipelines['check'] + self.assertEqual(check_pipeline.getAllItems(), []) + self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 2) + + self.executor_server.hold_jobs_in_build = False + self.executor_server.release() + self.waitUntilSettled() + + @simple_layout('layouts/basic-pagure.yaml', driver='pagure') + def test_client_enqueue_change_pagure(self): + + A = self.fake_pagure.openFakePullRequest('org/project', 'master', 'A') + + client = zuul.rpcclient.RPCClient('127.0.0.1', + self.gearman_server.port) + self.addCleanup(client.shutdown) + r = client.enqueue(tenant='tenant-one', + pipeline='check', + project='org/project', + trigger='pagure', + change='%s,%s' % (A.number, A.commit_stop)) + self.waitUntilSettled() + + self.assertEqual(self.getJobFromHistory('project-test1').result, + 'SUCCESS') + self.assertEqual(self.getJobFromHistory('project-test2').result, + 'SUCCESS') + self.assertEqual(r, True) + + @simple_layout('layouts/requirements-pagure.yaml', driver='pagure') + def test_pr_score_require_1_vote(self): + + A = self.fake_pagure.openFakePullRequest( + 'org/project1', 'master', 'A') + self.fake_pagure.emitEvent( + A.getPullRequestCommentedEvent("I like that change")) + self.waitUntilSettled() + self.assertEqual(0, len(self.history)) + + self.fake_pagure.emitEvent( + A.getPullRequestCommentedEvent(":thumbsup:")) + self.waitUntilSettled() + self.assertEqual(1, len(self.history)) + + self.assertEqual( + 'SUCCESS', + self.getJobFromHistory('project-test').result) + + @simple_layout('layouts/requirements-pagure.yaml', driver='pagure') + def test_pr_score_require_2_votes(self): + + A = self.fake_pagure.openFakePullRequest( + 'org/project2', 'master', 'A') + self.fake_pagure.emitEvent( + A.getPullRequestCommentedEvent("I like that change")) + self.waitUntilSettled() + self.assertEqual(0, len(self.history)) + + self.fake_pagure.emitEvent( + A.getPullRequestCommentedEvent(":thumbsup:")) + self.waitUntilSettled() + self.assertEqual(0, len(self.history)) + + self.fake_pagure.emitEvent( + A.getPullRequestCommentedEvent(":thumbsdown:")) + self.waitUntilSettled() + self.assertEqual(0, len(self.history)) + + self.fake_pagure.emitEvent( + A.getPullRequestCommentedEvent(":thumbsup:")) + self.waitUntilSettled() + self.assertEqual(0, len(self.history)) + + self.fake_pagure.emitEvent( + A.getPullRequestCommentedEvent(":thumbsup:")) + self.waitUntilSettled() + self.assertEqual(1, len(self.history)) + + @simple_layout('layouts/requirements-pagure.yaml', driver='pagure') + def test_status_trigger(self): + + A = self.fake_pagure.openFakePullRequest( + 'org/project3', 'master', 'A') + + self.fake_pagure.emitEvent( + A.getPullRequestStatusSetEvent("failure")) + self.waitUntilSettled() + self.assertEqual(0, len(self.history)) + + self.fake_pagure.emitEvent( + A.getPullRequestStatusSetEvent("success")) + self.waitUntilSettled() + self.assertEqual(1, len(self.history)) + + @simple_layout('layouts/merging-pagure.yaml', driver='pagure') + def test_merge_action_in_independent(self): + + A = self.fake_pagure.openFakePullRequest( + 'org/project1', 'master', 'A') + self.fake_pagure.emitEvent(A.getPullRequestOpenedEvent()) + self.waitUntilSettled() + + self.assertEqual(1, len(self.history)) + self.assertEqual('SUCCESS', + self.getJobFromHistory('project-test').result) + self.assertEqual('Merged', A.status) + + @simple_layout('layouts/merging-pagure.yaml', driver='pagure') + def test_merge_action_in_dependent(self): + + A = self.fake_pagure.openFakePullRequest( + 'org/project2', 'master', 'A') + self.fake_pagure.emitEvent(A.getPullRequestOpenedEvent()) + self.waitUntilSettled() + # connection.canMerge is not validated + self.assertEqual(0, len(self.history)) + + # Set the mergeable PR flag to a expected value + A.cached_merge_status = 'MERGE' + self.fake_pagure.emitEvent(A.getPullRequestOpenedEvent()) + self.waitUntilSettled() + # connection.canMerge is not validated + self.assertEqual(0, len(self.history)) + + # Set the score threshold as reached + A.threshold_reached = True + self.fake_pagure.emitEvent(A.getPullRequestOpenedEvent()) + self.waitUntilSettled() + # connection.canMerge is not validated + self.assertEqual(0, len(self.history)) + + # Set CI flag as passed CI + A.addFlag('success', 'https://url', 'Build passed') + self.fake_pagure.emitEvent(A.getPullRequestOpenedEvent()) + self.waitUntilSettled() + # connection.canMerge is validated + self.assertEqual(1, len(self.history)) + + self.assertEqual('SUCCESS', + self.getJobFromHistory('project-test').result) + self.assertEqual('Merged', A.status) + + @simple_layout('layouts/crd-pagure.yaml', driver='pagure') + def test_crd_independent(self): + + # Create a change in project1 that a project2 change will depend on + A = self.fake_pagure.openFakePullRequest('org/project1', 'master', 'A') + + # Create a commit in B that sets the dependency on A + msg = "Depends-On: %s" % A.url + B = self.fake_pagure.openFakePullRequest( + 'org/project2', 'master', 'B', initial_comment=msg) + + # Make an event to re-use + event = B.getPullRequestCommentedEvent('A comment') + self.fake_pagure.emitEvent(event) + self.waitUntilSettled() + + # The changes for the job from project2 should include the project1 + # PR content + changes = self.getJobFromHistory( + 'project2-test', 'org/project2').changes + + self.assertEqual(changes, "%s,%s %s,%s" % (A.number, + A.commit_stop, + B.number, + B.commit_stop)) + + # There should be no more changes in the queue + tenant = self.sched.abide.tenants.get('tenant-one') + self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0) + + @simple_layout('layouts/crd-pagure.yaml', driver='pagure') + def test_crd_dependent(self): + + # Create a change in project3 that a project4 change will depend on + A = self.fake_pagure.openFakePullRequest('org/project3', 'master', 'A') + + # Create a commit in B that sets the dependency on A + msg = "Depends-On: %s" % A.url + B = self.fake_pagure.openFakePullRequest( + 'org/project4', 'master', 'B', initial_comment=msg) + + # Make an event to re-use + event = B.getPullRequestCommentedEvent('A comment') + + self.fake_pagure.emitEvent(event) + self.waitUntilSettled() + + # Neither A and B can't merge (no flag, no score threshold) + self.assertEqual(0, len(self.history)) + + B.threshold_reached = True + B.addFlag('success', 'https://url', 'Build passed') + self.fake_pagure.emitEvent(event) + self.waitUntilSettled() + + # B can't merge as A got no flag, no score threshold + self.assertEqual(0, len(self.history)) + + A.threshold_reached = True + A.addFlag('success', 'https://url', 'Build passed') + self.fake_pagure.emitEvent(event) + self.waitUntilSettled() + + # The changes for the job from project4 should include the project3 + # PR content + changes = self.getJobFromHistory( + 'project4-test', 'org/project4').changes + + self.assertEqual(changes, "%s,%s %s,%s" % (A.number, + A.commit_stop, + B.number, + B.commit_stop)) + + self.assertTrue(A.is_merged) + self.assertTrue(B.is_merged) + + @simple_layout('layouts/crd-pagure.yaml', driver='pagure') + def test_crd_needed_changes(self): + + # Given change A and B, where B depends on A, when A + # completes B should be enqueued (using a shared queue) + + # Create a change in project3 that a project4 change will depend on + A = self.fake_pagure.openFakePullRequest('org/project3', 'master', 'A') + A.threshold_reached = True + A.addFlag('success', 'https://url', 'Build passed') + + # Set B to depend on A + msg = "Depends-On: %s" % A.url + B = self.fake_pagure.openFakePullRequest( + 'org/project4', 'master', 'B', initial_comment=msg) + # Make the driver aware of change B by sending an event + # At that moment B can't merge + self.fake_pagure.emitEvent(B.getPullRequestCommentedEvent('A comment')) + # Now set B mergeable + B.threshold_reached = True + B.addFlag('success', 'https://url', 'Build passed') + + # Enqueue A, which will make the scheduler detect that B is + # depending on so B will be enqueue as well. + self.fake_pagure.emitEvent(A.getPullRequestCommentedEvent('A comment')) + self.waitUntilSettled() + + # The changes for the job from project4 should include the project3 + # PR content + changes = self.getJobFromHistory( + 'project4-test', 'org/project4').changes + + self.assertEqual(changes, "%s,%s %s,%s" % (A.number, + A.commit_stop, + B.number, + B.commit_stop)) + + self.assertTrue(A.is_merged) + self.assertTrue(B.is_merged) + + +class TestPagureToGerritCRD(ZuulTestCase): + config_file = 'zuul-crd-pagure.conf' + tenant_config_file = 'config/cross-source-pagure/gerrit.yaml' + + def test_crd_gate(self): + "Test cross-repo dependencies" + A = self.fake_pagure.openFakePullRequest('pagure/project2', 'master', + 'A') + B = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'B') + + # A Depends-On: B + A.editInitialComment('Depends-On: %s\n' % (B.data['url'])) + + A.addFlag('success', 'https://url', 'Build passed') + A.threshold_reached = True + + B.addApproval('Code-Review', 2) + + # Make A enter the pipeline + self.fake_pagure.emitEvent( + A.getPullRequestCommentedEvent(":thumbsup:")) + self.waitUntilSettled() + + # Expect not merged as B not approved yet + self.assertFalse(A.is_merged) + self.assertEqual(B.data['status'], 'NEW') + + for connection in self.connections.connections.values(): + connection.maintainCache([]) + + B.addApproval('Approved', 1) + self.fake_pagure.emitEvent( + A.getPullRequestCommentedEvent(":thumbsup:")) + self.waitUntilSettled() + + self.assertTrue(A.is_merged) + self.assertEqual(B.data['status'], 'MERGED') + self.assertEqual(len(A.comments), 4) + self.assertEqual(B.reported, 2) + + changes = self.getJobFromHistory( + 'project-merge', 'pagure/project2').changes + self.assertEqual(changes, '1,1 1,%s' % A.commit_stop) + + def test_crd_check(self): + "Test cross-repo dependencies in independent pipelines" + A = self.fake_pagure.openFakePullRequest('pagure/project2', 'master', + 'A') + B = self.fake_gerrit.addFakeChange( + 'gerrit/project1', 'master', 'B') + + # A Depends-On: B + A.editInitialComment('Depends-On: %s\n' % (B.data['url'],)) + + self.executor_server.hold_jobs_in_build = True + + self.fake_pagure.emitEvent(A.getPullRequestUpdatedEvent()) + self.waitUntilSettled() + + self.assertTrue(self.builds[0].hasChanges(A, B)) + + self.executor_server.hold_jobs_in_build = False + self.executor_server.release() + self.waitUntilSettled() + + self.assertFalse(A.is_merged) + self.assertEqual(B.data['status'], 'NEW') + self.assertEqual(len(A.comments), 2) + self.assertEqual(B.reported, 0) + + changes = self.getJobFromHistory( + 'project-merge', 'pagure/project2').changes + self.assertEqual(changes, '1,1 1,%s' % A.commit_stop) + + +class TestGerritToPagureCRD(ZuulTestCase): + config_file = 'zuul-crd-pagure.conf' + tenant_config_file = 'config/cross-source-pagure/gerrit.yaml' + + def test_crd_gate(self): + "Test cross-repo dependencies" + A = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'A') + B = self.fake_pagure.openFakePullRequest('pagure/project2', 'master', + 'B') + + A.addApproval('Code-Review', 2) + + AM2 = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', + 'AM2') + AM1 = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', + 'AM1') + AM2.setMerged() + AM1.setMerged() + + # A -> AM1 -> AM2 + # A Depends-On: B + # M2 is here to make sure it is never queried. If it is, it + # means zuul is walking down the entire history of merged + # changes. + + A.setDependsOn(AM1, 1) + AM1.setDependsOn(AM2, 1) + + A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % ( + A.subject, B.url) + + self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) + self.waitUntilSettled() + + self.assertEqual(A.data['status'], 'NEW') + self.assertFalse(B.is_merged) + + for connection in self.connections.connections.values(): + connection.maintainCache([]) + + B.addFlag('success', 'https://url', 'Build passed') + B.threshold_reached = True + self.fake_pagure.emitEvent( + B.getPullRequestCommentedEvent(":thumbsup:")) + self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) + + self.waitUntilSettled() + + self.assertEqual(AM2.queried, 0) + self.assertEqual(A.data['status'], 'MERGED') + self.assertTrue(B.is_merged) + self.assertEqual(A.reported, 2) + self.assertEqual(len(B.comments), 3) + + changes = self.getJobFromHistory( + 'project-merge', 'gerrit/project1').changes + self.assertEqual(changes, '1,%s 1,1' % B.commit_stop) + + def test_crd_check(self): + "Test cross-repo dependencies in independent pipelines" + A = self.fake_gerrit.addFakeChange('gerrit/project1', 'master', 'A') + B = self.fake_pagure.openFakePullRequest( + 'pagure/project2', 'master', 'B') + + # A Depends-On: B + A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % ( + A.subject, B.url) + + self.executor_server.hold_jobs_in_build = True + + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + self.assertTrue(self.builds[0].hasChanges(A, B)) + + self.executor_server.hold_jobs_in_build = False + self.executor_server.release() + self.waitUntilSettled() + + self.assertEqual(A.data['status'], 'NEW') + self.assertFalse(B.is_merged) + self.assertEqual(A.reported, 1) + self.assertEqual(len(B.comments), 0) + + changes = self.getJobFromHistory( + 'project-merge', 'gerrit/project1').changes + self.assertEqual(changes, '1,%s 1,1' % B.commit_stop) + + +class TestPagureToGithubCRD(ZuulTestCase): + config_file = 'zuul-crd-pagure.conf' + tenant_config_file = 'config/cross-source-pagure/github.yaml' + + def test_crd_gate(self): + "Test cross-repo dependencies" + A = self.fake_pagure.openFakePullRequest('pagure/project2', 'master', + 'A') + B = self.fake_github.openFakePullRequest('github/project1', 'master', + 'B') + # A Depends-On: B + A.editInitialComment('Depends-On: %s\n' % (B.url)) + + A.addFlag('success', 'https://url', 'Build passed') + A.threshold_reached = True + + # Make A enter the pipeline + self.fake_pagure.emitEvent( + A.getPullRequestCommentedEvent(":thumbsup:")) + self.waitUntilSettled() + + # Expect not merged as B not approved yet + self.assertFalse(A.is_merged) + self.assertFalse(B.is_merged) + + for connection in self.connections.connections.values(): + connection.maintainCache([]) + + B.addLabel('approved') + self.fake_pagure.emitEvent( + A.getPullRequestCommentedEvent(":thumbsup:")) + self.waitUntilSettled() + + self.assertTrue(A.is_merged) + self.assertTrue(B.is_merged) + self.assertEqual(len(A.comments), 4) + self.assertEqual(len(B.comments), 2) + + changes = self.getJobFromHistory( + 'project-merge', 'pagure/project2').changes + self.assertEqual(changes, '1,%s 1,%s' % (B.head_sha, A.commit_stop)) + + def test_crd_check(self): + "Test cross-repo dependencies in independent pipelines" + A = self.fake_pagure.openFakePullRequest('pagure/project2', 'master', + 'A') + B = self.fake_github.openFakePullRequest('github/project1', 'master', + 'A') + + # A Depends-On: B + A.editInitialComment('Depends-On: %s\n' % B.url) + + self.executor_server.hold_jobs_in_build = True + + self.fake_pagure.emitEvent(A.getPullRequestUpdatedEvent()) + self.waitUntilSettled() + + self.assertTrue(self.builds[0].hasChanges(A, B)) + + self.executor_server.hold_jobs_in_build = False + self.executor_server.release() + self.waitUntilSettled() + + self.assertFalse(A.is_merged) + self.assertFalse(B.is_merged) + self.assertEqual(len(A.comments), 2) + self.assertEqual(len(A.comments), 2) + + changes = self.getJobFromHistory( + 'project-merge', 'pagure/project2').changes + self.assertEqual(changes, '1,%s 1,%s' % (B.head_sha, A.commit_stop)) + + +class TestGithubToPagureCRD(ZuulTestCase): + config_file = 'zuul-crd-pagure.conf' + tenant_config_file = 'config/cross-source-pagure/github.yaml' + + def test_crd_gate(self): + "Test cross-repo dependencies" + A = self.fake_github.openFakePullRequest('github/project1', 'master', + 'A') + B = self.fake_pagure.openFakePullRequest('pagure/project2', 'master', + 'B') + + # A Depends-On: B + A.editBody('Depends-On: %s\n' % B.url) + + event = A.addLabel('approved') + self.fake_github.emitEvent(event) + self.waitUntilSettled() + + self.assertFalse(A.is_merged) + self.assertFalse(B.is_merged) + + for connection in self.connections.connections.values(): + connection.maintainCache([]) + + B.addFlag('success', 'https://url', 'Build passed') + B.threshold_reached = True + self.fake_pagure.emitEvent( + B.getPullRequestCommentedEvent(":thumbsup:")) + + self.fake_github.emitEvent(event) + + self.waitUntilSettled() + + self.assertTrue(A.is_merged) + self.assertTrue(B.is_merged) + self.assertEqual(len(A.comments), 2) + self.assertEqual(len(B.comments), 3) + + changes = self.getJobFromHistory( + 'project-merge', 'github/project1').changes + self.assertEqual(changes, '1,%s 1,%s' % (B.commit_stop, A.head_sha)) + + def test_crd_check(self): + "Test cross-repo dependencies in independent pipelines" + A = self.fake_github.openFakePullRequest( + 'github/project1', 'master', 'A') + B = self.fake_pagure.openFakePullRequest( + 'pagure/project2', 'master', 'B') + + # A Depends-On: B + A.editBody('Depends-On: %s\n' % B.url) + + self.executor_server.hold_jobs_in_build = True + + self.fake_github.emitEvent(A.getPullRequestEditedEvent()) + self.waitUntilSettled() + + self.assertTrue(self.builds[0].hasChanges(A, B)) + + self.executor_server.hold_jobs_in_build = False + self.executor_server.release() + self.waitUntilSettled() + + self.assertFalse(A.is_merged) + self.assertFalse(B.is_merged) + self.assertEqual(len(A.comments), 1) + self.assertEqual(len(B.comments), 0) + + changes = self.getJobFromHistory( + 'project-merge', 'github/project1').changes + self.assertEqual(changes, '1,%s 1,%s' % (B.commit_stop, A.head_sha)) + + +class TestPagureWebhook(ZuulTestCase): + config_file = 'zuul-pagure-driver.conf' + + def setUp(self): + super(TestPagureWebhook, self).setUp() + + # Start the web server + self.web = self.useFixture( + ZuulWebFixture(self.gearman_server.port, + self.config)) + + host = '127.0.0.1' + # Wait until web server is started + while True: + port = self.web.port + try: + with socket.create_connection((host, port)): + break + except ConnectionRefusedError: + pass + + self.fake_pagure.setZuulWebPort(port) + + def tearDown(self): + super(TestPagureWebhook, self).tearDown() + + @simple_layout('layouts/basic-pagure.yaml', driver='pagure') + def test_webhook(self): + + A = self.fake_pagure.openFakePullRequest( + 'org/project', 'master', 'A') + self.fake_pagure.emitEvent(A.getPullRequestOpenedEvent(), + use_zuulweb=True, + project='org/project') + self.waitUntilSettled() + + self.assertEqual('SUCCESS', + self.getJobFromHistory('project-test1').result) + self.assertEqual('SUCCESS', + self.getJobFromHistory('project-test2').result) diff --git a/zuul/cmd/web.py b/zuul/cmd/web.py index b974a756ec..061bd19c63 100755 --- a/zuul/cmd/web.py +++ b/zuul/cmd/web.py @@ -97,7 +97,8 @@ class WebServer(zuul.cmd.ZuulDaemonApp): try: self.configure_connections( include_drivers=[zuul.driver.sql.SQLDriver, - zuul.driver.github.GithubDriver]) + zuul.driver.github.GithubDriver, + zuul.driver.pagure.PagureDriver]) self._run() except Exception: self.log.exception("Exception from WebServer:") diff --git a/zuul/driver/pagure/__init__.py b/zuul/driver/pagure/__init__.py new file mode 100644 index 0000000000..5514b1c628 --- /dev/null +++ b/zuul/driver/pagure/__init__.py @@ -0,0 +1,50 @@ +# Copyright 2018 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from zuul.driver import Driver, ConnectionInterface, TriggerInterface +from zuul.driver import SourceInterface, ReporterInterface +from zuul.driver.pagure import pagureconnection +from zuul.driver.pagure import paguresource +from zuul.driver.pagure import pagurereporter +from zuul.driver.pagure import paguretrigger + + +class PagureDriver(Driver, ConnectionInterface, TriggerInterface, + SourceInterface, ReporterInterface): + name = 'pagure' + + def getConnection(self, name, config): + return pagureconnection.PagureConnection(self, name, config) + + def getTrigger(self, connection, config=None): + return paguretrigger.PagureTrigger(self, connection, config) + + def getSource(self, connection): + return paguresource.PagureSource(self, connection) + + def getReporter(self, connection, pipeline, config=None): + return pagurereporter.PagureReporter( + self, connection, pipeline, config) + + def getTriggerSchema(self): + return paguretrigger.getSchema() + + def getReporterSchema(self): + return pagurereporter.getSchema() + + def getRequireSchema(self): + return paguresource.getRequireSchema() + + def getRejectSchema(self): + return paguresource.getRejectSchema() diff --git a/zuul/driver/pagure/pagureconnection.py b/zuul/driver/pagure/pagureconnection.py new file mode 100644 index 0000000000..1b2592b5bd --- /dev/null +++ b/zuul/driver/pagure/pagureconnection.py @@ -0,0 +1,886 @@ +# Copyright 2018 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import hmac +import hashlib +import queue +import threading +import time +import json +import requests +import cherrypy +import traceback +import voluptuous as v + +import gear + +from zuul.connection import BaseConnection +from zuul.web.handler import BaseWebController +from zuul.lib.config import get_default +from zuul.model import Ref, Branch, Tag +from zuul.lib import dependson + +from zuul.driver.pagure.paguremodel import PagureTriggerEvent, PullRequest + +# Minimal Pagure version supported 5.3.0 +# +# Pagure is similar to Github as it handles Pullrequest where PR is a branch +# composed of one or more commits. A PR can be commented, evaluated, updated, +# CI flagged, and merged. A PR can be flagged (success/failure/pending) and +# this driver use that capability. Code review (evaluation) is done via +# comments that contains a :thumbsup: or :thumbsdown:. Pagure computes a +# score based on that and allow or not the merge of PR if the "minimal score to +# merge" is set in repository settings. This driver uses that setting and need +# to be set. This driver expects to receive repository events via webhooks and +# expects to verify payload signature. The driver connection needs an user's +# API key with the "Modify an existing project" access. This user needs to be +# added as admin against projects to be gated by Zuul. +# +# The web hook target must be (in repository settings): +# - http:///zuul/api/connection//payload +# +# Repository settings (to be checked): +# - Always merge (Better to match internal merge strategy of Zuul) +# - Minimum score to merge pull-request +# - Notify on pull-request flag +# - Pull requests +# +# To define the connection in /etc/zuul/zuul.conf: +# [connection pagure.sftests.com] +# driver=pagure +# server=pagure.sftests.com +# baseurl=https://pagure.sftests.com/pagure +# cloneurl=https://pagure.sftests.com/pagure/git +# api_token=QX29SXAW96C2CTLUNA5JKEEU65INGWTO2B5NHBDBRMF67S7PYZWCS0L1AKHXXXXX +# +# Current Non blocking issues: +# - Pagure does not send event when git tag is added/removed +# https://pagure.io/pagure/issue/4400 (merged so need to be used) +# - Pagure does not send the oldrev info when a branch is updated/created +# https://pagure.io/pagure/issue/4401 +# - Pagure does not send an event when a branch is deleted +# https://pagure.io/pagure/issue/4399 (merged so need to be used) +# - Pagure does not reset the score when a PR code is updated +# https://pagure.io/pagure/issue/3985 +# Pagure does not send an event when initial_comment is updated +# https://pagure.io/pagure/issue/4398 (merged need to be used) +# - CI status flag updated field unit is second, better to have millisecond +# unit to avoid unpossible sorting to get last status if two status set the +# same second. +# https://pagure.io/pagure/issue/4402 +# - Zuul needs to be able to search commits that set a dependency (depends-on) +# to a specific commit to reset jobs run when a dependency is changed. On +# Gerrit and Github search through commits message is possible and used by +# Zuul. Pagure does not offer this capability. + +# Side notes +# - Idea would be to prevent PR merge by anybody else than Zuul. +# Pagure project option: "Activate Only assignee can merge pull-request" +# https://docs.pagure.org/pagure/usage/project_settings.html?highlight=score#activate-only-assignee-can-merge-pull-request + + +def _sign_request(body, secret): + signature = hmac.new( + secret.encode('utf-8'), body, hashlib.sha1).hexdigest() + return signature, body + + +class PagureGearmanWorker(object): + """A thread that answers gearman requests""" + log = logging.getLogger("zuul.PagureGearmanWorker") + + def __init__(self, connection): + self.config = connection.sched.config + self.connection = connection + self.thread = threading.Thread(target=self._run, + name='pagure-gearman-worker') + self._running = False + handler = "pagure:%s:payload" % self.connection.connection_name + self.jobs = { + handler: self.handle_payload, + } + + def _run(self): + while self._running: + try: + job = self.gearman.getJob() + try: + if job.name not in self.jobs: + self.log.exception("Exception while running job") + job.sendWorkException( + traceback.format_exc().encode('utf8')) + continue + output = self.jobs[job.name](json.loads(job.arguments)) + job.sendWorkComplete(json.dumps(output)) + except Exception: + self.log.exception("Exception while running job") + job.sendWorkException( + traceback.format_exc().encode('utf8')) + except gear.InterruptedError: + pass + except Exception: + self.log.exception("Exception while getting job") + + def handle_payload(self, args): + payload = args["payload"] + + self.log.info( + "Pagure Webhook Received (id: %(msg_id)s, topic: %(topic)s)" % ( + payload)) + + try: + self.__dispatch_event(payload) + output = {'return_code': 200} + except Exception: + output = {'return_code': 503} + self.log.exception("Exception handling Pagure event:") + + return output + + def __dispatch_event(self, payload): + event = payload['topic'] + try: + self.log.info("Dispatching event %s" % event) + self.connection.addEvent(payload, event) + except Exception as err: + message = 'Exception dispatching event: %s' % str(err) + self.log.exception(message) + raise Exception(message) + + def start(self): + self._running = True + server = self.config.get('gearman', 'server') + port = get_default(self.config, 'gearman', 'port', 4730) + ssl_key = get_default(self.config, 'gearman', 'ssl_key') + ssl_cert = get_default(self.config, 'gearman', 'ssl_cert') + ssl_ca = get_default(self.config, 'gearman', 'ssl_ca') + self.gearman = gear.TextWorker('Zuul Pagure Connector') + self.log.debug("Connect to gearman") + self.gearman.addServer(server, port, ssl_key, ssl_cert, ssl_ca) + self.log.debug("Waiting for server") + self.gearman.waitForServer() + self.log.debug("Registering") + for job in self.jobs: + self.gearman.registerFunction(job) + self.thread.start() + + def stop(self): + self._running = False + self.gearman.stopWaitingForJobs() + # We join here to avoid whitelisting the thread -- if it takes more + # than 5s to stop in tests, there's a problem. + self.thread.join(timeout=5) + self.gearman.shutdown() + + +class PagureEventConnector(threading.Thread): + """Move events from Pagure into the scheduler""" + + log = logging.getLogger("zuul.PagureEventConnector") + + def __init__(self, connection): + super(PagureEventConnector, self).__init__() + self.daemon = True + self.connection = connection + self._stopped = False + self.event_handler_mapping = { + 'pull-request.comment.added': self._event_issue_comment, + 'pull-request.new': self._event_pull_request, + 'pull-request.flag.added': self._event_flag_added, + 'git.receive': self._event_ref_updated, + } + + def stop(self): + self._stopped = True + self.connection.addEvent(None) + + def _handleEvent(self): + ts, json_body, event_type = self.connection.getEvent() + if self._stopped: + return + + self.log.info("Received event: %s" % str(event_type)) + # self.log.debug("Event payload: %s " % json_body) + + if event_type not in self.event_handler_mapping: + message = "Unhandled X-Pagure-Event: %s" % event_type + self.log.info(message) + return + + if event_type in self.event_handler_mapping: + self.log.debug("Handling event: %s" % event_type) + + try: + event = self.event_handler_mapping[event_type](json_body) + except Exception: + self.log.exception( + 'Exception when handling event: %s' % event_type) + event = None + + if event: + if event.change_number: + project = self.connection.source.getProject(event.project_name) + self.connection._getChange(project, + event.change_number, + event.patch_number, + refresh=True, + url=event.change_url, + event=event) + event.project_hostname = self.connection.canonical_hostname + self.connection.logEvent(event) + self.connection.sched.addEvent(event) + + def _event_base(self, body): + event = PagureTriggerEvent() + if 'pullrequest' in body['msg']: + data = body['msg']['pullrequest'] + data['flag'] = body['msg'].get('flag') + event.title = data.get('title') + event.project_name = data.get('project', {}).get('fullname') + event.change_number = data.get('id') + event.updated_at = data.get('date_created') + event.branch = data.get('branch') + event.change_url = self.connection.getPullUrl(event.project_name, + event.change_number) + event.ref = "refs/pull/%s/head" % event.change_number + # commit_stop is the tip of the PR branch + event.patch_number = data.get('commit_stop') + event.type = 'pg_pull_request' + else: + data = body['msg'] + event.type = 'pg_push' + return event, data + + def _event_issue_comment(self, body): + """ Handles pull request comments """ + # https://fedora-fedmsg.readthedocs.io/en/latest/topics.html#pagure-pull-request-comment-added + event, data = self._event_base(body) + last_comment = data.get('comments', [])[-1] + if last_comment.get('notification') is True: + # An updated PR (new commits) triggers the comment.added + # event. A message is added by pagure on the PR but notification + # is set to true. + event.action = 'changed' + else: + if last_comment.get('comment', '').find(':thumbsup:') >= 0: + event.action = 'thumbsup' + event.type = 'pg_pull_request_review' + elif last_comment.get('comment', '').find(':thumbsdown:') >= 0: + event.action = 'thumbsdown' + event.type = 'pg_pull_request_review' + else: + event.action = 'comment' + # Assume last comment is the one that have triggered the event + event.comment = last_comment.get('comment') + return event + + def _event_pull_request(self, body): + """ Handles pull request event """ + # https://fedora-fedmsg.readthedocs.io/en/latest/topics.html#pagure-pull-request-new + event, data = self._event_base(body) + event.action = 'opened' + return event + + def _event_flag_added(self, body): + """ Handles flag added event """ + # https://fedora-fedmsg.readthedocs.io/en/latest/topics.html#pagure-pull-request-flag-added + event, data = self._event_base(body) + event.status = data['flag']['status'] + event.action = 'status' + return event + + def _event_ref_updated(self, body): + """ Handles ref updated """ + # https://fedora-fedmsg.readthedocs.io/en/latest/topics.html#git-receive + event, data = self._event_base(body) + event.project_name = data.get('project_fullname') + event.branch = data.get('branch') + event.ref = 'refs/heads/%s' % event.branch + event.newrev = data.get('end_commit', data.get('stop_commit')) + # There is no concept of old rev (that is the previous branch tip) in + # pagure. end_commit is the new tip, start_commit is the oldest + # commit on the branch merged. stop_commit is the youngest commit + # on the branch. When a PR is merged (with a merge commit) end_commit + # is the merge commit sha, start and stop commits are the boundaries + # of the branch. + + # Then do not set oldrev as this information is missing + # event.oldrev = data.get('start_commit') + event.branch_updated = True + + # TODO(fbo): Pagure sends an event when a branch is created but the + # old rev info is not set by pagure. A new branch will be handled + # as ref updated. https://pagure.io/pagure/issue/4401 + # TODO(fbo): Pagure does not send an event when a branch is deleted + # https://pagure.io/pagure/issue/4399 + + # if event.oldrev == '0' * 40: + # event.branch_created = True + # if event.newrev == '0' * 40: + # event.branch_deleted = True + # + # if event.branch: + # project = self.connection.source.getProject(event.project_name) + # if event.branch_deleted: + # self.connection.project_branch_cache[project].remove( + # event.branch) + # elif event.branch_created: + # self.connection.project_branch_cache[project].append( + # event.branch) + # else: + # pass + + return event + + def run(self): + while True: + if self._stopped: + return + try: + self._handleEvent() + except Exception: + self.log.exception("Exception moving Pagure event:") + finally: + self.connection.eventDone() + + +class PagureAPIClient(): + log = logging.getLogger("zuul.PagureAPIClient") + + def __init__( + self, baseurl, api_token, project, token_exp_date=None): + self.session = requests.Session() + self.base_url = '%s/api/0/' % baseurl + self.api_token = api_token + self.project = project + self.headers = {'Authorization': 'token %s' % self.api_token} + self.token_exp_date = token_exp_date + + def is_expired(self): + if self.token_exp_date: + if int(time.time()) > (self.token_exp_date - 3600): + return True + return False + + def get(self, url): + self.log.debug("Getting resource %s ..." % url) + ret = self.session.get(url, headers=self.headers) + self.log.debug("GET returned (code: %s): %s" % ( + ret.status_code, ret.text)) + return ret.json() + + def post(self, url, params=None): + self.log.info( + "Posting on resource %s, params (%s) ..." % (url, params)) + ret = self.session.post(url, data=params, headers=self.headers) + self.log.debug("POST returned (code: %s): %s" % ( + ret.status_code, ret.text)) + return ret.json() + + def get_project_branches(self): + path = '%s/git/branches' % self.project + return self.get(self.base_url + path).get('branches', []) + + def get_pr(self, number): + path = '%s/pull-request/%s' % (self.project, number) + return self.get(self.base_url + path) + + def get_pr_diffstats(self, number): + path = '%s/pull-request/%s/diffstats' % (self.project, number) + return self.get(self.base_url + path) + + def get_pr_flags(self, number, last=False): + path = '%s/pull-request/%s/flag' % (self.project, number) + data = self.get(self.base_url + path) + if last: + if data['flags']: + return data['flags'][0] + else: + return {} + else: + return data['flags'] + + def set_pr_flag(self, number, status, url, description): + params = { + "username": "Zuul", + "comment": "Jobs result is %s" % status, + "status": status, + "url": url} + path = '%s/pull-request/%s/flag' % (self.project, number) + return self.post(self.base_url + path, params) + + def comment_pull(self, number, message): + params = {"comment": message} + path = '%s/pull-request/%s/comment' % (self.project, number) + return self.post(self.base_url + path, params) + + def merge_pr(self, number): + path = '%s/pull-request/%s/merge' % (self.project, number) + return self.post(self.base_url + path) + + def create_project_api_token(self): + """ A project admin user's api token must be use with that endpoint + """ + param = { + "description": "zuul-token-%s" % int(time.time()), + "acls": [ + "pull_request_merge", "pull_request_comment", + "pull_request_flag"] + } + path = '%s/token/new' % self.project + data = self.post(self.base_url + path, param) + # {"token": {"description": "mytoken", "id": "IED2HC...4QIXS6WPZDTET"}} + return data['token'] + + def get_connectors(self): + """ A project admin user's api token must be use with that endpoint + """ + def get_token_epoch(token): + return int(token['description'].split('-')[-1]) + + path = '%s/connector' % self.project + data = self.get(self.base_url + path) + # {"connector": { + # "hook_token": "WCL92MLWMRPGKBQ5LI0LZCSIS4TRQMHR0Q", + # "api_tokens": [ + # { + # "description": "zuul-token-123", + # "expired": false, + # "id": "X03J4DOJT7P3G4....3DNPPXN4G144BBIAJ" + # } + # ] + # }} + # Filter expired tokens + tokens = [ + token for token in data['connector'].get('api_tokens', {}) + if not token['expired']] + # Now following the pattern zuul-token-{epoch} find the last + # one created + api_token = None + for token in tokens: + if not token['description'].startswith('zuul-token-'): + continue + epoch = get_token_epoch(token) + if api_token: + if epoch > get_token_epoch(api_token): + api_token = token + else: + api_token = token + if not api_token: + # Let's create one + api_token = self.create_project_api_token() + api_token['created_at'] = get_token_epoch(api_token) + webhook_token = data['connector']['hook_token'] + return api_token, webhook_token + + +class PagureConnection(BaseConnection): + driver_name = 'pagure' + log = logging.getLogger("zuul.PagureConnection") + payload_path = 'payload' + + def __init__(self, driver, connection_name, connection_config): + super(PagureConnection, self).__init__( + driver, connection_name, connection_config) + self._change_cache = {} + self.project_branch_cache = {} + self.projects = {} + self.server = self.connection_config.get('server', 'pagure.io') + self.canonical_hostname = self.connection_config.get( + 'canonical_hostname', self.server) + self.git_ssh_key = self.connection_config.get('sshkey') + self.admin_api_token = self.connection_config.get('api_token') + self.baseurl = self.connection_config.get( + 'baseurl', 'https://%s' % self.server).rstrip('/') + self.cloneurl = self.connection_config.get( + 'cloneurl', self.baseurl).rstrip('/') + self.connectors = {} + self.source = driver.getSource(self) + self.event_queue = queue.Queue() + + self.sched = None + + def onLoad(self): + self.log.info('Starting Pagure connection: %s' % self.connection_name) + self.gearman_worker = PagureGearmanWorker(self) + self.log.info('Starting event connector') + self._start_event_connector() + self.log.info('Starting GearmanWorker') + self.gearman_worker.start() + + def _start_event_connector(self): + self.pagure_event_connector = PagureEventConnector(self) + self.pagure_event_connector.start() + + def _stop_event_connector(self): + if self.pagure_event_connector: + self.pagure_event_connector.stop() + self.pagure_event_connector.join() + + def onStop(self): + if hasattr(self, 'gearman_worker'): + self.gearman_worker.stop() + self._stop_event_connector() + + def addEvent(self, data, event=None): + return self.event_queue.put((time.time(), data, event)) + + def getEvent(self): + return self.event_queue.get() + + def eventDone(self): + self.event_queue.task_done() + + def _refresh_project_connectors(self, project): + pagure = PagureAPIClient( + self.baseurl, self.admin_api_token, project) + api_token, webhook_token = pagure.get_connectors() + connector = self.connectors.setdefault( + project, {'api_client': None, 'webhook_token': None}) + api_token_exp_date = api_token['created_at'] + 60 * 24 * 3600 + connector['api_client'] = PagureAPIClient( + self.baseurl, api_token['id'], project, + token_exp_date=api_token_exp_date) + connector['webhook_token'] = webhook_token + return connector + + def get_project_webhook_token(self, project): + token = self.connectors.get( + project, {}).get('webhook_token', None) + if token: + self.log.debug( + "Fetching project %s webhook_token from cache" % project) + return token + else: + self.log.debug( + "Fetching project %s webhook_token from API" % project) + return self._refresh_project_connectors(project)['webhook_token'] + + def get_project_api_client(self, project): + api_client = self.connectors.get( + project, {}).get('api_client', None) + if api_client: + if not api_client.is_expired(): + self.log.debug( + "Fetching project %s api_client from cache" % project) + return api_client + else: + self.log.debug( + "Project %s api token is expired (expiration date %s)" % ( + project, api_client.token_exp_date)) + self.log.debug("Building project %s api_client" % project) + return self._refresh_project_connectors(project)['api_client'] + + def maintainCache(self, relevant): + remove = set() + for key, change in self._change_cache.items(): + if change not in relevant: + remove.add(key) + for key in remove: + del self._change_cache[key] + + def clearBranchCache(self): + self.project_branch_cache = {} + + def getWebController(self, zuul_web): + return PagureWebController(zuul_web, self) + + def validateWebConfig(self, config, connections): + return True + + def getProject(self, name): + return self.projects.get(name) + + def addProject(self, project): + self.projects[project.name] = project + + def getPullUrl(self, project, number): + return '%s/pull-request/%s' % (self.getGitwebUrl(project), number) + + def getGitwebUrl(self, project, sha=None): + url = '%s/%s' % (self.baseurl, project) + if sha is not None: + url += '/commit/%s' % sha + return url + + def getProjectBranches(self, project, tenant): + branches = self.project_branch_cache.get(project.name) + + if branches is not None: + return branches + + pagure = self.get_project_api_client(project.name) + branches = pagure.get_project_branches() + self.project_branch_cache[project.name] = branches + + self.log.info("Got branches for %s" % project.name) + return branches + + def getGitUrl(self, project): + return '%s/%s' % (self.cloneurl, project.name) + + def getChange(self, event, refresh=False): + project = self.source.getProject(event.project_name) + if event.change_number: + self.log.info("Getting change for %s#%s" % ( + project, event.change_number)) + change = self._getChange( + project, event.change_number, event.patch_number, + refresh=refresh, event=event) + change.source_event = event + change.is_current_patchset = (change.pr.get('commit_stop') == + event.patch_number) + else: + self.log.info("Getting change for %s ref:%s" % ( + project, event.ref)) + if event.ref and event.ref.startswith('refs/tags/'): + change = Tag(project) + change.tag = event.ref[len('refs/tags/'):] + elif event.ref and event.ref.startswith('refs/heads/'): + change = Branch(project) + change.branch = event.ref[len('refs/heads/'):] + else: + change = Ref(project) + change.ref = event.ref + change.oldrev = event.oldrev + change.newrev = event.newrev + change.branch = event.branch + change.url = self.getGitwebUrl(project, sha=event.newrev) + + # Pagure does not send files details in the git-receive event. + # Explicitly set files to None and let the pipelines processor + # call the merger asynchronuously + change.files = None + + change.source_event = event + return change + + def _getChange(self, project, number, patchset=None, + refresh=False, url=None, event=None): + key = (project.name, number, patchset) + change = self._change_cache.get(key) + if change and not refresh: + self.log.debug("Getting change from cache %s" % str(key)) + return change + if not change: + change = PullRequest(project.name) + change.project = project + change.number = number + # patchset is the tips commit of the PR + change.patchset = patchset + change.url = url + change.uris = [ + '%s/%s/pull/%s' % (self.baseurl, project, number), + ] + self._change_cache[key] = change + try: + self.log.debug("Getting change pr#%s from project %s" % ( + number, project.name)) + self._updateChange(change, event) + except Exception: + if key in self._change_cache: + del self._change_cache[key] + raise + return change + + def _hasRequiredStatusChecks(self, change): + pagure = self.get_project_api_client(change.project.name) + flag = pagure.get_pr_flags(change.number, last=True) + return True if flag.get('status', '') == 'success' else False + + def canMerge(self, change, allow_needs): + pagure = self.get_project_api_client(change.project.name) + pr = pagure.get_pr(change.number) + + mergeable = False + if pr.get('cached_merge_status') in ('FFORWARD', 'MERGE'): + mergeable = True + + ci_flag = False + if self._hasRequiredStatusChecks(change): + ci_flag = True + + threshold = pr.get('threshold_reached') + if threshold is None: + self.log.debug("No threshold_reached attribute found") + + self.log.debug( + 'PR %s#%s mergeability details mergeable: %s ' + 'flag: %s threshold: %s' % ( + change.project.name, change.number, mergeable, + ci_flag, threshold)) + + can_merge = mergeable and ci_flag and threshold + + self.log.info('Check PR %s#%s mergeability can_merge: %s' % ( + change.project.name, change.number, can_merge)) + return can_merge + + def getPull(self, project_name, number): + pagure = self.get_project_api_client(project_name) + pr = pagure.get_pr(number) + diffstats = pagure.get_pr_diffstats(number) + pr['files'] = diffstats.keys() + self.log.info('Got PR %s#%s', project_name, number) + return pr + + def getStatus(self, project, number): + return self.getCommitStatus(project.name, number) + + def getScore(self, pr): + score_board = {} + last_pr_code_updated = 0 + # First get last PR updated date + for comment in pr.get('comments', []): + # PR updated are reported as comment but with the notification flag + if comment['notification']: + date = int(comment['date_created']) + if date > last_pr_code_updated: + last_pr_code_updated = date + # Now compute the score + # TODO(fbo): Pagure does not reset the score when a PR code is updated + # This code block computes the score based on votes after the last PR + # update. This should be proposed upstream + # https://pagure.io/pagure/issue/3985 + for comment in pr.get('comments', []): + author = comment['user']['fullname'] + date = int(comment['date_created']) + # Only handle score since the last PR update + if date >= last_pr_code_updated: + score_board.setdefault(author, 0) + # Use the same strategy to compute the score than Pagure + if comment.get('comment', '').find(':thumbsup:') >= 0: + score_board[author] += 1 + if comment.get('comment', '').find(':thumbsdown:') >= 0: + score_board[author] -= 1 + return sum(score_board.values()) + + def _updateChange(self, change, event): + self.log.info("Updating change from pagure %s" % change) + change.pr = self.getPull(change.project.name, change.number) + change.ref = "refs/pull/%s/head" % change.number + change.branch = change.pr.get('branch') + change.patchset = change.pr.get('commit_stop') + change.files = change.pr.get('files') + change.title = change.pr.get('title') + change.open = change.pr.get('status') == 'Open' + change.is_merged = change.pr.get('status') == 'Merged' + change.status = self.getStatus(change.project, change.number) + change.score = self.getScore(change.pr) + change.message = change.pr.get('initial_comment') or '' + # last_updated seems to be touch for comment changed/flags - that's OK + change.updated_at = change.pr.get('last_updated') + self.log.info("Updated change from pagure %s" % change) + + if self.sched: + self.sched.onChangeUpdated(change, event) + + return change + + def commentPull(self, project, number, message): + pagure = self.get_project_api_client(project) + pagure.comment_pull(number, message) + self.log.info("Commented on PR %s#%s", project, number) + + def setCommitStatus(self, project, number, state, url='', + description='', context=''): + pagure = self.get_project_api_client(project) + pagure.set_pr_flag(number, state, url, description) + self.log.info("Set pull-request CI flag status : %s" % description) + # Wait for 1 second as flag timestamp is by second + time.sleep(1) + + def getCommitStatus(self, project, number): + pagure = self.get_project_api_client(project) + flag = pagure.get_pr_flags(number, last=True) + self.log.info( + "Got pull-request CI status for PR %s on %s status: %s" % ( + number, project, flag.get('status'))) + return flag.get('status') + + def getChangesDependingOn(self, change, projects, tenant): + """ Reverse lookup of PR depending on this one + """ + # TODO(fbo) No way to Query pagure to search accross projects' PRs for + # a the depends-on string in PR initial message. Not a blocker + # for now, let's workaround using the local change cache ! + changes_dependencies = [] + for cached_change_id, _change in self._change_cache.items(): + for dep_header in dependson.find_dependency_headers( + _change.message): + if change.url in dep_header: + changes_dependencies.append(_change) + return changes_dependencies + + def mergePull(self, project, number): + pagure = self.get_project_api_client(project) + pagure.merge_pr(number) + self.log.debug("Merged PR %s#%s", project, number) + + +class PagureWebController(BaseWebController): + + log = logging.getLogger("zuul.PagureWebController") + + def __init__(self, zuul_web, connection): + self.connection = connection + self.zuul_web = zuul_web + + def _validate_signature(self, body, headers): + try: + request_signature = headers['x-pagure-signature'] + except KeyError: + raise cherrypy.HTTPError(401, 'x-pagure-signature header missing.') + + project = headers['x-pagure-project'] + token = self.connection.get_project_webhook_token(project) + if not token: + raise cherrypy.HTTPError( + 401, 'no webhook token for %s.' % project) + + signature, payload = _sign_request(body, token) + + if not hmac.compare_digest(str(signature), str(request_signature)): + self.log.debug( + "Missmatch (Payload Signature: %s, Request Signature: %s)" % ( + signature, request_signature)) + raise cherrypy.HTTPError( + 401, + 'Request signature does not match calculated payload ' + 'signature. Check that secret is correct.') + + return payload + + @cherrypy.expose + @cherrypy.tools.json_out(content_type='application/json; charset=utf-8') + def payload(self): + # https://docs.pagure.org/pagure/usage/using_webhooks.html + headers = dict() + for key, value in cherrypy.request.headers.items(): + headers[key.lower()] = value + body = cherrypy.request.body.read() + payload = self._validate_signature(body, headers) + json_payload = json.loads(payload.decode('utf-8')) + + job = self.zuul_web.rpc.submitJob( + 'pagure:%s:payload' % self.connection.connection_name, + {'payload': json_payload}) + + return json.loads(job.data[0]) + + +def getSchema(): + pagure_connection = v.Any(str, v.Schema(dict)) + return pagure_connection diff --git a/zuul/driver/pagure/paguremodel.py b/zuul/driver/pagure/paguremodel.py new file mode 100644 index 0000000000..5036fc8a74 --- /dev/null +++ b/zuul/driver/pagure/paguremodel.py @@ -0,0 +1,206 @@ +# Copyright 2018 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import re +from zuul.model import Change, TriggerEvent, EventFilter, RefFilter + +EMPTY_GIT_REF = '0' * 40 # git sha of all zeros, used during creates/deletes + + +class PullRequest(Change): + def __init__(self, project): + super(PullRequest, self).__init__(project) + self.project = None + self.pr = None + self.updated_at = None + self.title = None + self.score = 0 + self.files = [] + + def __repr__(self): + r = ['' + + def isUpdateOf(self, other): + if (self.project == other.project and + hasattr(other, 'number') and self.number == other.number and + hasattr(other, 'updated_at') and + self.updated_at > other.updated_at): + return True + return False + + +class PagureTriggerEvent(TriggerEvent): + def __init__(self): + super(PagureTriggerEvent, self).__init__() + self.trigger_name = 'pagure' + self.title = None + self.action = None + self.status = None + + def _repr(self): + r = [super(PagureTriggerEvent, self)._repr()] + if self.action: + r.append("action:%s" % self.action) + if self.status: + r.append("status:%s" % self.status) + r.append("project:%s" % self.canonical_project_name) + if self.change_number: + r.append("pr:%s" % self.change_number) + return ' '.join(r) + + def isPatchsetCreated(self): + if self.type == 'pg_pull_request': + return self.action in ['opened', 'changed'] + return False + + +class PagureEventFilter(EventFilter): + def __init__(self, trigger, types=[], refs=[], statuses=[], + comments=[], actions=[], ignore_deletes=True): + + EventFilter.__init__(self, trigger) + + self._types = types + self._refs = refs + self._comments = comments + self.types = [re.compile(x) for x in types] + self.refs = [re.compile(x) for x in refs] + self.comments = [re.compile(x) for x in comments] + self.actions = actions + self.statuses = statuses + self.ignore_deletes = ignore_deletes + + def __repr__(self): + ret = '