Browse Source
The canMerge check is executed whenever zuul tests if a change can enter a gate pipeline. This is part of the critical path in the event handling of the scheduler and therefore must be as fast as possible. Currently this takes five requests for doing its work and also transfers large amounts of data that is unneeded: * get pull request * get branch protection settings * get commits * get status of latest commit * get check runs of latest commit Especially when Github is busy this can slow down zuul's event processing considerably. This can be optimized using graphql to only query the data we need with a single request. This reduces requests and load on Github and speeds up event processing in the scheduler. Since this is the first usage of graphql this also sets up needed testing infrastructure using graphene to mock the github api with real test data. Change-Id: I77be4f16cf7eb5c8035ce0312f792f4e8d4c3e10changes/36/709836/5
Tobias Henkel
2 years ago
8 changed files with 376 additions and 56 deletions
@ -0,0 +1,166 @@
|
||||
# Copyright 2019 BMW Group |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
||||
# not use this file except in compliance with the License. You may obtain |
||||
# a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
||||
# License for the specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
from graphene import Boolean, Field, Int, List, ObjectType, String |
||||
|
||||
|
||||
class FakePageInfo(ObjectType): |
||||
end_cursor = String() |
||||
has_next_page = Boolean() |
||||
|
||||
def resolve_end_cursor(parent, info): |
||||
return 'testcursor' |
||||
|
||||
def resolve_has_next_page(parent, info): |
||||
return False |
||||
|
||||
|
||||
class FakeBranchProtectionRule(ObjectType): |
||||
pattern = String() |
||||
requiredStatusCheckContexts = List(String) |
||||
requiresApprovingReviews = Boolean() |
||||
requiresCodeOwnerReviews = Boolean() |
||||
|
||||
def resolve_pattern(parent, info): |
||||
return parent.pattern |
||||
|
||||
def resolve_requiredStatusCheckContexts(parent, info): |
||||
return parent.required_contexts |
||||
|
||||
def resolve_requiresApprovingReviews(parent, info): |
||||
return parent.require_reviews |
||||
|
||||
def resolve_requiresCodeOwnerReviews(parent, info): |
||||
return parent.require_codeowners_review |
||||
|
||||
|
||||
class FakeBranchProtectionRules(ObjectType): |
||||
nodes = List(FakeBranchProtectionRule) |
||||
|
||||
def resolve_nodes(parent, info): |
||||
return parent.values() |
||||
|
||||
|
||||
class FakeStatusContext(ObjectType): |
||||
state = String() |
||||
context = String() |
||||
|
||||
def resolve_state(parent, info): |
||||
state = parent.state.upper() |
||||
return state |
||||
|
||||
def resolve_context(parent, info): |
||||
return parent.context |
||||
|
||||
|
||||
class FakeStatus(ObjectType): |
||||
contexts = List(FakeStatusContext) |
||||
|
||||
def resolve_contexts(parent, info): |
||||
return parent |
||||
|
||||
|
||||
class FakeCheckRun(ObjectType): |
||||
name = String() |
||||
conclusion = String() |
||||
|
||||
def resolve_name(parent, info): |
||||
return parent.name |
||||
|
||||
def resolve_conclusion(parent, info): |
||||
return parent.conclusion.upper() |
||||
|
||||
|
||||
class FakeCheckRuns(ObjectType): |
||||
nodes = List(FakeCheckRun) |
||||
|
||||
def resolve_nodes(parent, info): |
||||
return parent |
||||
|
||||
|
||||
class FakeCheckSuite(ObjectType): |
||||
checkRuns = Field(FakeCheckRuns, first=Int()) |
||||
|
||||
def resolve_checkRuns(parent, info, first=None): |
||||
return parent |
||||
|
||||
|
||||
class FakeCheckSuites(ObjectType): |
||||
|
||||
nodes = List(FakeCheckSuite) |
||||
|
||||
def resolve_nodes(parent, info): |
||||
# Note: we only use a single check suite in the tests so return a |
||||
# single item to keep it simple. |
||||
return [parent] |
||||
|
||||
|
||||
class FakeCommit(ObjectType): |
||||
|
||||
class Meta: |
||||
# Graphql object type that defaults to the class name, but we require |
||||
# 'Commit'. |
||||
name = 'Commit' |
||||
|
||||
status = Field(FakeStatus) |
||||
checkSuites = Field(FakeCheckSuites, first=Int()) |
||||
|
||||
def resolve_status(parent, info): |
||||
seen = set() |
||||
result = [] |
||||
for status in parent._statuses: |
||||
if status.context not in seen: |
||||
seen.add(status.context) |
||||
result.append(status) |
||||
# Github returns None if there are no results |
||||
return result or None |
||||
|
||||
def resolve_checkSuites(parent, info, first=None): |
||||
# Tests only utilize one check suite so return all runs for that. |
||||
return parent._check_runs |
||||
|
||||
|
||||
class FakePullRequest(ObjectType): |
||||
isDraft = Boolean() |
||||
|
||||
def resolve_isDraft(parent, info): |
||||
return parent.draft |
||||
|
||||
|
||||
class FakeRepository(ObjectType): |
||||
name = String() |
||||
branchProtectionRules = Field(FakeBranchProtectionRules, first=Int()) |
||||
pullRequest = Field(FakePullRequest, number=Int(required=True)) |
||||
object = Field(FakeCommit, expression=String(required=True)) |
||||
|
||||
def resolve_name(parent, info): |
||||
org, name = parent.name.split('/') |
||||
return name |
||||
|
||||
def resolve_branchProtectionRules(parent, info, first): |
||||
return parent._branch_protection_rules |
||||
|
||||
def resolve_pullRequest(parent, info, number): |
||||
return parent.data.pull_requests.get(number) |
||||
|
||||
def resolve_object(parent, info, expression): |
||||
return parent._commits.get(expression) |
||||
|
||||
|
||||
class FakeGithubQuery(ObjectType): |
||||
repository = Field(FakeRepository, owner=String(required=True), |
||||
name=String(required=True)) |
||||
|
||||
def resolve_repository(root, info, owner, name): |
||||
return info.context.repos.get((owner, name)) |
||||
@ -0,0 +1,113 @@
|
||||
# Copyright 2020 BMW Group |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
||||
# not use this file except in compliance with the License. You may obtain |
||||
# a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
||||
# License for the specific language governing permissions and limitations |
||||
# under the License. |
||||
import logging |
||||
|
||||
import pathspec |
||||
from pkg_resources import resource_string |
||||
|
||||
|
||||
def nested_get(d, *keys, default=None): |
||||
temp = d |
||||
for key in keys[:-1]: |
||||
temp = temp.get(key, {}) if temp is not None else None |
||||
return temp.get(keys[-1], default) if temp is not None else default |
||||
|
||||
|
||||
class GraphQLClient: |
||||
log = logging.getLogger('zuul.github.graphql') |
||||
|
||||
def __init__(self, url): |
||||
self.url = url |
||||
self.queries = {} |
||||
self._load_queries() |
||||
|
||||
def _load_queries(self): |
||||
self.log.debug('Loading prepared graphql queries') |
||||
query_names = [ |
||||
'canmerge', |
||||
] |
||||
for query_name in query_names: |
||||
self.queries[query_name] = resource_string( |
||||
__name__, '%s.graphql' % query_name).decode('utf-8') |
||||
|
||||
@staticmethod |
||||
def _prepare_query(query, variables): |
||||
data = { |
||||
'query': query, |
||||
'variables': variables, |
||||
} |
||||
return data |
||||
|
||||
def _fetch_canmerge(self, github, owner, repo, pull, sha): |
||||
variables = { |
||||
'zuul_query': 'canmerge', # used for logging |
||||
'owner': owner, |
||||
'repo': repo, |
||||
'pull': pull, |
||||
'head_sha': sha, |
||||
} |
||||
query = self._prepare_query(self.queries['canmerge'], variables) |
||||
response = github.session.post(self.url, json=query) |
||||
return response.json() |
||||
|
||||
def fetch_canmerge(self, github, change): |
||||
owner, repo = change.project.name.split('/') |
||||
|
||||
data = self._fetch_canmerge(github, owner, repo, change.number, |
||||
change.patchset) |
||||
result = {} |
||||
|
||||
repository = nested_get(data, 'data', 'repository') |
||||
# Find corresponding rule to our branch |
||||
rules = nested_get(repository, 'branchProtectionRules', 'nodes', |
||||
default=[]) |
||||
matching_rule = None |
||||
for rule in rules: |
||||
pattern = pathspec.patterns.GitWildMatchPattern( |
||||
rule.get('pattern')) |
||||
match = pathspec.match_files([pattern], [change.branch]) |
||||
if match: |
||||
matching_rule = rule |
||||
break |
||||
|
||||
# If there is a matching rule, get required status checks |
||||
if matching_rule: |
||||
result['requiredStatusCheckContexts'] = matching_rule.get( |
||||
'requiredStatusCheckContexts', []) |
||||
result['requiresApprovingReviews'] = matching_rule.get( |
||||
'requiresApprovingReviews') |
||||
result['requiresCodeOwnerReviews'] = matching_rule.get( |
||||
'requiresCodeOwnerReviews') |
||||
else: |
||||
result['requiredStatusCheckContexts'] = [] |
||||
|
||||
# Check for draft |
||||
pull_request = nested_get(repository, 'pullRequest') |
||||
result['isDraft'] = nested_get(pull_request, 'isDraft', default=False) |
||||
|
||||
# Add status checks |
||||
result['status'] = {} |
||||
commit = nested_get(data, 'data', 'repository', 'object') |
||||
# Status can be explicit None so make sure we work with a dict |
||||
# afterwards |
||||
status = commit.get('status') or {} |
||||
for context in status.get('contexts', []): |
||||
result['status'][context['context']] = context['state'] |
||||
|
||||
# Add check runs |
||||
for suite in nested_get(commit, 'checkSuites', 'nodes', default=[]): |
||||
for run in nested_get(suite, 'checkRuns', 'nodes', default=[]): |
||||
result['status'][run['name']] = run['conclusion'] |
||||
|
||||
return result |
||||
@ -0,0 +1,40 @@
|
||||
query canMergeData( |
||||
$owner: String! |
||||
$repo: String! |
||||
$pull: Int! |
||||
$head_sha: String! |
||||
) { |
||||
repository(owner: $owner, name: $repo) { |
||||
branchProtectionRules(first: 100) { |
||||
nodes { |
||||
pattern |
||||
requiredStatusCheckContexts |
||||
requiresApprovingReviews |
||||
requiresCodeOwnerReviews |
||||
} |
||||
} |
||||
pullRequest(number: $pull) { |
||||
isDraft |
||||
} |
||||
object(expression: $head_sha) { |
||||
... on Commit { |
||||
checkSuites(first: 100) { |
||||
nodes { |
||||
checkRuns(first: 100) { |
||||
nodes { |
||||
name |
||||
conclusion |
||||
} |
||||
} |
||||
} |
||||
} |
||||
status { |
||||
contexts { |
||||
state |
||||
context |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
Loading…
Reference in new issue