Merge "Match tag items against containing branches"

This commit is contained in:
Zuul 2020-03-07 08:32:09 +00:00 committed by Gerrit Code Review
commit 288e50d35f
10 changed files with 125 additions and 33 deletions

View File

@ -782,6 +782,10 @@ Here is an example of two job definitions:
those projects. This can be used to run a job defined in one those projects. This can be used to run a job defined in one
project on another project without a matching branch. project on another project without a matching branch.
If a tag item is enqueued, we look up the branches which contain
the commit referenced by the tag. If any of those branches match a
branch matcher, the matcher is considered to have matched.
This example illustrates a job called *run-tests* which uses a This example illustrates a job called *run-tests* which uses a
nodeset based on the current release of an operating system to nodeset based on the current release of an operating system to
perform its tests, except when testing changes to the stable/2.0 perform its tests, except when testing changes to the stable/2.0

View File

@ -0,0 +1,6 @@
---
features:
- |
Zuul now matches tag items against their containing branches. This makes
it simpler to have release jobs in-tree. See :attr:`job.branches` for
details.

View File

@ -431,13 +431,67 @@ class TestBranchTag(ZuulTestCase):
event = self.fake_gerrit.addFakeTag('org/project', 'master', 'foo') event = self.fake_gerrit.addFakeTag('org/project', 'master', 'foo')
self.fake_gerrit.addEvent(event) self.fake_gerrit.addEvent(event)
self.waitUntilSettled() self.waitUntilSettled()
# test-job does not run in this case because it is defined in # test-job does run in this case because it is defined in a
# a branched repo with implied branch matchers. A release job # branched repo with implied branch matchers, and the tagged
# defined in a multi-branch repo would need at least one # commit is in both branches.
# top-level variant with no branch matcher in order to match a
# tag.
self.assertHistory([ self.assertHistory([
dict(name='central-job', result='SUCCESS', ref='refs/tags/foo')]) dict(name='central-job', result='SUCCESS', ref='refs/tags/foo'),
dict(name='test-job', result='SUCCESS', ref='refs/tags/foo')],
ordered=False)
def test_no_branch_match_divergent_multi_branch(self):
# Test that tag jobs from divergent branches run different job
# variants.
self.create_branch('org/project', 'stable/pike')
self.fake_gerrit.addEvent(
self.fake_gerrit.getFakeBranchCreatedEvent(
'org/project', 'stable/pike'))
self.waitUntilSettled()
# Add a new job to master
in_repo_conf = textwrap.dedent(
"""
- job:
name: test2-job
run: playbooks/test-job.yaml
- project:
name: org/project
tag:
jobs:
- central-job
- test2-job
""")
file_dict = {'.zuul.yaml': in_repo_conf}
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
files=file_dict)
A.setMerged()
self.fake_gerrit.addEvent(A.getChangeMergedEvent())
self.waitUntilSettled()
event = self.fake_gerrit.addFakeTag(
'org/project', 'stable/pike', 'foo')
self.fake_gerrit.addEvent(event)
self.waitUntilSettled()
# test-job runs because we tagged stable/pike, but test2-job does
# not, it only applied to master.
self.assertHistory([
dict(name='central-job', result='SUCCESS', ref='refs/tags/foo'),
dict(name='test-job', result='SUCCESS', ref='refs/tags/foo')],
ordered=False)
event = self.fake_gerrit.addFakeTag('org/project', 'master', 'bar')
self.fake_gerrit.addEvent(event)
self.waitUntilSettled()
# test2-job runs because we tagged master, but test-job does
# not, it only applied to stable/pike.
self.assertHistory([
dict(name='central-job', result='SUCCESS', ref='refs/tags/foo'),
dict(name='test-job', result='SUCCESS', ref='refs/tags/foo'),
dict(name='central-job', result='SUCCESS', ref='refs/tags/bar'),
dict(name='test2-job', result='SUCCESS', ref='refs/tags/bar')],
ordered=False)
class TestBranchNegative(ZuulTestCase): class TestBranchNegative(ZuulTestCase):

View File

@ -58,29 +58,29 @@ class ProjectMatcher(AbstractChangeMatcher):
class BranchMatcher(AbstractChangeMatcher): class BranchMatcher(AbstractChangeMatcher):
fullmatch = False
def matches(self, change): def matches(self, change):
if hasattr(change, 'branch'): if hasattr(change, 'branch'):
if self.regex.match(change.branch): # an implied branch matcher must do a fullmatch to work correctly
return True if self.fullmatch:
if self.regex.fullmatch(change.branch):
return True
else:
if self.regex.match(change.branch):
return True
return False return False
if self.regex.match(change.ref): if self.regex.match(change.ref):
return True return True
if hasattr(change, 'containing_branches'):
for branch in change.containing_branches:
if self.regex.fullmatch(branch):
return True
return False return False
class ImpliedBranchMatcher(AbstractChangeMatcher): class ImpliedBranchMatcher(BranchMatcher):
""" fullmatch = True
A branch matcher that only considers branch refs, and always
succeeds on other types (e.g., tags).
"""
def matches(self, change):
if hasattr(change, 'branch'):
if self.regex.fullmatch(change.branch):
return True
return False
return True
class FileMatcher(AbstractChangeMatcher): class FileMatcher(AbstractChangeMatcher):

View File

@ -1027,6 +1027,7 @@ class PipelineManager(object):
def onMergeCompleted(self, event): def onMergeCompleted(self, event):
build_set = event.build_set build_set = event.build_set
item = build_set.item item = build_set.item
item.change.containing_branches = event.item_in_branches
build_set.merge_state = build_set.COMPLETE build_set.merge_state = build_set.COMPLETE
build_set.repo_state = event.repo_state build_set.repo_state = event.repo_state
if event.merged: if event.merged:

View File

@ -186,9 +186,11 @@ class MergeClient(object):
commit = data.get('commit') commit = data.get('commit')
files = data.get('files', {}) files = data.get('files', {})
repo_state = data.get('repo_state', {}) repo_state = data.get('repo_state', {})
item_in_branches = data.get('item_in_branches', [])
job.files = files job.files = files
log.info("Merge %s complete, merged: %s, updated: %s, " log.info("Merge %s complete, merged: %s, updated: %s, "
"commit: %s", job, merged, job.updated, commit) "commit: %s, branches: %s", job, merged, job.updated, commit,
item_in_branches)
job.setComplete() job.setComplete()
if job.build_set: if job.build_set:
if job.name == 'merger:fileschanges': if job.name == 'merger:fileschanges':
@ -196,8 +198,7 @@ class MergeClient(object):
else: else:
self.sched.onMergeCompleted(job.build_set, self.sched.onMergeCompleted(job.build_set,
merged, job.updated, commit, files, merged, job.updated, commit, files,
repo_state) repo_state, item_in_branches)
# The test suite expects the job to be removed from the # The test suite expects the job to be removed from the
# internal account after the wake flag is set. # internal account after the wake flag is set.
self.jobs.remove(job) self.jobs.remove(job)

View File

@ -610,6 +610,20 @@ class Repo(object):
return int(m.group(1)) return int(m.group(1))
return None return None
def contains(self, hexsha, zuul_event_id=None):
repo = self.createRepoObject(zuul_event_id)
log = get_annotated_logger(self.log, zuul_event_id)
try:
branches = repo.git.branch(contains=hexsha, color='never')
except git.GitCommandError as e:
if e.status == 129:
log.debug("Found commit %s in no branches", hexsha)
return []
branches = [x.replace('*', '').strip() for x in branches.split('\n')]
branches = [x for x in branches if x != '(no branch)']
log.debug("Found commit %s in branches: %s", hexsha, branches)
return branches
class Merger(object): class Merger(object):
def __init__(self, working_root, connections, email, username, def __init__(self, working_root, connections, email, username,
@ -910,6 +924,8 @@ class Merger(object):
seen = set() seen = set()
recent = {} recent = {}
repo_state = {} repo_state = {}
# A list of branch names the last item appears in.
item_in_branches = []
for item in items: for item in items:
# If we're in the executor context we have the repo_locks object # If we're in the executor context we have the repo_locks object
# and perform per repo locking. # and perform per repo locking.
@ -926,7 +942,7 @@ class Merger(object):
repo.reset() repo.reset()
except Exception: except Exception:
self.log.exception("Unable to reset repo %s" % repo) self.log.exception("Unable to reset repo %s" % repo)
return (False, {}) return (False, {}, [])
self._saveRepoState(item['connection'], item['project'], self._saveRepoState(item['connection'], item['project'],
repo, repo_state, recent, branches) repo, repo_state, recent, branches)
@ -937,7 +953,10 @@ class Merger(object):
self._alterRepoState( self._alterRepoState(
item['connection'], item['project'], repo_state, item['connection'], item['project'], repo_state,
item['ref'], item['newrev']) item['ref'], item['newrev'])
return (True, repo_state) item = items[-1]
repo = self.getRepo(item['connection'], item['project'])
item_in_branches = repo.contains(item['newrev'])
return (True, repo_state, item_in_branches)
def getFiles(self, connection_name, project_name, branch, files, dirs=[]): def getFiles(self, connection_name, project_name, branch, files, dirs=[]):
repo = self.getRepo(connection_name, project_name) repo = self.getRepo(connection_name, project_name)

View File

@ -161,11 +161,13 @@ class BaseMergeServer(metaclass=ABCMeta):
self.log.debug("Got refstate job: %s" % job.unique) self.log.debug("Got refstate job: %s" % job.unique)
args = json.loads(job.arguments) args = json.loads(job.arguments)
zuul_event_id = args.get('zuul_event_id') zuul_event_id = args.get('zuul_event_id')
success, repo_state = self.merger.getRepoState( success, repo_state, item_in_branches = \
args['items'], branches=args.get('branches'), self.merger.getRepoState(
repo_locks=self.repo_locks) args['items'], branches=args.get('branches'),
repo_locks=self.repo_locks)
result = dict(updated=success, result = dict(updated=success,
repo_state=repo_state) repo_state=repo_state,
item_in_branches=item_in_branches)
result['zuul_event_id'] = zuul_event_id result['zuul_event_id'] = zuul_event_id
job.sendWorkComplete(json.dumps(result)) job.sendWorkComplete(json.dumps(result))

View File

@ -3136,6 +3136,7 @@ class Tag(Ref):
def __init__(self, project): def __init__(self, project):
super(Tag, self).__init__(project) super(Tag, self).__init__(project)
self.tag = None self.tag = None
self.containing_branches = []
class Change(Branch): class Change(Branch):

View File

@ -221,16 +221,19 @@ class MergeCompletedEvent(ResultEvent):
:arg bool updated: Whether the repo was updated (changes without refs). :arg bool updated: Whether the repo was updated (changes without refs).
:arg str commit: The SHA of the merged commit (changes with refs). :arg str commit: The SHA of the merged commit (changes with refs).
:arg dict repo_state: The starting repo state before the merge. :arg dict repo_state: The starting repo state before the merge.
:arg list item_in_branches: A list of branches in which the final
commit in the merge list appears (changes without refs).
""" """
def __init__(self, build_set, merged, updated, commit, def __init__(self, build_set, merged, updated, commit,
files, repo_state): files, repo_state, item_in_branches):
self.build_set = build_set self.build_set = build_set
self.merged = merged self.merged = merged
self.updated = updated self.updated = updated
self.commit = commit self.commit = commit
self.files = files self.files = files
self.repo_state = repo_state self.repo_state = repo_state
self.item_in_branches = item_in_branches
class FilesChangesCompletedEvent(ResultEvent): class FilesChangesCompletedEvent(ResultEvent):
@ -519,9 +522,10 @@ class Scheduler(threading.Thread):
self.wake_event.set() self.wake_event.set()
def onMergeCompleted(self, build_set, merged, updated, def onMergeCompleted(self, build_set, merged, updated,
commit, files, repo_state): commit, files, repo_state, item_in_branches):
event = MergeCompletedEvent(build_set, merged, event = MergeCompletedEvent(build_set, merged, updated,
updated, commit, files, repo_state) commit, files, repo_state,
item_in_branches)
self.result_event_queue.put(event) self.result_event_queue.put(event)
self.wake_event.set() self.wake_event.set()