Fix implied branch matchers and tags
Adding an implied branch matcher to jobs on in-repo project defs works great when an item *has* a branch. But some items, such as tags, don't. With recent changes, it is now impossible for a project to add a job in-repo that runs in a tag pipeline. To correct this, we need to drop some of the optimizations which assumed we could match the implied branch against existing branch matchers, and instead, when adding a job in-repo, simply add a new kind of branch matcher, an ImpliedBranchMatcher, that is evaluated in a boolean 'and' with any existing branch matchers. The ImpliedBranchMatcher only fails if the item has a branch, and the branch doesn't match. If the item doesn't have a branch, it always succeeds. This means that when a project adds a job to a tag pipeline in-repo, it will most likely only have the ImpliedBranchMatcher, which will simply succeed. It also means that the multiple project configurations present in the project's multiple branches can all add jobs to tag pipelines, and so to remove such a job, changes may need to be made to all branches of a project. However, there's not much that can be done about that at the moment. Change-Id: Id51ddfce7ef0a6d5e3273da784e407ac72a669db
This commit is contained in:
parent
9ab8db4e13
commit
1edfd97943
|
@ -486,6 +486,29 @@ class FakeGerritConnection(gerritconnection.GerritConnection):
|
||||||
self.changes[self.change_number] = c
|
self.changes[self.change_number] = c
|
||||||
return c
|
return c
|
||||||
|
|
||||||
|
def addFakeTag(self, project, branch, tag):
|
||||||
|
path = os.path.join(self.upstream_root, project)
|
||||||
|
repo = git.Repo(path)
|
||||||
|
commit = repo.heads[branch].commit
|
||||||
|
newrev = commit.hexsha
|
||||||
|
ref = 'refs/tags/' + tag
|
||||||
|
|
||||||
|
git.Tag.create(repo, tag, commit)
|
||||||
|
|
||||||
|
event = {
|
||||||
|
"type": "ref-updated",
|
||||||
|
"submitter": {
|
||||||
|
"name": "User Name",
|
||||||
|
},
|
||||||
|
"refUpdate": {
|
||||||
|
"oldRev": 40 * '0',
|
||||||
|
"newRev": newrev,
|
||||||
|
"refName": ref,
|
||||||
|
"project": project,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return event
|
||||||
|
|
||||||
def getFakeBranchCreatedEvent(self, project, branch):
|
def getFakeBranchCreatedEvent(self, project, branch):
|
||||||
path = os.path.join(self.upstream_root, project)
|
path = os.path.join(self.upstream_root, project)
|
||||||
repo = git.Repo(path)
|
repo = git.Repo(path)
|
||||||
|
|
|
@ -0,0 +1,9 @@
|
||||||
|
- job:
|
||||||
|
name: test-job
|
||||||
|
run: playbooks/test-job.yaml
|
||||||
|
|
||||||
|
- project:
|
||||||
|
name: org/project
|
||||||
|
tag:
|
||||||
|
jobs:
|
||||||
|
- test-job
|
|
@ -0,0 +1 @@
|
||||||
|
test
|
|
@ -0,0 +1,2 @@
|
||||||
|
- hosts: all
|
||||||
|
tasks: []
|
|
@ -0,0 +1,21 @@
|
||||||
|
- pipeline:
|
||||||
|
name: tag
|
||||||
|
manager: independent
|
||||||
|
trigger:
|
||||||
|
gerrit:
|
||||||
|
- event: ref-updated
|
||||||
|
ref: ^refs/tags/.*$
|
||||||
|
|
||||||
|
- job:
|
||||||
|
name: base
|
||||||
|
parent: null
|
||||||
|
|
||||||
|
- project:
|
||||||
|
name: project-config
|
||||||
|
tag:
|
||||||
|
jobs: []
|
||||||
|
|
||||||
|
- project:
|
||||||
|
name: org/project
|
||||||
|
tag:
|
||||||
|
jobs: []
|
|
@ -0,0 +1,8 @@
|
||||||
|
- tenant:
|
||||||
|
name: tenant-one
|
||||||
|
source:
|
||||||
|
gerrit:
|
||||||
|
config-projects:
|
||||||
|
- project-config
|
||||||
|
untrusted-projects:
|
||||||
|
- org/project
|
|
@ -157,6 +157,18 @@ class TestFinal(ZuulTestCase):
|
||||||
self.assertIn('Unable to modify final job', A.messages[0])
|
self.assertIn('Unable to modify final job', A.messages[0])
|
||||||
|
|
||||||
|
|
||||||
|
class TestBranchTag(ZuulTestCase):
|
||||||
|
tenant_config_file = 'config/branch-tag/main.yaml'
|
||||||
|
|
||||||
|
def test_negative_branch_match(self):
|
||||||
|
# Test that a negative branch matcher works with implied branches.
|
||||||
|
event = self.fake_gerrit.addFakeTag('org/project', 'master', 'foo')
|
||||||
|
self.fake_gerrit.addEvent(event)
|
||||||
|
self.waitUntilSettled()
|
||||||
|
self.assertHistory([
|
||||||
|
dict(name='test-job', result='SUCCESS', ref='refs/tags/foo')])
|
||||||
|
|
||||||
|
|
||||||
class TestBranchNegative(ZuulTestCase):
|
class TestBranchNegative(ZuulTestCase):
|
||||||
tenant_config_file = 'config/branch-negative/main.yaml'
|
tenant_config_file = 'config/branch-negative/main.yaml'
|
||||||
|
|
||||||
|
|
|
@ -69,6 +69,20 @@ class BranchMatcher(AbstractChangeMatcher):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
class ImpliedBranchMatcher(AbstractChangeMatcher):
|
||||||
|
"""
|
||||||
|
A branch matcher that only considers branch refs, and always
|
||||||
|
succeeds on other types (e.g., tags).
|
||||||
|
"""
|
||||||
|
|
||||||
|
def matches(self, change):
|
||||||
|
if hasattr(change, 'branch'):
|
||||||
|
if self.regex.match(change.branch):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
class FileMatcher(AbstractChangeMatcher):
|
class FileMatcher(AbstractChangeMatcher):
|
||||||
|
|
||||||
def matches(self, change):
|
def matches(self, change):
|
||||||
|
|
|
@ -853,20 +853,22 @@ class PipelineManager(object):
|
||||||
if dt:
|
if dt:
|
||||||
self.sched.statsd.timing(key + '.resident_time', dt)
|
self.sched.statsd.timing(key + '.resident_time', dt)
|
||||||
self.sched.statsd.incr(key + '.total_changes')
|
self.sched.statsd.incr(key + '.total_changes')
|
||||||
|
if hasattr(item.change, 'branch'):
|
||||||
hostname = (item.change.project.canonical_hostname.
|
hostname = (item.change.project.canonical_hostname.
|
||||||
replace('.', '_'))
|
replace('.', '_'))
|
||||||
projectname = (item.change.project.name.
|
projectname = (item.change.project.name.
|
||||||
replace('.', '_').replace('/', '.'))
|
replace('.', '_').replace('/', '.'))
|
||||||
projectname = projectname.replace('.', '_').replace('/', '.')
|
projectname = projectname.replace('.', '_').replace('/', '.')
|
||||||
branchname = item.change.branch.replace('.', '_').replace('/', '.')
|
branchname = item.change.branch.replace('.', '_').replace(
|
||||||
# stats.timers.zuul.tenant.<tenant>.pipeline.<pipeline>.
|
'/', '.')
|
||||||
# project.<host>.<project>.<branch>.resident_time
|
# stats.timers.zuul.tenant.<tenant>.pipeline.<pipeline>.
|
||||||
# stats_counts.zuul.tenant.<tenant>.pipeline.<pipeline>.
|
# project.<host>.<project>.<branch>.resident_time
|
||||||
# project.<host>.<project>.<branch>.total_changes
|
# stats_counts.zuul.tenant.<tenant>.pipeline.<pipeline>.
|
||||||
key += '.project.%s.%s.%s' % (hostname, projectname, branchname)
|
# project.<host>.<project>.<branch>.total_changes
|
||||||
if dt:
|
key += '.project.%s.%s.%s' % (hostname, projectname,
|
||||||
self.sched.statsd.timing(key + '.resident_time', dt)
|
branchname)
|
||||||
self.sched.statsd.incr(key + '.total_changes')
|
if dt:
|
||||||
|
self.sched.statsd.timing(key + '.resident_time', dt)
|
||||||
|
self.sched.statsd.incr(key + '.total_changes')
|
||||||
except Exception:
|
except Exception:
|
||||||
self.log.exception("Exception reporting pipeline stats")
|
self.log.exception("Exception reporting pipeline stats")
|
||||||
|
|
|
@ -963,10 +963,10 @@ class Job(object):
|
||||||
return None
|
return None
|
||||||
return m
|
return m
|
||||||
|
|
||||||
def addBranchMatcher(self, branch):
|
def addImpliedBranchMatcher(self, branch):
|
||||||
# Add a branch matcher that combines as a boolean *and* with
|
# Add a branch matcher that combines as a boolean *and* with
|
||||||
# existing branch matchers, if any.
|
# existing branch matchers, if any.
|
||||||
matchers = [change_matcher.BranchMatcher(branch)]
|
matchers = [change_matcher.ImpliedBranchMatcher(branch)]
|
||||||
if self.branch_matcher:
|
if self.branch_matcher:
|
||||||
matchers.append(self.branch_matcher)
|
matchers.append(self.branch_matcher)
|
||||||
self.branch_matcher = change_matcher.MatchAll(matchers)
|
self.branch_matcher = change_matcher.MatchAll(matchers)
|
||||||
|
@ -1121,26 +1121,8 @@ class JobList(object):
|
||||||
joblist = self.jobs.setdefault(jobname, [])
|
joblist = self.jobs.setdefault(jobname, [])
|
||||||
for job in jobs:
|
for job in jobs:
|
||||||
if implied_branch:
|
if implied_branch:
|
||||||
# If setting an implied branch and the current
|
job = job.copy()
|
||||||
# branch matcher is a simple match for a different
|
job.addImpliedBranchMatcher(implied_branch)
|
||||||
# branch, then simply do not add this job. If it
|
|
||||||
# is absent, set it to the implied branch.
|
|
||||||
# Otherwise, combine it with the implied branch to
|
|
||||||
# ensure that it still only affects this branch
|
|
||||||
# (whatever else it may do).
|
|
||||||
simple_branch = job.getSimpleBranchMatcher()
|
|
||||||
if simple_branch:
|
|
||||||
if not simple_branch.regex.match(implied_branch):
|
|
||||||
# This branch will never match, don't add it.
|
|
||||||
continue
|
|
||||||
if not simple_branch:
|
|
||||||
# The branch matcher could be complex, or
|
|
||||||
# missing. Add our implied matcher.
|
|
||||||
job = job.copy()
|
|
||||||
job.addBranchMatcher(implied_branch)
|
|
||||||
# Otherwise we have a simple branch matcher which
|
|
||||||
# is the same as our implied branch, the job can
|
|
||||||
# be added as-is.
|
|
||||||
if job not in joblist:
|
if job not in joblist:
|
||||||
joblist.append(job)
|
joblist.append(job)
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue