Fix implied branch matchers and tags

Adding an implied branch matcher to jobs on in-repo project defs
works great when an item *has* a branch.  But some items, such as
tags, don't.  With recent changes, it is now impossible for a
project to add a job in-repo that runs in a tag pipeline.

To correct this, we need to drop some of the optimizations which
assumed we could match the implied branch against existing branch
matchers, and instead, when adding a job in-repo, simply add a new
kind of branch matcher, an ImpliedBranchMatcher, that is evaluated
in a boolean 'and' with any existing branch matchers.

The ImpliedBranchMatcher only fails if the item has a branch, and
the branch doesn't match.  If the item doesn't have a branch, it
always succeeds.

This means that when a project adds a job to a tag pipeline in-repo,
it will most likely only have the ImpliedBranchMatcher, which will
simply succeed.

It also means that the multiple project configurations present in
the project's multiple branches can all add jobs to tag pipelines,
and so to remove such a job, changes may need to be made to all
branches of a project.  However, there's not much that can be done
about that at the moment.

Change-Id: Id51ddfce7ef0a6d5e3273da784e407ac72a669db
This commit is contained in:
James E. Blair 2017-12-01 15:54:24 -08:00
parent 9ab8db4e13
commit 1edfd97943
10 changed files with 111 additions and 37 deletions

View File

@ -486,6 +486,29 @@ class FakeGerritConnection(gerritconnection.GerritConnection):
self.changes[self.change_number] = c self.changes[self.change_number] = c
return c return c
def addFakeTag(self, project, branch, tag):
path = os.path.join(self.upstream_root, project)
repo = git.Repo(path)
commit = repo.heads[branch].commit
newrev = commit.hexsha
ref = 'refs/tags/' + tag
git.Tag.create(repo, tag, commit)
event = {
"type": "ref-updated",
"submitter": {
"name": "User Name",
},
"refUpdate": {
"oldRev": 40 * '0',
"newRev": newrev,
"refName": ref,
"project": project,
}
}
return event
def getFakeBranchCreatedEvent(self, project, branch): def getFakeBranchCreatedEvent(self, project, branch):
path = os.path.join(self.upstream_root, project) path = os.path.join(self.upstream_root, project)
repo = git.Repo(path) repo = git.Repo(path)

View File

@ -0,0 +1,9 @@
- job:
name: test-job
run: playbooks/test-job.yaml
- project:
name: org/project
tag:
jobs:
- test-job

View File

@ -0,0 +1 @@
test

View File

@ -0,0 +1,2 @@
- hosts: all
tasks: []

View File

@ -0,0 +1,21 @@
- pipeline:
name: tag
manager: independent
trigger:
gerrit:
- event: ref-updated
ref: ^refs/tags/.*$
- job:
name: base
parent: null
- project:
name: project-config
tag:
jobs: []
- project:
name: org/project
tag:
jobs: []

View File

@ -0,0 +1,8 @@
- tenant:
name: tenant-one
source:
gerrit:
config-projects:
- project-config
untrusted-projects:
- org/project

View File

@ -157,6 +157,18 @@ class TestFinal(ZuulTestCase):
self.assertIn('Unable to modify final job', A.messages[0]) self.assertIn('Unable to modify final job', A.messages[0])
class TestBranchTag(ZuulTestCase):
tenant_config_file = 'config/branch-tag/main.yaml'
def test_negative_branch_match(self):
# Test that a negative branch matcher works with implied branches.
event = self.fake_gerrit.addFakeTag('org/project', 'master', 'foo')
self.fake_gerrit.addEvent(event)
self.waitUntilSettled()
self.assertHistory([
dict(name='test-job', result='SUCCESS', ref='refs/tags/foo')])
class TestBranchNegative(ZuulTestCase): class TestBranchNegative(ZuulTestCase):
tenant_config_file = 'config/branch-negative/main.yaml' tenant_config_file = 'config/branch-negative/main.yaml'

View File

@ -69,6 +69,20 @@ class BranchMatcher(AbstractChangeMatcher):
return False return False
class ImpliedBranchMatcher(AbstractChangeMatcher):
"""
A branch matcher that only considers branch refs, and always
succeeds on other types (e.g., tags).
"""
def matches(self, change):
if hasattr(change, 'branch'):
if self.regex.match(change.branch):
return True
return False
return True
class FileMatcher(AbstractChangeMatcher): class FileMatcher(AbstractChangeMatcher):
def matches(self, change): def matches(self, change):

View File

@ -853,20 +853,22 @@ class PipelineManager(object):
if dt: if dt:
self.sched.statsd.timing(key + '.resident_time', dt) self.sched.statsd.timing(key + '.resident_time', dt)
self.sched.statsd.incr(key + '.total_changes') self.sched.statsd.incr(key + '.total_changes')
if hasattr(item.change, 'branch'):
hostname = (item.change.project.canonical_hostname. hostname = (item.change.project.canonical_hostname.
replace('.', '_')) replace('.', '_'))
projectname = (item.change.project.name. projectname = (item.change.project.name.
replace('.', '_').replace('/', '.')) replace('.', '_').replace('/', '.'))
projectname = projectname.replace('.', '_').replace('/', '.') projectname = projectname.replace('.', '_').replace('/', '.')
branchname = item.change.branch.replace('.', '_').replace('/', '.') branchname = item.change.branch.replace('.', '_').replace(
# stats.timers.zuul.tenant.<tenant>.pipeline.<pipeline>. '/', '.')
# project.<host>.<project>.<branch>.resident_time # stats.timers.zuul.tenant.<tenant>.pipeline.<pipeline>.
# stats_counts.zuul.tenant.<tenant>.pipeline.<pipeline>. # project.<host>.<project>.<branch>.resident_time
# project.<host>.<project>.<branch>.total_changes # stats_counts.zuul.tenant.<tenant>.pipeline.<pipeline>.
key += '.project.%s.%s.%s' % (hostname, projectname, branchname) # project.<host>.<project>.<branch>.total_changes
if dt: key += '.project.%s.%s.%s' % (hostname, projectname,
self.sched.statsd.timing(key + '.resident_time', dt) branchname)
self.sched.statsd.incr(key + '.total_changes') if dt:
self.sched.statsd.timing(key + '.resident_time', dt)
self.sched.statsd.incr(key + '.total_changes')
except Exception: except Exception:
self.log.exception("Exception reporting pipeline stats") self.log.exception("Exception reporting pipeline stats")

View File

@ -963,10 +963,10 @@ class Job(object):
return None return None
return m return m
def addBranchMatcher(self, branch): def addImpliedBranchMatcher(self, branch):
# Add a branch matcher that combines as a boolean *and* with # Add a branch matcher that combines as a boolean *and* with
# existing branch matchers, if any. # existing branch matchers, if any.
matchers = [change_matcher.BranchMatcher(branch)] matchers = [change_matcher.ImpliedBranchMatcher(branch)]
if self.branch_matcher: if self.branch_matcher:
matchers.append(self.branch_matcher) matchers.append(self.branch_matcher)
self.branch_matcher = change_matcher.MatchAll(matchers) self.branch_matcher = change_matcher.MatchAll(matchers)
@ -1121,26 +1121,8 @@ class JobList(object):
joblist = self.jobs.setdefault(jobname, []) joblist = self.jobs.setdefault(jobname, [])
for job in jobs: for job in jobs:
if implied_branch: if implied_branch:
# If setting an implied branch and the current job = job.copy()
# branch matcher is a simple match for a different job.addImpliedBranchMatcher(implied_branch)
# branch, then simply do not add this job. If it
# is absent, set it to the implied branch.
# Otherwise, combine it with the implied branch to
# ensure that it still only affects this branch
# (whatever else it may do).
simple_branch = job.getSimpleBranchMatcher()
if simple_branch:
if not simple_branch.regex.match(implied_branch):
# This branch will never match, don't add it.
continue
if not simple_branch:
# The branch matcher could be complex, or
# missing. Add our implied matcher.
job = job.copy()
job.addBranchMatcher(implied_branch)
# Otherwise we have a simple branch matcher which
# is the same as our implied branch, the job can
# be added as-is.
if job not in joblist: if job not in joblist:
joblist.append(job) joblist.append(job)