Merge "Add always-dynamic-branches option"
This commit is contained in:
commit
2f2e0ce28c
|
@ -232,6 +232,45 @@ configuration. Some examples of tenant definitions are:
|
|||
It will not exclude a branch which already matched
|
||||
*include-branches*.
|
||||
|
||||
.. attr:: always-dynamic-branches
|
||||
|
||||
A list of regular expressions matching branches which
|
||||
should be treated as if every change newly proposes
|
||||
dynamic Zuul configuration. In other words, the only time
|
||||
Zuul will realize any configuration related to these
|
||||
branches is during the time it is running jobs for a
|
||||
proposed change.
|
||||
|
||||
This is potentially useful for situations with large
|
||||
numbers of rarely used feature branches, but comes at the
|
||||
cost of a significant reduction in Zuul features for these
|
||||
branches.
|
||||
|
||||
Every regular expression listed here will also implicitly
|
||||
be included in *exclude-branches*, therefore Zuul will not
|
||||
load any static in-repo configuration from this branch.
|
||||
These branches will not be available for use in overriding
|
||||
checkouts of repos, nor will they be included in the git
|
||||
repos that Zuul prepares for *required-projects* (unless
|
||||
there is a change in the dependency tree for this branch).
|
||||
|
||||
In particular, this means that the only jobs which can be
|
||||
specified for these branches are pre-merge and gating jobs
|
||||
(such as :term:`check` and :term:`gate`). No post-merge
|
||||
or periodic jobs will run for these branches.
|
||||
|
||||
Using this setting also incurs additional processing for
|
||||
each change submitted for these branches as Zuul must
|
||||
recalculate the configuration layout it uses for such a
|
||||
change as if it included a change to a ``zuul.yaml`` file,
|
||||
even if the change does not alter the configuration).
|
||||
|
||||
With all these caveats in mind, this can be useful for
|
||||
repos with large numbers of rarely used branches as it
|
||||
allows Zuul to omit their configuration in most
|
||||
circumstances and only calculate the configuration of a
|
||||
single additional branch when it is used.
|
||||
|
||||
.. attr:: extra-config-paths
|
||||
|
||||
Normally Zuul loads in-repo configuration from the first
|
||||
|
|
|
@ -0,0 +1,10 @@
|
|||
---
|
||||
features:
|
||||
- |
|
||||
Added a new
|
||||
:attr:`tenant.untrusted-projects.<project>.always-dynamic-branches`
|
||||
tenant project configuration option. This may be used to specify
|
||||
branches from which Zuul should never load static configuration
|
||||
and instead treat every change as if it newly proposed dynamic
|
||||
configuration. This is potentially useful for large numbers of
|
||||
rarely-used feature branches.
|
|
@ -0,0 +1,10 @@
|
|||
- tenant:
|
||||
name: tenant-one
|
||||
source:
|
||||
gerrit:
|
||||
config-projects:
|
||||
- common-config
|
||||
untrusted-projects:
|
||||
- org/project:
|
||||
always-dynamic-branches:
|
||||
- "^feature/.*"
|
|
@ -0,0 +1,10 @@
|
|||
- tenant:
|
||||
name: tenant-one
|
||||
source:
|
||||
gerrit:
|
||||
config-projects:
|
||||
- common-config
|
||||
untrusted-projects:
|
||||
- org/project:
|
||||
exclude-branches:
|
||||
- "^feature/.*"
|
1
tests/fixtures/config/dynamic-only-project/git/common-config/playbooks/run.yaml
vendored
Normal file
1
tests/fixtures/config/dynamic-only-project/git/common-config/playbooks/run.yaml
vendored
Normal file
|
@ -0,0 +1 @@
|
|||
---
|
|
@ -0,0 +1,67 @@
|
|||
- pipeline:
|
||||
name: check
|
||||
manager: independent
|
||||
trigger:
|
||||
gerrit:
|
||||
- event: patchset-created
|
||||
- event: comment-added
|
||||
comment: '^(Patch Set [0-9]+:\n\n)?(?i:recheck)$'
|
||||
success:
|
||||
gerrit:
|
||||
Verified: 1
|
||||
failure:
|
||||
gerrit:
|
||||
Verified: -1
|
||||
|
||||
- pipeline:
|
||||
name: gate
|
||||
manager: dependent
|
||||
success-message: Build succeeded (gate).
|
||||
trigger:
|
||||
gerrit:
|
||||
- event: comment-added
|
||||
approval:
|
||||
- Approved: 1
|
||||
success:
|
||||
gerrit:
|
||||
Verified: 2
|
||||
submit: true
|
||||
failure:
|
||||
gerrit:
|
||||
Verified: -2
|
||||
start:
|
||||
gerrit:
|
||||
Verified: 0
|
||||
precedence: high
|
||||
|
||||
- pipeline:
|
||||
name: post
|
||||
manager: independent
|
||||
trigger:
|
||||
gerrit:
|
||||
- event: ref-updated
|
||||
ref: ^(?!refs/).*$
|
||||
precedence: low
|
||||
|
||||
- job:
|
||||
name: base
|
||||
parent: null
|
||||
run: playbooks/run.yaml
|
||||
|
||||
- job:
|
||||
name: central-test
|
||||
|
||||
- job:
|
||||
name: central-post
|
||||
|
||||
- project:
|
||||
name: "^org/project.*"
|
||||
check:
|
||||
jobs:
|
||||
- central-test
|
||||
gate:
|
||||
jobs:
|
||||
- central-test
|
||||
post:
|
||||
jobs:
|
||||
- central-post
|
|
@ -0,0 +1 @@
|
|||
test
|
|
@ -0,0 +1,17 @@
|
|||
- job:
|
||||
name: project-test
|
||||
|
||||
# Note: this job is not expected to run
|
||||
- job:
|
||||
name: project-post
|
||||
|
||||
- project:
|
||||
check:
|
||||
jobs:
|
||||
- project-test
|
||||
gate:
|
||||
jobs:
|
||||
- project-test
|
||||
post:
|
||||
jobs:
|
||||
- project-post
|
|
@ -0,0 +1,11 @@
|
|||
- tenant:
|
||||
name: tenant-one
|
||||
source:
|
||||
gerrit:
|
||||
config-projects:
|
||||
- common-config
|
||||
untrusted-projects:
|
||||
- org/project:
|
||||
include-branches:
|
||||
- master
|
||||
- stable
|
|
@ -8048,3 +8048,120 @@ class TestConnectionVars(AnsibleZuulTestCase):
|
|||
# job_output = self._get_file(job, 'work/logs/job-output.txt')
|
||||
# self.log.debug(job_output)
|
||||
# self.assertNotIn("/bin/du", job_output)
|
||||
|
||||
|
||||
class IncludeBranchesTestCase(ZuulTestCase):
|
||||
def _test_include_branches(self, history1, history2, history3, history4):
|
||||
self.create_branch('org/project', 'stable')
|
||||
self.create_branch('org/project', 'feature/foo')
|
||||
self.fake_gerrit.addEvent(
|
||||
self.fake_gerrit.getFakeBranchCreatedEvent(
|
||||
'org/project', 'stable'))
|
||||
self.fake_gerrit.addEvent(
|
||||
self.fake_gerrit.getFakeBranchCreatedEvent(
|
||||
'org/project', 'feature/foo'))
|
||||
self.waitUntilSettled()
|
||||
|
||||
# Test the jobs on the master branch.
|
||||
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
|
||||
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
|
||||
self.waitUntilSettled()
|
||||
|
||||
self.assertHistory(history1, ordered=False)
|
||||
|
||||
# Test the jobs on the excluded feature branch.
|
||||
B = self.fake_gerrit.addFakeChange('org/project', 'feature/foo', 'A')
|
||||
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
|
||||
self.waitUntilSettled()
|
||||
|
||||
self.assertHistory(history1 + history2, ordered=False)
|
||||
|
||||
# Test in-repo config proposed on the excluded feature branch.
|
||||
conf = textwrap.dedent(
|
||||
"""
|
||||
- job:
|
||||
name: project-dynamic
|
||||
|
||||
- project:
|
||||
check:
|
||||
jobs:
|
||||
- project-dynamic
|
||||
""")
|
||||
file_dict = {'zuul.yaml': conf}
|
||||
C = self.fake_gerrit.addFakeChange('org/project', 'feature/foo', 'A',
|
||||
files=file_dict)
|
||||
self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
|
||||
self.waitUntilSettled()
|
||||
|
||||
self.assertHistory(history1 + history2 + history3, ordered=False)
|
||||
|
||||
# Merge a change to the excluded feature branch.
|
||||
B.addApproval('Code-Review', 2)
|
||||
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
|
||||
self.waitUntilSettled()
|
||||
self.assertEqual(B.data['status'], 'MERGED')
|
||||
self.assertHistory(history1 + history2 + history3 + history4,
|
||||
ordered=False)
|
||||
|
||||
|
||||
class TestIncludeBranchesProject(IncludeBranchesTestCase):
|
||||
tenant_config_file = 'config/dynamic-only-project/include.yaml'
|
||||
|
||||
def test_include_branches(self):
|
||||
history1 = [
|
||||
dict(name='central-test', result='SUCCESS', changes='1,1'),
|
||||
dict(name='project-test', result='SUCCESS', changes='1,1'),
|
||||
]
|
||||
history2 = [
|
||||
dict(name='central-test', result='SUCCESS', changes='2,1'),
|
||||
]
|
||||
history3 = [
|
||||
dict(name='central-test', result='SUCCESS', changes='3,1'),
|
||||
]
|
||||
history4 = [
|
||||
dict(name='central-test', result='SUCCESS', changes='2,1'),
|
||||
]
|
||||
self._test_include_branches(history1, history2, history3, history4)
|
||||
|
||||
|
||||
class TestExcludeBranchesProject(IncludeBranchesTestCase):
|
||||
tenant_config_file = 'config/dynamic-only-project/exclude.yaml'
|
||||
|
||||
def test_exclude_branches(self):
|
||||
history1 = [
|
||||
dict(name='central-test', result='SUCCESS', changes='1,1'),
|
||||
dict(name='project-test', result='SUCCESS', changes='1,1'),
|
||||
]
|
||||
history2 = [
|
||||
dict(name='central-test', result='SUCCESS', changes='2,1'),
|
||||
]
|
||||
history3 = [
|
||||
dict(name='central-test', result='SUCCESS', changes='3,1'),
|
||||
]
|
||||
history4 = [
|
||||
dict(name='central-test', result='SUCCESS', changes='2,1'),
|
||||
]
|
||||
self._test_include_branches(history1, history2, history3, history4)
|
||||
|
||||
|
||||
class TestDynamicBranchesProject(IncludeBranchesTestCase):
|
||||
tenant_config_file = 'config/dynamic-only-project/dynamic.yaml'
|
||||
|
||||
def test_dynamic_branches(self):
|
||||
history1 = [
|
||||
dict(name='central-test', result='SUCCESS', changes='1,1'),
|
||||
dict(name='project-test', result='SUCCESS', changes='1,1'),
|
||||
]
|
||||
history2 = [
|
||||
dict(name='central-test', result='SUCCESS', changes='2,1'),
|
||||
dict(name='project-test', result='SUCCESS', changes='2,1'),
|
||||
]
|
||||
history3 = [
|
||||
dict(name='central-test', result='SUCCESS', changes='3,1'),
|
||||
dict(name='project-dynamic', result='SUCCESS', changes='3,1'),
|
||||
]
|
||||
history4 = [
|
||||
dict(name='central-test', result='SUCCESS', changes='2,1'),
|
||||
dict(name='project-test', result='SUCCESS', changes='2,1'),
|
||||
]
|
||||
self._test_include_branches(history1, history2, history3, history4)
|
||||
|
|
|
@ -1554,6 +1554,7 @@ class TenantParser(object):
|
|||
'load-branch': str,
|
||||
'include-branches': to_list(str),
|
||||
'exclude-branches': to_list(str),
|
||||
'always-dynamic-branches': to_list(str),
|
||||
'allow-circular-dependencies': bool,
|
||||
}}
|
||||
|
||||
|
@ -1742,11 +1743,18 @@ class TenantParser(object):
|
|||
min_ltime = -1
|
||||
branches = sorted(tpc.project.source.getProjectBranches(
|
||||
tpc.project, tenant, min_ltime))
|
||||
branches = [b for b in branches if tpc.includesBranch(b)]
|
||||
if 'master' in branches:
|
||||
branches.remove('master')
|
||||
branches = ['master'] + branches
|
||||
tpc.branches = branches
|
||||
static_branches = []
|
||||
always_dynamic_branches = []
|
||||
for b in branches:
|
||||
if tpc.includesBranch(b):
|
||||
static_branches.append(b)
|
||||
elif tpc.isAlwaysDynamicBranch(b):
|
||||
always_dynamic_branches.append(b)
|
||||
tpc.branches = static_branches
|
||||
tpc.dynamic_branches = always_dynamic_branches
|
||||
|
||||
def _loadProjectKeys(self, connection_name, project):
|
||||
project.private_secrets_key, project.public_secrets_key = (
|
||||
|
@ -1772,6 +1780,7 @@ class TenantParser(object):
|
|||
project_exclude_unprotected_branches = None
|
||||
project_include_branches = None
|
||||
project_exclude_branches = None
|
||||
project_always_dynamic_branches = None
|
||||
project_load_branch = None
|
||||
else:
|
||||
project_name = list(conf.keys())[0]
|
||||
|
@ -1796,12 +1805,28 @@ class TenantParser(object):
|
|||
project_include_branches = [
|
||||
re.compile(b) for b in as_list(project_include_branches)
|
||||
]
|
||||
project_exclude_branches = conf[project_name].get(
|
||||
exclude_branches = conf[project_name].get(
|
||||
'exclude-branches', None)
|
||||
if project_exclude_branches is not None:
|
||||
if exclude_branches is not None:
|
||||
project_exclude_branches = [
|
||||
re.compile(b) for b in as_list(project_exclude_branches)
|
||||
re.compile(b) for b in as_list(exclude_branches)
|
||||
]
|
||||
else:
|
||||
project_exclude_branches = None
|
||||
always_dynamic_branches = conf[project_name].get(
|
||||
'always-dynamic-branches', None)
|
||||
if always_dynamic_branches is not None:
|
||||
if project_exclude_branches is None:
|
||||
project_exclude_branches = []
|
||||
exclude_branches = []
|
||||
project_always_dynamic_branches = []
|
||||
for b in always_dynamic_branches:
|
||||
rb = re.compile(b)
|
||||
if b not in exclude_branches:
|
||||
project_exclude_branches.append(rb)
|
||||
project_always_dynamic_branches.append(rb)
|
||||
else:
|
||||
project_always_dynamic_branches = None
|
||||
if conf[project_name].get('extra-config-paths') is not None:
|
||||
extra_config_paths = as_list(
|
||||
conf[project_name]['extra-config-paths'])
|
||||
|
@ -1819,6 +1844,8 @@ class TenantParser(object):
|
|||
project_exclude_unprotected_branches
|
||||
tenant_project_config.include_branches = project_include_branches
|
||||
tenant_project_config.exclude_branches = project_exclude_branches
|
||||
tenant_project_config.always_dynamic_branches = \
|
||||
project_always_dynamic_branches
|
||||
tenant_project_config.extra_config_files = extra_config_files
|
||||
tenant_project_config.extra_config_dirs = extra_config_dirs
|
||||
tenant_project_config.load_branch = project_load_branch
|
||||
|
@ -2631,7 +2658,8 @@ class ConfigLoader(object):
|
|||
else:
|
||||
# Use the cached branch list; since this is a dynamic
|
||||
# reconfiguration there should not be any branch changes.
|
||||
branches = tenant.getProjectBranches(project.canonical_name)
|
||||
branches = tenant.getProjectBranches(project.canonical_name,
|
||||
include_always_dynamic=True)
|
||||
|
||||
for branch in branches:
|
||||
fns1 = []
|
||||
|
|
|
@ -5544,6 +5544,9 @@ class Ref(object):
|
|||
tpc = tenant.project_configs.get(self.project.canonical_name)
|
||||
if tpc is None:
|
||||
return False
|
||||
if hasattr(self, 'branch'):
|
||||
if tpc.isAlwaysDynamicBranch(self.branch):
|
||||
return True
|
||||
if self.files is None:
|
||||
# If self.files is None we don't know if this change updates the
|
||||
# config so assume it does as this is a safe default if we don't
|
||||
|
@ -6466,11 +6469,13 @@ class TenantProjectConfig(object):
|
|||
self.load_classes = set()
|
||||
self.shadow_projects = set()
|
||||
self.branches = []
|
||||
self.dynamic_branches = []
|
||||
# The tenant's default setting of exclude_unprotected_branches will
|
||||
# be overridden by this one if not None.
|
||||
self.exclude_unprotected_branches = None
|
||||
self.include_branches = None
|
||||
self.exclude_branches = None
|
||||
self.always_dynamic_branches = None
|
||||
self.parsed_branch_config = {} # branch -> ParsedConfig
|
||||
# The list of paths to look for extra zuul config files
|
||||
self.extra_config_files = ()
|
||||
|
@ -6479,6 +6484,13 @@ class TenantProjectConfig(object):
|
|||
# Load config from a different branch if this is a config project
|
||||
self.load_branch = None
|
||||
|
||||
def isAlwaysDynamicBranch(self, branch):
|
||||
if self.always_dynamic_branches is None:
|
||||
return False
|
||||
for r in self.always_dynamic_branches:
|
||||
if r.fullmatch(branch):
|
||||
return True
|
||||
|
||||
def includesBranch(self, branch):
|
||||
if self.include_branches is not None:
|
||||
included = False
|
||||
|
@ -7755,16 +7767,21 @@ class Tenant(object):
|
|||
(project,))
|
||||
return result
|
||||
|
||||
def getProjectBranches(self, project_canonical_name):
|
||||
def getProjectBranches(self, project_canonical_name,
|
||||
include_always_dynamic=False):
|
||||
"""Return a project's branches (filtered by this tenant config)
|
||||
|
||||
:arg str project_canonical: The project's canonical name.
|
||||
:arg bool include_always_dynamic: Whether to include
|
||||
always-dynamic-branches
|
||||
|
||||
:returns: A list of branch names.
|
||||
:rtype: [str]
|
||||
|
||||
"""
|
||||
tpc = self.project_configs[project_canonical_name]
|
||||
if include_always_dynamic:
|
||||
return tpc.branches + tpc.dynamic_branches
|
||||
return tpc.branches
|
||||
|
||||
def getExcludeUnprotectedBranches(self, project):
|
||||
|
|
Loading…
Reference in New Issue