diff --git a/doc/source/tenants.rst b/doc/source/tenants.rst index a82031366a..1b0469ea6c 100644 --- a/doc/source/tenants.rst +++ b/doc/source/tenants.rst @@ -232,6 +232,45 @@ configuration. Some examples of tenant definitions are: It will not exclude a branch which already matched *include-branches*. + .. attr:: always-dynamic-branches + + A list of regular expressions matching branches which + should be treated as if every change newly proposes + dynamic Zuul configuration. In other words, the only time + Zuul will realize any configuration related to these + branches is during the time it is running jobs for a + proposed change. + + This is potentially useful for situations with large + numbers of rarely used feature branches, but comes at the + cost of a significant reduction in Zuul features for these + branches. + + Every regular expression listed here will also implicitly + be included in *exclude-branches*, therefore Zuul will not + load any static in-repo configuration from this branch. + These branches will not be available for use in overriding + checkouts of repos, nor will they be included in the git + repos that Zuul prepares for *required-projects* (unless + there is a change in the dependency tree for this branch). + + In particular, this means that the only jobs which can be + specified for these branches are pre-merge and gating jobs + (such as :term:`check` and :term:`gate`). No post-merge + or periodic jobs will run for these branches. + + Using this setting also incurs additional processing for + each change submitted for these branches as Zuul must + recalculate the configuration layout it uses for such a + change as if it included a change to a ``zuul.yaml`` file, + even if the change does not alter the configuration). + + With all these caveats in mind, this can be useful for + repos with large numbers of rarely used branches as it + allows Zuul to omit their configuration in most + circumstances and only calculate the configuration of a + single additional branch when it is used. + .. attr:: extra-config-paths Normally Zuul loads in-repo configuration from the first diff --git a/releasenotes/notes/always-dynamic-dce165ca8b6e212f.yaml b/releasenotes/notes/always-dynamic-dce165ca8b6e212f.yaml new file mode 100644 index 0000000000..0d0fe82260 --- /dev/null +++ b/releasenotes/notes/always-dynamic-dce165ca8b6e212f.yaml @@ -0,0 +1,10 @@ +--- +features: + - | + Added a new + :attr:`tenant.untrusted-projects..always-dynamic-branches` + tenant project configuration option. This may be used to specify + branches from which Zuul should never load static configuration + and instead treat every change as if it newly proposed dynamic + configuration. This is potentially useful for large numbers of + rarely-used feature branches. diff --git a/tests/fixtures/config/dynamic-only-project/dynamic.yaml b/tests/fixtures/config/dynamic-only-project/dynamic.yaml new file mode 100644 index 0000000000..d11451181b --- /dev/null +++ b/tests/fixtures/config/dynamic-only-project/dynamic.yaml @@ -0,0 +1,10 @@ +- tenant: + name: tenant-one + source: + gerrit: + config-projects: + - common-config + untrusted-projects: + - org/project: + always-dynamic-branches: + - "^feature/.*" diff --git a/tests/fixtures/config/dynamic-only-project/exclude.yaml b/tests/fixtures/config/dynamic-only-project/exclude.yaml new file mode 100644 index 0000000000..58a0b03f10 --- /dev/null +++ b/tests/fixtures/config/dynamic-only-project/exclude.yaml @@ -0,0 +1,10 @@ +- tenant: + name: tenant-one + source: + gerrit: + config-projects: + - common-config + untrusted-projects: + - org/project: + exclude-branches: + - "^feature/.*" diff --git a/tests/fixtures/config/dynamic-only-project/git/common-config/playbooks/run.yaml b/tests/fixtures/config/dynamic-only-project/git/common-config/playbooks/run.yaml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/tests/fixtures/config/dynamic-only-project/git/common-config/playbooks/run.yaml @@ -0,0 +1 @@ +--- diff --git a/tests/fixtures/config/dynamic-only-project/git/common-config/zuul.yaml b/tests/fixtures/config/dynamic-only-project/git/common-config/zuul.yaml new file mode 100644 index 0000000000..331abc54e3 --- /dev/null +++ b/tests/fixtures/config/dynamic-only-project/git/common-config/zuul.yaml @@ -0,0 +1,67 @@ +- pipeline: + name: check + manager: independent + trigger: + gerrit: + - event: patchset-created + - event: comment-added + comment: '^(Patch Set [0-9]+:\n\n)?(?i:recheck)$' + success: + gerrit: + Verified: 1 + failure: + gerrit: + Verified: -1 + +- pipeline: + name: gate + manager: dependent + success-message: Build succeeded (gate). + trigger: + gerrit: + - event: comment-added + approval: + - Approved: 1 + success: + gerrit: + Verified: 2 + submit: true + failure: + gerrit: + Verified: -2 + start: + gerrit: + Verified: 0 + precedence: high + +- pipeline: + name: post + manager: independent + trigger: + gerrit: + - event: ref-updated + ref: ^(?!refs/).*$ + precedence: low + +- job: + name: base + parent: null + run: playbooks/run.yaml + +- job: + name: central-test + +- job: + name: central-post + +- project: + name: "^org/project.*" + check: + jobs: + - central-test + gate: + jobs: + - central-test + post: + jobs: + - central-post diff --git a/tests/fixtures/config/dynamic-only-project/git/org_project/README b/tests/fixtures/config/dynamic-only-project/git/org_project/README new file mode 100644 index 0000000000..9daeafb986 --- /dev/null +++ b/tests/fixtures/config/dynamic-only-project/git/org_project/README @@ -0,0 +1 @@ +test diff --git a/tests/fixtures/config/dynamic-only-project/git/org_project/zuul.yaml b/tests/fixtures/config/dynamic-only-project/git/org_project/zuul.yaml new file mode 100644 index 0000000000..9712c08280 --- /dev/null +++ b/tests/fixtures/config/dynamic-only-project/git/org_project/zuul.yaml @@ -0,0 +1,17 @@ +- job: + name: project-test + +# Note: this job is not expected to run +- job: + name: project-post + +- project: + check: + jobs: + - project-test + gate: + jobs: + - project-test + post: + jobs: + - project-post diff --git a/tests/fixtures/config/dynamic-only-project/include.yaml b/tests/fixtures/config/dynamic-only-project/include.yaml new file mode 100644 index 0000000000..315eccc547 --- /dev/null +++ b/tests/fixtures/config/dynamic-only-project/include.yaml @@ -0,0 +1,11 @@ +- tenant: + name: tenant-one + source: + gerrit: + config-projects: + - common-config + untrusted-projects: + - org/project: + include-branches: + - master + - stable diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py index 37a9c092fa..24943b4ecb 100644 --- a/tests/unit/test_v3.py +++ b/tests/unit/test_v3.py @@ -8048,3 +8048,120 @@ class TestConnectionVars(AnsibleZuulTestCase): # job_output = self._get_file(job, 'work/logs/job-output.txt') # self.log.debug(job_output) # self.assertNotIn("/bin/du", job_output) + + +class IncludeBranchesTestCase(ZuulTestCase): + def _test_include_branches(self, history1, history2, history3, history4): + self.create_branch('org/project', 'stable') + self.create_branch('org/project', 'feature/foo') + self.fake_gerrit.addEvent( + self.fake_gerrit.getFakeBranchCreatedEvent( + 'org/project', 'stable')) + self.fake_gerrit.addEvent( + self.fake_gerrit.getFakeBranchCreatedEvent( + 'org/project', 'feature/foo')) + self.waitUntilSettled() + + # Test the jobs on the master branch. + A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + self.assertHistory(history1, ordered=False) + + # Test the jobs on the excluded feature branch. + B = self.fake_gerrit.addFakeChange('org/project', 'feature/foo', 'A') + self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + self.assertHistory(history1 + history2, ordered=False) + + # Test in-repo config proposed on the excluded feature branch. + conf = textwrap.dedent( + """ + - job: + name: project-dynamic + + - project: + check: + jobs: + - project-dynamic + """) + file_dict = {'zuul.yaml': conf} + C = self.fake_gerrit.addFakeChange('org/project', 'feature/foo', 'A', + files=file_dict) + self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + self.assertHistory(history1 + history2 + history3, ordered=False) + + # Merge a change to the excluded feature branch. + B.addApproval('Code-Review', 2) + self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) + self.waitUntilSettled() + self.assertEqual(B.data['status'], 'MERGED') + self.assertHistory(history1 + history2 + history3 + history4, + ordered=False) + + +class TestIncludeBranchesProject(IncludeBranchesTestCase): + tenant_config_file = 'config/dynamic-only-project/include.yaml' + + def test_include_branches(self): + history1 = [ + dict(name='central-test', result='SUCCESS', changes='1,1'), + dict(name='project-test', result='SUCCESS', changes='1,1'), + ] + history2 = [ + dict(name='central-test', result='SUCCESS', changes='2,1'), + ] + history3 = [ + dict(name='central-test', result='SUCCESS', changes='3,1'), + ] + history4 = [ + dict(name='central-test', result='SUCCESS', changes='2,1'), + ] + self._test_include_branches(history1, history2, history3, history4) + + +class TestExcludeBranchesProject(IncludeBranchesTestCase): + tenant_config_file = 'config/dynamic-only-project/exclude.yaml' + + def test_exclude_branches(self): + history1 = [ + dict(name='central-test', result='SUCCESS', changes='1,1'), + dict(name='project-test', result='SUCCESS', changes='1,1'), + ] + history2 = [ + dict(name='central-test', result='SUCCESS', changes='2,1'), + ] + history3 = [ + dict(name='central-test', result='SUCCESS', changes='3,1'), + ] + history4 = [ + dict(name='central-test', result='SUCCESS', changes='2,1'), + ] + self._test_include_branches(history1, history2, history3, history4) + + +class TestDynamicBranchesProject(IncludeBranchesTestCase): + tenant_config_file = 'config/dynamic-only-project/dynamic.yaml' + + def test_dynamic_branches(self): + history1 = [ + dict(name='central-test', result='SUCCESS', changes='1,1'), + dict(name='project-test', result='SUCCESS', changes='1,1'), + ] + history2 = [ + dict(name='central-test', result='SUCCESS', changes='2,1'), + dict(name='project-test', result='SUCCESS', changes='2,1'), + ] + history3 = [ + dict(name='central-test', result='SUCCESS', changes='3,1'), + dict(name='project-dynamic', result='SUCCESS', changes='3,1'), + ] + history4 = [ + dict(name='central-test', result='SUCCESS', changes='2,1'), + dict(name='project-test', result='SUCCESS', changes='2,1'), + ] + self._test_include_branches(history1, history2, history3, history4) diff --git a/zuul/configloader.py b/zuul/configloader.py index b83a87acb2..70eb8d35ce 100644 --- a/zuul/configloader.py +++ b/zuul/configloader.py @@ -1518,6 +1518,7 @@ class TenantParser(object): 'load-branch': str, 'include-branches': to_list(str), 'exclude-branches': to_list(str), + 'always-dynamic-branches': to_list(str), 'allow-circular-dependencies': bool, }} @@ -1700,11 +1701,18 @@ class TenantParser(object): min_ltime = -1 branches = sorted(tpc.project.source.getProjectBranches( tpc.project, tenant, min_ltime)) - branches = [b for b in branches if tpc.includesBranch(b)] if 'master' in branches: branches.remove('master') branches = ['master'] + branches - tpc.branches = branches + static_branches = [] + always_dynamic_branches = [] + for b in branches: + if tpc.includesBranch(b): + static_branches.append(b) + elif tpc.isAlwaysDynamicBranch(b): + always_dynamic_branches.append(b) + tpc.branches = static_branches + tpc.dynamic_branches = always_dynamic_branches def _loadProjectKeys(self, connection_name, project): project.private_secrets_key, project.public_secrets_key = ( @@ -1730,6 +1738,7 @@ class TenantParser(object): project_exclude_unprotected_branches = None project_include_branches = None project_exclude_branches = None + project_always_dynamic_branches = None project_load_branch = None else: project_name = list(conf.keys())[0] @@ -1754,12 +1763,28 @@ class TenantParser(object): project_include_branches = [ re.compile(b) for b in as_list(project_include_branches) ] - project_exclude_branches = conf[project_name].get( + exclude_branches = conf[project_name].get( 'exclude-branches', None) - if project_exclude_branches is not None: + if exclude_branches is not None: project_exclude_branches = [ - re.compile(b) for b in as_list(project_exclude_branches) + re.compile(b) for b in as_list(exclude_branches) ] + else: + project_exclude_branches = None + always_dynamic_branches = conf[project_name].get( + 'always-dynamic-branches', None) + if always_dynamic_branches is not None: + if project_exclude_branches is None: + project_exclude_branches = [] + exclude_branches = [] + project_always_dynamic_branches = [] + for b in always_dynamic_branches: + rb = re.compile(b) + if b not in exclude_branches: + project_exclude_branches.append(rb) + project_always_dynamic_branches.append(rb) + else: + project_always_dynamic_branches = None if conf[project_name].get('extra-config-paths') is not None: extra_config_paths = as_list( conf[project_name]['extra-config-paths']) @@ -1777,6 +1802,8 @@ class TenantParser(object): project_exclude_unprotected_branches tenant_project_config.include_branches = project_include_branches tenant_project_config.exclude_branches = project_exclude_branches + tenant_project_config.always_dynamic_branches = \ + project_always_dynamic_branches tenant_project_config.extra_config_files = extra_config_files tenant_project_config.extra_config_dirs = extra_config_dirs tenant_project_config.load_branch = project_load_branch @@ -2577,7 +2604,8 @@ class ConfigLoader(object): else: # Use the cached branch list; since this is a dynamic # reconfiguration there should not be any branch changes. - branches = tenant.getProjectBranches(project.canonical_name) + branches = tenant.getProjectBranches(project.canonical_name, + include_always_dynamic=True) for branch in branches: fns1 = [] diff --git a/zuul/model.py b/zuul/model.py index 50482138db..aaa324251f 100644 --- a/zuul/model.py +++ b/zuul/model.py @@ -5387,6 +5387,9 @@ class Ref(object): tpc = tenant.project_configs.get(self.project.canonical_name) if tpc is None: return False + if hasattr(self, 'branch'): + if tpc.isAlwaysDynamicBranch(self.branch): + return True if self.files is None: # If self.files is None we don't know if this change updates the # config so assume it does as this is a safe default if we don't @@ -6309,11 +6312,13 @@ class TenantProjectConfig(object): self.load_classes = set() self.shadow_projects = set() self.branches = [] + self.dynamic_branches = [] # The tenant's default setting of exclude_unprotected_branches will # be overridden by this one if not None. self.exclude_unprotected_branches = None self.include_branches = None self.exclude_branches = None + self.always_dynamic_branches = None self.parsed_branch_config = {} # branch -> ParsedConfig # The list of paths to look for extra zuul config files self.extra_config_files = () @@ -6322,6 +6327,13 @@ class TenantProjectConfig(object): # Load config from a different branch if this is a config project self.load_branch = None + def isAlwaysDynamicBranch(self, branch): + if self.always_dynamic_branches is None: + return False + for r in self.always_dynamic_branches: + if r.fullmatch(branch): + return True + def includesBranch(self, branch): if self.include_branches is not None: included = False @@ -7574,16 +7586,21 @@ class Tenant(object): (project,)) return result - def getProjectBranches(self, project_canonical_name): + def getProjectBranches(self, project_canonical_name, + include_always_dynamic=False): """Return a project's branches (filtered by this tenant config) :arg str project_canonical: The project's canonical name. + :arg bool include_always_dynamic: Whether to include + always-dynamic-branches :returns: A list of branch names. :rtype: [str] """ tpc = self.project_configs[project_canonical_name] + if include_always_dynamic: + return tpc.branches + tpc.dynamic_branches return tpc.branches def getExcludeUnprotectedBranches(self, project):