diff --git a/doc/source/config/project.rst b/doc/source/config/project.rst index e61571781b..e7b809af75 100644 --- a/doc/source/config/project.rst +++ b/doc/source/config/project.rst @@ -15,12 +15,9 @@ Multiple project definitions may appear for the same project (for example, in a central :term:`config projects ` as well as in a repo's own ``.zuul.yaml``). In this case, all of the project definitions for the relevant branch are combined (the jobs listed in -all of the matching definitions will be run). If a project definition -appears in a :term:`config-project`, it will apply to all branches of -the project. If it appears in a branch of an -:term:`untrusted-project` it will only apply to changes on that -branch. In the case of an item which does not have a branch (for -example, a tag), all of the project definitions will be combined. +all of the matching definitions will be run). In the case of an item +which does not have a branch (for example, a tag), all of the project +definitions will be combined. Consider the following project definition:: @@ -89,6 +86,47 @@ pipeline. jobs specified in project-pipeline definitions on the project itself. + .. attr:: branches + + A list of branches to which this `project` stanza should apply. + + If omitted on a `project` stanza within an + :term:`untrusted-project` that is configuring its own project, + the current branch will be used (:attr:`pragma` settings are + ignored). That means that in the typical case where this option + is omitted on an untrusted project, the stanza is always + interpreted as configuring the project on the branch where the + definition is found. + + If omitted on a `project` stanza within a + :term:`config-project`, the stanza will be interpreted as + applying to all branches (but :attr:`pragma` settings are + effective in this case, see below). + + If omitted when configuring a project other than the current + project, if the current project is branched, then the current + branch will be used (but :attr:`pragma` settings are effective + in this case, see below). If the current project has only one + branch, then the stanza will be interpreted as applying to all + branches. + + In the cases where :attr:`pragma` settings are effective, if + :attr:`pragma.implied-branch-matchers` is in effect then + :attr:`pragma.implied-branches` will be used. + + In all cases, explicit configuration of branches overrides + implied branches. + + Note that use of this attribute when configuring the jobs run on + the current project can produce undesirable behavior when + combined with common project branching paradigms. In + particular, note that when a project is branched, the project + stanzas are effectively copied onto that branch, and therefore + additional explicit stanzas will be in effect. It is + recommended to only use this attribute inside unbranched + projects and instead use the default implicit branch behavior + for branched projects. + .. attr:: default-branch :default: master diff --git a/doc/source/tenants.rst b/doc/source/tenants.rst index 4bedc25fbe..71b1456678 100644 --- a/doc/source/tenants.rst +++ b/doc/source/tenants.rst @@ -327,6 +327,22 @@ configuration. Some examples of tenant definitions are: in-repo configuration for its own testing (which may not be relevant to other users of the project). + .. attr:: configure-projects + + A list of project names (or :ref:`regular expressions + ` to match project names) that this project is + permitted to configure. The use of this setting will + allow this project to specify :attr:`project` stanzas that + apply to untrusted-projects specified here. This is an + advanced and potentially dangerous configuration setting + since it would allow one project to cause another project + to run certain jobs. This behavior is normally reserved + for :term:`config projects `. + + This should only be used in situations where there is a + strong trust relationship between this project and the + projects it is permitted to configure. + .. attr:: The items in the list are dictionaries with the following diff --git a/releasenotes/notes/configure-projects-52094812f855dc5b.yaml b/releasenotes/notes/configure-projects-52094812f855dc5b.yaml new file mode 100644 index 0000000000..48b19adf54 --- /dev/null +++ b/releasenotes/notes/configure-projects-52094812f855dc5b.yaml @@ -0,0 +1,16 @@ +--- +features: + - | + Untrusted projects may now be allowed (via explicit configuration + in the tenant config file) to configure jobs that are run by + certain specified other untrusted projects. This allows a + "super-project" to configure the jobs run by its "sub-projects". + + See :attr:`tenant.untrusted-projects..configure-projects` + for details. + - | + Project stanzas may now include an explicit branch configuration + via the :attr:`project.branches` attribute. This enables projects + that configure other projects (whether the configuring projects + are trusted or untrusted) to better control what jobs run on certain + branches. diff --git a/tests/fixtures/config/config-project-branch-matcher/git/common-config/playbooks/run.yaml b/tests/fixtures/config/config-project-branch-matcher/git/common-config/playbooks/run.yaml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/tests/fixtures/config/config-project-branch-matcher/git/common-config/playbooks/run.yaml @@ -0,0 +1 @@ +--- diff --git a/tests/fixtures/config/config-project-branch-matcher/git/common-config/zuul.d/jobs.yaml b/tests/fixtures/config/config-project-branch-matcher/git/common-config/zuul.d/jobs.yaml new file mode 100644 index 0000000000..ab5f414498 --- /dev/null +++ b/tests/fixtures/config/config-project-branch-matcher/git/common-config/zuul.d/jobs.yaml @@ -0,0 +1,7 @@ +- job: + name: base + parent: null + run: playbooks/run.yaml + +- job: + name: testjob diff --git a/tests/fixtures/config/config-project-branch-matcher/git/common-config/zuul.d/pipelines.yaml b/tests/fixtures/config/config-project-branch-matcher/git/common-config/zuul.d/pipelines.yaml new file mode 100644 index 0000000000..c2ae71d96d --- /dev/null +++ b/tests/fixtures/config/config-project-branch-matcher/git/common-config/zuul.d/pipelines.yaml @@ -0,0 +1,51 @@ +- pipeline: + name: check + manager: independent + trigger: + gerrit: + - event: patchset-created + - event: comment-added + comment: '^(Patch Set [0-9]+:\n\n)?(?i:recheck)$' + success: + gerrit: + Verified: 1 + failure: + gerrit: + Verified: -1 + +- pipeline: + name: gate + manager: dependent + success-message: Build succeeded (gate). + trigger: + gerrit: + - event: comment-added + approval: + - Approved: 1 + success: + gerrit: + Verified: 2 + submit: true + failure: + gerrit: + Verified: -2 + start: + gerrit: + Verified: 0 + precedence: high + +- pipeline: + name: post + manager: independent + trigger: + gerrit: + - event: ref-updated + ref: ^(?!refs/).*$ + precedence: low + +- pipeline: + name: periodic + manager: independent + trigger: + timer: + - time: '0 0 * * *' diff --git a/tests/fixtures/config/config-project-branch-matcher/git/common-config/zuul.d/projects.yaml b/tests/fixtures/config/config-project-branch-matcher/git/common-config/zuul.d/projects.yaml new file mode 100644 index 0000000000..7b62dfc6ef --- /dev/null +++ b/tests/fixtures/config/config-project-branch-matcher/git/common-config/zuul.d/projects.yaml @@ -0,0 +1,32 @@ +- pragma: + implied-branch-matchers: True + implied-branches: + - stable/implied + +- project: + name: org/project1 + check: + jobs: + - testjob + +- project: + name: org/project2 + branches: + - stable/explicit + check: + jobs: + - testjob + +- project: + name: ^org/reproject3.*$ + check: + jobs: + - testjob + +- project: + name: ^org/reproject4.*$ + branches: + - stable/explicit + check: + jobs: + - testjob diff --git a/tests/fixtures/config/config-project-branch-matcher/git/org_project1/README b/tests/fixtures/config/config-project-branch-matcher/git/org_project1/README new file mode 100644 index 0000000000..9daeafb986 --- /dev/null +++ b/tests/fixtures/config/config-project-branch-matcher/git/org_project1/README @@ -0,0 +1 @@ +test diff --git a/tests/fixtures/config/config-project-branch-matcher/git/org_project2/README b/tests/fixtures/config/config-project-branch-matcher/git/org_project2/README new file mode 100644 index 0000000000..9daeafb986 --- /dev/null +++ b/tests/fixtures/config/config-project-branch-matcher/git/org_project2/README @@ -0,0 +1 @@ +test diff --git a/tests/fixtures/config/config-project-branch-matcher/git/org_reproject3/README b/tests/fixtures/config/config-project-branch-matcher/git/org_reproject3/README new file mode 100644 index 0000000000..9daeafb986 --- /dev/null +++ b/tests/fixtures/config/config-project-branch-matcher/git/org_reproject3/README @@ -0,0 +1 @@ +test diff --git a/tests/fixtures/config/config-project-branch-matcher/git/org_reproject4/README b/tests/fixtures/config/config-project-branch-matcher/git/org_reproject4/README new file mode 100644 index 0000000000..9daeafb986 --- /dev/null +++ b/tests/fixtures/config/config-project-branch-matcher/git/org_reproject4/README @@ -0,0 +1 @@ +test diff --git a/tests/fixtures/config/config-project-branch-matcher/main.yaml b/tests/fixtures/config/config-project-branch-matcher/main.yaml new file mode 100644 index 0000000000..6fb3fc4f0e --- /dev/null +++ b/tests/fixtures/config/config-project-branch-matcher/main.yaml @@ -0,0 +1,11 @@ +- tenant: + name: tenant-one + source: + gerrit: + config-projects: + - common-config + untrusted-projects: + - org/project1 + - org/project2 + - org/reproject3 + - org/reproject4 diff --git a/tests/fixtures/config/superproject/git/common-config/playbooks/run.yaml b/tests/fixtures/config/superproject/git/common-config/playbooks/run.yaml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/tests/fixtures/config/superproject/git/common-config/playbooks/run.yaml @@ -0,0 +1 @@ +--- diff --git a/tests/fixtures/config/superproject/git/common-config/zuul.yaml b/tests/fixtures/config/superproject/git/common-config/zuul.yaml new file mode 100644 index 0000000000..c6ddfddffb --- /dev/null +++ b/tests/fixtures/config/superproject/git/common-config/zuul.yaml @@ -0,0 +1,56 @@ +- pipeline: + name: check + manager: independent + trigger: + gerrit: + - event: patchset-created + - event: comment-added + comment: '^(Patch Set [0-9]+:\n\n)?(?i:recheck)$' + success: + gerrit: + Verified: 1 + failure: + gerrit: + Verified: -1 + +- pipeline: + name: gate + manager: dependent + success-message: Build succeeded (gate). + trigger: + gerrit: + - event: comment-added + approval: + - Approved: 1 + success: + gerrit: + Verified: 2 + submit: true + failure: + gerrit: + Verified: -2 + start: + gerrit: + Verified: 0 + precedence: high + +- pipeline: + name: post + manager: independent + trigger: + gerrit: + - event: ref-updated + ref: ^(?!refs/).*$ + precedence: low + +- pipeline: + name: periodic + manager: independent + trigger: + timer: + - time: '0 0 * * *' + +- job: + name: base + parent: null + run: playbooks/run.yaml diff --git a/tests/fixtures/config/superproject/git/othermodule/README b/tests/fixtures/config/superproject/git/othermodule/README new file mode 100644 index 0000000000..9daeafb986 --- /dev/null +++ b/tests/fixtures/config/superproject/git/othermodule/README @@ -0,0 +1 @@ +test diff --git a/tests/fixtures/config/superproject/git/submodule1/README b/tests/fixtures/config/superproject/git/submodule1/README new file mode 100644 index 0000000000..9daeafb986 --- /dev/null +++ b/tests/fixtures/config/superproject/git/submodule1/README @@ -0,0 +1 @@ +test diff --git a/tests/fixtures/config/superproject/git/submodules_foo/README b/tests/fixtures/config/superproject/git/submodules_foo/README new file mode 100644 index 0000000000..9daeafb986 --- /dev/null +++ b/tests/fixtures/config/superproject/git/submodules_foo/README @@ -0,0 +1 @@ +test diff --git a/tests/fixtures/config/superproject/git/superproject/README b/tests/fixtures/config/superproject/git/superproject/README new file mode 100644 index 0000000000..9daeafb986 --- /dev/null +++ b/tests/fixtures/config/superproject/git/superproject/README @@ -0,0 +1 @@ +test diff --git a/tests/fixtures/config/superproject/git/superproject/zuul.yaml b/tests/fixtures/config/superproject/git/superproject/zuul.yaml new file mode 100644 index 0000000000..a6dc2c4f7a --- /dev/null +++ b/tests/fixtures/config/superproject/git/superproject/zuul.yaml @@ -0,0 +1,28 @@ +- job: + name: superproject-job + +- job: + name: integration-job + +- project-template: + name: submodule-jobs + check: + jobs: + - integration-job + +- project: + name: submodule1 + templates: [submodule-jobs] +- project: + name: othermodule + templates: [submodule-jobs] +- project: + name: ^submodules/.*$ + templates: [submodule-jobs] + +# This project +- project: + templates: [submodule-jobs] + check: + jobs: + - superproject-job diff --git a/tests/fixtures/config/superproject/git/unrelated-project/README b/tests/fixtures/config/superproject/git/unrelated-project/README new file mode 100644 index 0000000000..9daeafb986 --- /dev/null +++ b/tests/fixtures/config/superproject/git/unrelated-project/README @@ -0,0 +1 @@ +test diff --git a/tests/fixtures/config/superproject/main.yaml b/tests/fixtures/config/superproject/main.yaml new file mode 100644 index 0000000000..09f83d8740 --- /dev/null +++ b/tests/fixtures/config/superproject/main.yaml @@ -0,0 +1,16 @@ +- tenant: + name: tenant-one + source: + gerrit: + config-projects: + - common-config + untrusted-projects: + - superproject: + configure-projects: + - submodule1 + - othermodule + - ^submodules/.*$ + - submodule1 + - othermodule + - submodules/foo + - unrelated-project diff --git a/tests/fixtures/config/tenant-parser/config-superproject.yaml b/tests/fixtures/config/tenant-parser/config-superproject.yaml new file mode 100644 index 0000000000..2b0e29c3ff --- /dev/null +++ b/tests/fixtures/config/tenant-parser/config-superproject.yaml @@ -0,0 +1,11 @@ +- tenant: + name: tenant-one + source: + gerrit: + config-projects: + - common-config: + configure-projects: + - org/project1 + untrusted-projects: + - org/project1 + - org/project2 diff --git a/tests/fixtures/config/tenant-parser/superproject-config-project-regex.yaml b/tests/fixtures/config/tenant-parser/superproject-config-project-regex.yaml new file mode 100644 index 0000000000..c4c17c5591 --- /dev/null +++ b/tests/fixtures/config/tenant-parser/superproject-config-project-regex.yaml @@ -0,0 +1,10 @@ +- tenant: + name: tenant-one + source: + gerrit: + config-projects: + - common-config + untrusted-projects: + - org/project1: + configure-projects: + - ^.*$ diff --git a/tests/fixtures/config/tenant-parser/superproject-config-project.yaml b/tests/fixtures/config/tenant-parser/superproject-config-project.yaml new file mode 100644 index 0000000000..6a445eef2b --- /dev/null +++ b/tests/fixtures/config/tenant-parser/superproject-config-project.yaml @@ -0,0 +1,11 @@ +- tenant: + name: tenant-one + source: + gerrit: + config-projects: + - common-config + untrusted-projects: + - org/project1: + configure-projects: + common-config + - org/project2 diff --git a/tests/fixtures/config/untrusted-project-branch-matcher/git/common-config/playbooks/run.yaml b/tests/fixtures/config/untrusted-project-branch-matcher/git/common-config/playbooks/run.yaml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/tests/fixtures/config/untrusted-project-branch-matcher/git/common-config/playbooks/run.yaml @@ -0,0 +1 @@ +--- diff --git a/tests/fixtures/config/untrusted-project-branch-matcher/git/common-config/zuul.d/jobs.yaml b/tests/fixtures/config/untrusted-project-branch-matcher/git/common-config/zuul.d/jobs.yaml new file mode 100644 index 0000000000..ab5f414498 --- /dev/null +++ b/tests/fixtures/config/untrusted-project-branch-matcher/git/common-config/zuul.d/jobs.yaml @@ -0,0 +1,7 @@ +- job: + name: base + parent: null + run: playbooks/run.yaml + +- job: + name: testjob diff --git a/tests/fixtures/config/untrusted-project-branch-matcher/git/common-config/zuul.d/pipelines.yaml b/tests/fixtures/config/untrusted-project-branch-matcher/git/common-config/zuul.d/pipelines.yaml new file mode 100644 index 0000000000..c2ae71d96d --- /dev/null +++ b/tests/fixtures/config/untrusted-project-branch-matcher/git/common-config/zuul.d/pipelines.yaml @@ -0,0 +1,51 @@ +- pipeline: + name: check + manager: independent + trigger: + gerrit: + - event: patchset-created + - event: comment-added + comment: '^(Patch Set [0-9]+:\n\n)?(?i:recheck)$' + success: + gerrit: + Verified: 1 + failure: + gerrit: + Verified: -1 + +- pipeline: + name: gate + manager: dependent + success-message: Build succeeded (gate). + trigger: + gerrit: + - event: comment-added + approval: + - Approved: 1 + success: + gerrit: + Verified: 2 + submit: true + failure: + gerrit: + Verified: -2 + start: + gerrit: + Verified: 0 + precedence: high + +- pipeline: + name: post + manager: independent + trigger: + gerrit: + - event: ref-updated + ref: ^(?!refs/).*$ + precedence: low + +- pipeline: + name: periodic + manager: independent + trigger: + timer: + - time: '0 0 * * *' diff --git a/tests/fixtures/config/untrusted-project-branch-matcher/git/org_project1/README b/tests/fixtures/config/untrusted-project-branch-matcher/git/org_project1/README new file mode 100644 index 0000000000..9daeafb986 --- /dev/null +++ b/tests/fixtures/config/untrusted-project-branch-matcher/git/org_project1/README @@ -0,0 +1 @@ +test diff --git a/tests/fixtures/config/untrusted-project-branch-matcher/git/org_project2/README b/tests/fixtures/config/untrusted-project-branch-matcher/git/org_project2/README new file mode 100644 index 0000000000..9daeafb986 --- /dev/null +++ b/tests/fixtures/config/untrusted-project-branch-matcher/git/org_project2/README @@ -0,0 +1 @@ +test diff --git a/tests/fixtures/config/untrusted-project-branch-matcher/git/org_reproject3/README b/tests/fixtures/config/untrusted-project-branch-matcher/git/org_reproject3/README new file mode 100644 index 0000000000..9daeafb986 --- /dev/null +++ b/tests/fixtures/config/untrusted-project-branch-matcher/git/org_reproject3/README @@ -0,0 +1 @@ +test diff --git a/tests/fixtures/config/untrusted-project-branch-matcher/git/org_reproject4/README b/tests/fixtures/config/untrusted-project-branch-matcher/git/org_reproject4/README new file mode 100644 index 0000000000..9daeafb986 --- /dev/null +++ b/tests/fixtures/config/untrusted-project-branch-matcher/git/org_reproject4/README @@ -0,0 +1 @@ +test diff --git a/tests/fixtures/config/untrusted-project-branch-matcher/git/superproject/zuul.yaml b/tests/fixtures/config/untrusted-project-branch-matcher/git/superproject/zuul.yaml new file mode 100644 index 0000000000..1c8d1cfb13 --- /dev/null +++ b/tests/fixtures/config/untrusted-project-branch-matcher/git/superproject/zuul.yaml @@ -0,0 +1,38 @@ +- pragma: + implied-branch-matchers: True + implied-branches: + - stable/implied + +- project: + name: org/project1 + check: + jobs: + - testjob + +- project: + name: org/project2 + branches: + - stable/explicit + check: + jobs: + - testjob + +- project: + name: ^org/reproject3.*$ + check: + jobs: + - testjob + +- project: + name: ^org/reproject4.*$ + branches: + - stable/explicit + check: + jobs: + - testjob + +# This project (should run on master) +- project: + check: + jobs: + - testjob diff --git a/tests/fixtures/config/untrusted-project-branch-matcher/main.yaml b/tests/fixtures/config/untrusted-project-branch-matcher/main.yaml new file mode 100644 index 0000000000..b3b31f1870 --- /dev/null +++ b/tests/fixtures/config/untrusted-project-branch-matcher/main.yaml @@ -0,0 +1,16 @@ +- tenant: + name: tenant-one + source: + gerrit: + config-projects: + - common-config + untrusted-projects: + - superproject: + configure-projects: + - org/project1 + - org/project2 + - ^org/reproject.*$ + - org/project1 + - org/project2 + - org/reproject3 + - org/reproject4 diff --git a/tests/unit/test_configloader.py b/tests/unit/test_configloader.py index 0498edddfe..0cc6ccc04c 100644 --- a/tests/unit/test_configloader.py +++ b/tests/unit/test_configloader.py @@ -88,13 +88,13 @@ class TestTenantSimple(TenantParserTestCase): self.assertEqual(['org/project1', 'org/project2'], [x.name for x in tenant.untrusted_projects]) - project = tenant.config_projects[0] + project = list(tenant.config_projects)[0] tpc = tenant.project_configs[project.canonical_name] self.assertEqual(self.CONFIG_SET, tpc.load_classes) - project = tenant.untrusted_projects[0] + project = list(tenant.untrusted_projects)[0] tpc = tenant.project_configs[project.canonical_name] self.assertEqual(self.UNTRUSTED_SET, tpc.load_classes) - project = tenant.untrusted_projects[1] + project = list(tenant.untrusted_projects)[1] tpc = tenant.project_configs[project.canonical_name] self.assertEqual(self.UNTRUSTED_SET, tpc.load_classes) self.assertTrue('common-config-job' in tenant.layout.jobs) @@ -277,7 +277,7 @@ class TestTenantSimple(TenantParserTestCase): <<: *docker_vars """) tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') - project = tenant.config_projects[0] + project = list(tenant.config_projects)[0] source_context = SourceContext( project.canonical_name, project.name, project.connection_name, 'master', 'zuul.yaml', True) @@ -346,14 +346,14 @@ class TestTenantOverride(TenantParserTestCase): [x.name for x in tenant.config_projects]) self.assertEqual(['org/project1', 'org/project2', 'org/project4'], [x.name for x in tenant.untrusted_projects]) - project = tenant.config_projects[0] + project = list(tenant.config_projects)[0] tpc = tenant.project_configs[project.canonical_name] self.assertEqual(self.CONFIG_SET, tpc.load_classes) - project = tenant.untrusted_projects[0] + project = list(tenant.untrusted_projects)[0] tpc = tenant.project_configs[project.canonical_name] self.assertEqual(self.UNTRUSTED_SET - set(['project']), tpc.load_classes) - project = tenant.untrusted_projects[1] + project = list(tenant.untrusted_projects)[1] tpc = tenant.project_configs[project.canonical_name] self.assertEqual(set(['job']), tpc.load_classes) self.assertTrue('common-config-job' in tenant.layout.jobs) @@ -382,14 +382,14 @@ class TestTenantGroups(TenantParserTestCase): [x.name for x in tenant.config_projects]) self.assertEqual(['org/project1', 'org/project2'], [x.name for x in tenant.untrusted_projects]) - project = tenant.config_projects[0] + project = list(tenant.config_projects)[0] tpc = tenant.project_configs[project.canonical_name] self.assertEqual(self.CONFIG_SET, tpc.load_classes) - project = tenant.untrusted_projects[0] + project = list(tenant.untrusted_projects)[0] tpc = tenant.project_configs[project.canonical_name] self.assertEqual(self.UNTRUSTED_SET - set(['project']), tpc.load_classes) - project = tenant.untrusted_projects[1] + project = list(tenant.untrusted_projects)[1] tpc = tenant.project_configs[project.canonical_name] self.assertEqual(self.UNTRUSTED_SET - set(['project']), tpc.load_classes) @@ -419,14 +419,14 @@ class TestTenantGroups2(TenantParserTestCase): [x.name for x in tenant.config_projects]) self.assertEqual(['org/project1', 'org/project2', 'org/project3'], [x.name for x in tenant.untrusted_projects]) - project = tenant.config_projects[0] + project = list(tenant.config_projects)[0] tpc = tenant.project_configs[project.canonical_name] self.assertEqual(self.CONFIG_SET, tpc.load_classes) - project = tenant.untrusted_projects[0] + project = list(tenant.untrusted_projects)[0] tpc = tenant.project_configs[project.canonical_name] self.assertEqual(self.UNTRUSTED_SET - set(['project']), tpc.load_classes) - project = tenant.untrusted_projects[1] + project = list(tenant.untrusted_projects)[1] tpc = tenant.project_configs[project.canonical_name] self.assertEqual(self.UNTRUSTED_SET - set(['project', 'job']), tpc.load_classes) @@ -457,13 +457,13 @@ class TestTenantGroups3(TenantParserTestCase): [x.name for x in tenant.config_projects]) self.assertEqual(['org/project1', 'org/project2'], [x.name for x in tenant.untrusted_projects]) - project = tenant.config_projects[0] + project = list(tenant.config_projects)[0] tpc = tenant.project_configs[project.canonical_name] self.assertEqual(self.CONFIG_SET, tpc.load_classes) - project = tenant.untrusted_projects[0] + project = list(tenant.untrusted_projects)[0] tpc = tenant.project_configs[project.canonical_name] self.assertEqual(set(['job']), tpc.load_classes) - project = tenant.untrusted_projects[1] + project = list(tenant.untrusted_projects)[1] tpc = tenant.project_configs[project.canonical_name] self.assertEqual(set(['project', 'job']), tpc.load_classes) self.assertTrue('common-config-job' in tenant.layout.jobs) @@ -492,14 +492,14 @@ class TestTenantGroups4(TenantParserTestCase): [x.name for x in tenant.config_projects]) self.assertEqual(['org/project1', 'org/project2'], [x.name for x in tenant.untrusted_projects]) - project = tenant.config_projects[0] + project = list(tenant.config_projects)[0] tpc = tenant.project_configs[project.canonical_name] self.assertEqual(self.CONFIG_SET, tpc.load_classes) - project = tenant.untrusted_projects[0] + project = list(tenant.untrusted_projects)[0] tpc = tenant.project_configs[project.canonical_name] self.assertEqual(set([]), tpc.load_classes) - project = tenant.untrusted_projects[1] + project = list(tenant.untrusted_projects)[1] tpc = tenant.project_configs[project.canonical_name] self.assertEqual(set([]), tpc.load_classes) @@ -529,10 +529,10 @@ class TestTenantGroups5(TenantParserTestCase): [x.name for x in tenant.config_projects]) self.assertEqual(['org/project1'], [x.name for x in tenant.untrusted_projects]) - project = tenant.config_projects[0] + project = list(tenant.config_projects)[0] tpc = tenant.project_configs[project.canonical_name] self.assertEqual(self.CONFIG_SET, tpc.load_classes) - project = tenant.untrusted_projects[0] + project = list(tenant.untrusted_projects)[0] tpc = tenant.project_configs[project.canonical_name] self.assertEqual(set([]), tpc.load_classes) @@ -562,13 +562,13 @@ class TestTenantUnprotectedBranches(TenantParserTestCase): [x.name for x in tenant.untrusted_projects]) tpc = tenant.project_configs - project_name = tenant.config_projects[0].canonical_name + project_name = list(tenant.config_projects)[0].canonical_name self.assertEqual(False, tpc[project_name].exclude_unprotected_branches) - project_name = tenant.untrusted_projects[0].canonical_name + project_name = list(tenant.untrusted_projects)[0].canonical_name self.assertIsNone(tpc[project_name].exclude_unprotected_branches) - project_name = tenant.untrusted_projects[1].canonical_name + project_name = list(tenant.untrusted_projects)[1].canonical_name self.assertIsNone(tpc[project_name].exclude_unprotected_branches) @@ -584,11 +584,11 @@ class TestTenantIncludeBranches(TenantParserTestCase): [x.name for x in tenant.untrusted_projects]) tpc = tenant.project_configs - project_name = tenant.config_projects[0].canonical_name + project_name = list(tenant.config_projects)[0].canonical_name self.assertEqual(['master'], tpc[project_name].branches) # No branches pass the filter at the start - project_name = tenant.untrusted_projects[0].canonical_name + project_name = list(tenant.untrusted_projects)[0].canonical_name self.assertEqual([], tpc[project_name].branches) # Create the foo branch @@ -601,7 +601,7 @@ class TestTenantIncludeBranches(TenantParserTestCase): # It should pass the filter tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') tpc = tenant.project_configs - project_name = tenant.untrusted_projects[0].canonical_name + project_name = list(tenant.untrusted_projects)[0].canonical_name self.assertEqual(['foo'], tpc[project_name].branches) # Create the baz branch @@ -614,7 +614,7 @@ class TestTenantIncludeBranches(TenantParserTestCase): # It should not pass the filter tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') tpc = tenant.project_configs - project_name = tenant.untrusted_projects[0].canonical_name + project_name = list(tenant.untrusted_projects)[0].canonical_name self.assertEqual(['foo'], tpc[project_name].branches) @@ -1245,6 +1245,56 @@ class TestTenantDuplicate(TenantParserTestCase): pass +class TestTenantSuperprojectConfigProject(TenantParserTestCase): + tenant_config_file = ('config/tenant-parser/' + 'superproject-config-project.yaml') + scheduler_count = 1 + + def setUp(self): + # Test that we get an error trying to configure a + # config-project + err = ".*may not configure config-project.*" + with testtools.ExpectedException(Exception, err): + super().setUp() + + def test_tenant_superproject_config_project(self): + # The magic is in setUp + pass + + +class TestTenantSuperprojectConfigProjectRegex(TenantParserTestCase): + tenant_config_file = ('config/tenant-parser/' + 'superproject-config-project-regex.yaml') + scheduler_count = 1 + + def setUp(self): + # Test that we get an error trying to configure a + # config-project via regex + err = ".*may not configure config-project.*" + with testtools.ExpectedException(Exception, err): + super().setUp() + + def test_tenant_superproject_config_project_regex(self): + # The magic is in setUp + pass + + +class TestTenantConfigSuperproject(TenantParserTestCase): + tenant_config_file = ('config/tenant-parser/' + 'config-superproject.yaml') + scheduler_count = 1 + + def setUp(self): + # Test that we get an error trying to use configure-projects + # on a config-project + with testtools.ExpectedException(vs.MultipleInvalid): + super().setUp() + + def test_tenant_config_superproject(self): + # The magic is in setUp + pass + + class TestMergeMode(ZuulTestCase): config_file = 'zuul-connections-gerrit-and-github.conf' diff --git a/tests/unit/test_connection.py b/tests/unit/test_connection.py index 052e41a9f1..0637503933 100644 --- a/tests/unit/test_connection.py +++ b/tests/unit/test_connection.py @@ -997,6 +997,6 @@ class TestConnectionsBranchCache(ZuulTestCase): connection.addProject(newproject) tpc = zuul.model.TenantProjectConfig(newproject) tpc.exclude_unprotected_branches = True - tenant.addUntrustedProject(tpc) + tenant.addTPC(tpc) branches = connection.getProjectBranches(newproject, tenant) self.assertEqual([], branches) diff --git a/tests/unit/test_git_driver.py b/tests/unit/test_git_driver.py index 95fca30d30..a8dba71327 100644 --- a/tests/unit/test_git_driver.py +++ b/tests/unit/test_git_driver.py @@ -38,10 +38,12 @@ class TestGitDriver(ZuulTestCase): tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') # Check that we have the git source for common-config and the # gerrit source for the project. - self.assertEqual('git', tenant.config_projects[0].source.name) - self.assertEqual('common-config', tenant.config_projects[0].name) - self.assertEqual('gerrit', tenant.untrusted_projects[0].source.name) - self.assertEqual('org/project', tenant.untrusted_projects[0].name) + self.assertEqual('git', list(tenant.config_projects)[0].source.name) + self.assertEqual('common-config', list(tenant.config_projects)[0].name) + self.assertEqual('gerrit', + list(tenant.untrusted_projects)[0].source.name) + self.assertEqual('org/project', + list(tenant.untrusted_projects)[0].name) # The configuration for this test is accessed via the git # driver (in common-config), rather than the gerrit driver, so diff --git a/tests/unit/test_github_driver.py b/tests/unit/test_github_driver.py index a51d4f7247..1861b22efa 100644 --- a/tests/unit/test_github_driver.py +++ b/tests/unit/test_github_driver.py @@ -1570,8 +1570,8 @@ class TestGithubUnprotectedBranches(ZuulTestCase): tenant = self.scheds.first.sched.abide.tenants\ .get('tenant-one') - project1 = tenant.untrusted_projects[0] - project2 = tenant.untrusted_projects[1] + project1 = list(tenant.untrusted_projects)[0] + project2 = list(tenant.untrusted_projects)[1] tpc1 = tenant.project_configs[project1.canonical_name] tpc2 = tenant.project_configs[project2.canonical_name] @@ -1972,9 +1972,9 @@ class TestGithubLockedBranches(ZuulTestCase): tenant = self.scheds.first.sched.abide.tenants\ .get('tenant-one') - project1 = tenant.untrusted_projects[0] - project2 = tenant.untrusted_projects[1] - project3 = tenant.untrusted_projects[2] + project1 = list(tenant.untrusted_projects)[0] + project2 = list(tenant.untrusted_projects)[1] + project3 = list(tenant.untrusted_projects)[2] tpc1 = tenant.project_configs[project1.canonical_name] tpc2 = tenant.project_configs[project2.canonical_name] diff --git a/tests/unit/test_gitlab_driver.py b/tests/unit/test_gitlab_driver.py index e885d2a602..48a642b5e1 100644 --- a/tests/unit/test_gitlab_driver.py +++ b/tests/unit/test_gitlab_driver.py @@ -1000,8 +1000,8 @@ class TestGitlabUnprotectedBranches(ZuulTestCase): tenant = self.scheds.first.sched.abide.tenants\ .get('tenant-one') - project1 = tenant.untrusted_projects[0] - project2 = tenant.untrusted_projects[1] + project1 = list(tenant.untrusted_projects)[0] + project2 = list(tenant.untrusted_projects)[1] tpc1 = tenant.project_configs[project1.canonical_name] tpc2 = tenant.project_configs[project2.canonical_name] diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py index 386384bfdd..36ac6d48e2 100644 --- a/tests/unit/test_model.py +++ b/tests/unit/test_model.py @@ -74,7 +74,7 @@ class TestJob(BaseTestCase): self.project.canonical_name, self.project.name, self.project.connection_name, 'master', 'test', False) self.tpc = model.TenantProjectConfig(self.project) - self.tenant.addUntrustedProject(self.tpc) + self.tenant.addTPC(self.tpc) self.pipeline = model.Pipeline('gate', self.tenant) self.pipeline.source_context = self.context self.pipeline.manager = mock.Mock() @@ -318,7 +318,7 @@ class TestJob(BaseTestCase): base_project.canonical_name, base_project.name, base_project.connection_name, 'master', 'test', True) tpc = model.TenantProjectConfig(base_project) - self.tenant.addUntrustedProject(tpc) + self.tenant.addTPC(tpc) base = self.pcontext.job_parser.fromYaml({ '_source_context': base_context, @@ -333,7 +333,7 @@ class TestJob(BaseTestCase): other_project.canonical_name, other_project.name, other_project.connection_name, 'master', 'test', True) tpc = model.TenantProjectConfig(other_project) - self.tenant.addUntrustedProject(tpc) + self.tenant.addTPC(tpc) base2 = self.pcontext.job_parser.fromYaml({ '_source_context': other_context, '_start_mark': self.start_mark, @@ -658,7 +658,8 @@ class TestTenant(BaseTestCase): source1_project1 = model.Project('project1', source1) source1_project1_tpc = model.TenantProjectConfig(source1_project1) - tenant.addConfigProject(source1_project1_tpc) + source1_project1_tpc.trusted = True + tenant.addTPC(source1_project1_tpc) d = {'project1': {'git1.example.com': source1_project1}} self.assertEqual(d, tenant.projects) @@ -669,7 +670,7 @@ class TestTenant(BaseTestCase): source1_project2 = model.Project('project2', source1) tpc = model.TenantProjectConfig(source1_project2) - tenant.addUntrustedProject(tpc) + tenant.addTPC(tpc) d = {'project1': {'git1.example.com': source1_project1}, 'project2': @@ -687,7 +688,7 @@ class TestTenant(BaseTestCase): source2_project1 = model.Project('project1', source2) tpc = model.TenantProjectConfig(source2_project1) - tenant.addUntrustedProject(tpc) + tenant.addTPC(tpc) d = {'project1': {'git1.example.com': source1_project1, 'git2.example.com': source2_project1}, @@ -707,7 +708,8 @@ class TestTenant(BaseTestCase): source2_project2 = model.Project('project2', source2) tpc = model.TenantProjectConfig(source2_project2) - tenant.addConfigProject(tpc) + tpc.trusted = True + tenant.addTPC(tpc) d = {'project1': {'git1.example.com': source1_project1, 'git2.example.com': source2_project1}, @@ -734,7 +736,8 @@ class TestTenant(BaseTestCase): source1_project2b = model.Project('subpath/project2', source1) tpc = model.TenantProjectConfig(source1_project2b) - tenant.addConfigProject(tpc) + tpc.trusted = True + tenant.addTPC(tpc) d = {'project1': {'git1.example.com': source1_project1, 'git2.example.com': source2_project1}, @@ -756,7 +759,8 @@ class TestTenant(BaseTestCase): source2_project2b = model.Project('subpath/project2', source2) tpc = model.TenantProjectConfig(source2_project2b) - tenant.addConfigProject(tpc) + tpc.trusted = True + tenant.addTPC(tpc) d = {'project1': {'git1.example.com': source1_project1, 'git2.example.com': source2_project1}, diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py index adaea89436..f9f555feaf 100644 --- a/tests/unit/test_v3.py +++ b/tests/unit/test_v3.py @@ -10481,3 +10481,170 @@ class TestBlobStorePipelineProcessing(ZuulTestCase): break self.waitUntilSettled() + + +class TestConfigProjectBranchMatcher(ZuulTestCase): + tenant_config_file = 'config/config-project-branch-matcher/main.yaml' + + def test_config_project_branch_matcher(self): + for project in ['org/project1', 'org/project2', + 'org/reproject3', 'org/reproject4']: + for branch in ['stable/implied', 'stable/explicit']: + self.create_branch(project, branch) + self.fake_gerrit.addEvent( + self.fake_gerrit.getFakeBranchCreatedEvent( + project, branch)) + self.waitUntilSettled("initial reconfig") + + # Test project1: implied only + self.fake_gerrit.addEvent(self.fake_gerrit.addFakeChange( + 'org/project1', 'stable/implied', 'A' + ).getPatchsetCreatedEvent(1)) + self.fake_gerrit.addEvent(self.fake_gerrit.addFakeChange( + 'org/project1', 'stable/explicit', 'B' + ).getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + # Test project2: explicit only + self.fake_gerrit.addEvent(self.fake_gerrit.addFakeChange( + 'org/project2', 'stable/implied', 'C' + ).getPatchsetCreatedEvent(1)) + self.fake_gerrit.addEvent(self.fake_gerrit.addFakeChange( + 'org/project2', 'stable/explicit', 'D' + ).getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + # Test reproject3: implied only + self.fake_gerrit.addEvent(self.fake_gerrit.addFakeChange( + 'org/reproject3', 'stable/implied', 'E' + ).getPatchsetCreatedEvent(1)) + self.fake_gerrit.addEvent(self.fake_gerrit.addFakeChange( + 'org/reproject3', 'stable/explicit', 'F' + ).getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + # Test reproject4: explicit only + self.fake_gerrit.addEvent(self.fake_gerrit.addFakeChange( + 'org/reproject4', 'stable/implied', 'G' + ).getPatchsetCreatedEvent(1)) + self.fake_gerrit.addEvent(self.fake_gerrit.addFakeChange( + 'org/reproject4', 'stable/explicit', 'H' + ).getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + self.assertHistory([ + dict(name='testjob', result='SUCCESS', changes='1,1'), # A + dict(name='testjob', result='SUCCESS', changes='4,1'), # D + dict(name='testjob', result='SUCCESS', changes='5,1'), # E + dict(name='testjob', result='SUCCESS', changes='8,1'), # H + ], ordered=False) + + +class TestUntrustedProjectBranchMatcher(ZuulTestCase): + tenant_config_file = 'config/untrusted-project-branch-matcher/main.yaml' + + def test_untrusted_project_branch_matcher(self): + for project in ['org/project1', 'org/project2', + 'org/reproject3', 'org/reproject4', 'superproject']: + for branch in ['stable/implied', 'stable/explicit']: + self.create_branch(project, branch) + self.fake_gerrit.addEvent( + self.fake_gerrit.getFakeBranchCreatedEvent( + project, branch)) + self.waitUntilSettled("initial reconfig") + + # Test project1: implied only + self.fake_gerrit.addEvent(self.fake_gerrit.addFakeChange( + 'org/project1', 'stable/implied', 'A' + ).getPatchsetCreatedEvent(1)) + self.fake_gerrit.addEvent(self.fake_gerrit.addFakeChange( + 'org/project1', 'stable/explicit', 'B' + ).getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + # Test project2: explicit only + self.fake_gerrit.addEvent(self.fake_gerrit.addFakeChange( + 'org/project2', 'stable/implied', 'C' + ).getPatchsetCreatedEvent(1)) + self.fake_gerrit.addEvent(self.fake_gerrit.addFakeChange( + 'org/project2', 'stable/explicit', 'D' + ).getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + # Test reproject3: implied only + self.fake_gerrit.addEvent(self.fake_gerrit.addFakeChange( + 'org/reproject3', 'stable/implied', 'E' + ).getPatchsetCreatedEvent(1)) + self.fake_gerrit.addEvent(self.fake_gerrit.addFakeChange( + 'org/reproject3', 'stable/explicit', 'F' + ).getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + # Test reproject4: explicit only + self.fake_gerrit.addEvent(self.fake_gerrit.addFakeChange( + 'org/reproject4', 'stable/implied', 'G' + ).getPatchsetCreatedEvent(1)) + self.fake_gerrit.addEvent(self.fake_gerrit.addFakeChange( + 'org/reproject4', 'stable/explicit', 'H' + ).getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + # Test superproject: master (the implied branch matcher is + # forced to the same branch for the same project). + self.fake_gerrit.addEvent(self.fake_gerrit.addFakeChange( + 'superproject', 'master', 'I' + ).getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + self.assertHistory([ + dict(name='testjob', result='SUCCESS', changes='1,1'), # A + dict(name='testjob', result='SUCCESS', changes='4,1'), # D + dict(name='testjob', result='SUCCESS', changes='5,1'), # E + dict(name='testjob', result='SUCCESS', changes='8,1'), # H + dict(name='testjob', result='SUCCESS', changes='9,1'), # I + ], ordered=False) + + +class TestSuperproject(ZuulTestCase): + tenant_config_file = 'config/superproject/main.yaml' + + def test_project_configs(self): + A = self.fake_gerrit.addFakeChange('superproject', 'master', 'A') + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + B = self.fake_gerrit.addFakeChange('submodule1', 'master', 'B') + self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) + C = self.fake_gerrit.addFakeChange('othermodule', 'master', 'C') + self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1)) + D = self.fake_gerrit.addFakeChange('submodules/foo', 'master', 'D') + self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + self.assertHistory([ + dict(name='integration-job', result='SUCCESS', changes='1,1'), + dict(name='superproject-job', result='SUCCESS', changes='1,1'), + dict(name='integration-job', result='SUCCESS', changes='2,1'), + dict(name='integration-job', result='SUCCESS', changes='3,1'), + dict(name='integration-job', result='SUCCESS', changes='4,1'), + ], ordered=False) + + def test_configure_unrelated_project(self): + # Ensure we can't configure an unpermitted project + in_repo_conf = textwrap.dedent( + """ + - project: + name: unrelated-project + check: + jobs: + - integration-job + """) + + file_dict = {'zuul.yaml': in_repo_conf} + A = self.fake_gerrit.addFakeChange('superproject', 'master', 'A', + files=file_dict) + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + self.assertEqual(A.reported, 1) + self.assertIn('the only project definition permitted is', + A.messages[0]) + self.assertHistory([]) diff --git a/tests/unit/test_web.py b/tests/unit/test_web.py index 2d19a479ec..1b68472ec2 100644 --- a/tests/unit/test_web.py +++ b/tests/unit/test_web.py @@ -3865,7 +3865,7 @@ class TestWebUnprotectedBranches(BaseWithWeb): self.startWebServer() tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') - project2 = tenant.untrusted_projects[1] + project2 = list(tenant.untrusted_projects)[1] tpc2 = tenant.project_configs[project2.canonical_name] # project2 should have no parsed branch diff --git a/zuul/configloader.py b/zuul/configloader.py index 82040a5969..11eac860ef 100644 --- a/zuul/configloader.py +++ b/zuul/configloader.py @@ -17,7 +17,6 @@ from contextlib import contextmanager from concurrent.futures import ThreadPoolExecutor, as_completed import copy import io -import itertools import logging import math import os @@ -1158,7 +1157,8 @@ class ProjectTemplateParser(object): self.schema = self.getSchema() self.not_pipelines = ['name', 'description', 'templates', 'merge-mode', 'default-branch', 'vars', - 'queue', '_source_context', '_start_mark'] + 'queue', 'branches', + '_source_context', '_start_mark'] def getSchema(self): job = {str: vs.Any(str, JobParser.job_attributes)} @@ -1167,7 +1167,7 @@ class ProjectTemplateParser(object): pipeline_contents = { 'debug': bool, 'fail-fast': bool, - 'jobs': job_list + 'jobs': job_list, } project = { @@ -1260,6 +1260,7 @@ class ProjectParser(object): project = { 'name': str, 'description': str, + 'branches': to_list(vs.Any(ZUUL_REGEX, str)), 'vars': ansible_vars_dict, 'templates': [str], 'merge-mode': vs.Any('merge', 'merge-resolve', @@ -1285,11 +1286,13 @@ class ProjectParser(object): # of the project where it is defined. project_name = (source_context.project_canonical_name) + source_tpc = self.pcontext.tenant.getTPC( + source_context.project_canonical_name) if project_name.startswith('^'): - # regex matching is designed to match other projects so disallow - # in untrusted contexts - if not source_context.trusted: - raise ProjectNotPermittedError() + for other_tpc in \ + self.pcontext.tenant.getTPCsByRegex(project_name): + if not source_tpc.canConfigureProject(other_tpc): + raise ProjectNotPermittedError() # Parse the project as a template since they're mostly the # same. @@ -1298,14 +1301,18 @@ class ProjectParser(object): project_config.name = project_name else: - (trusted, project) = self.pcontext.tenant.getProject(project_name) - if project is None: + other_tpc = self.pcontext.tenant.getTPC(project_name) + if other_tpc is None: raise ProjectNotFoundError(project_name) + project = other_tpc.project - if not source_context.trusted: - if project.canonical_name != \ - source_context.project_canonical_name: - raise ProjectNotPermittedError() + same_project = (project.canonical_name == + source_context.project_canonical_name) + if not ( + same_project or + source_tpc.canConfigureProject(other_tpc) + ): + raise ProjectNotPermittedError() # Parse the project as a template since they're mostly the # same. @@ -1321,13 +1328,27 @@ class ProjectParser(object): # Pragmas can cause templates to end up with implied # branch matchers for arbitrary branches, but project # stanzas should not. They should either have the current - # branch or no branch matcher. - if source_context.trusted: - project_config.setImpliedBranchMatchers([]) - else: - project_config.setImpliedBranchMatchers( - [change_matcher.ImpliedBranchMatcher( - ZuulRegex(source_context.branch))]) + # branch or no branch matcher. But if we're configuring a + # different project, then we'll allow the pragma-supplied + # branch matchers. + if same_project: + if source_context.trusted: + project_config.setImpliedBranchMatchers([]) + else: + project_config.setImpliedBranchMatchers( + [change_matcher.ImpliedBranchMatcher( + ZuulRegex(source_context.branch))]) + + branches = None + if 'branches' in conf: + with self.pcontext.confAttr(conf, 'branches') as conf_branches: + branches = [ + change_matcher.BranchMatcher( + make_regex(x, self.pcontext)) + for x in as_list(conf_branches) + ] + if branches: + project_config.setImpliedBranchMatchers(branches) # Add templates for name in conf.get('templates', []): @@ -1829,7 +1850,7 @@ class TenantParser(object): 'project-template', 'nodeset', 'secret', 'queue', 'image', 'flavor', 'label', 'section', 'provider') - project_dict = {str: { + inner_config_project_dict = { 'include': to_list(classes), 'exclude': to_list(classes), 'shadow': to_list(str), @@ -1842,21 +1863,33 @@ class TenantParser(object): 'always-dynamic-branches': to_list(str), 'allow-circular-dependencies': bool, 'implied-branch-matchers': bool, - }} + } + config_project_dict = {str: inner_config_project_dict} - project = vs.Any(str, project_dict) + inner_untrusted_project_dict = inner_config_project_dict.copy() + inner_untrusted_project_dict['configure-projects'] = to_list(str) + untrusted_project_dict = {str: inner_untrusted_project_dict} - group = { + config_project = vs.Any(str, config_project_dict) + untrusted_project = vs.Any(str, untrusted_project_dict) + + config_group = { 'include': to_list(classes), 'exclude': to_list(classes), - vs.Required('projects'): to_list(project), + vs.Required('projects'): to_list(config_project), + } + untrusted_group = { + 'include': to_list(classes), + 'exclude': to_list(classes), + vs.Required('projects'): to_list(untrusted_project), } - project_or_group = vs.Any(project, group) + config_project_or_group = vs.Any(config_project, config_group) + untrusted_project_or_group = vs.Any(untrusted_project, untrusted_group) tenant_source = vs.Schema({ - 'config-projects': to_list(project_or_group), - 'untrusted-projects': to_list(project_or_group), + 'config-projects': to_list(config_project_or_group), + 'untrusted-projects': to_list(untrusted_project_or_group), }) def validateTenantSources(self): @@ -1948,17 +1981,12 @@ class TenantParser(object): tenant.unparsed_config = conf # tpcs is TenantProjectConfigs - tpc_registry = abide.getTPCRegistry(tenant.name) - config_tpcs = tpc_registry.getConfigTPCs() - for tpc in config_tpcs: - tenant.addConfigProject(tpc) - untrusted_tpcs = tpc_registry.getUntrustedTPCs() - for tpc in untrusted_tpcs: - tenant.addUntrustedProject(tpc) + for tpc in abide.getAllTPCs(tenant.name): + tenant.addTPC(tpc) # Get branches in parallel branch_futures = {} - for tpc in config_tpcs + untrusted_tpcs: + for tpc in abide.getAllTPCs(tenant.name): future = executor.submit(self._getProjectBranches, tenant, tpc, branch_cache_min_ltimes) branch_futures[future] = tpc @@ -2098,6 +2126,7 @@ class TenantParser(object): project_always_dynamic_branches = None project_load_branch = None project_implied_branch_matchers = None + project_configure_projects = None else: project_name = list(conf.keys())[0] project = source.getProject(project_name) @@ -2156,6 +2185,15 @@ class TenantParser(object): 'load-branch', None) project_implied_branch_matchers = conf[project_name].get( 'implied-branch-matchers', None) + configure_projects = as_list(conf[project_name].get( + 'configure-projects', None)) + if configure_projects is not None: + project_configure_projects = [] + for p in configure_projects: + rp = re.compile(p) + project_configure_projects.append(rp) + else: + project_configure_projects = None tenant_project_config = model.TenantProjectConfig(project) tenant_project_config.load_classes = frozenset(project_include) @@ -2173,7 +2211,8 @@ class TenantParser(object): tenant_project_config.load_branch = project_load_branch tenant_project_config.implied_branch_matchers = \ project_implied_branch_matchers - + tenant_project_config.configure_projects = \ + project_configure_projects return tenant_project_config def _getProjects(self, source, conf, current_include): @@ -2222,6 +2261,7 @@ class TenantParser(object): # tpcs = TenantProjectConfigs tpcs = self._getProjects(source, conf_repo, current_include) for tpc in tpcs: + tpc.trusted = True futures.append(executor.submit( self._loadProjectKeys, source_name, tpc.project)) config_projects.append(tpc) @@ -2231,12 +2271,23 @@ class TenantParser(object): tpcs = self._getProjects(source, conf_repo, current_include) for tpc in tpcs: + tpc.trusted = False futures.append(executor.submit( self._loadProjectKeys, source_name, tpc.project)) untrusted_projects.append(tpc) for f in futures: f.result() + for tpc in untrusted_projects: + if tpc.configure_projects: + for config_tpc in config_projects: + if tpc.canConfigureProject( + config_tpc, validation_only=True): + raise Exception( + f"Untrusted-project {tpc.project.name} may not " + f"configure " + f"config-project {config_tpc.project.name}") + return config_projects, untrusted_projects def _cacheTenantYAML(self, abide, tenant, parse_context, min_ltimes, @@ -2305,9 +2356,8 @@ class TenantParser(object): jobs = [] futures = [] - for project in itertools.chain( - tenant.config_projects, tenant.untrusted_projects): - tpc = tenant.project_configs[project.canonical_name] + for tpc in tenant.all_tpcs: + project = tpc.project # For each branch in the repo, get the zuul.yaml for that # branch. Remember the branch and then implicitly add a # branch selector to each job there. This makes the diff --git a/zuul/model.py b/zuul/model.py index 10c28f5d6a..13a30863f9 100644 --- a/zuul/model.py +++ b/zuul/model.py @@ -7869,6 +7869,24 @@ class TenantProjectConfig(object): self.load_branch = None self.merge_modes = None self.implied_branch_matchers = None + self.trusted = False + # A list of project names/regexes that this project is allowed + # to configure + self.configure_projects = None + + def canConfigureProject(self, other_tpc, validation_only=False): + if self.trusted: + return True + if not self.configure_projects: + return False + if not validation_only and other_tpc.trusted: + return False + for r in self.configure_projects: + for name in (other_tpc.project.name, + other_tpc.project.canonical_name): + if r.fullmatch(name): + return True + return False def isAlwaysDynamicBranch(self, branch): if self.always_dynamic_branches is None: @@ -7876,6 +7894,7 @@ class TenantProjectConfig(object): for r in self.always_dynamic_branches: if r.fullmatch(branch): return True + return False def includesBranch(self, branch): if self.include_branches is not None: @@ -9114,15 +9133,9 @@ class Tenant(object): # The unparsed configuration from the main zuul config for # this tenant. self.unparsed_config = None - # The list of projects from which we will read full - # configuration. - self.config_projects = [] - # The parsed config from those projects. + # The parsed config from the config-projects self.config_projects_config = None - # The list of projects from which we will read untrusted - # in-repo configuration. - self.untrusted_projects = [] - # The parsed config from those projects. + # The parsed config from untrusted-projects self.untrusted_projects_config = None self.semaphore_handler = None # Metadata about projects for this tenant @@ -9155,6 +9168,32 @@ class Tenant(object): for project in hostname_dict.values(): yield project + @property + def all_tpcs(self): + """ + Return a generator for all tenant TPCS. + """ + for tpc in self.project_configs.values(): + yield tpc + + @property + def config_projects(self): + """ + Return a generator for all config projects. + """ + for tpc in self.project_configs.values(): + if tpc.trusted: + yield tpc.project + + @property + def untrusted_projects(self): + """ + Return a generator for all tenant TPCS. + """ + for tpc in self.project_configs.values(): + if not tpc.trusted: + yield tpc.project + def _addProject(self, tpc): """Add a project to the project index @@ -9208,20 +9247,19 @@ class Tenant(object): "with a hostname" % (name,)) if project is None: return (None, None) - if project in self.config_projects: - return (True, project) - if project in self.untrusted_projects: - return (False, project) - # This should never happen: - raise Exception("Project %s is neither trusted nor untrusted" % - (project,)) + tpc = self.project_configs.get(project.canonical_name) + if tpc is None: + # This should never happen: + raise Exception("Project %s is neither trusted nor untrusted" % + (project,)) + return (tpc.trusted, project) - def getProjectsByRegex(self, regex): - """Return all projects with a full match to either project name or + def getTPCsByRegex(self, regex): + """Return all TPCs with a full match to either project name or canonical project name. :arg str regex: The regex to match - :returns: A list of tuples (trusted, project) describing the found + :returns: A list of TenantProjectConfigs describing the found projects. """ @@ -9241,15 +9279,25 @@ class Tenant(object): projects.append(project) for project in projects: - if project in self.config_projects: - result.append((True, project)) - elif project in self.untrusted_projects: - result.append((False, project)) - else: + tpc = self.project_configs.get(project.canonical_name) + if tpc is None: + # This should never happen: raise Exception("Project %s is neither trusted nor untrusted" % (project,)) + result.append(tpc) return result + def getProjectsByRegex(self, regex): + """Return all projects with a full match to either project name or + canonical project name. + + :arg str regex: The regex to match + :returns: A list of tuples (trusted, project) describing the found + projects. + """ + return [(tpc.trusted, tpc.project) + for tpc in self.getTPCsByRegex(regex)] + def getProjectBranches(self, project_canonical_name, include_always_dynamic=False): """Return a project's branches (filtered by this tenant config) @@ -9283,13 +9331,14 @@ class Tenant(object): return project_config.exclude_locked_branches return self.exclude_locked_branches - def addConfigProject(self, tpc): - self.config_projects.append(tpc.project) + def addTPC(self, tpc): self._addProject(tpc) - def addUntrustedProject(self, tpc): - self.untrusted_projects.append(tpc.project) - self._addProject(tpc) + def getTPC(self, name): + (_, project) = self.getProject(name) + if project is None: + return None + return self.project_configs[project.canonical_name] def getSafeAttributes(self): return Attributes(name=self.name) diff --git a/zuul/web/__init__.py b/zuul/web/__init__.py index 32944f0d5b..8d2b5edc66 100755 --- a/zuul/web/__init__.py +++ b/zuul/web/__init__.py @@ -1355,7 +1355,7 @@ class ZuulWebAPI(object): result.append({ 'name': tenant_name, - 'projects': len(tenant.untrusted_projects), + 'projects': len(list(tenant.untrusted_projects)), 'queue': queue_size, }) return result @@ -1656,15 +1656,11 @@ class ZuulWebAPI(object): @cherrypy.tools.check_tenant_auth() def projects(self, tenant_name, tenant, auth): result = [] - for project in tenant.config_projects: + for tpc in tenant.all_tpcs: + project = tpc.project pobj = project.toDict() - pobj['type'] = "config" + pobj['type'] = "config" if tpc.trusted else "untrusted" result.append(pobj) - for project in tenant.untrusted_projects: - pobj = project.toDict() - pobj['type'] = "untrusted" - result.append(pobj) - return sorted(result, key=lambda project: project["name"]) @cherrypy.expose