1016 lines
43 KiB
Python
1016 lines
43 KiB
Python
# Copyright 2017 Red Hat, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
import fixtures
|
|
import logging
|
|
import textwrap
|
|
import testtools
|
|
import voluptuous as vs
|
|
from collections import defaultdict
|
|
from configparser import ConfigParser
|
|
|
|
from zuul import model
|
|
from zuul.lib.ansible import AnsibleManager
|
|
from zuul.configloader import (
|
|
AuthorizationRuleParser, ConfigLoader, safe_load_yaml
|
|
)
|
|
from zuul.model import Abide, MergeRequest, SourceContext
|
|
from zuul.zk.locks import tenant_read_lock
|
|
|
|
from tests.base import ZuulTestCase
|
|
|
|
|
|
class TestConfigLoader(ZuulTestCase):
|
|
tenant_config_file = 'config/single-tenant/main.yaml'
|
|
|
|
def test_update_system_config(self):
|
|
"""Test if the system config can be updated without a scheduler."""
|
|
sched = self.scheds.first.sched
|
|
|
|
# Get the current system config before instantiating a ConfigLoader.
|
|
unparsed_abide, zuul_globals = sched.system_config_cache.get()
|
|
ansible_manager = AnsibleManager(
|
|
default_version=zuul_globals.default_ansible_version)
|
|
loader = ConfigLoader(
|
|
sched.connections, self.zk_client, zuul_globals, sched.statsd,
|
|
keystorage=sched.keystore)
|
|
abide = Abide()
|
|
loader.loadTPCs(abide, unparsed_abide)
|
|
loader.loadAdminRules(abide, unparsed_abide)
|
|
|
|
for tenant_name in unparsed_abide.tenants:
|
|
tlock = tenant_read_lock(self.zk_client, tenant_name)
|
|
# Consider all caches valid (min. ltime -1)
|
|
min_ltimes = defaultdict(lambda: defaultdict(lambda: -1))
|
|
with tlock:
|
|
tenant = loader.loadTenant(
|
|
abide, tenant_name, ansible_manager, unparsed_abide,
|
|
min_ltimes=min_ltimes)
|
|
|
|
self.assertEqual(tenant.name, tenant_name)
|
|
|
|
|
|
class TenantParserTestCase(ZuulTestCase):
|
|
create_project_keys = True
|
|
|
|
CONFIG_SET = set(['pipeline', 'job', 'semaphore', 'project',
|
|
'project-template', 'nodeset', 'secret', 'queue'])
|
|
UNTRUSTED_SET = CONFIG_SET - set(['pipeline'])
|
|
|
|
def setupAllProjectKeys(self, config: ConfigParser):
|
|
for project in ['common-config', 'org/project1', 'org/project2']:
|
|
self.setupProjectKeys('gerrit', project)
|
|
|
|
|
|
class TestTenantSimple(TenantParserTestCase):
|
|
tenant_config_file = 'config/tenant-parser/simple.yaml'
|
|
|
|
def test_tenant_simple(self):
|
|
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
|
|
self.assertEqual(['common-config'],
|
|
[x.name for x in tenant.config_projects])
|
|
self.assertEqual(['org/project1', 'org/project2'],
|
|
[x.name for x in tenant.untrusted_projects])
|
|
|
|
project = tenant.config_projects[0]
|
|
tpc = tenant.project_configs[project.canonical_name]
|
|
self.assertEqual(self.CONFIG_SET, tpc.load_classes)
|
|
project = tenant.untrusted_projects[0]
|
|
tpc = tenant.project_configs[project.canonical_name]
|
|
self.assertEqual(self.UNTRUSTED_SET, tpc.load_classes)
|
|
project = tenant.untrusted_projects[1]
|
|
tpc = tenant.project_configs[project.canonical_name]
|
|
self.assertEqual(self.UNTRUSTED_SET, tpc.load_classes)
|
|
self.assertTrue('common-config-job' in tenant.layout.jobs)
|
|
self.assertTrue('project1-job' in tenant.layout.jobs)
|
|
self.assertTrue('project2-job' in tenant.layout.jobs)
|
|
project1_config = tenant.layout.project_configs.get(
|
|
'review.example.com/org/project1')
|
|
self.assertTrue('common-config-job' in
|
|
project1_config[0].pipelines['check'].job_list.jobs)
|
|
self.assertTrue('project1-job' in
|
|
project1_config[1].pipelines['check'].job_list.jobs)
|
|
project2_config = tenant.layout.project_configs.get(
|
|
'review.example.com/org/project2')
|
|
self.assertTrue('common-config-job' in
|
|
project2_config[0].pipelines['check'].job_list.jobs)
|
|
self.assertTrue('project2-job' in
|
|
project2_config[1].pipelines['check'].job_list.jobs)
|
|
|
|
def test_variant_description(self):
|
|
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
|
|
job = tenant.layout.jobs.get("project2-job")
|
|
self.assertEqual(job[0].variant_description, "")
|
|
self.assertEqual(job[1].variant_description, "stable")
|
|
|
|
def test_merge_anchor(self):
|
|
to_parse = textwrap.dedent(
|
|
"""
|
|
- job:
|
|
name: job1
|
|
vars: &docker_vars
|
|
registry: 'registry.example.org'
|
|
|
|
- job:
|
|
name: job2
|
|
vars:
|
|
<<: &buildenv_vars
|
|
image_name: foo
|
|
<<: *docker_vars
|
|
|
|
- job:
|
|
name: job3
|
|
vars:
|
|
<<: *buildenv_vars
|
|
<<: *docker_vars
|
|
""")
|
|
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
|
|
project = tenant.config_projects[0]
|
|
source_context = SourceContext(
|
|
project.canonical_name, project.name, project.connection_name,
|
|
'master', 'zuul.yaml', True)
|
|
|
|
data = safe_load_yaml(to_parse, source_context)
|
|
self.assertEqual(len(data), 3)
|
|
job_vars = [i['job']['vars'] for i in data]
|
|
# Test that merging worked
|
|
self.assertEqual(job_vars, [
|
|
{'registry': 'registry.example.org'},
|
|
{'registry': 'registry.example.org', 'image_name': 'foo'},
|
|
{'registry': 'registry.example.org', 'image_name': 'foo'},
|
|
])
|
|
|
|
def test_deny_localhost_nodeset(self):
|
|
in_repo_conf = textwrap.dedent(
|
|
"""
|
|
- nodeset:
|
|
name: localhost
|
|
nodes:
|
|
- name: localhost
|
|
label: ubuntu
|
|
""")
|
|
file_dict = {'zuul.yaml': in_repo_conf}
|
|
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A',
|
|
files=file_dict)
|
|
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
|
|
self.waitUntilSettled()
|
|
|
|
# No job should have run due to the change introducing a config error
|
|
self.assertHistory([])
|
|
self.assertTrue(A.reported)
|
|
self.assertTrue("Nodes named 'localhost' are not allowed."
|
|
in A.messages[0])
|
|
|
|
in_repo_conf = textwrap.dedent(
|
|
"""
|
|
- nodeset:
|
|
name: localhost-group
|
|
nodes:
|
|
- name: ubuntu
|
|
label: ubuntu
|
|
groups:
|
|
- name: localhost
|
|
nodes: ubuntu
|
|
""")
|
|
file_dict = {'zuul.yaml': in_repo_conf}
|
|
B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A',
|
|
files=file_dict)
|
|
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
|
|
self.waitUntilSettled()
|
|
|
|
# No job should have run due to the change introducing a config error
|
|
self.assertHistory([])
|
|
self.assertTrue(B.reported)
|
|
self.assertTrue("Groups named 'localhost' are not allowed."
|
|
in B.messages[0])
|
|
|
|
|
|
class TestTenantOverride(TenantParserTestCase):
|
|
tenant_config_file = 'config/tenant-parser/override.yaml'
|
|
|
|
def test_tenant_override(self):
|
|
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
|
|
self.assertEqual(['common-config'],
|
|
[x.name for x in tenant.config_projects])
|
|
self.assertEqual(['org/project1', 'org/project2', 'org/project4'],
|
|
[x.name for x in tenant.untrusted_projects])
|
|
project = tenant.config_projects[0]
|
|
tpc = tenant.project_configs[project.canonical_name]
|
|
self.assertEqual(self.CONFIG_SET, tpc.load_classes)
|
|
project = tenant.untrusted_projects[0]
|
|
tpc = tenant.project_configs[project.canonical_name]
|
|
self.assertEqual(self.UNTRUSTED_SET - set(['project']),
|
|
tpc.load_classes)
|
|
project = tenant.untrusted_projects[1]
|
|
tpc = tenant.project_configs[project.canonical_name]
|
|
self.assertEqual(set(['job']), tpc.load_classes)
|
|
self.assertTrue('common-config-job' in tenant.layout.jobs)
|
|
self.assertTrue('project1-job' in tenant.layout.jobs)
|
|
self.assertTrue('project2-job' in tenant.layout.jobs)
|
|
project1_config = tenant.layout.project_configs.get(
|
|
'review.example.com/org/project1')
|
|
self.assertTrue('common-config-job' in
|
|
project1_config[0].pipelines['check'].job_list.jobs)
|
|
self.assertFalse('project1-job' in
|
|
project1_config[0].pipelines['check'].job_list.jobs)
|
|
project2_config = tenant.layout.project_configs.get(
|
|
'review.example.com/org/project2')
|
|
self.assertTrue('common-config-job' in
|
|
project2_config[0].pipelines['check'].job_list.jobs)
|
|
self.assertFalse('project2-job' in
|
|
project2_config[0].pipelines['check'].job_list.jobs)
|
|
|
|
|
|
class TestTenantGroups(TenantParserTestCase):
|
|
tenant_config_file = 'config/tenant-parser/groups.yaml'
|
|
|
|
def test_tenant_groups(self):
|
|
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
|
|
self.assertEqual(['common-config'],
|
|
[x.name for x in tenant.config_projects])
|
|
self.assertEqual(['org/project1', 'org/project2'],
|
|
[x.name for x in tenant.untrusted_projects])
|
|
project = tenant.config_projects[0]
|
|
tpc = tenant.project_configs[project.canonical_name]
|
|
self.assertEqual(self.CONFIG_SET, tpc.load_classes)
|
|
project = tenant.untrusted_projects[0]
|
|
tpc = tenant.project_configs[project.canonical_name]
|
|
self.assertEqual(self.UNTRUSTED_SET - set(['project']),
|
|
tpc.load_classes)
|
|
project = tenant.untrusted_projects[1]
|
|
tpc = tenant.project_configs[project.canonical_name]
|
|
self.assertEqual(self.UNTRUSTED_SET - set(['project']),
|
|
tpc.load_classes)
|
|
self.assertTrue('common-config-job' in tenant.layout.jobs)
|
|
self.assertTrue('project1-job' in tenant.layout.jobs)
|
|
self.assertTrue('project2-job' in tenant.layout.jobs)
|
|
project1_config = tenant.layout.project_configs.get(
|
|
'review.example.com/org/project1')
|
|
self.assertTrue('common-config-job' in
|
|
project1_config[0].pipelines['check'].job_list.jobs)
|
|
self.assertFalse('project1-job' in
|
|
project1_config[0].pipelines['check'].job_list.jobs)
|
|
project2_config = tenant.layout.project_configs.get(
|
|
'review.example.com/org/project2')
|
|
self.assertTrue('common-config-job' in
|
|
project2_config[0].pipelines['check'].job_list.jobs)
|
|
self.assertFalse('project2-job' in
|
|
project2_config[0].pipelines['check'].job_list.jobs)
|
|
|
|
|
|
class TestTenantGroups2(TenantParserTestCase):
|
|
tenant_config_file = 'config/tenant-parser/groups2.yaml'
|
|
|
|
def test_tenant_groups2(self):
|
|
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
|
|
self.assertEqual(['common-config'],
|
|
[x.name for x in tenant.config_projects])
|
|
self.assertEqual(['org/project1', 'org/project2', 'org/project3'],
|
|
[x.name for x in tenant.untrusted_projects])
|
|
project = tenant.config_projects[0]
|
|
tpc = tenant.project_configs[project.canonical_name]
|
|
self.assertEqual(self.CONFIG_SET, tpc.load_classes)
|
|
project = tenant.untrusted_projects[0]
|
|
tpc = tenant.project_configs[project.canonical_name]
|
|
self.assertEqual(self.UNTRUSTED_SET - set(['project']),
|
|
tpc.load_classes)
|
|
project = tenant.untrusted_projects[1]
|
|
tpc = tenant.project_configs[project.canonical_name]
|
|
self.assertEqual(self.UNTRUSTED_SET - set(['project', 'job']),
|
|
tpc.load_classes)
|
|
self.assertTrue('common-config-job' in tenant.layout.jobs)
|
|
self.assertTrue('project1-job' in tenant.layout.jobs)
|
|
self.assertFalse('project2-job' in tenant.layout.jobs)
|
|
project1_config = tenant.layout.project_configs.get(
|
|
'review.example.com/org/project1')
|
|
self.assertTrue('common-config-job' in
|
|
project1_config[0].pipelines['check'].job_list.jobs)
|
|
self.assertFalse('project1-job' in
|
|
project1_config[0].pipelines['check'].job_list.jobs)
|
|
project2_config = tenant.layout.project_configs.get(
|
|
'review.example.com/org/project2')
|
|
self.assertTrue('common-config-job' in
|
|
project2_config[0].pipelines['check'].job_list.jobs)
|
|
self.assertFalse('project2-job' in
|
|
project2_config[0].pipelines['check'].job_list.jobs)
|
|
|
|
|
|
class TestTenantGroups3(TenantParserTestCase):
|
|
tenant_config_file = 'config/tenant-parser/groups3.yaml'
|
|
|
|
def test_tenant_groups3(self):
|
|
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
|
|
self.assertEqual(False, tenant.exclude_unprotected_branches)
|
|
self.assertEqual(['common-config'],
|
|
[x.name for x in tenant.config_projects])
|
|
self.assertEqual(['org/project1', 'org/project2'],
|
|
[x.name for x in tenant.untrusted_projects])
|
|
project = tenant.config_projects[0]
|
|
tpc = tenant.project_configs[project.canonical_name]
|
|
self.assertEqual(self.CONFIG_SET, tpc.load_classes)
|
|
project = tenant.untrusted_projects[0]
|
|
tpc = tenant.project_configs[project.canonical_name]
|
|
self.assertEqual(set(['job']), tpc.load_classes)
|
|
project = tenant.untrusted_projects[1]
|
|
tpc = tenant.project_configs[project.canonical_name]
|
|
self.assertEqual(set(['project', 'job']), tpc.load_classes)
|
|
self.assertTrue('common-config-job' in tenant.layout.jobs)
|
|
self.assertTrue('project1-job' in tenant.layout.jobs)
|
|
self.assertTrue('project2-job' in tenant.layout.jobs)
|
|
project1_config = tenant.layout.project_configs.get(
|
|
'review.example.com/org/project1')
|
|
self.assertTrue('common-config-job' in
|
|
project1_config[0].pipelines['check'].job_list.jobs)
|
|
self.assertFalse('project1-job' in
|
|
project1_config[0].pipelines['check'].job_list.jobs)
|
|
project2_config = tenant.layout.project_configs.get(
|
|
'review.example.com/org/project2')
|
|
self.assertTrue('common-config-job' in
|
|
project2_config[0].pipelines['check'].job_list.jobs)
|
|
self.assertTrue('project2-job' in
|
|
project2_config[1].pipelines['check'].job_list.jobs)
|
|
|
|
|
|
class TestTenantGroups4(TenantParserTestCase):
|
|
tenant_config_file = 'config/tenant-parser/groups4.yaml'
|
|
|
|
def test_tenant_groups(self):
|
|
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
|
|
self.assertEqual(['common-config'],
|
|
[x.name for x in tenant.config_projects])
|
|
self.assertEqual(['org/project1', 'org/project2'],
|
|
[x.name for x in tenant.untrusted_projects])
|
|
project = tenant.config_projects[0]
|
|
tpc = tenant.project_configs[project.canonical_name]
|
|
self.assertEqual(self.CONFIG_SET, tpc.load_classes)
|
|
project = tenant.untrusted_projects[0]
|
|
tpc = tenant.project_configs[project.canonical_name]
|
|
self.assertEqual(set([]),
|
|
tpc.load_classes)
|
|
project = tenant.untrusted_projects[1]
|
|
tpc = tenant.project_configs[project.canonical_name]
|
|
self.assertEqual(set([]),
|
|
tpc.load_classes)
|
|
# Check that only one merger:cat job was requested
|
|
# org/project1 and org/project2 have an empty load_classes
|
|
cat_jobs = [job for job in self.merge_job_history.values()
|
|
if job.job_type == MergeRequest.CAT]
|
|
self.assertEqual(1, len(cat_jobs))
|
|
old_layout = tenant.layout
|
|
|
|
# Check that creating a change in project1 doesn't cause a
|
|
# reconfiguration (due to a mistaken belief that we need to
|
|
# load config from it since there is none in memory).
|
|
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
|
|
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
|
|
self.waitUntilSettled()
|
|
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
|
|
new_layout = tenant.layout
|
|
|
|
self.assertEqual(old_layout, new_layout)
|
|
|
|
|
|
class TestTenantGroups5(TenantParserTestCase):
|
|
tenant_config_file = 'config/tenant-parser/groups5.yaml'
|
|
|
|
def test_tenant_single_projet_exclude(self):
|
|
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
|
|
self.assertEqual(['common-config'],
|
|
[x.name for x in tenant.config_projects])
|
|
self.assertEqual(['org/project1'],
|
|
[x.name for x in tenant.untrusted_projects])
|
|
project = tenant.config_projects[0]
|
|
tpc = tenant.project_configs[project.canonical_name]
|
|
self.assertEqual(self.CONFIG_SET, tpc.load_classes)
|
|
project = tenant.untrusted_projects[0]
|
|
tpc = tenant.project_configs[project.canonical_name]
|
|
self.assertEqual(set([]),
|
|
tpc.load_classes)
|
|
# Check that only one merger:cat job was requested
|
|
# org/project1 and org/project2 have an empty load_classes
|
|
cat_jobs = [job for job in self.merge_job_history.values()
|
|
if job.job_type == MergeRequest.CAT]
|
|
self.assertEqual(1, len(cat_jobs))
|
|
|
|
|
|
class TestTenantFromScript(TestTenantSimple):
|
|
tenant_config_file = None
|
|
tenant_config_script_file = 'config/tenant-parser/tenant_config_script.py'
|
|
|
|
def test_tenant_simple(self):
|
|
TestTenantSimple.test_tenant_simple(self)
|
|
|
|
|
|
class TestTenantUnprotectedBranches(TenantParserTestCase):
|
|
tenant_config_file = 'config/tenant-parser/unprotected-branches.yaml'
|
|
|
|
def test_tenant_unprotected_branches(self):
|
|
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
|
|
self.assertEqual(True, tenant.exclude_unprotected_branches)
|
|
|
|
self.assertEqual(['common-config'],
|
|
[x.name for x in tenant.config_projects])
|
|
self.assertEqual(['org/project1', 'org/project2'],
|
|
[x.name for x in tenant.untrusted_projects])
|
|
|
|
tpc = tenant.project_configs
|
|
project_name = tenant.config_projects[0].canonical_name
|
|
self.assertEqual(False, tpc[project_name].exclude_unprotected_branches)
|
|
|
|
project_name = tenant.untrusted_projects[0].canonical_name
|
|
self.assertIsNone(tpc[project_name].exclude_unprotected_branches)
|
|
|
|
project_name = tenant.untrusted_projects[1].canonical_name
|
|
self.assertIsNone(tpc[project_name].exclude_unprotected_branches)
|
|
|
|
|
|
class TestTenantExcludeAll(TenantParserTestCase):
|
|
tenant_config_file = 'config/tenant-parser/exclude-all.yaml'
|
|
|
|
def test_tenant_exclude_all(self):
|
|
"""
|
|
Tests that excluding all configuration of project1 in tenant-one
|
|
doesn't remove the configuration of project1 in tenant-two.
|
|
"""
|
|
# The config in org/project5 depends on config in org/project1 so
|
|
# validate that there are no config errors in that tenant.
|
|
tenant_two = self.scheds.first.sched.abide.tenants.get('tenant-two')
|
|
self.assertEquals(
|
|
len(tenant_two.layout.loading_errors), 0,
|
|
"No error should have been accumulated")
|
|
|
|
|
|
class TestTenantConfigBranches(ZuulTestCase):
|
|
tenant_config_file = 'config/tenant-parser/simple.yaml'
|
|
|
|
def _validate_job(self, job, branch):
|
|
tenant_one = self.scheds.first.sched.abide.tenants.get('tenant-one')
|
|
jobs = tenant_one.layout.getJobs(job)
|
|
self.assertEquals(len(jobs), 1)
|
|
self.assertIn(jobs[0].source_context.branch, branch)
|
|
|
|
def test_tenant_config_load_branch(self):
|
|
"""
|
|
Tests that when specifying branches for a project only those branches
|
|
are parsed.
|
|
"""
|
|
# Job must be defined in master
|
|
common_job = 'common-config-job'
|
|
self._validate_job(common_job, 'master')
|
|
|
|
self.log.debug('Creating branches')
|
|
self.create_branch('common-config', 'stable')
|
|
self.create_branch('common-config', 'feat_x')
|
|
self.fake_gerrit.addEvent(
|
|
self.fake_gerrit.getFakeBranchCreatedEvent(
|
|
'common-config', 'stable'))
|
|
self.fake_gerrit.addEvent(
|
|
self.fake_gerrit.getFakeBranchCreatedEvent(
|
|
'common-config', 'feat_x'))
|
|
self.waitUntilSettled()
|
|
|
|
# Job must be defined in master
|
|
self._validate_job(common_job, 'master')
|
|
|
|
# Reconfigure with load-branch stable for common-config
|
|
self.newTenantConfig('config/tenant-parser/branch.yaml')
|
|
self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
|
|
|
|
# Now job must be defined on stable branch
|
|
self._validate_job(common_job, 'stable')
|
|
|
|
# Now try to break the config in common-config on stable
|
|
in_repo_conf = textwrap.dedent(
|
|
"""
|
|
- job:
|
|
name: base
|
|
parent: non-existing
|
|
""")
|
|
file_dict = {'zuul.yaml': in_repo_conf}
|
|
A = self.fake_gerrit.addFakeChange('common-config', 'stable', 'A',
|
|
files=file_dict)
|
|
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
|
|
self.waitUntilSettled()
|
|
|
|
# No job should have run due to the change introducing a config error
|
|
self.assertHistory([])
|
|
self.assertTrue(A.reported)
|
|
self.assertTrue('Job non-existing not defined' in A.messages[0])
|
|
|
|
|
|
class TestSplitConfig(ZuulTestCase):
|
|
tenant_config_file = 'config/split-config/main.yaml'
|
|
|
|
def test_split_config(self):
|
|
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
|
|
self.assertIn('project-test1', tenant.layout.jobs)
|
|
self.assertIn('project-test2', tenant.layout.jobs)
|
|
test1 = tenant.layout.getJob('project-test1')
|
|
self.assertEqual(test1.source_context.project_name, 'common-config')
|
|
self.assertEqual(test1.source_context.branch, 'master')
|
|
self.assertEqual(test1.source_context.path, 'zuul.d/jobs.yaml')
|
|
self.assertEqual(test1.source_context.trusted, True)
|
|
test2 = tenant.layout.getJob('project-test2')
|
|
self.assertEqual(test2.source_context.project_name, 'common-config')
|
|
self.assertEqual(test2.source_context.branch, 'master')
|
|
self.assertEqual(test2.source_context.path, 'zuul.d/more-jobs.yaml')
|
|
self.assertEqual(test2.source_context.trusted, True)
|
|
|
|
self.assertNotEqual(test1.source_context, test2.source_context)
|
|
self.assertTrue(test1.source_context.isSameProject(
|
|
test2.source_context))
|
|
|
|
project_config = tenant.layout.project_configs.get(
|
|
'review.example.com/org/project')
|
|
self.assertIn('project-test1',
|
|
project_config[0].pipelines['check'].job_list.jobs)
|
|
project1_config = tenant.layout.project_configs.get(
|
|
'review.example.com/org/project1')
|
|
self.assertIn('project1-project2-integration',
|
|
project1_config[0].pipelines['check'].job_list.jobs)
|
|
|
|
# This check ensures the .zuul.ignore flag file is working in
|
|
# the config directory.
|
|
self.assertEquals(
|
|
len(tenant.layout.loading_errors), 0)
|
|
|
|
def test_dynamic_split_config(self):
|
|
in_repo_conf = textwrap.dedent(
|
|
"""
|
|
- project:
|
|
name: org/project1
|
|
check:
|
|
jobs:
|
|
- project-test1
|
|
""")
|
|
file_dict = {'.zuul.d/gate.yaml': in_repo_conf}
|
|
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A',
|
|
files=file_dict)
|
|
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
|
|
self.waitUntilSettled()
|
|
# project1-project2-integration test removed, only want project-test1
|
|
self.assertHistory([
|
|
dict(name='project-test1', result='SUCCESS', changes='1,1')])
|
|
|
|
def test_config_path_conflict(self):
|
|
def add_file(project, path):
|
|
new_file = textwrap.dedent(
|
|
"""
|
|
- job:
|
|
name: test-job
|
|
"""
|
|
)
|
|
file_dict = {path: new_file}
|
|
A = self.fake_gerrit.addFakeChange(project, 'master', 'A',
|
|
files=file_dict)
|
|
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
|
|
self.waitUntilSettled()
|
|
return A
|
|
|
|
log_fixture = self.useFixture(
|
|
fixtures.FakeLogger(level=logging.WARNING))
|
|
|
|
log_fixture._output.truncate(0)
|
|
A = add_file("common-config", "zuul.yaml")
|
|
self.assertIn("Configuration in common-config/zuul.d/jobs.yaml@master "
|
|
"ignored because project-branch is already configured",
|
|
log_fixture.output)
|
|
self.assertIn("Configuration in common-config/zuul.d/jobs.yaml@master "
|
|
"ignored because project-branch is already configured",
|
|
A.messages[0])
|
|
|
|
log_fixture._output.truncate(0)
|
|
add_file("org/project1", ".zuul.yaml")
|
|
self.assertIn("Configuration in org/project1/.zuul.d/gate.yaml@master "
|
|
"ignored because project-branch is already configured",
|
|
log_fixture.output)
|
|
|
|
|
|
class TestConfigConflict(ZuulTestCase):
|
|
tenant_config_file = 'config/conflict-config/main.yaml'
|
|
|
|
def test_conflict_config(self):
|
|
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
|
|
jobs = sorted(tenant.layout.jobs.keys())
|
|
self.assertEqual(
|
|
['base', 'noop', 'trusted-zuul.yaml-job',
|
|
'untrusted-zuul.yaml-job'],
|
|
jobs)
|
|
|
|
|
|
class TestUnparsedConfigCache(ZuulTestCase):
|
|
tenant_config_file = 'config/single-tenant/main.yaml'
|
|
|
|
def test_config_caching(self):
|
|
sched = self.scheds.first.sched
|
|
cache = sched.unparsed_config_cache
|
|
tenant = sched.abide.tenants["tenant-one"]
|
|
|
|
common_cache = cache.getFilesCache("review.example.com/common-config",
|
|
"master")
|
|
upb_common_cache = sched.abide.getUnparsedBranchCache(
|
|
"review.example.com/common-config", "master")
|
|
tpc = tenant.project_configs["review.example.com/common-config"]
|
|
self.assertTrue(common_cache.isValidFor(tpc, min_ltime=-1))
|
|
self.assertEqual(len(common_cache), 1)
|
|
self.assertIn("zuul.yaml", common_cache)
|
|
self.assertTrue(len(common_cache["zuul.yaml"]) > 0)
|
|
self.assertEqual(upb_common_cache.ltime, common_cache.ltime)
|
|
|
|
project_cache = cache.getFilesCache("review.example.com/org/project",
|
|
"master")
|
|
upb_project_cache = sched.abide.getUnparsedBranchCache(
|
|
"review.example.com/org/project", "master")
|
|
# Cache of org/project should be valid but empty (no in-repo config)
|
|
tpc = tenant.project_configs["review.example.com/org/project"]
|
|
self.assertTrue(project_cache.isValidFor(tpc, min_ltime=-1))
|
|
self.assertEqual(len(project_cache), 0)
|
|
self.assertEqual(upb_project_cache.ltime, project_cache.ltime)
|
|
|
|
def test_cache_use(self):
|
|
sched = self.scheds.first.sched
|
|
# Stop cleanup jobs so it's not removing projects from
|
|
# the cache during the test.
|
|
sched.apsched.shutdown()
|
|
tenant = sched.abide.tenants['tenant-one']
|
|
_, project = tenant.getProject('org/project2')
|
|
|
|
cache = self.scheds.first.sched.unparsed_config_cache
|
|
files_cache = cache.getFilesCache(
|
|
"review.example.com/org/project2", "master")
|
|
zk_initial_ltime = files_cache.ltime
|
|
upb_cache = sched.abide.getUnparsedBranchCache(
|
|
"review.example.com/org/project2", "master")
|
|
self.assertEqual(zk_initial_ltime, upb_cache.ltime)
|
|
|
|
# Get the current ltime from Zookeeper and run a full reconfiguration,
|
|
# so that we know all items in the cache have a larger ltime.
|
|
ltime = self.zk_client.getCurrentLtime()
|
|
self.scheds.first.fullReconfigure()
|
|
self.assertGreater(files_cache.ltime, zk_initial_ltime)
|
|
upb_cache = sched.abide.getUnparsedBranchCache(
|
|
"review.example.com/org/project2", "master")
|
|
self.assertEqual(files_cache.ltime, upb_cache.ltime)
|
|
|
|
# Clear the unparsed branch cache so all projects (except for
|
|
# org/project2) are retrieved from the cache in Zookeeper.
|
|
sched.abide.unparsed_project_branch_cache.clear()
|
|
del self.merge_job_history
|
|
|
|
# Create a tenant reconfiguration event with a known ltime that is
|
|
# smaller than the ltime of the items in the cache.
|
|
event = model.TenantReconfigureEvent(
|
|
tenant.name, project.canonical_name, branch_name=None)
|
|
event.zuul_event_ltime = ltime
|
|
sched.management_events[tenant.name].put(event, needs_result=False)
|
|
self.waitUntilSettled()
|
|
|
|
# As the cache should be valid (cache ltime of org/project2 newer than
|
|
# event ltime) we don't expect any cat jobs.
|
|
cat_jobs = [job for job in self.merge_job_history.values()
|
|
if job.job_type == MergeRequest.CAT]
|
|
self.assertEqual(len(cat_jobs), 0)
|
|
|
|
# Set canary value so we can detect if the configloader used
|
|
# the cache in Zookeeper (it shouldn't).
|
|
common_cache = cache.getFilesCache("review.example.com/common-config",
|
|
"master")
|
|
common_cache.setValidFor({"CANARY"}, set(), common_cache.ltime)
|
|
del self.merge_job_history
|
|
|
|
# Create a tenant reconfiguration event with a known ltime that is
|
|
# smaller than the ltime of the items in the cache.
|
|
event = model.TenantReconfigureEvent(
|
|
tenant.name, project.canonical_name, branch_name=None)
|
|
event.zuul_event_ltime = ltime
|
|
sched.management_events[tenant.name].put(event, needs_result=False)
|
|
self.waitUntilSettled()
|
|
|
|
upb_cache = sched.abide.getUnparsedBranchCache(
|
|
"review.example.com/common-config", "master")
|
|
self.assertEqual(common_cache.ltime, upb_cache.ltime)
|
|
self.assertNotIn("CANARY", upb_cache.extra_files_searched)
|
|
|
|
# As the cache should be valid (cache ltime of org/project2 newer than
|
|
# event ltime) we don't expect any cat jobs.
|
|
cat_jobs = [job for job in self.merge_job_history.values()
|
|
if job.job_type == MergeRequest.CAT]
|
|
self.assertEqual(len(cat_jobs), 0)
|
|
sched.apsched.start()
|
|
|
|
|
|
class TestAuthorizationRuleParser(ZuulTestCase):
|
|
tenant_config_file = 'config/tenant-parser/authorizations.yaml'
|
|
|
|
def test_rules_are_loaded(self):
|
|
rules = self.scheds.first.sched.abide.admin_rules
|
|
self.assertTrue('auth-rule-one' in rules,
|
|
self.scheds.first.sched.abide)
|
|
self.assertTrue('auth-rule-two' in rules,
|
|
self.scheds.first.sched.abide)
|
|
claims_1 = {'sub': 'venkman'}
|
|
claims_2 = {'sub': 'gozer',
|
|
'iss': 'another_dimension'}
|
|
self.assertTrue(rules['auth-rule-one'](claims_1))
|
|
self.assertTrue(not rules['auth-rule-one'](claims_2))
|
|
self.assertTrue(not rules['auth-rule-two'](claims_1))
|
|
self.assertTrue(rules['auth-rule-two'](claims_2))
|
|
|
|
def test_parse_simplest_rule_from_yaml(self):
|
|
rule_d = {'name': 'my-rule',
|
|
'conditions': {'sub': 'user1'}
|
|
}
|
|
rule = AuthorizationRuleParser().fromYaml(rule_d)
|
|
self.assertEqual('my-rule', rule.name)
|
|
claims = {'iss': 'my-idp',
|
|
'sub': 'user1',
|
|
'groups': ['admin', 'ghostbusters']}
|
|
self.assertTrue(rule(claims))
|
|
claims = {'iss': 'my-2nd-idp',
|
|
'sub': 'user2',
|
|
'groups': ['admin', 'ghostbusters']}
|
|
self.assertFalse(rule(claims))
|
|
|
|
def test_parse_AND_rule_from_yaml(self):
|
|
rule_d = {'name': 'my-rule',
|
|
'conditions': {'sub': 'user1',
|
|
'iss': 'my-idp'}
|
|
}
|
|
rule = AuthorizationRuleParser().fromYaml(rule_d)
|
|
self.assertEqual('my-rule', rule.name)
|
|
claims = {'iss': 'my-idp',
|
|
'sub': 'user1',
|
|
'groups': ['admin', 'ghostbusters']}
|
|
self.assertTrue(rule(claims))
|
|
claims = {'iss': 'my-2nd-idp',
|
|
'sub': 'user1',
|
|
'groups': ['admin', 'ghostbusters']}
|
|
self.assertFalse(rule(claims))
|
|
|
|
def test_parse_OR_rule_from_yaml(self):
|
|
rule_d = {'name': 'my-rule',
|
|
'conditions': [{'sub': 'user1',
|
|
'iss': 'my-idp'},
|
|
{'sub': 'user2',
|
|
'iss': 'my-2nd-idp'}
|
|
]
|
|
}
|
|
rule = AuthorizationRuleParser().fromYaml(rule_d)
|
|
self.assertEqual('my-rule', rule.name)
|
|
claims = {'iss': 'my-idp',
|
|
'sub': 'user1',
|
|
'groups': ['admin', 'ghostbusters']}
|
|
self.assertTrue(rule(claims))
|
|
claims = {'iss': 'my-2nd-idp',
|
|
'sub': 'user1',
|
|
'groups': ['admin', 'ghostbusters']}
|
|
self.assertFalse(rule(claims))
|
|
claims = {'iss': 'my-2nd-idp',
|
|
'sub': 'user2',
|
|
'groups': ['admin', 'ghostbusters']}
|
|
self.assertTrue(rule(claims))
|
|
|
|
def test_parse_rule_with_list_claim_from_yaml(self):
|
|
rule_d = {'name': 'my-rule',
|
|
'conditions': [{'groups': 'ghostbusters',
|
|
'iss': 'my-idp'},
|
|
{'sub': 'user2',
|
|
'iss': 'my-2nd-idp'}
|
|
],
|
|
}
|
|
rule = AuthorizationRuleParser().fromYaml(rule_d)
|
|
self.assertEqual('my-rule', rule.name)
|
|
claims = {'iss': 'my-idp',
|
|
'sub': 'user1',
|
|
'groups': ['admin', 'ghostbusters']}
|
|
self.assertTrue(rule(claims))
|
|
claims = {'iss': 'my-idp',
|
|
'sub': 'user1',
|
|
'groups': ['admin', 'ghostbeaters']}
|
|
self.assertFalse(rule(claims))
|
|
claims = {'iss': 'my-2nd-idp',
|
|
'sub': 'user2',
|
|
'groups': ['admin', 'ghostbusters']}
|
|
self.assertTrue(rule(claims))
|
|
|
|
def test_check_complex_rule_from_yaml_jsonpath(self):
|
|
rule_d = {'name': 'my-rule',
|
|
'conditions': [{'hello.this.is': 'a complex value'},
|
|
],
|
|
}
|
|
rule = AuthorizationRuleParser().fromYaml(rule_d)
|
|
self.assertEqual('my-rule', rule.name)
|
|
claims = {'iss': 'my-idp',
|
|
'hello': {
|
|
'this': {
|
|
'is': 'a complex value'
|
|
},
|
|
'and': {
|
|
'this one': 'too'
|
|
}
|
|
}
|
|
}
|
|
self.assertTrue(rule(claims))
|
|
|
|
def test_check_complex_rule_from_yaml_nested_dict(self):
|
|
rule_d = {'name': 'my-rule',
|
|
'conditions': [{'hello': {'this': {'is': 'a complex value'
|
|
}
|
|
}
|
|
},
|
|
],
|
|
}
|
|
rule = AuthorizationRuleParser().fromYaml(rule_d)
|
|
self.assertEqual('my-rule', rule.name)
|
|
claims = {'iss': 'my-idp',
|
|
'hello': {
|
|
'this': {
|
|
'is': 'a complex value'
|
|
},
|
|
'and': {
|
|
'this one': 'too'
|
|
}
|
|
}
|
|
}
|
|
self.assertTrue(rule(claims))
|
|
|
|
|
|
class TestAuthorizationRuleParserWithTemplating(ZuulTestCase):
|
|
tenant_config_file = 'config/tenant-parser/authorizations-templating.yaml'
|
|
|
|
def test_rules_are_loaded(self):
|
|
rules = self.scheds.first.sched.abide.admin_rules
|
|
self.assertTrue('tenant-admin' in rules, self.scheds.first.sched.abide)
|
|
self.assertTrue('tenant-admin-complex' in rules,
|
|
self.scheds.first.sched.abide)
|
|
|
|
def test_tenant_substitution(self):
|
|
claims_1 = {'group': 'tenant-one-admin'}
|
|
claims_2 = {'group': 'tenant-two-admin'}
|
|
rules = self.scheds.first.sched.abide.admin_rules
|
|
tenant_one = self.scheds.first.sched.abide.tenants.get('tenant-one')
|
|
tenant_two = self.scheds.first.sched.abide.tenants.get('tenant-two')
|
|
self.assertTrue(rules['tenant-admin'](claims_1, tenant_one))
|
|
self.assertTrue(rules['tenant-admin'](claims_2, tenant_two))
|
|
self.assertTrue(not rules['tenant-admin'](claims_1, tenant_two))
|
|
self.assertTrue(not rules['tenant-admin'](claims_2, tenant_one))
|
|
|
|
def test_tenant_substitution_in_list(self):
|
|
claims_1 = {'group': ['tenant-one-admin', 'some-other-tenant']}
|
|
claims_2 = {'group': ['tenant-two-admin', 'some-other-tenant']}
|
|
rules = self.scheds.first.sched.abide.admin_rules
|
|
tenant_one = self.scheds.first.sched.abide.tenants.get('tenant-one')
|
|
tenant_two = self.scheds.first.sched.abide.tenants.get('tenant-two')
|
|
self.assertTrue(rules['tenant-admin'](claims_1, tenant_one))
|
|
self.assertTrue(rules['tenant-admin'](claims_2, tenant_two))
|
|
self.assertTrue(not rules['tenant-admin'](claims_1, tenant_two))
|
|
self.assertTrue(not rules['tenant-admin'](claims_2, tenant_one))
|
|
|
|
def test_tenant_substitution_in_dict(self):
|
|
claims_2 = {
|
|
'path': {
|
|
'to': {
|
|
'group': 'tenant-two-admin'
|
|
}
|
|
}
|
|
}
|
|
rules = self.scheds.first.sched.abide.admin_rules
|
|
tenant_one = self.scheds.first.sched.abide.tenants.get('tenant-one')
|
|
tenant_two = self.scheds.first.sched.abide.tenants.get('tenant-two')
|
|
self.assertTrue(not rules['tenant-admin-complex'](claims_2,
|
|
tenant_one))
|
|
self.assertTrue(rules['tenant-admin-complex'](claims_2, tenant_two))
|
|
|
|
|
|
class TestTenantExtra(TenantParserTestCase):
|
|
tenant_config_file = 'config/tenant-parser/extra.yaml'
|
|
|
|
def test_tenant_extra(self):
|
|
tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
|
|
self.assertTrue('project2-extra-file' in tenant.layout.jobs)
|
|
self.assertTrue('project2-extra-dir' in tenant.layout.jobs)
|
|
|
|
def test_dynamic_extra(self):
|
|
in_repo_conf = textwrap.dedent(
|
|
"""
|
|
- job:
|
|
name: project2-extra-file2
|
|
parent: common-config-job
|
|
- project:
|
|
name: org/project2
|
|
check:
|
|
jobs:
|
|
- project2-extra-file2
|
|
""")
|
|
file_dict = {'extra.yaml': in_repo_conf, '.zuul.yaml': ''}
|
|
A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A',
|
|
files=file_dict)
|
|
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
|
|
self.waitUntilSettled()
|
|
self.assertHistory([
|
|
dict(name='common-config-job', result='SUCCESS', changes='1,1'),
|
|
dict(name='project2-extra-file2', result='SUCCESS', changes='1,1'),
|
|
], ordered=False)
|
|
|
|
def test_extra_reconfigure(self):
|
|
in_repo_conf = textwrap.dedent(
|
|
"""
|
|
- job:
|
|
name: project2-extra-file2
|
|
parent: common-config-job
|
|
- project:
|
|
name: org/project2
|
|
check:
|
|
jobs:
|
|
- project2-extra-file2
|
|
""")
|
|
file_dict = {'extra.yaml': in_repo_conf}
|
|
A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A',
|
|
files=file_dict)
|
|
A.setMerged()
|
|
self.fake_gerrit.addEvent(A.getChangeMergedEvent())
|
|
self.waitUntilSettled()
|
|
self.fake_gerrit.addEvent(A.getRefUpdatedEvent())
|
|
self.waitUntilSettled()
|
|
|
|
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
|
|
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
|
|
self.waitUntilSettled()
|
|
self.assertHistory([
|
|
dict(name='common-config-job', result='SUCCESS', changes='2,1'),
|
|
dict(name='project2-job', result='SUCCESS', changes='2,1'),
|
|
dict(name='project2-extra-file2', result='SUCCESS', changes='2,1'),
|
|
], ordered=False)
|
|
|
|
|
|
class TestTenantExtraConfigsInvalidType(TenantParserTestCase):
|
|
tenant_config_file = 'config/tenant-parser/extra_invalid_type.yaml'
|
|
|
|
# This test raises a config error during the startup of the test
|
|
# case which makes the first scheduler fail during its startup.
|
|
# The second (or any additional) scheduler won't even run as the
|
|
# startup is serialized in tests/base.py.
|
|
# Thus it doesn't make sense to execute this test with multiple
|
|
# schedulers.
|
|
scheduler_count = 1
|
|
|
|
def setUp(self):
|
|
err = "Expected str or list of str for extra-config-paths.*"
|
|
with testtools.ExpectedException(vs.MultipleInvalid, err):
|
|
super().setUp()
|
|
|
|
def test_tenant_extra_configs_invalid_type(self):
|
|
# The magic is in setUp
|
|
pass
|
|
|
|
|
|
class TestTenantExtraConfigsInvalidValue(TenantParserTestCase):
|
|
tenant_config_file = 'config/tenant-parser/extra_invalid_value.yaml'
|
|
|
|
# This test raises a config error during the startup of the test
|
|
# case which makes the first scheduler fail during its startup.
|
|
# The second (or any additional) scheduler won't even run as the
|
|
# startup is serialized in tests/base.py.
|
|
# Thus it doesn't make sense to execute this test with multiple
|
|
# schedulers.
|
|
scheduler_count = 1
|
|
|
|
def setUp(self):
|
|
err = "Default zuul configs are not allowed in extra-config-paths.*"
|
|
with testtools.ExpectedException(vs.MultipleInvalid, err):
|
|
super().setUp()
|
|
|
|
def test_tenant_extra_configs_invalid_value(self):
|
|
# The magic is in setUp
|
|
pass
|
|
|
|
|
|
class TestTenantDuplicate(TenantParserTestCase):
|
|
tenant_config_file = 'config/tenant-parser/duplicate.yaml'
|
|
|
|
# This test raises a config error during the startup of the test
|
|
# case which makes the first scheduler fail during its startup.
|
|
# The second (or any additional) scheduler won't even run as the
|
|
# startup is serialized in tests/base.py.
|
|
# Thus it doesn't make sense to execute this test with multiple
|
|
# schedulers.
|
|
scheduler_count = 1
|
|
|
|
def setUp(self):
|
|
with testtools.ExpectedException(Exception, 'Duplicate configuration'):
|
|
super().setUp()
|
|
|
|
def test_tenant_dupe(self):
|
|
# The magic is in setUp
|
|
pass
|