Add TenantProjectConfig object
We were attaching tenant-specific metadata (include/exclude) to Project objects which is incorrect since those objects may span tenants. Instead, create a new TenantProjectConfig class which holds such metadata, and attach it to the Tenant class. Change-Id: Id69f9ec3a5116460beef2f83e065f5a1021dc147
This commit is contained in:
James E. Blair
parent
aa4199cd2a
commit
08d9b7801d
|
|
@ -37,12 +37,16 @@ class TestTenantSimple(TenantParserTestCase):
|
|||
[x.name for x in tenant.config_projects])
|
||||
self.assertEqual(['org/project1', 'org/project2'],
|
||||
[x.name for x in tenant.untrusted_projects])
|
||||
self.assertEqual(self.CONFIG_SET,
|
||||
tenant.config_projects[0].load_classes)
|
||||
self.assertEqual(self.UNTRUSTED_SET,
|
||||
tenant.untrusted_projects[0].load_classes)
|
||||
self.assertEqual(self.UNTRUSTED_SET,
|
||||
tenant.untrusted_projects[1].load_classes)
|
||||
|
||||
project = tenant.config_projects[0]
|
||||
tpc = tenant.project_configs[project.canonical_name]
|
||||
self.assertEqual(self.CONFIG_SET, tpc.load_classes)
|
||||
project = tenant.untrusted_projects[0]
|
||||
tpc = tenant.project_configs[project.canonical_name]
|
||||
self.assertEqual(self.UNTRUSTED_SET, tpc.load_classes)
|
||||
project = tenant.untrusted_projects[1]
|
||||
tpc = tenant.project_configs[project.canonical_name]
|
||||
self.assertEqual(self.UNTRUSTED_SET, tpc.load_classes)
|
||||
self.assertTrue('common-config-job' in tenant.layout.jobs)
|
||||
self.assertTrue('project1-job' in tenant.layout.jobs)
|
||||
self.assertTrue('project2-job' in tenant.layout.jobs)
|
||||
|
|
@ -69,12 +73,16 @@ class TestTenantOverride(TenantParserTestCase):
|
|||
[x.name for x in tenant.config_projects])
|
||||
self.assertEqual(['org/project1', 'org/project2'],
|
||||
[x.name for x in tenant.untrusted_projects])
|
||||
self.assertEqual(self.CONFIG_SET,
|
||||
tenant.config_projects[0].load_classes)
|
||||
project = tenant.config_projects[0]
|
||||
tpc = tenant.project_configs[project.canonical_name]
|
||||
self.assertEqual(self.CONFIG_SET, tpc.load_classes)
|
||||
project = tenant.untrusted_projects[0]
|
||||
tpc = tenant.project_configs[project.canonical_name]
|
||||
self.assertEqual(self.UNTRUSTED_SET - set(['project']),
|
||||
tenant.untrusted_projects[0].load_classes)
|
||||
self.assertEqual(set(['job']),
|
||||
tenant.untrusted_projects[1].load_classes)
|
||||
tpc.load_classes)
|
||||
project = tenant.untrusted_projects[1]
|
||||
tpc = tenant.project_configs[project.canonical_name]
|
||||
self.assertEqual(set(['job']), tpc.load_classes)
|
||||
self.assertTrue('common-config-job' in tenant.layout.jobs)
|
||||
self.assertTrue('project1-job' in tenant.layout.jobs)
|
||||
self.assertTrue('project2-job' in tenant.layout.jobs)
|
||||
|
|
@ -101,12 +109,17 @@ class TestTenantGroups(TenantParserTestCase):
|
|||
[x.name for x in tenant.config_projects])
|
||||
self.assertEqual(['org/project1', 'org/project2'],
|
||||
[x.name for x in tenant.untrusted_projects])
|
||||
self.assertEqual(self.CONFIG_SET,
|
||||
tenant.config_projects[0].load_classes)
|
||||
project = tenant.config_projects[0]
|
||||
tpc = tenant.project_configs[project.canonical_name]
|
||||
self.assertEqual(self.CONFIG_SET, tpc.load_classes)
|
||||
project = tenant.untrusted_projects[0]
|
||||
tpc = tenant.project_configs[project.canonical_name]
|
||||
self.assertEqual(self.UNTRUSTED_SET - set(['project']),
|
||||
tenant.untrusted_projects[0].load_classes)
|
||||
tpc.load_classes)
|
||||
project = tenant.untrusted_projects[1]
|
||||
tpc = tenant.project_configs[project.canonical_name]
|
||||
self.assertEqual(self.UNTRUSTED_SET - set(['project']),
|
||||
tenant.untrusted_projects[1].load_classes)
|
||||
tpc.load_classes)
|
||||
self.assertTrue('common-config-job' in tenant.layout.jobs)
|
||||
self.assertTrue('project1-job' in tenant.layout.jobs)
|
||||
self.assertTrue('project2-job' in tenant.layout.jobs)
|
||||
|
|
@ -133,12 +146,17 @@ class TestTenantGroups2(TenantParserTestCase):
|
|||
[x.name for x in tenant.config_projects])
|
||||
self.assertEqual(['org/project1', 'org/project2'],
|
||||
[x.name for x in tenant.untrusted_projects])
|
||||
self.assertEqual(self.CONFIG_SET,
|
||||
tenant.config_projects[0].load_classes)
|
||||
project = tenant.config_projects[0]
|
||||
tpc = tenant.project_configs[project.canonical_name]
|
||||
self.assertEqual(self.CONFIG_SET, tpc.load_classes)
|
||||
project = tenant.untrusted_projects[0]
|
||||
tpc = tenant.project_configs[project.canonical_name]
|
||||
self.assertEqual(self.UNTRUSTED_SET - set(['project']),
|
||||
tenant.untrusted_projects[0].load_classes)
|
||||
tpc.load_classes)
|
||||
project = tenant.untrusted_projects[1]
|
||||
tpc = tenant.project_configs[project.canonical_name]
|
||||
self.assertEqual(self.UNTRUSTED_SET - set(['project', 'job']),
|
||||
tenant.untrusted_projects[1].load_classes)
|
||||
tpc.load_classes)
|
||||
self.assertTrue('common-config-job' in tenant.layout.jobs)
|
||||
self.assertTrue('project1-job' in tenant.layout.jobs)
|
||||
self.assertFalse('project2-job' in tenant.layout.jobs)
|
||||
|
|
@ -165,12 +183,15 @@ class TestTenantGroups3(TenantParserTestCase):
|
|||
[x.name for x in tenant.config_projects])
|
||||
self.assertEqual(['org/project1', 'org/project2'],
|
||||
[x.name for x in tenant.untrusted_projects])
|
||||
self.assertEqual(self.CONFIG_SET,
|
||||
tenant.config_projects[0].load_classes)
|
||||
self.assertEqual(set(['job']),
|
||||
tenant.untrusted_projects[0].load_classes)
|
||||
self.assertEqual(set(['project', 'job']),
|
||||
tenant.untrusted_projects[1].load_classes)
|
||||
project = tenant.config_projects[0]
|
||||
tpc = tenant.project_configs[project.canonical_name]
|
||||
self.assertEqual(self.CONFIG_SET, tpc.load_classes)
|
||||
project = tenant.untrusted_projects[0]
|
||||
tpc = tenant.project_configs[project.canonical_name]
|
||||
self.assertEqual(set(['job']), tpc.load_classes)
|
||||
project = tenant.untrusted_projects[1]
|
||||
tpc = tenant.project_configs[project.canonical_name]
|
||||
self.assertEqual(set(['project', 'job']), tpc.load_classes)
|
||||
self.assertTrue('common-config-job' in tenant.layout.jobs)
|
||||
self.assertTrue('project1-job' in tenant.layout.jobs)
|
||||
self.assertTrue('project2-job' in tenant.layout.jobs)
|
||||
|
|
|
|||
|
|
@ -42,7 +42,8 @@ class TestJob(BaseTestCase):
|
|||
self.tenant = model.Tenant('tenant')
|
||||
self.layout = model.Layout()
|
||||
self.project = model.Project('project', self.source)
|
||||
self.tenant.addUntrustedProject(self.project)
|
||||
self.tpc = model.TenantProjectConfig(self.project)
|
||||
self.tenant.addUntrustedProject(self.tpc)
|
||||
self.pipeline = model.Pipeline('gate', self.layout)
|
||||
self.layout.addPipeline(self.pipeline)
|
||||
self.queue = model.ChangeQueue(self.pipeline)
|
||||
|
|
@ -175,7 +176,8 @@ class TestJob(BaseTestCase):
|
|||
layout.addPipeline(pipeline)
|
||||
queue = model.ChangeQueue(pipeline)
|
||||
project = model.Project('project', self.source)
|
||||
tenant.addUntrustedProject(project)
|
||||
tpc = model.TenantProjectConfig(project)
|
||||
tenant.addUntrustedProject(tpc)
|
||||
|
||||
base = configloader.JobParser.fromYaml(tenant, layout, {
|
||||
'_source_context': self.context,
|
||||
|
|
@ -442,7 +444,8 @@ class TestJob(BaseTestCase):
|
|||
def test_job_inheritance_job_tree(self):
|
||||
tenant = model.Tenant('tenant')
|
||||
layout = model.Layout()
|
||||
tenant.addUntrustedProject(self.project)
|
||||
tpc = model.TenantProjectConfig(self.project)
|
||||
tenant.addUntrustedProject(tpc)
|
||||
|
||||
pipeline = model.Pipeline('gate', layout)
|
||||
layout.addPipeline(pipeline)
|
||||
|
|
@ -523,7 +526,8 @@ class TestJob(BaseTestCase):
|
|||
layout.addPipeline(pipeline)
|
||||
queue = model.ChangeQueue(pipeline)
|
||||
project = model.Project('project', self.source)
|
||||
tenant.addUntrustedProject(project)
|
||||
tpc = model.TenantProjectConfig(project)
|
||||
tenant.addUntrustedProject(tpc)
|
||||
|
||||
base = configloader.JobParser.fromYaml(tenant, layout, {
|
||||
'_source_context': self.context,
|
||||
|
|
@ -604,7 +608,8 @@ class TestJob(BaseTestCase):
|
|||
self.layout.addJob(job)
|
||||
|
||||
project2 = model.Project('project2', self.source)
|
||||
self.tenant.addUntrustedProject(project2)
|
||||
tpc2 = model.TenantProjectConfig(project2)
|
||||
self.tenant.addUntrustedProject(tpc2)
|
||||
context2 = model.SourceContext(project2, 'master',
|
||||
'test', True)
|
||||
|
||||
|
|
@ -805,7 +810,8 @@ class TestTenant(BaseTestCase):
|
|||
connection=connection1)
|
||||
|
||||
source1_project1 = model.Project('project1', source1)
|
||||
tenant.addConfigProject(source1_project1)
|
||||
source1_project1_tpc = model.TenantProjectConfig(source1_project1)
|
||||
tenant.addConfigProject(source1_project1_tpc)
|
||||
d = {'project1':
|
||||
{'git1.example.com': source1_project1}}
|
||||
self.assertEqual(d, tenant.projects)
|
||||
|
|
@ -815,7 +821,8 @@ class TestTenant(BaseTestCase):
|
|||
tenant.getProject('git1.example.com/project1'))
|
||||
|
||||
source1_project2 = model.Project('project2', source1)
|
||||
tenant.addUntrustedProject(source1_project2)
|
||||
tpc = model.TenantProjectConfig(source1_project2)
|
||||
tenant.addUntrustedProject(tpc)
|
||||
d = {'project1':
|
||||
{'git1.example.com': source1_project1},
|
||||
'project2':
|
||||
|
|
@ -832,7 +839,8 @@ class TestTenant(BaseTestCase):
|
|||
connection=connection2)
|
||||
|
||||
source2_project1 = model.Project('project1', source2)
|
||||
tenant.addUntrustedProject(source2_project1)
|
||||
tpc = model.TenantProjectConfig(source2_project1)
|
||||
tenant.addUntrustedProject(tpc)
|
||||
d = {'project1':
|
||||
{'git1.example.com': source1_project1,
|
||||
'git2.example.com': source2_project1},
|
||||
|
|
@ -851,7 +859,8 @@ class TestTenant(BaseTestCase):
|
|||
tenant.getProject('git2.example.com/project1'))
|
||||
|
||||
source2_project2 = model.Project('project2', source2)
|
||||
tenant.addConfigProject(source2_project2)
|
||||
tpc = model.TenantProjectConfig(source2_project2)
|
||||
tenant.addConfigProject(tpc)
|
||||
d = {'project1':
|
||||
{'git1.example.com': source1_project1,
|
||||
'git2.example.com': source2_project1},
|
||||
|
|
@ -877,7 +886,8 @@ class TestTenant(BaseTestCase):
|
|||
tenant.getProject('git2.example.com/project2'))
|
||||
|
||||
source1_project2b = model.Project('subpath/project2', source1)
|
||||
tenant.addConfigProject(source1_project2b)
|
||||
tpc = model.TenantProjectConfig(source1_project2b)
|
||||
tenant.addConfigProject(tpc)
|
||||
d = {'project1':
|
||||
{'git1.example.com': source1_project1,
|
||||
'git2.example.com': source2_project1},
|
||||
|
|
@ -898,7 +908,8 @@ class TestTenant(BaseTestCase):
|
|||
tenant.getProject('git1.example.com/subpath/project2'))
|
||||
|
||||
source2_project2b = model.Project('subpath/project2', source2)
|
||||
tenant.addConfigProject(source2_project2b)
|
||||
tpc = model.TenantProjectConfig(source2_project2b)
|
||||
tenant.addConfigProject(tpc)
|
||||
d = {'project1':
|
||||
{'git1.example.com': source1_project1,
|
||||
'git2.example.com': source2_project1},
|
||||
|
|
@ -927,4 +938,4 @@ class TestTenant(BaseTestCase):
|
|||
with testtools.ExpectedException(
|
||||
Exception,
|
||||
"Project project1 is already in project index"):
|
||||
tenant._addProject(source1_project1)
|
||||
tenant._addProject(source1_project1_tpc)
|
||||
|
|
|
|||
|
|
@ -932,13 +932,14 @@ class TenantParser(object):
|
|||
tenant = model.Tenant(conf['name'])
|
||||
tenant.unparsed_config = conf
|
||||
unparsed_config = model.UnparsedTenantConfig()
|
||||
config_projects, untrusted_projects = \
|
||||
# tpcs is TenantProjectConfigs
|
||||
config_tpcs, untrusted_tpcs = \
|
||||
TenantParser._loadTenantProjects(
|
||||
project_key_dir, connections, conf)
|
||||
for project in config_projects:
|
||||
tenant.addConfigProject(project)
|
||||
for project in untrusted_projects:
|
||||
tenant.addUntrustedProject(project)
|
||||
for tpc in config_tpcs:
|
||||
tenant.addConfigProject(tpc)
|
||||
for tpc in untrusted_tpcs:
|
||||
tenant.addUntrustedProject(tpc)
|
||||
tenant.config_projects_config, tenant.untrusted_projects_config = \
|
||||
TenantParser._loadTenantInRepoLayouts(merger, connections,
|
||||
tenant.config_projects,
|
||||
|
|
@ -1020,8 +1021,10 @@ class TenantParser(object):
|
|||
if project_exclude:
|
||||
project_include = frozenset(project_include - project_exclude)
|
||||
|
||||
project.load_classes = frozenset(project_include)
|
||||
return project
|
||||
tenant_project_config = model.TenantProjectConfig(project)
|
||||
tenant_project_config.load_classes = frozenset(project_include)
|
||||
|
||||
return tenant_project_config
|
||||
|
||||
@staticmethod
|
||||
def _getProjects(source, conf, current_include):
|
||||
|
|
@ -1065,21 +1068,22 @@ class TenantParser(object):
|
|||
|
||||
current_include = default_include
|
||||
for conf_repo in conf_source.get('config-projects', []):
|
||||
projects = TenantParser._getProjects(source, conf_repo,
|
||||
current_include)
|
||||
for project in projects:
|
||||
# tpcs = TenantProjectConfigs
|
||||
tpcs = TenantParser._getProjects(source, conf_repo,
|
||||
current_include)
|
||||
for tpc in tpcs:
|
||||
TenantParser._loadProjectKeys(
|
||||
project_key_dir, source_name, project)
|
||||
config_projects.append(project)
|
||||
project_key_dir, source_name, tpc.project)
|
||||
config_projects.append(tpc)
|
||||
|
||||
current_include = frozenset(default_include - set(['pipeline']))
|
||||
for conf_repo in conf_source.get('untrusted-projects', []):
|
||||
projects = TenantParser._getProjects(source, conf_repo,
|
||||
current_include)
|
||||
for project in projects:
|
||||
tpcs = TenantParser._getProjects(source, conf_repo,
|
||||
current_include)
|
||||
for tpc in tpcs:
|
||||
TenantParser._loadProjectKeys(
|
||||
project_key_dir, source_name, project)
|
||||
untrusted_projects.append(project)
|
||||
project_key_dir, source_name, tpc.project)
|
||||
untrusted_projects.append(tpc)
|
||||
|
||||
return config_projects, untrusted_projects
|
||||
|
||||
|
|
@ -1191,13 +1195,19 @@ class TenantParser(object):
|
|||
raise PipelineNotPermittedError()
|
||||
return config
|
||||
|
||||
@staticmethod
|
||||
def _getLoadClasses(tenant, conf_object):
|
||||
project = conf_object['_source_context'].project
|
||||
tpc = tenant.project_configs[project.canonical_name]
|
||||
return tpc.load_classes
|
||||
|
||||
@staticmethod
|
||||
def _parseLayoutItems(layout, tenant, data, scheduler, connections,
|
||||
skip_pipelines=False, skip_semaphores=False):
|
||||
if not skip_pipelines:
|
||||
for config_pipeline in data.pipelines:
|
||||
classes = config_pipeline['_source_context'].\
|
||||
project.load_classes
|
||||
classes = TenantParser._getLoadClasses(
|
||||
tenant, config_pipeline)
|
||||
if 'pipeline' not in classes:
|
||||
continue
|
||||
layout.addPipeline(PipelineParser.fromYaml(
|
||||
|
|
@ -1205,7 +1215,7 @@ class TenantParser(object):
|
|||
scheduler, config_pipeline))
|
||||
|
||||
for config_nodeset in data.nodesets:
|
||||
classes = config_nodeset['_source_context'].project.load_classes
|
||||
classes = TenantParser._getLoadClasses(tenant, config_nodeset)
|
||||
if 'nodeset' not in classes:
|
||||
continue
|
||||
with configuration_exceptions('nodeset', config_nodeset):
|
||||
|
|
@ -1213,13 +1223,13 @@ class TenantParser(object):
|
|||
layout, config_nodeset))
|
||||
|
||||
for config_secret in data.secrets:
|
||||
classes = config_secret['_source_context'].project.load_classes
|
||||
classes = TenantParser._getLoadClasses(tenant, config_secret)
|
||||
if 'secret' not in classes:
|
||||
continue
|
||||
layout.addSecret(SecretParser.fromYaml(layout, config_secret))
|
||||
|
||||
for config_job in data.jobs:
|
||||
classes = config_job['_source_context'].project.load_classes
|
||||
classes = TenantParser._getLoadClasses(tenant, config_job)
|
||||
if 'job' not in classes:
|
||||
continue
|
||||
with configuration_exceptions('job', config_job):
|
||||
|
|
@ -1228,14 +1238,14 @@ class TenantParser(object):
|
|||
|
||||
if not skip_semaphores:
|
||||
for config_semaphore in data.semaphores:
|
||||
classes = config_semaphore['_source_context'].\
|
||||
project.load_classes
|
||||
classes = TenantParser._getLoadClasses(
|
||||
tenant, config_semaphore)
|
||||
if 'semaphore' not in classes:
|
||||
continue
|
||||
layout.addSemaphore(SemaphoreParser.fromYaml(config_semaphore))
|
||||
|
||||
for config_template in data.project_templates:
|
||||
classes = config_template['_source_context'].project.load_classes
|
||||
classes = TenantParser._getLoadClasses(tenant, config_template)
|
||||
if 'project-template' not in classes:
|
||||
continue
|
||||
layout.addProjectTemplate(ProjectTemplateParser.fromYaml(
|
||||
|
|
@ -1249,10 +1259,11 @@ class TenantParser(object):
|
|||
# each of the project stanzas. Each one may be (should
|
||||
# be!) from a different repo, so filter them according to
|
||||
# the include/exclude rules before parsing them.
|
||||
filtered_projects = [
|
||||
p for p in config_projects if
|
||||
'project' in p['_source_context'].project.load_classes
|
||||
]
|
||||
filtered_projects = []
|
||||
for config_project in config_projects:
|
||||
classes = TenantParser._getLoadClasses(tenant, config_project)
|
||||
if 'project' in classes:
|
||||
filtered_projects.append(config_project)
|
||||
|
||||
if not filtered_projects:
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -331,9 +331,6 @@ class Project(object):
|
|||
self.foreign = foreign
|
||||
self.unparsed_config = None
|
||||
self.unparsed_branch_config = {} # branch -> UnparsedTenantConfig
|
||||
# Configuration object classes to include or exclude when
|
||||
# loading zuul config files.
|
||||
self.load_classes = frozenset()
|
||||
|
||||
def __str__(self):
|
||||
return self.name
|
||||
|
|
@ -1998,6 +1995,19 @@ class ProjectPipelineConfig(object):
|
|||
self.merge_mode = None
|
||||
|
||||
|
||||
class TenantProjectConfig(object):
|
||||
"""A project in the context of a tenant.
|
||||
|
||||
A Project is globally unique in the system, however, when used in
|
||||
a tenant, some metadata about the project local to the tenant is
|
||||
stored in a TenantProjectConfig.
|
||||
"""
|
||||
|
||||
def __init__(self, project):
|
||||
self.project = project
|
||||
self.load_classes = set()
|
||||
|
||||
|
||||
class ProjectConfig(object):
|
||||
# Represents a project cofiguration
|
||||
def __init__(self, name):
|
||||
|
|
@ -2009,6 +2019,7 @@ class ProjectConfig(object):
|
|||
|
||||
|
||||
class UnparsedAbideConfig(object):
|
||||
|
||||
"""A collection of yaml lists that has not yet been parsed into objects.
|
||||
|
||||
An Abide is a collection of tenants.
|
||||
|
|
@ -2356,6 +2367,9 @@ class Tenant(object):
|
|||
# The unparsed config from those projects.
|
||||
self.untrusted_projects_config = None
|
||||
self.semaphore_handler = SemaphoreHandler()
|
||||
# Metadata about projects for this tenant
|
||||
# canonical project name -> TenantProjectConfig
|
||||
self.project_configs = {}
|
||||
|
||||
# A mapping of project names to projects. project_name ->
|
||||
# VALUE where VALUE is a further dictionary of
|
||||
|
|
@ -2363,17 +2377,21 @@ class Tenant(object):
|
|||
self.projects = {}
|
||||
self.canonical_hostnames = set()
|
||||
|
||||
def _addProject(self, project):
|
||||
def _addProject(self, tpc):
|
||||
"""Add a project to the project index
|
||||
|
||||
:arg Project project: The project to add.
|
||||
:arg TenantProjectConfig tpc: The TenantProjectConfig (with
|
||||
associated project) to add.
|
||||
|
||||
"""
|
||||
project = tpc.project
|
||||
self.canonical_hostnames.add(project.canonical_hostname)
|
||||
hostname_dict = self.projects.setdefault(project.name, {})
|
||||
if project.canonical_hostname in hostname_dict:
|
||||
raise Exception("Project %s is already in project index" %
|
||||
(project,))
|
||||
hostname_dict[project.canonical_hostname] = project
|
||||
self.project_configs[project.canonical_name] = tpc
|
||||
|
||||
def getProject(self, name):
|
||||
"""Return a project given its name.
|
||||
|
|
@ -2420,13 +2438,13 @@ class Tenant(object):
|
|||
raise Exception("Project %s is neither trusted nor untrusted" %
|
||||
(project,))
|
||||
|
||||
def addConfigProject(self, project):
|
||||
self.config_projects.append(project)
|
||||
self._addProject(project)
|
||||
def addConfigProject(self, tpc):
|
||||
self.config_projects.append(tpc.project)
|
||||
self._addProject(tpc)
|
||||
|
||||
def addUntrustedProject(self, project):
|
||||
self.untrusted_projects.append(project)
|
||||
self._addProject(project)
|
||||
def addUntrustedProject(self, tpc):
|
||||
self.untrusted_projects.append(tpc.project)
|
||||
self._addProject(tpc)
|
||||
|
||||
def getSafeAttributes(self):
|
||||
return Attributes(name=self.name)
|
||||
|
|
|
|||
Loading…
Reference in New Issue