zuul
/
zuul
Code Issues Proposed changes

Replace config/project repos with config/untrusted projects 

The config-repo / project-repo terminology is confusing and we've
generally been using trusted-project and untrusted-project instead.
The term "untrusted-project" is straightforward, but "trusted-project"
is a little misleading, since there are other characteristics which
define it (notably, that the configuration held in it is branchless).
A better term for that might be "config-project".

Since the main config now directly refers to projects rather than
repos, complete the transition by replacing all config-repo
occurances with config-project and likewise project-repo with
untrusted-project.

Change-Id: I2341aa94e3622e2647b0506b78bc84dbcdec1901
Story: 2000953
This commit is contained in:
James E. Blair 2017-04-04 14:39:43 -07:00
parent 0ffa010656
commit 109da3f9a7
34 changed files with 148 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
tests/base.py
View File

@ -1475,9 +1475,9 @@ class ZuulTestCase(BaseTestCase):
for tenant in tenant_config:
sources = tenant['tenant']['source']
for source, conf in sources.items():
for project in conf.get('config-repos', []):
for project in conf.get('config-projects', []):
self.setupProjectKeys(source, project)
for project in conf.get('project-repos', []):
for project in conf.get('untrusted-projects', []):
self.setupProjectKeys(source, project)
def setupProjectKeys(self, source, project):
@ -1927,9 +1927,9 @@ class ZuulTestCase(BaseTestCase):
def getPipeline(self, name):
return self.sched.abide.tenants.values()[0].layout.pipelines.get(name)
def updateConfigLayout(self, path, project_repos=None):
if project_repos is None:
project_repos = []
def updateConfigLayout(self, path, untrusted_projects=None):
if untrusted_projects is None:
untrusted_projects = []
root = os.path.join(self.test_root, "config")
if not os.path.exists(root):
os.makedirs(root)
@ -1939,9 +1939,9 @@ class ZuulTestCase(BaseTestCase):
name: openstack
source:
gerrit:
config-repos:
config-projects:
- %s
project-repos:
untrusted-projects:
- org/project
- org/project1
- org/project2
@ -1959,7 +1959,7 @@ class ZuulTestCase(BaseTestCase):
- org/experimental-project
- org/no-jobs-project\n""" % path)
for repo in project_repos:
for repo in untrusted_projects:
f.write(" - %s\n" % repo)
f.close()
self.config.set('zuul', 'tenant_config',

4
tests/fixtures/config/ansible/main.yaml vendored
View File

@ -2,8 +2,8 @@
name: tenant-one
source:
gerrit:
config-repos:
config-projects:
- common-config
project-repos:
untrusted-projects:
- org/project
- bare-role

2
tests/fixtures/config/broken/main.yaml vendored
View File

@ -2,5 +2,5 @@
name: tenant-one
source:
gerrit:
config-repos:
config-projects:
- common-config

4
tests/fixtures/config/dependency-graph/main.yaml vendored
View File

@ -2,7 +2,7 @@
name: tenant-one
source:
gerrit:
config-repos:
config-projects:
- common-config
project-repos:
untrusted-projects:
- org/project

4
tests/fixtures/config/duplicate-pipeline/main.yaml vendored
View File

@ -2,7 +2,7 @@
name: tenant-duplicate
source:
gerrit:
config-repos:
config-projects:
- common-config
project-repos:
untrusted-projects:
- org/project

4
tests/fixtures/config/git-driver/main.yaml vendored
View File

@ -2,8 +2,8 @@
name: tenant-one
source:
git:
config-repos:
config-projects:
- common-config
gerrit:
project-repos:
untrusted-projects:
- org/project

4
tests/fixtures/config/in-repo/main.yaml vendored
View File

@ -2,7 +2,7 @@
name: tenant-one
source:
gerrit:
config-repos:
config-projects:
- common-config
project-repos:
untrusted-projects:
- org/project

4
tests/fixtures/config/merges/main.yaml vendored
View File

@ -2,9 +2,9 @@
name: tenant-one
source:
gerrit:
config-repos:
config-projects:
- common-config
project-repos:
untrusted-projects:
- org/project-cherry-pick
- org/project-merge
- org/project-merge-branches

8
tests/fixtures/config/multi-tenant-semaphore/main.yaml vendored
View File

@ -2,10 +2,10 @@
name: tenant-one
source:
gerrit:
config-repos:
config-projects:
- common-config
- tenant-one-config
project-repos:
untrusted-projects:
- org/project1
- org/project2
@ -13,9 +13,9 @@
name: tenant-two
source:
gerrit:
config-repos:
config-projects:
- common-config
- tenant-two-config
project-repos:
untrusted-projects:
- org/project1
- org/project2

8
tests/fixtures/config/multi-tenant/main.yaml vendored
View File

@ -2,18 +2,18 @@
name: tenant-one
source:
gerrit:
config-repos:
config-projects:
- common-config
- tenant-one-config
project-repos:
untrusted-projects:
- org/project1
- tenant:
name: tenant-two
source:
gerrit:
config-repos:
config-projects:
- common-config
- tenant-two-config
project-repos:
untrusted-projects:
- org/project2

4
tests/fixtures/config/one-job-project/main.yaml vendored
View File

@ -2,7 +2,7 @@
name: tenant-one
source:
gerrit:
config-repos:
config-projects:
- common-config
project-repos:
untrusted-projects:
- org/one-job-project

4
tests/fixtures/config/openstack/main.yaml vendored
View File

@ -2,8 +2,8 @@
name: openstack
source:
gerrit:
config-repos:
config-projects:
- project-config
project-repos:
untrusted-projects:
- openstack/nova
- openstack/keystone

4
tests/fixtures/config/requirements/email/main.yaml vendored
View File

@ -2,8 +2,8 @@
name: tenant-one
source:
gerrit:
config-repos:
config-projects:
- common-config
project-repos:
untrusted-projects:
- org/project1
- org/project2

4
tests/fixtures/config/requirements/newer-than/main.yaml vendored
View File

@ -2,8 +2,8 @@
name: tenant-one
source:
gerrit:
config-repos:
config-projects:
- common-config
project-repos:
untrusted-projects:
- org/project1
- org/project2

4
tests/fixtures/config/requirements/older-than/main.yaml vendored
View File

@ -2,8 +2,8 @@
name: tenant-one
source:
gerrit:
config-repos:
config-projects:
- common-config
project-repos:
untrusted-projects:
- org/project1
- org/project2

4
tests/fixtures/config/requirements/reject-username/main.yaml vendored
View File

@ -2,8 +2,8 @@
name: tenant-one
source:
gerrit:
config-repos:
config-projects:
- common-config
project-repos:
untrusted-projects:
- org/project1
- org/project2

4
tests/fixtures/config/requirements/reject/main.yaml vendored
View File

@ -2,8 +2,8 @@
name: tenant-one
source:
gerrit:
config-repos:
config-projects:
- common-config
project-repos:
untrusted-projects:
- org/project1
- org/project2

4
tests/fixtures/config/requirements/state/main.yaml vendored
View File

@ -2,9 +2,9 @@
name: tenant-one
source:
gerrit:
config-repos:
config-projects:
- common-config
project-repos:
untrusted-projects:
- current-project
- open-project
- status-project

4
tests/fixtures/config/requirements/username/main.yaml vendored
View File

@ -2,8 +2,8 @@
name: tenant-one
source:
gerrit:
config-repos:
config-projects:
- common-config
project-repos:
untrusted-projects:
- org/project1
- org/project2

4
tests/fixtures/config/requirements/vote1/main.yaml vendored
View File

@ -2,8 +2,8 @@
name: tenant-one
source:
gerrit:
config-repos:
config-projects:
- common-config
project-repos:
untrusted-projects:
- org/project1
- org/project2

4
tests/fixtures/config/requirements/vote2/main.yaml vendored
View File

@ -2,8 +2,8 @@
name: tenant-one
source:
gerrit:
config-repos:
config-projects:
- common-config
project-repos:
untrusted-projects:
- org/project1
- org/project2

4
tests/fixtures/config/single-tenant/main.yaml vendored
View File

@ -2,9 +2,9 @@
name: tenant-one
source:
gerrit:
config-repos:
config-projects:
- common-config
project-repos:
untrusted-projects:
- org/project
- org/project1
- org/project2

4
tests/fixtures/config/sql-driver/main.yaml vendored
View File

@ -2,7 +2,7 @@
name: tenant-one
source:
gerrit:
config-repos:
config-projects:
- common-config
project-repos:
untrusted-projects:
- org/project

4
tests/fixtures/config/success-url/main.yaml vendored
View File

@ -2,7 +2,7 @@
name: tenant-one
source:
gerrit:
config-repos:
config-projects:
- common-config
project-repos:
untrusted-projects:
- org/docs

4
tests/fixtures/config/templated-project/main.yaml vendored
View File

@ -2,8 +2,8 @@
name: tenant-one
source:
gerrit:
config-repos:
config-projects:
- common-config
project-repos:
untrusted-projects:
- org/templated-project
- org/layered-project

6
tests/fixtures/config/zuul-connections-multiple-gerrits/main.yaml vendored
View File

@ -2,10 +2,10 @@
name: tenant-one
source:
review_gerrit:
config-repos:
config-projects:
- common-config
project-repos:
untrusted-projects:
- org/project1
another_gerrit:
project-repos:
untrusted-projects:
- org/project1

4
tests/fixtures/config/zuul-connections-same-gerrit/main.yaml vendored
View File

@ -2,7 +2,7 @@
name: tenant-one
source:
review_gerrit:
config-repos:
config-projects:
- common-config
project-repos:
untrusted-projects:
- org/project

4
tests/fixtures/config/zuultrigger/parent-change-enqueued/main.yaml vendored
View File

@ -2,7 +2,7 @@
name: tenant-one
source:
gerrit:
config-repos:
config-projects:
- common-config
project-repos:
untrusted-projects:
- org/project

2
tests/fixtures/config/zuultrigger/project-change-merged/main.yaml vendored
View File

@ -2,5 +2,5 @@
name: tenant-one
source:
gerrit:
config-repos:
config-projects:
- common-config

8
tests/unit/test_git_driver.py
View File

@ -27,10 +27,10 @@ class TestGitDriver(ZuulTestCase):
tenant = self.sched.abide.tenants.get('tenant-one')
# Check that we have the git source for common-config and the
# gerrit source for the project.
self.assertEqual('git', tenant.config_repos[0].source.name)
self.assertEqual('common-config', tenant.config_repos[0].name)
self.assertEqual('gerrit', tenant.project_repos[0].source.name)
self.assertEqual('org/project', tenant.project_repos[0].name)
self.assertEqual('git', tenant.config_projects[0].source.name)
self.assertEqual('common-config', tenant.config_projects[0].name)
self.assertEqual('gerrit', tenant.untrusted_projects[0].source.name)
self.assertEqual('org/project', tenant.untrusted_projects[0].name)
# The configuration for this test is accessed via the git
# driver (in common-config), rather than the gerrit driver, so

22
tests/unit/test_model.py
View File

@ -42,7 +42,7 @@ class TestJob(BaseTestCase):
self.tenant = model.Tenant('tenant')
self.layout = model.Layout()
self.project = model.Project('project', self.source)
self.tenant.addProjectRepo(self.project)
self.tenant.addUntrustedProject(self.project)
self.pipeline = model.Pipeline('gate', self.layout)
self.layout.addPipeline(self.pipeline)
self.queue = model.ChangeQueue(self.pipeline)
@ -165,7 +165,7 @@ class TestJob(BaseTestCase):
layout.addPipeline(pipeline)
queue = model.ChangeQueue(pipeline)
project = model.Project('project', self.source)
tenant.addProjectRepo(project)
tenant.addUntrustedProject(project)
base = configloader.JobParser.fromYaml(tenant, layout, {
'_source_context': self.context,
@ -432,7 +432,7 @@ class TestJob(BaseTestCase):
def test_job_inheritance_job_tree(self):
tenant = model.Tenant('tenant')
layout = model.Layout()
tenant.addProjectRepo(self.project)
tenant.addUntrustedProject(self.project)
pipeline = model.Pipeline('gate', layout)
layout.addPipeline(pipeline)
@ -513,7 +513,7 @@ class TestJob(BaseTestCase):
layout.addPipeline(pipeline)
queue = model.ChangeQueue(pipeline)
project = model.Project('project', self.source)
tenant.addProjectRepo(project)
tenant.addUntrustedProject(project)
base = configloader.JobParser.fromYaml(tenant, layout, {
'_source_context': self.context,
@ -594,7 +594,7 @@ class TestJob(BaseTestCase):
self.layout.addJob(job)
project2 = model.Project('project2', self.source)
self.tenant.addProjectRepo(project2)
self.tenant.addUntrustedProject(project2)
context2 = model.SourceContext(project2, 'master',
'test', True)
@ -795,7 +795,7 @@ class TestTenant(BaseTestCase):
connection=connection1)
source1_project1 = model.Project('project1', source1)
tenant.addConfigRepo(source1_project1)
tenant.addConfigProject(source1_project1)
d = {'project1':
{'git1.example.com': source1_project1}}
self.assertEqual(d, tenant.projects)
@ -805,7 +805,7 @@ class TestTenant(BaseTestCase):
tenant.getProject('git1.example.com/project1'))
source1_project2 = model.Project('project2', source1)
tenant.addProjectRepo(source1_project2)
tenant.addUntrustedProject(source1_project2)
d = {'project1':
{'git1.example.com': source1_project1},
'project2':
@ -822,7 +822,7 @@ class TestTenant(BaseTestCase):
connection=connection2)
source2_project1 = model.Project('project1', source2)
tenant.addProjectRepo(source2_project1)
tenant.addUntrustedProject(source2_project1)
d = {'project1':
{'git1.example.com': source1_project1,
'git2.example.com': source2_project1},
@ -841,7 +841,7 @@ class TestTenant(BaseTestCase):
tenant.getProject('git2.example.com/project1'))
source2_project2 = model.Project('project2', source2)
tenant.addConfigRepo(source2_project2)
tenant.addConfigProject(source2_project2)
d = {'project1':
{'git1.example.com': source1_project1,
'git2.example.com': source2_project1},
@ -867,7 +867,7 @@ class TestTenant(BaseTestCase):
tenant.getProject('git2.example.com/project2'))
source1_project2b = model.Project('subpath/project2', source1)
tenant.addConfigRepo(source1_project2b)
tenant.addConfigProject(source1_project2b)
d = {'project1':
{'git1.example.com': source1_project1,
'git2.example.com': source2_project1},
@ -888,7 +888,7 @@ class TestTenant(BaseTestCase):
tenant.getProject('git1.example.com/subpath/project2'))
source2_project2b = model.Project('subpath/project2', source2)
tenant.addConfigRepo(source2_project2b)
tenant.addConfigProject(source2_project2b)
d = {'project1':
{'git1.example.com': source1_project1,
'git2.example.com': source2_project1},

98
zuul/configloader.py
View File

@ -754,8 +754,8 @@ class SemaphoreParser(object):
class TenantParser(object):
log = logging.getLogger("zuul.TenantParser")
tenant_source = vs.Schema({'config-repos': [str],
'project-repos': [str]})
tenant_source = vs.Schema({'config-projects': [str],
'untrusted-projects': [str]})
@staticmethod
def validateTenantSources(connections):
@ -785,20 +785,20 @@ class TenantParser(object):
tenant = model.Tenant(conf['name'])
tenant.unparsed_config = conf
unparsed_config = model.UnparsedTenantConfig()
config_repos, project_repos = \
TenantParser._loadTenantConfigRepos(
config_projects, untrusted_projects = \
TenantParser._loadTenantProjects(
project_key_dir, connections, conf)
for repo in config_repos:
tenant.addConfigRepo(repo)
for repo in project_repos:
tenant.addProjectRepo(repo)
tenant.config_repos_config, tenant.project_repos_config = \
for project in config_projects:
tenant.addConfigProject(project)
for project in untrusted_projects:
tenant.addUntrustedProject(project)
tenant.config_projects_config, tenant.untrusted_projects_config = \
TenantParser._loadTenantInRepoLayouts(merger, connections,
tenant.config_repos,
tenant.project_repos,
tenant.config_projects,
tenant.untrusted_projects,
cached)
unparsed_config.extend(tenant.config_repos_config)
unparsed_config.extend(tenant.project_repos_config)
unparsed_config.extend(tenant.config_projects_config)
unparsed_config.extend(tenant.untrusted_projects_config)
tenant.layout = TenantParser._parseLayout(base, tenant,
unparsed_config,
scheduler,
@ -852,41 +852,41 @@ class TenantParser(object):
encryption.deserialize_rsa_keypair(f.read())
@staticmethod
def _loadTenantConfigRepos(project_key_dir, connections, conf_tenant):
config_repos = []
project_repos = []
def _loadTenantProjects(project_key_dir, connections, conf_tenant):
config_projects = []
untrusted_projects = []
for source_name, conf_source in conf_tenant.get('source', {}).items():
source = connections.getSource(source_name)
for conf_repo in conf_source.get('config-repos', []):
for conf_repo in conf_source.get('config-projects', []):
project = source.getProject(conf_repo)
TenantParser._loadProjectKeys(
project_key_dir, source_name, project)
config_repos.append(project)
config_projects.append(project)
for conf_repo in conf_source.get('project-repos', []):
for conf_repo in conf_source.get('untrusted-projects', []):
project = source.getProject(conf_repo)
TenantParser._loadProjectKeys(
project_key_dir, source_name, project)
project_repos.append(project)
untrusted_projects.append(project)
return config_repos, project_repos
return config_projects, untrusted_projects
@staticmethod
def _loadTenantInRepoLayouts(merger, connections, config_repos,
project_repos, cached):
config_repos_config = model.UnparsedTenantConfig()
project_repos_config = model.UnparsedTenantConfig()
def _loadTenantInRepoLayouts(merger, connections, config_projects,
untrusted_projects, cached):
config_projects_config = model.UnparsedTenantConfig()
untrusted_projects_config = model.UnparsedTenantConfig()
jobs = []
for project in config_repos:
for project in config_projects:
# If we have cached data (this is a reconfiguration) use it.
if cached and project.unparsed_config:
TenantParser.log.info(
"Loading previously parsed configuration from %s" %
(project,))
config_repos_config.extend(project.unparsed_config)
config_projects_config.extend(project.unparsed_config)
continue
# Otherwise, prepare an empty unparsed config object to
# hold cached data later.
@ -900,13 +900,13 @@ class TenantParser(object):
'', True)
jobs.append(job)
for project in project_repos:
for project in untrusted_projects:
# If we have cached data (this is a reconfiguration) use it.
if cached and project.unparsed_config:
TenantParser.log.info(
"Loading previously parsed configuration from %s" %
(project,))
project_repos_config.extend(project.unparsed_config)
untrusted_projects_config.extend(project.unparsed_config)
continue
# Otherwise, prepare an empty unparsed config object to
# hold cached data later.
@ -951,27 +951,27 @@ class TenantParser(object):
project = job.source_context.project
branch = job.source_context.branch
if job.source_context.trusted:
incdata = TenantParser._parseConfigRepoLayout(
incdata = TenantParser._parseConfigProjectLayout(
job.files[fn], job.source_context)
config_repos_config.extend(incdata)
config_projects_config.extend(incdata)
else:
incdata = TenantParser._parseProjectRepoLayout(
incdata = TenantParser._parseUntrustedProjectLayout(
job.files[fn], job.source_context)
project_repos_config.extend(incdata)
untrusted_projects_config.extend(incdata)
project.unparsed_config.extend(incdata)
if branch in project.unparsed_branch_config:
project.unparsed_branch_config[branch].extend(incdata)
return config_repos_config, project_repos_config
return config_projects_config, untrusted_projects_config
@staticmethod
def _parseConfigRepoLayout(data, source_context):
def _parseConfigProjectLayout(data, source_context):
# This is the top-level configuration for a tenant.
config = model.UnparsedTenantConfig()
config.extend(safe_load_yaml(data, source_context))
return config
@staticmethod
def _parseProjectRepoLayout(data, source_context):
def _parseUntrustedProjectLayout(data, source_context):
# TODOv3(jeblair): this should implement some rules to protect
# aspects of the config that should not be changed in-repo
config = model.UnparsedTenantConfig()
@ -1061,9 +1061,8 @@ class ConfigLoader(object):
new_abide.tenants[tenant.name] = new_tenant
return new_abide
def _loadDynamicProjectData(self, config, project, files,
config_repo):
if config_repo:
def _loadDynamicProjectData(self, config, project, files, trusted):
if trusted:
branches = ['master']
fn = 'zuul.yaml'
else:
@ -1075,29 +1074,30 @@ class ConfigLoader(object):
data = files.getFile(project.name, branch, fn)
if data:
source_context = model.SourceContext(project, branch,
fn, config_repo)
if config_repo:
incdata = TenantParser._parseConfigRepoLayout(
fn, trusted)
if trusted:
incdata = TenantParser._parseConfigProjectLayout(
data, source_context)
else:
incdata = TenantParser._parseProjectRepoLayout(
incdata = TenantParser._parseUntrustedProjectLayout(
data, source_context)
else:
if config_repo:
if trusted:
incdata = project.unparsed_config
else:
incdata = project.unparsed_branch_config.get(branch)
if incdata:
config.extend(incdata)
def createDynamicLayout(self, tenant, files, include_config_repos=False):
if include_config_repos:
def createDynamicLayout(self, tenant, files,
include_config_projects=False):
if include_config_projects:
config = model.UnparsedTenantConfig()
for project in tenant.config_repos:
for project in tenant.config_projects:
self._loadDynamicProjectData(config, project, files, True)
else:
config = tenant.config_repos_config.copy()
for project in tenant.project_repos:
config = tenant.config_projects_config.copy()
for project in tenant.untrusted_projects:
self._loadDynamicProjectData(config, project, files, False)
layout = model.Layout()

4
zuul/manager/__init__.py
View File

@ -487,14 +487,14 @@ class PipelineManager(object):
loader.createDynamicLayout(
item.pipeline.layout.tenant,
build_set.files,
include_config_repos=True)
include_config_projects=True)
# Then create the config a second time but without changes
# to config repos so that we actually use this config.
layout = loader.createDynamicLayout(
item.pipeline.layout.tenant,
build_set.files,
include_config_repos=False)
include_config_projects=False)
except zuul.configloader.ConfigurationSyntaxError as e:
self.log.info("Configuration syntax error "
"in dynamic layout %s" %

30
zuul/model.py
View File

@ -2512,16 +2512,16 @@ class Tenant(object):
# The unparsed configuration from the main zuul config for
# this tenant.
self.unparsed_config = None
# The list of repos from which we will read main
# The list of projects from which we will read full
# configuration.
self.config_repos = []
# The unparsed config from those repos.
self.config_repos_config = None
# The list of projects from which we will read in-repo
# configuration.
self.project_repos = []
# The unparsed config from those repos.
self.project_repos_config = None
self.config_projects = []
# The unparsed config from those projects.
self.config_projects_config = None
# The list of projects from which we will read untrusted
# in-repo configuration.
self.untrusted_projects = []
# The unparsed config from those projects.
self.untrusted_projects_config = None
self.semaphore_handler = SemaphoreHandler()
# A mapping of project names to projects. project_name ->
@ -2579,20 +2579,20 @@ class Tenant(object):
"with a hostname" % (name,))
if project is None:
return (None, None)
if project in self.config_repos:
if project in self.config_projects:
return (True, project)
if project in self.project_repos:
if project in self.untrusted_projects:
return (False, project)
# This should never happen:
raise Exception("Project %s is neither trusted nor untrusted" %
(project,))
def addConfigRepo(self, project):
self.config_repos.append(project)
def addConfigProject(self, project):
self.config_projects.append(project)
self._addProject(project)
def addProjectRepo(self, project):
self.project_repos.append(project)
def addUntrustedProject(self, project):
self.untrusted_projects.append(project)
self._addProject(project)