zuul
/
zuul
Code Issues Proposed changes

Fully qualify project configuration names 

The layout stores the configuration of a project in the ProjectConfig
class (not the Project class -- that represents the abstract idea
of a project independent of Zuul, the ProjectConfig represents a
particular Zuul configuration operating on that project).  Therefore,
in the continuing effort to fully qualify project names, index
ProjectConfig objects by their canonical project name.  Use that
name when looking for a ProjectConfig to find the jobs to run for
a given change.

Story: 2000953
Change-Id: I733a66369c969770e57c2fa8b30822bd15e1aca7
This commit is contained in:
James E. Blair 2017-03-30 13:11:33 -07:00
parent 6f284b451f
commit 0ffa010656
31 changed files with 156 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
tests/base.py
View File

@ -1927,7 +1927,9 @@ class ZuulTestCase(BaseTestCase):
def getPipeline(self, name):
return self.sched.abide.tenants.values()[0].layout.pipelines.get(name)
def updateConfigLayout(self, path):
def updateConfigLayout(self, path, project_repos=None):
if project_repos is None:
project_repos = []
root = os.path.join(self.test_root, "config")
if not os.path.exists(root):
os.makedirs(root)
@ -1939,7 +1941,26 @@ class ZuulTestCase(BaseTestCase):
gerrit:
config-repos:
- %s
""" % path)
project-repos:
- org/project
- org/project1
- org/project2
- org/project3
- org/project4
- org/project5
- org/project6
- org/one-job-project
- org/nonvoting-project
- org/templated-project
- org/layered-project
- org/node-project
- org/conflict-project
- org/noop-project
- org/experimental-project
- org/no-jobs-project\n""" % path)
for repo in project_repos:
f.write(" - %s\n" % repo)
f.close()
self.config.set('zuul', 'tenant_config',
os.path.join(FIXTURE_DIR, f.name))

2
tests/fixtures/config/duplicate-pipeline/main.yaml vendored
View File

@ -4,3 +4,5 @@
gerrit:
config-repos:
- common-config
project-repos:
- org/project

6
tests/fixtures/config/merges/main.yaml vendored
View File

@ -4,3 +4,9 @@
gerrit:
config-repos:
- common-config
project-repos:
- org/project-cherry-pick
- org/project-merge
- org/project-merge-branches
- org/project-merge-resolve

6
tests/fixtures/config/multi-tenant-semaphore/main.yaml vendored
View File

@ -5,6 +5,9 @@
config-repos:
- common-config
- tenant-one-config
project-repos:
- org/project1
- org/project2
- tenant:
name: tenant-two
@ -13,3 +16,6 @@
config-repos:
- common-config
- tenant-two-config
project-repos:
- org/project1
- org/project2

4
tests/fixtures/config/multi-tenant/main.yaml vendored
View File

@ -5,6 +5,8 @@
config-repos:
- common-config
- tenant-one-config
project-repos:
- org/project1
- tenant:
name: tenant-two
@ -13,3 +15,5 @@
config-repos:
- common-config
- tenant-two-config
project-repos:
- org/project2

2
tests/fixtures/config/one-job-project/main.yaml vendored
View File

@ -4,3 +4,5 @@
gerrit:
config-repos:
- common-config
project-repos:
- org/one-job-project

3
tests/fixtures/config/openstack/main.yaml vendored
View File

@ -4,3 +4,6 @@
gerrit:
config-repos:
- project-config
project-repos:
- openstack/nova
- openstack/keystone

3
tests/fixtures/config/requirements/email/main.yaml vendored
View File

@ -4,3 +4,6 @@
gerrit:
config-repos:
- common-config
project-repos:
- org/project1
- org/project2

3
tests/fixtures/config/requirements/newer-than/main.yaml vendored
View File

@ -4,3 +4,6 @@
gerrit:
config-repos:
- common-config
project-repos:
- org/project1
- org/project2

3
tests/fixtures/config/requirements/older-than/main.yaml vendored
View File

@ -4,3 +4,6 @@
gerrit:
config-repos:
- common-config
project-repos:
- org/project1
- org/project2

3
tests/fixtures/config/requirements/reject-username/main.yaml vendored
View File

@ -4,3 +4,6 @@
gerrit:
config-repos:
- common-config
project-repos:
- org/project1
- org/project2

3
tests/fixtures/config/requirements/reject/main.yaml vendored
View File

@ -4,3 +4,6 @@
gerrit:
config-repos:
- common-config
project-repos:
- org/project1
- org/project2

4
tests/fixtures/config/requirements/state/main.yaml vendored
View File

@ -4,3 +4,7 @@
gerrit:
config-repos:
- common-config
project-repos:
- current-project
- open-project
- status-project

3
tests/fixtures/config/requirements/username/main.yaml vendored
View File

@ -4,3 +4,6 @@
gerrit:
config-repos:
- common-config
project-repos:
- org/project1
- org/project2

3
tests/fixtures/config/requirements/vote1/main.yaml vendored
View File

@ -4,3 +4,6 @@
gerrit:
config-repos:
- common-config
project-repos:
- org/project1
- org/project2

3
tests/fixtures/config/requirements/vote2/main.yaml vendored
View File

@ -4,3 +4,6 @@
gerrit:
config-repos:
- common-config
project-repos:
- org/project1
- org/project2

15
tests/fixtures/config/single-tenant/main.yaml vendored
View File

@ -6,3 +6,18 @@
- common-config
project-repos:
- org/project
- org/project1
- org/project2
- org/project3
- org/project4
- org/project5
- org/project6
- org/one-job-project
- org/nonvoting-project
- org/templated-project
- org/layered-project
- org/node-project
- org/conflict-project
- org/noop-project
- org/experimental-project
- org/no-jobs-project

2
tests/fixtures/config/success-url/main.yaml vendored
View File

@ -4,3 +4,5 @@
gerrit:
config-repos:
- common-config
project-repos:
- org/docs

3
tests/fixtures/config/templated-project/main.yaml vendored
View File

@ -4,3 +4,6 @@
gerrit:
config-repos:
- common-config
project-repos:
- org/templated-project
- org/layered-project

5
tests/fixtures/config/zuul-connections-multiple-gerrits/git/common-config/zuul.yaml vendored
View File

@ -33,10 +33,13 @@
name: project-test2
- project:
name: org/project1
name: review.example.com/org/project1
review_check:
jobs:
- project-test1
- project:
name: another.example.com/org/project1
another_check:
jobs:
- project-test2

5
tests/fixtures/config/zuul-connections-multiple-gerrits/main.yaml vendored
View File

@ -4,3 +4,8 @@
review_gerrit:
config-repos:
- common-config
project-repos:
- org/project1
another_gerrit:
project-repos:
- org/project1

2
tests/fixtures/config/zuultrigger/parent-change-enqueued/main.yaml vendored
View File

@ -4,3 +4,5 @@
gerrit:
config-repos:
- common-config
project-repos:
- org/project

4
tests/unit/test_model.py
View File

@ -165,6 +165,7 @@ class TestJob(BaseTestCase):
layout.addPipeline(pipeline)
queue = model.ChangeQueue(pipeline)
project = model.Project('project', self.source)
tenant.addProjectRepo(project)
base = configloader.JobParser.fromYaml(tenant, layout, {
'_source_context': self.context,
@ -431,6 +432,7 @@ class TestJob(BaseTestCase):
def test_job_inheritance_job_tree(self):
tenant = model.Tenant('tenant')
layout = model.Layout()
tenant.addProjectRepo(self.project)
pipeline = model.Pipeline('gate', layout)
layout.addPipeline(pipeline)
@ -511,6 +513,7 @@ class TestJob(BaseTestCase):
layout.addPipeline(pipeline)
queue = model.ChangeQueue(pipeline)
project = model.Project('project', self.source)
tenant.addProjectRepo(project)
base = configloader.JobParser.fromYaml(tenant, layout, {
'_source_context': self.context,
@ -591,6 +594,7 @@ class TestJob(BaseTestCase):
self.layout.addJob(job)
project2 = model.Project('project2', self.source)
self.tenant.addProjectRepo(project2)
context2 = model.SourceContext(project2, 'master',
'test', True)

6
tests/unit/test_scheduler.py
View File

@ -1498,8 +1498,8 @@ class TestScheduler(ZuulTestCase):
# https://bugs.executepad.net/zuul/+bug/1078946
# This test assumes the repo is already cloned; make sure it is
tenant = self.sched.abide.tenants.get('tenant-one')
url = self.fake_gerrit.getGitUrl(
tenant.layout.project_configs.get('org/project1'))
trusted, project = tenant.getProject('org/project1')
url = self.fake_gerrit.getGitUrl(project)
self.merge_server.merger.addProject('org/project1', url)
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
A.addPatchset(large=True)
@ -2881,7 +2881,7 @@ class TestScheduler(ZuulTestCase):
self.assertEqual(A.reported, 2)
def test_repo_deleted(self):
self.updateConfigLayout('layout-repo-deleted')
self.updateConfigLayout('layout-repo-deleted', ['org/delete-project'])
self.sched.reconfigure(self.config)
self.init_repo("org/delete-project")

37
zuul/configloader.py
View File

@ -46,6 +46,17 @@ class ConfigurationSyntaxError(Exception):
pass
class ProjectNotFoundError(Exception):
def __init__(self, project):
message = textwrap.dedent("""\
The project {project} was not found. All projects
referenced within a Zuul configuration must first be
added to the main configuration file by the Zuul
administrator.""")
message = textwrap.fill(message.format(project=project))
super(ProjectNotFoundError, self).__init__(message)
def indent(s):
return '\n'.join([' ' + x for x in s.split('\n')])
@ -54,7 +65,7 @@ def indent(s):
def configuration_exceptions(stanza, conf):
try:
yield
except vs.Invalid as e:
except Exception as e:
conf = copy.deepcopy(conf)
context = conf.pop('_source_context')
start_mark = conf.pop('_start_mark')
@ -488,7 +499,13 @@ class ProjectParser(object):
for conf in conf_list:
with configuration_exceptions('project', conf):
ProjectParser.getSchema(layout)(conf)
project = model.ProjectConfig(conf_list[0]['name'])
with configuration_exceptions('project', conf_list[0]):
project_name = conf_list[0]['name']
(trusted, project) = tenant.getProject(project_name)
if project is None:
raise ProjectNotFoundError(project_name)
project_config = model.ProjectConfig(project.canonical_name)
configs = []
for conf in conf_list:
@ -504,14 +521,14 @@ class ProjectParser(object):
for name in conf_templates])
configs.append(project_template)
mode = conf.get('merge-mode')
if mode and project.merge_mode is None:
if mode and project_config.merge_mode is None:
# Set the merge mode to the first one that we find and
# ignore subsequent settings.
project.merge_mode = model.MERGER_MAP[mode]
if project.merge_mode is None:
project_config.merge_mode = model.MERGER_MAP[mode]
if project_config.merge_mode is None:
# If merge mode was not specified in any project stanza,
# set it to the default.
project.merge_mode = model.MERGER_MAP['merge-resolve']
project_config.merge_mode = model.MERGER_MAP['merge-resolve']
for pipeline in layout.pipelines.values():
project_pipeline = model.ProjectPipelineConfig()
queue_name = None
@ -532,9 +549,8 @@ class ProjectParser(object):
if queue_name:
project_pipeline.queue_name = queue_name
if pipeline_defined:
project.pipelines[pipeline.name] = project_pipeline
return project
project_config.pipelines[pipeline.name] = project_pipeline
return project_config
class PipelineParser(object):
@ -787,7 +803,6 @@ class TenantParser(object):
unparsed_config,
scheduler,
connections)
tenant.layout.tenant = tenant
return tenant
@staticmethod
@ -992,6 +1007,8 @@ class TenantParser(object):
layout.addProjectConfig(ProjectParser.fromYaml(
tenant, layout, config_project))
layout.tenant = tenant
for pipeline in layout.pipelines.values():
pipeline.manager._postConfig(layout)

4
zuul/driver/timer/__init__.py
View File

@ -76,12 +76,12 @@ class TimerDriver(Driver, TriggerInterface):
def _onTrigger(self, tenant, pipeline_name, timespec):
for project_name in tenant.layout.project_configs.keys():
project_hostname, project_name = project_name.split('/', 1)
event = TriggerEvent()
event.type = 'timer'
event.timespec = timespec
event.forced_pipeline = pipeline_name
# TODOv3(jeblair): add project hostname in future change
event.project_hostname = ''
event.project_hostname = project_hostname
event.project_name = project_name
self.log.debug("Adding event %s" % event)
self.sched.addEvent(event)

2
zuul/executor/client.py
View File

@ -52,7 +52,7 @@ def make_merger_item(item):
url=item.pipeline.source.getGitUrl(
item.change.project),
connection_name=connection_name,
merge_mode=item.current_build_set.getMergeMode(project),
merge_mode=item.current_build_set.getMergeMode(),
refspec=refspec,
branch=branch,
ref=item.current_build_set.ref,

2
zuul/manager/__init__.py
View File

@ -461,7 +461,7 @@ class PipelineManager(object):
url=self.pipeline.source.getGitUrl(
item.change.project),
connection_name=connection_name,
merge_mode=item.current_build_set.getMergeMode(project),
merge_mode=item.current_build_set.getMergeMode(),
refspec=refspec,
branch=branch,
ref=item.current_build_set.ref,

3
zuul/manager/dependent.py
View File

@ -38,13 +38,14 @@ class DependentPipelineManager(PipelineManager):
self.log.debug("Building shared change queues")
change_queues = {}
project_configs = self.pipeline.layout.project_configs
tenant = self.pipeline.layout.tenant
for project_config in project_configs.values():
project_pipeline_config = project_config.pipelines.get(
self.pipeline.name)
if project_pipeline_config is None:
continue
project = self.pipeline.source.getProject(project_config.name)
(trusted, project) = tenant.getProject(project_config.name)
queue_name = project_pipeline_config.queue_name
if queue_name and queue_name in change_queues:
change_queue = change_queues[queue_name]

16
zuul/model.py
View File

@ -1239,10 +1239,14 @@ class BuildSet(object):
def getTries(self, job_name):
return self.tries.get(job_name)
def getMergeMode(self, job_name):
if not self.layout or job_name not in self.layout.project_configs:
return MERGER_MERGE_RESOLVE
return self.layout.project_configs[job_name].merge_mode
def getMergeMode(self):
if self.layout:
project = self.item.change.project
project_config = self.layout.project_configs.get(
project.canonical_name)
if project_config:
return project_config.merge_mode
return MERGER_MERGE_RESOLVE
class QueueItem(object):
@ -1843,7 +1847,7 @@ class TriggerEvent(object):
return self.project_hostname + '/' + self.project_name
def __repr__(self):
ret = '<TriggerEvent %s %s' % (self.type, self.project_name)
ret = '<TriggerEvent %s %s' % (self.type, self.canonical_project_name)
if self.branch:
ret += " %s" % self.branch
@ -2401,7 +2405,7 @@ class Layout(object):
def createJobGraph(self, item):
project_config = self.project_configs.get(
item.change.project.name, None)
item.change.project.canonical_name, None)
ret = JobGraph()
# NOTE(pabelanger): It is possible for a foreign project not to have a
# configured pipeline, if so return an empty JobGraph.

2
zuul/scheduler.py
View File

@ -611,7 +611,7 @@ class Scheduler(threading.Thread):
def _doEnqueueEvent(self, event):
tenant = self.abide.tenants.get(event.tenant_name)
project = tenant.layout.project_configs.get(event.project_name)
(trusted, project) = tenant.getProject(event.project_name)
pipeline = tenant.layout.pipelines[event.forced_pipeline]
change = pipeline.source.getChange(event, project)
self.log.debug("Event %s for change %s was directly assigned "