Add support for roles in zuul
This adds support for Ansible roles in Zuul-managed repos. It is currently limited to repos within the same source, which is something we should fix. We also plan to add support for roles from Ansible Galaxy in a future change. Change-Id: I7af4dc1333db0dcb9d4a8318a4a95b9564cd1dd8
This commit is contained in:
parent
646322fa50
commit
5ac9384d90
3
tests/fixtures/config/ansible/git/bare-role/tasks/main.yaml
vendored
Normal file
3
tests/fixtures/config/ansible/git/bare-role/tasks/main.yaml
vendored
Normal file
@ -0,0 +1,3 @@
|
||||
- file:
|
||||
path: "{{zuul._test.test_root}}/{{zuul.uuid}}.bare-role.flag"
|
||||
state: touch
|
@ -6,3 +6,5 @@
|
||||
- copy:
|
||||
src: "{{zuul._test.test_root}}/{{zuul.uuid}}.flag"
|
||||
dest: "{{zuul._test.test_root}}/{{zuul.uuid}}.copied"
|
||||
roles:
|
||||
- bare-role
|
||||
|
@ -40,3 +40,5 @@
|
||||
name: python27
|
||||
pre-run: pre
|
||||
post-run: post
|
||||
roles:
|
||||
- zuul: bare-role
|
||||
|
1
tests/fixtures/config/ansible/main.yaml
vendored
1
tests/fixtures/config/ansible/main.yaml
vendored
@ -6,3 +6,4 @@
|
||||
- common-config
|
||||
project-repos:
|
||||
- org/project
|
||||
- bare-role
|
||||
|
@ -34,10 +34,11 @@ class TestJob(BaseTestCase):
|
||||
|
||||
@property
|
||||
def job(self):
|
||||
tenant = model.Tenant('tenant')
|
||||
layout = model.Layout()
|
||||
project = model.Project('project', None)
|
||||
context = model.SourceContext(project, 'master', True)
|
||||
job = configloader.JobParser.fromYaml(layout, {
|
||||
job = configloader.JobParser.fromYaml(tenant, layout, {
|
||||
'_source_context': context,
|
||||
'name': 'job',
|
||||
'irrelevant-files': [
|
||||
@ -134,6 +135,7 @@ class TestJob(BaseTestCase):
|
||||
|
||||
def test_job_inheritance_configloader(self):
|
||||
# TODO(jeblair): move this to a configloader test
|
||||
tenant = model.Tenant('tenant')
|
||||
layout = model.Layout()
|
||||
|
||||
pipeline = model.Pipeline('gate', layout)
|
||||
@ -142,7 +144,7 @@ class TestJob(BaseTestCase):
|
||||
project = model.Project('project', None)
|
||||
context = model.SourceContext(project, 'master', True)
|
||||
|
||||
base = configloader.JobParser.fromYaml(layout, {
|
||||
base = configloader.JobParser.fromYaml(tenant, layout, {
|
||||
'_source_context': context,
|
||||
'name': 'base',
|
||||
'timeout': 30,
|
||||
@ -154,7 +156,7 @@ class TestJob(BaseTestCase):
|
||||
}],
|
||||
})
|
||||
layout.addJob(base)
|
||||
python27 = configloader.JobParser.fromYaml(layout, {
|
||||
python27 = configloader.JobParser.fromYaml(tenant, layout, {
|
||||
'_source_context': context,
|
||||
'name': 'python27',
|
||||
'parent': 'base',
|
||||
@ -167,7 +169,7 @@ class TestJob(BaseTestCase):
|
||||
'timeout': 40,
|
||||
})
|
||||
layout.addJob(python27)
|
||||
python27diablo = configloader.JobParser.fromYaml(layout, {
|
||||
python27diablo = configloader.JobParser.fromYaml(tenant, layout, {
|
||||
'_source_context': context,
|
||||
'name': 'python27',
|
||||
'branches': [
|
||||
@ -184,7 +186,7 @@ class TestJob(BaseTestCase):
|
||||
})
|
||||
layout.addJob(python27diablo)
|
||||
|
||||
python27essex = configloader.JobParser.fromYaml(layout, {
|
||||
python27essex = configloader.JobParser.fromYaml(tenant, layout, {
|
||||
'_source_context': context,
|
||||
'name': 'python27',
|
||||
'branches': [
|
||||
@ -195,7 +197,7 @@ class TestJob(BaseTestCase):
|
||||
})
|
||||
layout.addJob(python27essex)
|
||||
|
||||
project_config = configloader.ProjectParser.fromYaml(layout, {
|
||||
project_config = configloader.ProjectParser.fromYaml(tenant, layout, {
|
||||
'_source_context': context,
|
||||
'name': 'project',
|
||||
'gate': {
|
||||
@ -291,43 +293,46 @@ class TestJob(BaseTestCase):
|
||||
'playbooks/base'])
|
||||
|
||||
def test_job_auth_inheritance(self):
|
||||
tenant = model.Tenant('tenant')
|
||||
layout = model.Layout()
|
||||
project = model.Project('project', None)
|
||||
context = model.SourceContext(project, 'master', True)
|
||||
|
||||
base = configloader.JobParser.fromYaml(layout, {
|
||||
base = configloader.JobParser.fromYaml(tenant, layout, {
|
||||
'_source_context': context,
|
||||
'name': 'base',
|
||||
'timeout': 30,
|
||||
})
|
||||
layout.addJob(base)
|
||||
pypi_upload_without_inherit = configloader.JobParser.fromYaml(layout, {
|
||||
'_source_context': context,
|
||||
'name': 'pypi-upload-without-inherit',
|
||||
'parent': 'base',
|
||||
'timeout': 40,
|
||||
'auth': {
|
||||
'secrets': [
|
||||
'pypi-credentials',
|
||||
]
|
||||
}
|
||||
})
|
||||
pypi_upload_without_inherit = configloader.JobParser.fromYaml(
|
||||
tenant, layout, {
|
||||
'_source_context': context,
|
||||
'name': 'pypi-upload-without-inherit',
|
||||
'parent': 'base',
|
||||
'timeout': 40,
|
||||
'auth': {
|
||||
'secrets': [
|
||||
'pypi-credentials',
|
||||
]
|
||||
}
|
||||
})
|
||||
layout.addJob(pypi_upload_without_inherit)
|
||||
pypi_upload_with_inherit = configloader.JobParser.fromYaml(layout, {
|
||||
'_source_context': context,
|
||||
'name': 'pypi-upload-with-inherit',
|
||||
'parent': 'base',
|
||||
'timeout': 40,
|
||||
'auth': {
|
||||
'inherit': True,
|
||||
'secrets': [
|
||||
'pypi-credentials',
|
||||
]
|
||||
}
|
||||
})
|
||||
pypi_upload_with_inherit = configloader.JobParser.fromYaml(
|
||||
tenant, layout, {
|
||||
'_source_context': context,
|
||||
'name': 'pypi-upload-with-inherit',
|
||||
'parent': 'base',
|
||||
'timeout': 40,
|
||||
'auth': {
|
||||
'inherit': True,
|
||||
'secrets': [
|
||||
'pypi-credentials',
|
||||
]
|
||||
}
|
||||
})
|
||||
layout.addJob(pypi_upload_with_inherit)
|
||||
pypi_upload_with_inherit_false = configloader.JobParser.fromYaml(
|
||||
layout, {
|
||||
tenant, layout, {
|
||||
'_source_context': context,
|
||||
'name': 'pypi-upload-with-inherit-false',
|
||||
'parent': 'base',
|
||||
@ -340,20 +345,22 @@ class TestJob(BaseTestCase):
|
||||
}
|
||||
})
|
||||
layout.addJob(pypi_upload_with_inherit_false)
|
||||
in_repo_job_without_inherit = configloader.JobParser.fromYaml(layout, {
|
||||
'_source_context': context,
|
||||
'name': 'in-repo-job-without-inherit',
|
||||
'parent': 'pypi-upload-without-inherit',
|
||||
})
|
||||
in_repo_job_without_inherit = configloader.JobParser.fromYaml(
|
||||
tenant, layout, {
|
||||
'_source_context': context,
|
||||
'name': 'in-repo-job-without-inherit',
|
||||
'parent': 'pypi-upload-without-inherit',
|
||||
})
|
||||
layout.addJob(in_repo_job_without_inherit)
|
||||
in_repo_job_with_inherit = configloader.JobParser.fromYaml(layout, {
|
||||
'_source_context': context,
|
||||
'name': 'in-repo-job-with-inherit',
|
||||
'parent': 'pypi-upload-with-inherit',
|
||||
})
|
||||
in_repo_job_with_inherit = configloader.JobParser.fromYaml(
|
||||
tenant, layout, {
|
||||
'_source_context': context,
|
||||
'name': 'in-repo-job-with-inherit',
|
||||
'parent': 'pypi-upload-with-inherit',
|
||||
})
|
||||
layout.addJob(in_repo_job_with_inherit)
|
||||
in_repo_job_with_inherit_false = configloader.JobParser.fromYaml(
|
||||
layout, {
|
||||
tenant, layout, {
|
||||
'_source_context': context,
|
||||
'name': 'in-repo-job-with-inherit-false',
|
||||
'parent': 'pypi-upload-with-inherit-false',
|
||||
@ -367,6 +374,7 @@ class TestJob(BaseTestCase):
|
||||
self.assertNotIn('auth', in_repo_job_with_inherit_false.auth)
|
||||
|
||||
def test_job_inheritance_job_tree(self):
|
||||
tenant = model.Tenant('tenant')
|
||||
layout = model.Layout()
|
||||
|
||||
pipeline = model.Pipeline('gate', layout)
|
||||
@ -375,20 +383,20 @@ class TestJob(BaseTestCase):
|
||||
project = model.Project('project', None)
|
||||
context = model.SourceContext(project, 'master', True)
|
||||
|
||||
base = configloader.JobParser.fromYaml(layout, {
|
||||
base = configloader.JobParser.fromYaml(tenant, layout, {
|
||||
'_source_context': context,
|
||||
'name': 'base',
|
||||
'timeout': 30,
|
||||
})
|
||||
layout.addJob(base)
|
||||
python27 = configloader.JobParser.fromYaml(layout, {
|
||||
python27 = configloader.JobParser.fromYaml(tenant, layout, {
|
||||
'_source_context': context,
|
||||
'name': 'python27',
|
||||
'parent': 'base',
|
||||
'timeout': 40,
|
||||
})
|
||||
layout.addJob(python27)
|
||||
python27diablo = configloader.JobParser.fromYaml(layout, {
|
||||
python27diablo = configloader.JobParser.fromYaml(tenant, layout, {
|
||||
'_source_context': context,
|
||||
'name': 'python27',
|
||||
'branches': [
|
||||
@ -398,7 +406,7 @@ class TestJob(BaseTestCase):
|
||||
})
|
||||
layout.addJob(python27diablo)
|
||||
|
||||
project_config = configloader.ProjectParser.fromYaml(layout, {
|
||||
project_config = configloader.ProjectParser.fromYaml(tenant, layout, {
|
||||
'_source_context': context,
|
||||
'name': 'project',
|
||||
'gate': {
|
||||
@ -439,6 +447,7 @@ class TestJob(BaseTestCase):
|
||||
self.assertEqual(job.timeout, 70)
|
||||
|
||||
def test_inheritance_keeps_matchers(self):
|
||||
tenant = model.Tenant('tenant')
|
||||
layout = model.Layout()
|
||||
|
||||
pipeline = model.Pipeline('gate', layout)
|
||||
@ -447,13 +456,13 @@ class TestJob(BaseTestCase):
|
||||
project = model.Project('project', None)
|
||||
context = model.SourceContext(project, 'master', True)
|
||||
|
||||
base = configloader.JobParser.fromYaml(layout, {
|
||||
base = configloader.JobParser.fromYaml(tenant, layout, {
|
||||
'_source_context': context,
|
||||
'name': 'base',
|
||||
'timeout': 30,
|
||||
})
|
||||
layout.addJob(base)
|
||||
python27 = configloader.JobParser.fromYaml(layout, {
|
||||
python27 = configloader.JobParser.fromYaml(tenant, layout, {
|
||||
'_source_context': context,
|
||||
'name': 'python27',
|
||||
'parent': 'base',
|
||||
@ -462,7 +471,7 @@ class TestJob(BaseTestCase):
|
||||
})
|
||||
layout.addJob(python27)
|
||||
|
||||
project_config = configloader.ProjectParser.fromYaml(layout, {
|
||||
project_config = configloader.ProjectParser.fromYaml(tenant, layout, {
|
||||
'_source_context': context,
|
||||
'name': 'project',
|
||||
'gate': {
|
||||
@ -486,11 +495,12 @@ class TestJob(BaseTestCase):
|
||||
self.assertEqual([], item.getJobs())
|
||||
|
||||
def test_job_source_project(self):
|
||||
tenant = model.Tenant('tenant')
|
||||
layout = model.Layout()
|
||||
base_project = model.Project('base_project', None)
|
||||
base_context = model.SourceContext(base_project, 'master', True)
|
||||
|
||||
base = configloader.JobParser.fromYaml(layout, {
|
||||
base = configloader.JobParser.fromYaml(tenant, layout, {
|
||||
'_source_context': base_context,
|
||||
'name': 'base',
|
||||
})
|
||||
@ -498,7 +508,7 @@ class TestJob(BaseTestCase):
|
||||
|
||||
other_project = model.Project('other_project', None)
|
||||
other_context = model.SourceContext(other_project, 'master', True)
|
||||
base2 = configloader.JobParser.fromYaml(layout, {
|
||||
base2 = configloader.JobParser.fromYaml(tenant, layout, {
|
||||
'_source_context': other_context,
|
||||
'name': 'base',
|
||||
})
|
||||
|
@ -155,3 +155,6 @@ class TestAnsible(AnsibleZuulTestCase):
|
||||
post_flag_path = os.path.join(self.test_root, build.uuid +
|
||||
'.post.flag')
|
||||
self.assertTrue(os.path.exists(post_flag_path))
|
||||
bare_role_flag_path = os.path.join(self.test_root,
|
||||
build.uuid + '.bare-role.flag')
|
||||
self.assertTrue(os.path.exists(bare_role_flag_path))
|
||||
|
@ -84,6 +84,14 @@ class JobParser(object):
|
||||
vs.Required('image'): str,
|
||||
}
|
||||
|
||||
zuul_role = {vs.Required('zuul'): str,
|
||||
'name': str}
|
||||
|
||||
galaxy_role = {vs.Required('galaxy'): str,
|
||||
'name': str}
|
||||
|
||||
role = vs.Any(zuul_role, galaxy_role)
|
||||
|
||||
job = {vs.Required('name'): str,
|
||||
'parent': str,
|
||||
'queue-name': str,
|
||||
@ -106,6 +114,7 @@ class JobParser(object):
|
||||
'post-run': to_list(str),
|
||||
'run': str,
|
||||
'_source_context': model.SourceContext,
|
||||
'roles': to_list(role),
|
||||
}
|
||||
|
||||
return vs.Schema(job)
|
||||
@ -124,7 +133,7 @@ class JobParser(object):
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def fromYaml(layout, conf):
|
||||
def fromYaml(tenant, layout, conf):
|
||||
JobParser.getSchema()(conf)
|
||||
|
||||
# NB: The default detection system in the Job class requires
|
||||
@ -183,6 +192,14 @@ class JobParser(object):
|
||||
# accumulate onto any previously applied tags.
|
||||
job.tags = job.tags.union(set(tags))
|
||||
|
||||
roles = []
|
||||
for role in conf.get('roles', []):
|
||||
if 'zuul' in role:
|
||||
r = JobParser._makeZuulRole(tenant, job, role)
|
||||
if r:
|
||||
roles.append(r)
|
||||
job.roles = job.roles.union(set(roles))
|
||||
|
||||
# If the definition for this job came from a project repo,
|
||||
# implicitly apply a branch matcher for the branch it was on.
|
||||
if (not job.source_context.secure):
|
||||
@ -209,6 +226,20 @@ class JobParser(object):
|
||||
matchers)
|
||||
return job
|
||||
|
||||
@staticmethod
|
||||
def _makeZuulRole(tenant, job, role):
|
||||
name = role['zuul'].split('/')[-1]
|
||||
|
||||
# TODOv3(jeblair): this limits roles to the same
|
||||
# source; we should remove that limitation.
|
||||
source = job.source_context.project.connection_name
|
||||
(secure, project) = tenant.getRepo(source, role['zuul'])
|
||||
if project is None:
|
||||
return None
|
||||
|
||||
return model.ZuulRole(role.get('name', name), source,
|
||||
project.name, secure)
|
||||
|
||||
|
||||
class ProjectTemplateParser(object):
|
||||
log = logging.getLogger("zuul.ProjectTemplateParser")
|
||||
@ -229,7 +260,7 @@ class ProjectTemplateParser(object):
|
||||
return vs.Schema(project_template)
|
||||
|
||||
@staticmethod
|
||||
def fromYaml(layout, conf):
|
||||
def fromYaml(tenant, layout, conf):
|
||||
ProjectTemplateParser.getSchema(layout)(conf)
|
||||
# Make a copy since we modify this later via pop
|
||||
conf = copy.deepcopy(conf)
|
||||
@ -243,12 +274,12 @@ class ProjectTemplateParser(object):
|
||||
project_template.pipelines[pipeline.name] = project_pipeline
|
||||
project_pipeline.queue_name = conf_pipeline.get('queue')
|
||||
project_pipeline.job_tree = ProjectTemplateParser._parseJobTree(
|
||||
layout, conf_pipeline.get('jobs', []),
|
||||
tenant, layout, conf_pipeline.get('jobs', []),
|
||||
source_context)
|
||||
return project_template
|
||||
|
||||
@staticmethod
|
||||
def _parseJobTree(layout, conf, source_context, tree=None):
|
||||
def _parseJobTree(tenant, layout, conf, source_context, tree=None):
|
||||
if not tree:
|
||||
tree = model.JobTree(None)
|
||||
for conf_job in conf:
|
||||
@ -264,7 +295,8 @@ class ProjectTemplateParser(object):
|
||||
# We are overriding params, so make a new job def
|
||||
attrs['name'] = jobname
|
||||
attrs['_source_context'] = source_context
|
||||
subtree = tree.addJob(JobParser.fromYaml(layout, attrs))
|
||||
subtree = tree.addJob(JobParser.fromYaml(
|
||||
tenant, layout, attrs))
|
||||
else:
|
||||
# Not overriding, so add a blank job
|
||||
job = model.Job(jobname)
|
||||
@ -272,9 +304,8 @@ class ProjectTemplateParser(object):
|
||||
|
||||
if jobs:
|
||||
# This is the root of a sub tree
|
||||
ProjectTemplateParser._parseJobTree(layout, jobs,
|
||||
source_context,
|
||||
subtree)
|
||||
ProjectTemplateParser._parseJobTree(
|
||||
tenant, layout, jobs, source_context, subtree)
|
||||
else:
|
||||
raise Exception("Job must be a string or dictionary")
|
||||
return tree
|
||||
@ -299,7 +330,7 @@ class ProjectParser(object):
|
||||
return vs.Schema(project)
|
||||
|
||||
@staticmethod
|
||||
def fromYaml(layout, conf):
|
||||
def fromYaml(tenant, layout, conf):
|
||||
# TODOv3(jeblair): This may need some branch-specific
|
||||
# configuration for in-repo configs.
|
||||
ProjectParser.getSchema(layout)(conf)
|
||||
@ -309,7 +340,7 @@ class ProjectParser(object):
|
||||
# The way we construct a project definition is by parsing the
|
||||
# definition as a template, then applying all of the
|
||||
# templates, including the newly parsed one, in order.
|
||||
project_template = ProjectTemplateParser.fromYaml(layout, conf)
|
||||
project_template = ProjectTemplateParser.fromYaml(tenant, layout, conf)
|
||||
configs = [layout.project_templates[name] for name in conf_templates]
|
||||
configs.append(project_template)
|
||||
project = model.ProjectConfig(conf['name'])
|
||||
@ -551,6 +582,10 @@ class TenantParser(object):
|
||||
unparsed_config = model.UnparsedTenantConfig()
|
||||
tenant.config_repos, tenant.project_repos = \
|
||||
TenantParser._loadTenantConfigRepos(connections, conf)
|
||||
for source, repo in tenant.config_repos:
|
||||
tenant.addConfigRepo(source, repo)
|
||||
for source, repo in tenant.project_repos:
|
||||
tenant.addProjectRepo(source, repo)
|
||||
tenant.config_repos_config, tenant.project_repos_config = \
|
||||
TenantParser._loadTenantInRepoLayouts(merger, connections,
|
||||
tenant.config_repos,
|
||||
@ -558,8 +593,10 @@ class TenantParser(object):
|
||||
cached)
|
||||
unparsed_config.extend(tenant.config_repos_config)
|
||||
unparsed_config.extend(tenant.project_repos_config)
|
||||
tenant.layout = TenantParser._parseLayout(base, unparsed_config,
|
||||
scheduler, connections)
|
||||
tenant.layout = TenantParser._parseLayout(base, tenant,
|
||||
unparsed_config,
|
||||
scheduler,
|
||||
connections)
|
||||
tenant.layout.tenant = tenant
|
||||
return tenant
|
||||
|
||||
@ -672,7 +709,7 @@ class TenantParser(object):
|
||||
return config
|
||||
|
||||
@staticmethod
|
||||
def _parseLayout(base, data, scheduler, connections):
|
||||
def _parseLayout(base, tenant, data, scheduler, connections):
|
||||
layout = model.Layout()
|
||||
|
||||
for config_pipeline in data.pipelines:
|
||||
@ -684,15 +721,15 @@ class TenantParser(object):
|
||||
layout.addNodeSet(NodeSetParser.fromYaml(layout, config_nodeset))
|
||||
|
||||
for config_job in data.jobs:
|
||||
layout.addJob(JobParser.fromYaml(layout, config_job))
|
||||
layout.addJob(JobParser.fromYaml(tenant, layout, config_job))
|
||||
|
||||
for config_template in data.project_templates:
|
||||
layout.addProjectTemplate(ProjectTemplateParser.fromYaml(
|
||||
layout, config_template))
|
||||
tenant, layout, config_template))
|
||||
|
||||
for config_project in data.projects:
|
||||
layout.addProjectConfig(ProjectParser.fromYaml(
|
||||
layout, config_project))
|
||||
tenant, layout, config_project))
|
||||
|
||||
for pipeline in layout.pipelines.values():
|
||||
pipeline.manager._postConfig(layout)
|
||||
@ -765,14 +802,14 @@ class ConfigLoader(object):
|
||||
layout.pipelines = tenant.layout.pipelines
|
||||
|
||||
for config_job in config.jobs:
|
||||
layout.addJob(JobParser.fromYaml(layout, config_job))
|
||||
layout.addJob(JobParser.fromYaml(tenant, layout, config_job))
|
||||
|
||||
for config_template in config.project_templates:
|
||||
layout.addProjectTemplate(ProjectTemplateParser.fromYaml(
|
||||
layout, config_template))
|
||||
tenant, layout, config_template))
|
||||
|
||||
for config_project in config.projects:
|
||||
layout.addProjectConfig(ProjectParser.fromYaml(
|
||||
layout, config_project), update_pipeline=False)
|
||||
tenant, layout, config_project), update_pipeline=False)
|
||||
|
||||
return layout
|
||||
|
@ -334,6 +334,7 @@ class LaunchClient(object):
|
||||
params['playbooks'] = [x.toDict() for x in job.run]
|
||||
params['pre_playbooks'] = [x.toDict() for x in job.pre_run]
|
||||
params['post_playbooks'] = [x.toDict() for x in job.post_run]
|
||||
params['roles'] = [x.toDict() for x in job.roles]
|
||||
|
||||
nodes = []
|
||||
for node in item.current_build_set.getJobNodeSet(job.name).getNodes():
|
||||
|
@ -95,6 +95,8 @@ class JobDir(object):
|
||||
self.playbook = None # A pointer to the candidate we have chosen
|
||||
self.pre_playbooks = []
|
||||
self.post_playbooks = []
|
||||
self.roles = []
|
||||
self.roles_path = []
|
||||
self.config = os.path.join(self.ansible_root, 'ansible.cfg')
|
||||
self.secure_config = os.path.join(
|
||||
self.secure_ansible_root, 'ansible.cfg')
|
||||
@ -124,6 +126,13 @@ class JobDir(object):
|
||||
self.playbooks.append(playbook)
|
||||
return playbook
|
||||
|
||||
def addRole(self):
|
||||
count = len(self.roles)
|
||||
root = os.path.join(self.ansible_root, 'role_%i' % (count,))
|
||||
os.makedirs(root)
|
||||
self.roles.append(root)
|
||||
return root
|
||||
|
||||
def cleanup(self):
|
||||
if not self.keep:
|
||||
shutil.rmtree(self.root)
|
||||
@ -518,6 +527,8 @@ class AnsibleJob(object):
|
||||
# is the playbook in a repo that we have already prepared?
|
||||
self.preparePlaybookRepos(args)
|
||||
|
||||
self.prepareRoles(args)
|
||||
|
||||
# TODOv3: Ansible the ansible thing here.
|
||||
self.prepareAnsibleFiles(args)
|
||||
|
||||
@ -592,28 +603,27 @@ class AnsibleJob(object):
|
||||
hosts.append((node['name'], dict(ansible_connection='local')))
|
||||
return hosts
|
||||
|
||||
def _blockPluginDirs(self, fn):
|
||||
'''Prevent execution of playbooks with plugins
|
||||
def _blockPluginDirs(self, path):
|
||||
'''Prevent execution of playbooks or roles with plugins
|
||||
|
||||
Plugins are loaded from roles and also if there is a plugin
|
||||
dir adjacent to the playbook. Throw an error if the path
|
||||
contains a location that would cause a plugin to get loaded.
|
||||
|
||||
Plugins are loaded from roles and also if there is a plugin dir
|
||||
adjacent to the playbook. Role exclusion will be handled elsewhere,
|
||||
but while we're looking for playbooks, throw an error if the playbook
|
||||
exists in a location that would cause a plugin to get loaded if the
|
||||
playbook is not in a secure repository.
|
||||
'''
|
||||
playbook_dir = os.path.dirname(os.path.abspath(fn))
|
||||
for entry in os.listdir(playbook_dir):
|
||||
for entry in os.listdir(path):
|
||||
if os.path.isdir(entry) and entry.endswith('_plugins'):
|
||||
raise Exception(
|
||||
"Ansible plugin dir %s found adjacent to playbook %s in"
|
||||
" non-secure repo." % (entry, fn))
|
||||
" non-secure repo." % (entry, path))
|
||||
|
||||
def findPlaybook(self, path, required=False, secure=False):
|
||||
for ext in ['.yaml', '.yml']:
|
||||
fn = path + ext
|
||||
if os.path.exists(fn):
|
||||
if not secure:
|
||||
self._blockPluginDirs(fn)
|
||||
playbook_dir = os.path.dirname(os.path.abspath(fn))
|
||||
self._blockPluginDirs(playbook_dir)
|
||||
return fn
|
||||
if required:
|
||||
raise Exception("Unable to find playbook %s" % path)
|
||||
@ -681,6 +691,75 @@ class AnsibleJob(object):
|
||||
required=required,
|
||||
secure=playbook['secure'])
|
||||
|
||||
def prepareRoles(self, args):
|
||||
for role in args['roles']:
|
||||
if role['type'] == 'zuul':
|
||||
root = self.jobdir.addRole()
|
||||
self.prepareZuulRole(args, role, root)
|
||||
|
||||
def findRole(self, path, secure=False):
|
||||
d = os.path.join(path, 'tasks')
|
||||
if os.path.isdir(d):
|
||||
# This is a bare role
|
||||
if not secure:
|
||||
self._blockPluginDirs(path)
|
||||
# None signifies that the repo is a bare role
|
||||
return None
|
||||
d = os.path.join(path, 'roles')
|
||||
if os.path.isdir(d):
|
||||
# This repo has a collection of roles
|
||||
if not secure:
|
||||
for entry in os.listdir(d):
|
||||
self._blockPluginDirs(os.path.join(d, entry))
|
||||
return d
|
||||
# We assume the repository itself is a collection of roles
|
||||
if not secure:
|
||||
for entry in os.listdir(path):
|
||||
self._blockPluginDirs(os.path.join(path, entry))
|
||||
return path
|
||||
|
||||
def prepareZuulRole(self, args, role, root):
|
||||
self.log.debug("Prepare zuul role for %s" % (role,))
|
||||
# Check out the role repo if needed
|
||||
source = self.launcher_server.connections.getSource(
|
||||
role['connection'])
|
||||
project = source.getProject(role['project'])
|
||||
# TODO(jeblair): construct the url in the merger itself
|
||||
url = source.getGitUrl(project)
|
||||
role_repo = None
|
||||
if not role['secure']:
|
||||
# This is a project repo, so it is safe to use the already
|
||||
# checked out version (from speculative merging) of the
|
||||
# role
|
||||
|
||||
for i in args['items']:
|
||||
if (i['connection_name'] == role['connection'] and
|
||||
i['project'] == role['project']):
|
||||
# We already have this repo prepared;
|
||||
# copy it into location.
|
||||
|
||||
path = os.path.join(self.jobdir.git_root,
|
||||
project.name)
|
||||
link = os.path.join(root, role['name'])
|
||||
os.symlink(path, link)
|
||||
role_repo = link
|
||||
break
|
||||
|
||||
# The role repo is either a config repo, or it isn't in
|
||||
# the stack of changes we are testing, so check out the branch
|
||||
# tip into a dedicated space.
|
||||
|
||||
if not role_repo:
|
||||
merger = self.launcher_server._getMerger(root)
|
||||
merger.checkoutBranch(project.name, url, 'master')
|
||||
role_repo = os.path.join(root, project.name)
|
||||
|
||||
role_path = self.findRole(role_repo, secure=role['secure'])
|
||||
if role_path is None:
|
||||
# In the case of a bare role, add the containing directory
|
||||
role_path = root
|
||||
self.jobdir.roles_path.append(role_path)
|
||||
|
||||
def prepareAnsibleFiles(self, args):
|
||||
with open(self.jobdir.inventory, 'w') as inventory:
|
||||
for host_name, host_vars in self.getHostList(args):
|
||||
@ -710,6 +789,9 @@ class AnsibleJob(object):
|
||||
config.write('gathering = explicit\n')
|
||||
config.write('library = %s\n'
|
||||
% self.launcher_server.library_dir)
|
||||
if self.jobdir.roles_path:
|
||||
config.write('roles_path = %s\n' %
|
||||
':'.join(self.jobdir.roles_path))
|
||||
# bump the timeout because busy nodes may take more than
|
||||
# 10s to respond
|
||||
config.write('timeout = 30\n')
|
||||
|
@ -12,6 +12,7 @@
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import abc
|
||||
import copy
|
||||
import os
|
||||
import re
|
||||
@ -20,6 +21,8 @@ import time
|
||||
from uuid import uuid4
|
||||
import extras
|
||||
|
||||
import six
|
||||
|
||||
OrderedDict = extras.try_imports(['collections.OrderedDict',
|
||||
'ordereddict.OrderedDict'])
|
||||
|
||||
@ -593,6 +596,62 @@ class PlaybookContext(object):
|
||||
path=self.path)
|
||||
|
||||
|
||||
@six.add_metaclass(abc.ABCMeta)
|
||||
class Role(object):
|
||||
"""A reference to an ansible role."""
|
||||
|
||||
def __init__(self, target_name):
|
||||
self.target_name = target_name
|
||||
|
||||
@abc.abstractmethod
|
||||
def __repr__(self):
|
||||
pass
|
||||
|
||||
def __ne__(self, other):
|
||||
return not self.__eq__(other)
|
||||
|
||||
@abc.abstractmethod
|
||||
def __eq__(self, other):
|
||||
if not isinstance(other, Role):
|
||||
return False
|
||||
return (self.target_name == other.target_name)
|
||||
|
||||
@abc.abstractmethod
|
||||
def toDict(self):
|
||||
# Render to a dict to use in passing json to the launcher
|
||||
return dict(target_name=self.target_name)
|
||||
|
||||
|
||||
class ZuulRole(Role):
|
||||
"""A reference to an ansible role in a Zuul project."""
|
||||
|
||||
def __init__(self, target_name, connection_name, project_name, secure):
|
||||
super(ZuulRole, self).__init__(target_name)
|
||||
self.connection_name = connection_name
|
||||
self.project_name = project_name
|
||||
self.secure = secure
|
||||
|
||||
def __repr__(self):
|
||||
return '<ZuulRole %s %s>' % (self.project_name, self.target_name)
|
||||
|
||||
def __eq__(self, other):
|
||||
if not isinstance(other, ZuulRole):
|
||||
return False
|
||||
return (super(ZuulRole, self).__eq__(other) and
|
||||
self.connection_name == other.connection_name,
|
||||
self.project_name == other.project_name,
|
||||
self.secure == other.secure)
|
||||
|
||||
def toDict(self):
|
||||
# Render to a dict to use in passing json to the launcher
|
||||
d = super(ZuulRole, self).toDict()
|
||||
d['type'] = 'zuul'
|
||||
d['connection'] = self.connection_name
|
||||
d['project'] = self.project_name
|
||||
d['secure'] = self.secure
|
||||
return d
|
||||
|
||||
|
||||
class Job(object):
|
||||
|
||||
"""A Job represents the defintion of actions to perform.
|
||||
@ -638,6 +697,7 @@ class Job(object):
|
||||
mutex=None,
|
||||
attempts=3,
|
||||
final=False,
|
||||
roles=frozenset(),
|
||||
)
|
||||
|
||||
# These are generally internal attributes which are not
|
||||
@ -734,7 +794,7 @@ class Job(object):
|
||||
"%s=%s with variant %s" % (
|
||||
repr(self), k, other._get(k),
|
||||
repr(other)))
|
||||
if k not in set(['pre_run', 'post_run']):
|
||||
if k not in set(['pre_run', 'post_run', 'roles']):
|
||||
setattr(self, k, copy.deepcopy(other._get(k)))
|
||||
|
||||
# Don't set final above so that we don't trip an error halfway
|
||||
@ -746,6 +806,8 @@ class Job(object):
|
||||
self.pre_run = self.pre_run + other.pre_run
|
||||
if other._get('post_run') is not None:
|
||||
self.post_run = other.post_run + self.post_run
|
||||
if other._get('roles') is not None:
|
||||
self.roles = self.roles.union(other.roles)
|
||||
|
||||
for k in self.context_attributes:
|
||||
if (other._get(k) is not None and
|
||||
@ -2146,6 +2208,39 @@ class Tenant(object):
|
||||
self.project_repos = []
|
||||
# The unparsed config from those repos.
|
||||
self.project_repos_config = None
|
||||
# A mapping of source -> {config_repos: {}, project_repos: {}}
|
||||
self.sources = {}
|
||||
|
||||
def addConfigRepo(self, source, project):
|
||||
sd = self.sources.setdefault(source.name,
|
||||
{'config_repos': {},
|
||||
'project_repos': {}})
|
||||
sd['config_repos'][project.name] = project
|
||||
|
||||
def addProjectRepo(self, source, project):
|
||||
sd = self.sources.setdefault(source.name,
|
||||
{'config_repos': {},
|
||||
'project_repos': {}})
|
||||
sd['project_repos'][project.name] = project
|
||||
|
||||
def getRepo(self, source, project_name):
|
||||
"""Get a project given a source and project name
|
||||
|
||||
Returns a tuple (secure, project) or (None, None) if the
|
||||
project is not found.
|
||||
|
||||
Secure indicates the project is a config repo.
|
||||
|
||||
"""
|
||||
|
||||
sd = self.sources.get(source)
|
||||
if not sd:
|
||||
return (None, None)
|
||||
if project_name in sd['config_repos']:
|
||||
return (True, sd['config_repos'][project_name])
|
||||
if project_name in sd['project_repos']:
|
||||
return (False, sd['project_repos'][project_name])
|
||||
return (None, None)
|
||||
|
||||
|
||||
class Abide(object):
|
||||
|
Loading…
Reference in New Issue
Block a user