jjb/jenkins-job-builder
Code Issues Proposed changes

Basic folder support

Browse Source 
Allows specifying a folder attribute for each job generated, which in
turn is used when creating or uploading to place the job under the
requested folder.

The job name is expanded after defaults are applied, to support the
attribute being defined within a set of defaults applied to a number of
jobs.

This in turn allows for multiple jobs with the same basename to exist,
provided they are targeted at different folders.

Does not support creating the folders if they do not exist.

Change-Id: I8c2157c4c81087cc972a048d1b88d5f08ac65361
This commit is contained in:
Darragh Bailey 2015-11-30 13:20:15 +01:00 committed by Thanh Ha
parent 16a307188e
commit af9d984baa
No known key found for this signature in database
GPG Key ID: B0CB27E00DA095AA
19 changed files with 281 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
doc/source/definition.rst
View File

@ -329,6 +329,35 @@ always use ``{{`` to achieve a literal ``{``. A generic builder will need
to consider the correct quoting based on its use of parameters.
.. _folders:
Folders
^^^^^^^
Jenkins supports organising jobs, views, and slaves using a folder hierarchy.
This allows for easier separation of access as well credentials and resources
which can be assigned to only be available for a specific folder.
JJB has two methods of supporting uploading jobs to a specific folder:
* Name the job to contain the desired folder ``<folder>/my-job-name``
* Use the ``folder`` attribute on a job definition, via a template, or through
`Defaults`_.
Supporting both an attributed and use of it directly in job names allows for
teams to have all jobs using their defaults automatically use a top-level
folder, while still allowing for them to additionally nest jobs for their
own preferences.
Job Name Example:
.. literalinclude:: /../../tests/yamlparser/fixtures/folders-job-name.yaml
Folder Attribute Example:
.. literalinclude:: /../../tests/yamlparser/fixtures/folders-attribute.yaml
.. _ids:
Item ID's

68
jenkins_jobs/builder.py
View File

@ -65,18 +65,43 @@ class JenkinsManager(object):
self._view_list = None
self._jjb_config = jjb_config
def _setup_output(self, output, item, config_xml=False):
output_dir = output
output_fn = os.path.join(output, item)
if '/' in item:
# in item folder
output_fn = os.path.join(output, os.path.normpath(item))
output_dir = os.path.dirname(output_fn)
# if in a folder, re-adding name to the directory here
if config_xml:
output_dir = os.path.join(
output_dir, os.path.basename(item))
output_fn = os.path.join(output_dir, 'config.xml')
if output_dir != output:
logger.info("Creating directory %s" % output_dir)
try:
os.makedirs(output_dir)
except OSError:
if not os.path.isdir(output_dir):
raise
return output_fn
@property
def jobs(self):
if self._jobs is None:
# populate jobs
self._jobs = self.jenkins.get_jobs()
self._jobs = self.jenkins.get_all_jobs()
return self._jobs
@property
def job_list(self):
if self._job_list is None:
self._job_list = set(job['name'] for job in self.jobs)
# python-jenkins uses 'fullname' for folder/name combination
self._job_list = set(job['fullname'] for job in self.jobs)
return self._job_list
def update_job(self, job_name, xml):
@ -154,17 +179,18 @@ class JenkinsManager(object):
if keep is None:
keep = []
for job in jobs:
if job['name'] not in keep:
if self.is_managed(job['name']):
# python-jenkins stores the folder and name as 'fullname'
if job['fullname'] not in keep:
if self.is_managed(job['fullname']):
logger.info("Removing obsolete jenkins job {0}"
.format(job['name']))
self.delete_job(job['name'])
.format(job['fullname']))
self.delete_job(job['fullname'])
deleted_jobs += 1
else:
logger.info("Not deleting unmanaged jenkins job %s",
job['name'])
job['fullname'])
else:
logger.debug("Keeping job %s", job['name'])
logger.debug("Keeping job %s", job['fullname'])
return deleted_jobs
def delete_jobs(self, jobs):
@ -231,17 +257,8 @@ class JenkinsManager(object):
raise
continue
if config_xml:
output_dir = os.path.join(output, job.name)
logger.info("Creating directory %s" % output_dir)
try:
os.makedirs(output_dir)
except OSError:
if not os.path.isdir(output_dir):
raise
output_fn = os.path.join(output_dir, 'config.xml')
else:
output_fn = os.path.join(output, job.name)
output_fn = self._setup_output(output, job.name, config_xml)
logger.debug("Writing XML to '{0}'".format(output_fn))
with io.open(output_fn, 'w', encoding='utf-8') as f:
f.write(job.output().decode('utf-8'))
@ -383,17 +400,8 @@ class JenkinsManager(object):
raise
continue
if config_xml:
output_dir = os.path.join(output, view.name)
logger.info("Creating directory %s" % output_dir)
try:
os.makedirs(output_dir)
except OSError:
if not os.path.isdir(output_dir):
raise
output_fn = os.path.join(output_dir, 'config.xml')
else:
output_fn = os.path.join(output, view.name)
output_fn = self._setup_output(output, view.name, config_xml)
logger.debug("Writing XML to '{0}'".format(output_fn))
with io.open(output_fn, 'w', encoding='utf-8') as f:
f.write(view.output().decode('utf-8'))

7
jenkins_jobs/modules/general.py
View File

@ -56,6 +56,12 @@ Example:
Path for a custom workspace. Defaults to Jenkins default
configuration.
* **folder**:
The folder attribute provides an alternative to using '<path>/<name>' as
the job name to specify which Jenkins folder to upload the job to.
Requires the `CloudBees Folders Plugin.
<https://wiki.jenkins-ci.org/display/JENKINS/CloudBees+Folders+Plugin>`_
* **child-workspace**:
Path for a child custom workspace. Defaults to Jenkins default
configuration. This parameter is only valid for matrix type jobs.
@ -103,7 +109,6 @@ Example:
* **raw**:
If present, this section should contain a single **xml** entry. This XML
will be inserted at the top-level of the :ref:`Job` definition.
"""
import logging

11
jenkins_jobs/parser.py
View File

@ -226,6 +226,12 @@ class YamlParser(object):
for macro in self.data.get(component_type, {}).values():
self._macro_registry.register(component_type, macro)
def _getfullname(self, data):
if 'folder' in data:
return "%s/%s" % (data['folder'], data['name'])
return data['name']
def expandYaml(self, registry, jobs_glob=None):
changed = True
while changed:
@ -240,15 +246,17 @@ class YamlParser(object):
self._macro_registry.expand_macros(default)
for job in self.data.get('job', {}).values():
self._macro_registry.expand_macros(job)
job = self._applyDefaults(job)
job['name'] = self._getfullname(job)
if jobs_glob and not matches(job['name'], jobs_glob):
logger.debug("Ignoring job {0}".format(job['name']))
continue
logger.debug("Expanding job '{0}'".format(job['name']))
job = self._applyDefaults(job)
self._formatDescription(job)
self.jobs.append(job)
for view in self.data.get('view', {}).values():
view['name'] = self._getfullname(view)
logger.debug("Expanding view '{0}'".format(view['name']))
self._formatDescription(view)
self.views.append(view)
@ -400,6 +408,7 @@ class YamlParser(object):
"Failure formatting template '%s', containing '%s' with "
"params '%s'", template_name, template, params)
raise
expanded['name'] = self._getfullname(expanded)
self._macro_registry.expand_macros(expanded, params)
job_name = expanded.get('name')

2
requirements.txt
View File

@ -5,6 +5,6 @@ six>=1.9.0 # MIT
PyYAML>=3.10.0 # MIT
pbr>=1.8 # Apache-2.0
stevedore>=1.17.1 # Apache-2.0
python-jenkins>=0.4.8
python-jenkins>=0.4.9
fasteners
Jinja2

76
tests/base.py
View File

@ -26,6 +26,7 @@ import re
import xml.etree.ElementTree as XML
import fixtures
import six
from six.moves import StringIO
import testtools
from testtools.content import text_content
@ -64,11 +65,16 @@ def get_scenarios(fixtures_path, in_ext='yaml', out_ext='xml',
- content of the fixture output file (aka expected)
"""
scenarios = []
files = []
for dirpath, dirs, fs in os.walk(fixtures_path):
files.extend([os.path.join(dirpath, f) for f in fs])
files = {}
for dirpath, _, fs in os.walk(fixtures_path):
for fn in fs:
if fn in files:
files[fn].append(os.path.join(dirpath, fn))
else:
files[fn] = [os.path.join(dirpath, fn)]
input_files = [f for f in files if re.match(r'.*\.{0}$'.format(in_ext), f)]
input_files = [files[f][0] for f in files if
re.match(r'.*\.{0}$'.format(in_ext), f)]
for input_filename in input_files:
if input_filename.endswith(plugins_info_ext):
@ -80,24 +86,27 @@ def get_scenarios(fixtures_path, in_ext='yaml', out_ext='xml',
output_candidate = re.sub(r'\.{0}$'.format(in_ext),
'.{0}'.format(out_ext), input_filename)
# assume empty file if no output candidate found
if output_candidate not in files:
output_candidate = None
if os.path.basename(output_candidate) in files:
out_filenames = files[os.path.basename(output_candidate)]
else:
out_filenames = None
plugins_info_candidate = re.sub(r'\.{0}$'.format(in_ext),
'.{0}'.format(plugins_info_ext),
input_filename)
if plugins_info_candidate not in files:
if os.path.basename(plugins_info_candidate) not in files:
plugins_info_candidate = None
conf_candidate = re.sub(r'\.yaml$|\.json$', '.conf', input_filename)
# If present, add the configuration file
if conf_candidate not in files:
conf_candidate = None
conf_filename = files.get(os.path.basename(conf_candidate), None)
if conf_filename is not None:
conf_filename = conf_filename[0]
scenarios.append((input_filename, {
'in_filename': input_filename,
'out_filename': output_candidate,
'conf_filename': conf_candidate,
'out_filenames': out_filenames,
'conf_filename': conf_filename,
'plugins_info_filename': plugins_info_candidate,
}))
@ -117,12 +126,13 @@ class BaseTestCase(testtools.TestCase):
def _read_utf8_content(self):
# if None assume empty file
if self.out_filename is None:
if not self.out_filenames:
return u""
# Read XML content, assuming it is unicode encoded
xml_content = u"%s" % io.open(self.out_filename,
'r', encoding='utf-8').read()
xml_content = ""
for f in sorted(self.out_filenames):
xml_content += u"%s" % io.open(f, 'r', encoding='utf-8').read()
return xml_content
def _read_yaml_content(self, filename):
@ -195,6 +205,23 @@ class BaseScenariosTestCase(testscenarios.TestWithScenarios, BaseTestCase):
# Generate the XML tree directly with modules/general
pub.gen_xml(xml_project, yaml_content)
# check output file is under correct path
if 'name' in yaml_content:
prefix = os.path.dirname(self.in_filename)
# split using '/' since fullname uses URL path separator
expected_folders = [os.path.normpath(
os.path.join(prefix,
'/'.join(parser._getfullname(yaml_content).
split('/')[:-1])))]
actual_folders = [os.path.dirname(f) for f in self.out_filenames]
self.assertEquals(
expected_folders, actual_folders,
"Output file under wrong path, was '%s', should be '%s'" %
(self.out_filenames[0],
os.path.join(expected_folders[0],
os.path.basename(self.out_filenames[0]))))
# Prettify generated XML
pretty_xml = XmlJob(xml_project, 'fixturejob').output().decode('utf-8')
@ -226,6 +253,25 @@ class SingleJobTestCase(BaseScenariosTestCase):
xml_jobs.sort(key=AlphanumSort)
# check reference files are under correct path for folders
prefix = os.path.dirname(self.in_filename)
# split using '/' since fullname uses URL path separator
expected_folders = list(set([
os.path.normpath(
os.path.join(prefix,
'/'.join(job_data['name'].split('/')[:-1])))
for job_data in job_data_list
]))
actual_folders = [os.path.dirname(f) for f in self.out_filenames]
six.assertCountEqual(
self,
expected_folders, actual_folders,
"Output file under wrong path, was '%s', should be '%s'" %
(self.out_filenames[0],
os.path.join(expected_folders[0],
os.path.basename(self.out_filenames[0]))))
# Prettify generated XML
pretty_xml = u"\n".join(job.output().decode('utf-8')
for job in xml_jobs)

12
tests/cmd/subcommands/test_update.py
View File

@ -30,7 +30,7 @@ from tests.cmd.test_cmd import CmdTestsBase
class UpdateTests(CmdTestsBase):
@mock.patch('jenkins_jobs.builder.jenkins.Jenkins.job_exists')
@mock.patch('jenkins_jobs.builder.jenkins.Jenkins.get_jobs')
@mock.patch('jenkins_jobs.builder.jenkins.Jenkins.get_all_jobs')
@mock.patch('jenkins_jobs.builder.jenkins.Jenkins.reconfig_job')
def test_update_jobs(self,
jenkins_reconfig_job,
@ -72,14 +72,14 @@ class UpdateTests(CmdTestsBase):
six.text_type))
@mock.patch('jenkins_jobs.builder.jenkins.Jenkins.job_exists')
@mock.patch('jenkins_jobs.builder.jenkins.Jenkins.get_jobs')
@mock.patch('jenkins_jobs.builder.jenkins.Jenkins.get_all_jobs')
@mock.patch('jenkins_jobs.builder.jenkins.Jenkins.reconfig_job')
@mock.patch('jenkins_jobs.builder.jenkins.Jenkins.delete_job')
def test_update_jobs_and_delete_old(self,
jenkins_delete_job,
jenkins_reconfig_job,
jenkins_get_jobs,
jenkins_job_exists, ):
jenkins_get_all_jobs,
jenkins_job_exists):
"""
Test update behaviour with --delete-old option
@ -99,8 +99,8 @@ class UpdateTests(CmdTestsBase):
args = ['--conf', self.default_config_file, 'update', '--delete-old',
path]
jenkins_get_jobs.return_value = [{'name': name}
for name in yaml_jobs + extra_jobs]
jenkins_get_all_jobs.return_value = [
{'fullname': name} for name in yaml_jobs + extra_jobs]
with mock.patch('jenkins_jobs.builder.JenkinsManager.is_managed',
side_effect=(lambda name: name != 'unmanaged')):

9
tests/general/fixtures/folders/folders001.xml Normal file
View File

@ -0,0 +1,9 @@
<?xml version="1.0" encoding="utf-8"?>
<project>
<actions/>
<keepDependencies>false</keepDependencies>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<concurrentBuild>false</concurrentBuild>
<canRoam>true</canRoam>
</project>

9
tests/general/fixtures/folders/test-nested-folder/folders002.xml Normal file
View File

@ -0,0 +1,9 @@
<?xml version="1.0" encoding="utf-8"?>
<project>
<actions/>
<keepDependencies>false</keepDependencies>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<concurrentBuild>false</concurrentBuild>
<canRoam>true</canRoam>
</project>

3
tests/general/fixtures/folders001.yaml Normal file
View File

@ -0,0 +1,3 @@
name: test-folder
project-type: freestyle
folder: folders

3
tests/general/fixtures/folders002.yaml Normal file
View File

@ -0,0 +1,3 @@
name: test-folder
project-type: freestyle
folder: folders/test-nested-folder

4
tests/jenkins_manager/test_manager.py
View File

@ -62,8 +62,8 @@ class TestCaseTestJenkinsManager(base.BaseTestCase):
get_jobs=mock.DEFAULT,
is_managed=mock.DEFAULT,
delete_job=mock.DEFAULT) as patches:
patches['get_jobs'].return_value = [{'name': 'job1'},
{'name': 'job2'}]
patches['get_jobs'].return_value = [{'fullname': 'job1'},
{'fullname': 'job2'}]
patches['is_managed'].side_effect = [True, True]
self.builder.delete_old_managed()

12
tests/yamlparser/fixtures/folders-attribute.yaml Normal file
View File

@ -0,0 +1,12 @@
- defaults:
name: team1
folder: team1-jobs
- job:
name: ruby-jobs/rspec
defaults: team1
builders:
- shell: |
rvm use --create ruby-2.3.0@rspec
bundle install
bundle exec rspec

5
tests/yamlparser/fixtures/folders-job-name.yaml Normal file
View File

@ -0,0 +1,5 @@
- job:
name: python-jobs/tox-py27
builders:
- shell: |
tox -e py27

16
tests/yamlparser/fixtures/folders/folders001.xml Normal file
View File

@ -0,0 +1,16 @@
<?xml version="1.0" encoding="utf-8"?>
<project>
<actions/>
<description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
<keepDependencies>false</keepDependencies>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<concurrentBuild>false</concurrentBuild>
<canRoam>true</canRoam>
<properties/>
<scm class="hudson.scm.NullSCM"/>
<builders/>
<publishers/>
<buildWrappers/>
</project>

15
tests/yamlparser/fixtures/folders001.xml Normal file
View File

@ -0,0 +1,15 @@
<?xml version="1.0" encoding="utf-8"?>
<project>
<actions/>
<description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
<keepDependencies>false</keepDependencies>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<concurrentBuild>false</concurrentBuild>
<canRoam>true</canRoam>
<properties/>
<scm class="hudson.scm.NullSCM"/>
<builders/>
<publishers/>
<buildWrappers/>
</project>

14
tests/yamlparser/fixtures/folders001.yaml Normal file
View File

@ -0,0 +1,14 @@
- job-template:
name: 'same-{text}-name'
folder: '{path}'
- project:
name: test-same-job-name
path: folders
text: job
jobs:
- 'same-{text}-name'
- job:
name: same-job-name

20
tests/yamlparser/fixtures/python-jobs/folders-job-name.xml Normal file
View File

@ -0,0 +1,20 @@
<?xml version="1.0" encoding="utf-8"?>
<project>
<actions/>
<description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
<keepDependencies>false</keepDependencies>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<concurrentBuild>false</concurrentBuild>
<canRoam>true</canRoam>
<properties/>
<scm class="hudson.scm.NullSCM"/>
<builders>
<hudson.tasks.Shell>
<command>tox -e py27
</command>
</hudson.tasks.Shell>
</builders>
<publishers/>
<buildWrappers/>
</project>

22
tests/yamlparser/fixtures/team1-jobs/ruby-jobs/folders-attribute.xml Normal file
View File

@ -0,0 +1,22 @@
<?xml version="1.0" encoding="utf-8"?>
<project>
<actions/>
<description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
<keepDependencies>false</keepDependencies>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<concurrentBuild>false</concurrentBuild>
<canRoam>true</canRoam>
<properties/>
<scm class="hudson.scm.NullSCM"/>
<builders>
<hudson.tasks.Shell>
<command>rvm use --create ruby-2.3.0@rspec
bundle install
bundle exec rspec
</command>
</hudson.tasks.Shell>
</builders>
<publishers/>
<buildWrappers/>
</project>