Basic folder support

Allows specifying a folder attribute for each job generated, which in
turn is used when creating or uploading to place the job under the
requested folder.

The job name is expanded after defaults are applied, to support the
attribute being defined within a set of defaults applied to a number of
jobs.

This in turn allows for multiple jobs with the same basename to exist,
provided they are targeted at different folders.

Does not support creating the folders if they do not exist.

Change-Id: I8c2157c4c81087cc972a048d1b88d5f08ac65361

doc/source/definition.rst

@@ -329,6 +329,35 @@ always use ``{{`` to achieve a literal ``{``.  A generic builder will need
 to consider the correct quoting based on its use of parameters.
 
 
+.. _folders:
+
+Folders
+^^^^^^^
+
+Jenkins supports organising jobs, views, and slaves using a folder hierarchy.
+This allows for easier separation of access as well credentials and resources
+which can be assigned to only be available for a specific folder.
+
+JJB has two methods of supporting uploading jobs to a specific folder:
+
+* Name the job to contain the desired folder ``<folder>/my-job-name``
+* Use the ``folder`` attribute on a job definition, via a template, or through
+  `Defaults`_.
+
+Supporting both an attributed and use of it directly in job names allows for
+teams to have all jobs using their defaults automatically use a top-level
+folder, while still allowing for them to additionally nest jobs for their
+own preferences.
+
+Job Name Example:
+
+.. literalinclude:: /../../tests/yamlparser/fixtures/folders-job-name.yaml
+
+Folder Attribute Example:
+
+.. literalinclude:: /../../tests/yamlparser/fixtures/folders-attribute.yaml
+
+
 .. _ids:
 
 Item ID's

jenkins_jobs/builder.py

@@ -65,18 +65,43 @@ class JenkinsManager(object):
         self._view_list = None
         self._jjb_config = jjb_config
 
+    def _setup_output(self, output, item, config_xml=False):
+        output_dir = output
+        output_fn = os.path.join(output, item)
+        if '/' in item:
+            # in item folder
+            output_fn = os.path.join(output, os.path.normpath(item))
+            output_dir = os.path.dirname(output_fn)
+
+        # if in a folder, re-adding name to the directory here
+        if config_xml:
+            output_dir = os.path.join(
+                output_dir, os.path.basename(item))
+            output_fn = os.path.join(output_dir, 'config.xml')
+
+        if output_dir != output:
+            logger.info("Creating directory %s" % output_dir)
+            try:
+                os.makedirs(output_dir)
+            except OSError:
+                if not os.path.isdir(output_dir):
+                    raise
+
+        return output_fn
+
     @property
     def jobs(self):
         if self._jobs is None:
             # populate jobs
-            self._jobs = self.jenkins.get_jobs()
+            self._jobs = self.jenkins.get_all_jobs()
 
         return self._jobs
 
     @property
     def job_list(self):
         if self._job_list is None:
-            self._job_list = set(job['name'] for job in self.jobs)
+            # python-jenkins uses 'fullname' for folder/name combination
+            self._job_list = set(job['fullname'] for job in self.jobs)
         return self._job_list
 
     def update_job(self, job_name, xml):
@@ -154,17 +179,18 @@ class JenkinsManager(object):
         if keep is None:
             keep = []
         for job in jobs:
-            if job['name'] not in keep:
+            # python-jenkins stores the folder and name as 'fullname'
-                if self.is_managed(job['name']):
+            if job['fullname'] not in keep:
+                if self.is_managed(job['fullname']):
                     logger.info("Removing obsolete jenkins job {0}"
-                                .format(job['name']))
+                                .format(job['fullname']))
-                    self.delete_job(job['name'])
+                    self.delete_job(job['fullname'])
                     deleted_jobs += 1
                 else:
                     logger.info("Not deleting unmanaged jenkins job %s",
-                                job['name'])
+                                job['fullname'])
             else:
-                logger.debug("Keeping job %s", job['name'])
+                logger.debug("Keeping job %s", job['fullname'])
         return deleted_jobs
 
     def delete_jobs(self, jobs):
@@ -231,17 +257,8 @@ class JenkinsManager(object):
                         raise
                     continue
 
-                if config_xml:
+                output_fn = self._setup_output(output, job.name, config_xml)
-                    output_dir = os.path.join(output, job.name)
+
-                    logger.info("Creating directory %s" % output_dir)
-                    try:
-                        os.makedirs(output_dir)
-                    except OSError:
-                        if not os.path.isdir(output_dir):
-                            raise
-                    output_fn = os.path.join(output_dir, 'config.xml')
-                else:
-                    output_fn = os.path.join(output, job.name)
                 logger.debug("Writing XML to '{0}'".format(output_fn))
                 with io.open(output_fn, 'w', encoding='utf-8') as f:
                     f.write(job.output().decode('utf-8'))
@@ -383,17 +400,8 @@ class JenkinsManager(object):
                         raise
                     continue
 
-                if config_xml:
+                output_fn = self._setup_output(output, view.name, config_xml)
-                    output_dir = os.path.join(output, view.name)
+
-                    logger.info("Creating directory %s" % output_dir)
-                    try:
-                        os.makedirs(output_dir)
-                    except OSError:
-                        if not os.path.isdir(output_dir):
-                            raise
-                    output_fn = os.path.join(output_dir, 'config.xml')
-                else:
-                    output_fn = os.path.join(output, view.name)
                 logger.debug("Writing XML to '{0}'".format(output_fn))
                 with io.open(output_fn, 'w', encoding='utf-8') as f:
                     f.write(view.output().decode('utf-8'))

jenkins_jobs/modules/general.py

@@ -56,6 +56,12 @@ Example:
       Path for a custom workspace. Defaults to Jenkins default
       configuration.
 
+    * **folder**:
+      The folder attribute provides an alternative to using '<path>/<name>' as
+      the job name to specify which Jenkins folder to upload the job to.
+      Requires the `CloudBees Folders Plugin.
+      <https://wiki.jenkins-ci.org/display/JENKINS/CloudBees+Folders+Plugin>`_
+
     * **child-workspace**:
       Path for a child custom workspace. Defaults to Jenkins default
       configuration. This parameter is only valid for matrix type jobs.
@@ -103,7 +109,6 @@ Example:
     * **raw**:
       If present, this section should contain a single **xml** entry. This XML
       will be inserted at the top-level of the :ref:`Job` definition.
-
 """
 
 import logging

jenkins_jobs/parser.py

@@ -226,6 +226,12 @@ class YamlParser(object):
             for macro in self.data.get(component_type, {}).values():
                 self._macro_registry.register(component_type, macro)
 
+    def _getfullname(self, data):
+        if 'folder' in data:
+            return "%s/%s" % (data['folder'], data['name'])
+
+        return data['name']
+
     def expandYaml(self, registry, jobs_glob=None):
         changed = True
         while changed:
@@ -240,15 +246,17 @@ class YamlParser(object):
             self._macro_registry.expand_macros(default)
         for job in self.data.get('job', {}).values():
             self._macro_registry.expand_macros(job)
+            job = self._applyDefaults(job)
+            job['name'] = self._getfullname(job)
             if jobs_glob and not matches(job['name'], jobs_glob):
                 logger.debug("Ignoring job {0}".format(job['name']))
                 continue
             logger.debug("Expanding job '{0}'".format(job['name']))
-            job = self._applyDefaults(job)
             self._formatDescription(job)
             self.jobs.append(job)
 
         for view in self.data.get('view', {}).values():
+            view['name'] = self._getfullname(view)
             logger.debug("Expanding view '{0}'".format(view['name']))
             self._formatDescription(view)
             self.views.append(view)
@@ -400,6 +408,7 @@ class YamlParser(object):
                     "Failure formatting template '%s', containing '%s' with "
                     "params '%s'", template_name, template, params)
                 raise
+            expanded['name'] = self._getfullname(expanded)
 
             self._macro_registry.expand_macros(expanded, params)
             job_name = expanded.get('name')

requirements.txt

@@ -5,6 +5,6 @@ six>=1.9.0 # MIT
 PyYAML>=3.10.0 # MIT
 pbr>=1.8 # Apache-2.0
 stevedore>=1.17.1 # Apache-2.0
-python-jenkins>=0.4.8
+python-jenkins>=0.4.9
 fasteners
 Jinja2

tests/base.py

@@ -26,6 +26,7 @@ import re
 import xml.etree.ElementTree as XML
 
 import fixtures
+import six
 from six.moves import StringIO
 import testtools
 from testtools.content import text_content
@@ -64,11 +65,16 @@ def get_scenarios(fixtures_path, in_ext='yaml', out_ext='xml',
         - content of the fixture output file (aka expected)
     """
     scenarios = []
-    files = []
+    files = {}
-    for dirpath, dirs, fs in os.walk(fixtures_path):
+    for dirpath, _, fs in os.walk(fixtures_path):
-        files.extend([os.path.join(dirpath, f) for f in fs])
+        for fn in fs:
+            if fn in files:
+                files[fn].append(os.path.join(dirpath, fn))
+            else:
+                files[fn] = [os.path.join(dirpath, fn)]
 
-    input_files = [f for f in files if re.match(r'.*\.{0}$'.format(in_ext), f)]
+    input_files = [files[f][0] for f in files if
+                   re.match(r'.*\.{0}$'.format(in_ext), f)]
 
     for input_filename in input_files:
         if input_filename.endswith(plugins_info_ext):
@@ -80,24 +86,27 @@ def get_scenarios(fixtures_path, in_ext='yaml', out_ext='xml',
         output_candidate = re.sub(r'\.{0}$'.format(in_ext),
                                   '.{0}'.format(out_ext), input_filename)
         # assume empty file if no output candidate found
-        if output_candidate not in files:
+        if os.path.basename(output_candidate) in files:
-            output_candidate = None
+            out_filenames = files[os.path.basename(output_candidate)]
+        else:
+            out_filenames = None
 
         plugins_info_candidate = re.sub(r'\.{0}$'.format(in_ext),
                                         '.{0}'.format(plugins_info_ext),
                                         input_filename)
-        if plugins_info_candidate not in files:
+        if os.path.basename(plugins_info_candidate) not in files:
             plugins_info_candidate = None
 
         conf_candidate = re.sub(r'\.yaml$|\.json$', '.conf', input_filename)
-        # If present, add the configuration file
+        conf_filename = files.get(os.path.basename(conf_candidate), None)
-        if conf_candidate not in files:
+
-            conf_candidate = None
+        if conf_filename is not None:
+            conf_filename = conf_filename[0]
 
         scenarios.append((input_filename, {
             'in_filename': input_filename,
-            'out_filename': output_candidate,
+            'out_filenames': out_filenames,
-            'conf_filename': conf_candidate,
+            'conf_filename': conf_filename,
             'plugins_info_filename': plugins_info_candidate,
         }))
 
@@ -117,12 +126,13 @@ class BaseTestCase(testtools.TestCase):
 
     def _read_utf8_content(self):
         # if None assume empty file
-        if self.out_filename is None:
+        if not self.out_filenames:
             return u""
 
         # Read XML content, assuming it is unicode encoded
-        xml_content = u"%s" % io.open(self.out_filename,
+        xml_content = ""
-                                      'r', encoding='utf-8').read()
+        for f in sorted(self.out_filenames):
+            xml_content += u"%s" % io.open(f, 'r', encoding='utf-8').read()
         return xml_content
 
     def _read_yaml_content(self, filename):
@@ -195,6 +205,23 @@ class BaseScenariosTestCase(testscenarios.TestWithScenarios, BaseTestCase):
         # Generate the XML tree directly with modules/general
         pub.gen_xml(xml_project, yaml_content)
 
+        # check output file is under correct path
+        if 'name' in yaml_content:
+            prefix = os.path.dirname(self.in_filename)
+            # split using '/' since fullname uses URL path separator
+            expected_folders = [os.path.normpath(
+                os.path.join(prefix,
+                             '/'.join(parser._getfullname(yaml_content).
+                                      split('/')[:-1])))]
+            actual_folders = [os.path.dirname(f) for f in self.out_filenames]
+
+            self.assertEquals(
+                expected_folders, actual_folders,
+                "Output file under wrong path, was '%s', should be '%s'" %
+                (self.out_filenames[0],
+                 os.path.join(expected_folders[0],
+                              os.path.basename(self.out_filenames[0]))))
+
         # Prettify generated XML
         pretty_xml = XmlJob(xml_project, 'fixturejob').output().decode('utf-8')
 
@@ -226,6 +253,25 @@ class SingleJobTestCase(BaseScenariosTestCase):
 
         xml_jobs.sort(key=AlphanumSort)
 
+        # check reference files are under correct path for folders
+        prefix = os.path.dirname(self.in_filename)
+        # split using '/' since fullname uses URL path separator
+        expected_folders = list(set([
+            os.path.normpath(
+                os.path.join(prefix,
+                             '/'.join(job_data['name'].split('/')[:-1])))
+            for job_data in job_data_list
+        ]))
+        actual_folders = [os.path.dirname(f) for f in self.out_filenames]
+
+        six.assertCountEqual(
+            self,
+            expected_folders, actual_folders,
+            "Output file under wrong path, was '%s', should be '%s'" %
+            (self.out_filenames[0],
+                os.path.join(expected_folders[0],
+                             os.path.basename(self.out_filenames[0]))))
+
         # Prettify generated XML
         pretty_xml = u"\n".join(job.output().decode('utf-8')
                                 for job in xml_jobs)

tests/cmd/subcommands/test_update.py

@@ -30,7 +30,7 @@ from tests.cmd.test_cmd import CmdTestsBase
 class UpdateTests(CmdTestsBase):
 
     @mock.patch('jenkins_jobs.builder.jenkins.Jenkins.job_exists')
-    @mock.patch('jenkins_jobs.builder.jenkins.Jenkins.get_jobs')
+    @mock.patch('jenkins_jobs.builder.jenkins.Jenkins.get_all_jobs')
     @mock.patch('jenkins_jobs.builder.jenkins.Jenkins.reconfig_job')
     def test_update_jobs(self,
                          jenkins_reconfig_job,
@@ -72,14 +72,14 @@ class UpdateTests(CmdTestsBase):
                                    six.text_type))
 
     @mock.patch('jenkins_jobs.builder.jenkins.Jenkins.job_exists')
-    @mock.patch('jenkins_jobs.builder.jenkins.Jenkins.get_jobs')
+    @mock.patch('jenkins_jobs.builder.jenkins.Jenkins.get_all_jobs')
     @mock.patch('jenkins_jobs.builder.jenkins.Jenkins.reconfig_job')
     @mock.patch('jenkins_jobs.builder.jenkins.Jenkins.delete_job')
     def test_update_jobs_and_delete_old(self,
                                         jenkins_delete_job,
                                         jenkins_reconfig_job,
-                                        jenkins_get_jobs,
+                                        jenkins_get_all_jobs,
-                                        jenkins_job_exists, ):
+                                        jenkins_job_exists):
         """
         Test update behaviour with --delete-old option
 
@@ -99,8 +99,8 @@ class UpdateTests(CmdTestsBase):
         args = ['--conf', self.default_config_file, 'update', '--delete-old',
                 path]
 
-        jenkins_get_jobs.return_value = [{'name': name}
+        jenkins_get_all_jobs.return_value = [
-                                         for name in yaml_jobs + extra_jobs]
+            {'fullname': name} for name in yaml_jobs + extra_jobs]
 
         with mock.patch('jenkins_jobs.builder.JenkinsManager.is_managed',
                         side_effect=(lambda name: name != 'unmanaged')):

tests/general/fixtures/folders/folders001.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+</project>

tests/general/fixtures/folders/test-nested-folder/folders002.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+</project>

tests/general/fixtures/folders001.yaml

@@ -0,0 +1,3 @@
+name: test-folder
+project-type: freestyle
+folder: folders

tests/general/fixtures/folders002.yaml

@@ -0,0 +1,3 @@
+name: test-folder
+project-type: freestyle
+folder: folders/test-nested-folder

tests/jenkins_manager/test_manager.py

@@ -62,8 +62,8 @@ class TestCaseTestJenkinsManager(base.BaseTestCase):
                                  get_jobs=mock.DEFAULT,
                                  is_managed=mock.DEFAULT,
                                  delete_job=mock.DEFAULT) as patches:
-            patches['get_jobs'].return_value = [{'name': 'job1'},
+            patches['get_jobs'].return_value = [{'fullname': 'job1'},
-                                                {'name': 'job2'}]
+                                                {'fullname': 'job2'}]
             patches['is_managed'].side_effect = [True, True]
 
             self.builder.delete_old_managed()

tests/yamlparser/fixtures/folders-attribute.yaml

@@ -0,0 +1,12 @@
+- defaults:
+    name: team1
+    folder: team1-jobs
+
+- job:
+    name: ruby-jobs/rspec
+    defaults: team1
+    builders:
+      - shell: |
+          rvm use --create ruby-2.3.0@rspec
+          bundle install
+          bundle exec rspec

tests/yamlparser/fixtures/folders-job-name.yaml

@@ -0,0 +1,5 @@
+- job:
+    name: python-jobs/tox-py27
+    builders:
+      - shell: |
+          tox -e py27

tests/yamlparser/fixtures/folders/folders001.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties/>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders/>
+  <publishers/>
+  <buildWrappers/>
+</project>
+

tests/yamlparser/fixtures/folders001.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties/>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders/>
+  <publishers/>
+  <buildWrappers/>
+</project>

tests/yamlparser/fixtures/folders001.yaml

@@ -0,0 +1,14 @@
+
+- job-template:
+    name: 'same-{text}-name'
+    folder: '{path}'
+
+- project:
+    name: test-same-job-name
+    path: folders
+    text: job
+    jobs:
+        - 'same-{text}-name'
+
+- job:
+    name: same-job-name

tests/yamlparser/fixtures/python-jobs/folders-job-name.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties/>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders>
+    <hudson.tasks.Shell>
+      <command>tox -e py27
+</command>
+    </hudson.tasks.Shell>
+  </builders>
+  <publishers/>
+  <buildWrappers/>
+</project>

tests/yamlparser/fixtures/team1-jobs/ruby-jobs/folders-attribute.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties/>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders>
+    <hudson.tasks.Shell>
+      <command>rvm use --create ruby-2.3.0@rspec
+bundle install
+bundle exec rspec
+</command>
+    </hudson.tasks.Shell>
+  </builders>
+  <publishers/>
+  <buildWrappers/>
+</project>