diff --git a/doc/source/definition.rst b/doc/source/definition.rst index 4cd8b0ea6..b9315b89e 100644 --- a/doc/source/definition.rst +++ b/doc/source/definition.rst @@ -329,6 +329,35 @@ always use ``{{`` to achieve a literal ``{``. A generic builder will need to consider the correct quoting based on its use of parameters. +.. _folders: + +Folders +^^^^^^^ + +Jenkins supports organising jobs, views, and slaves using a folder hierarchy. +This allows for easier separation of access as well credentials and resources +which can be assigned to only be available for a specific folder. + +JJB has two methods of supporting uploading jobs to a specific folder: + +* Name the job to contain the desired folder ``/my-job-name`` +* Use the ``folder`` attribute on a job definition, via a template, or through + `Defaults`_. + +Supporting both an attributed and use of it directly in job names allows for +teams to have all jobs using their defaults automatically use a top-level +folder, while still allowing for them to additionally nest jobs for their +own preferences. + +Job Name Example: + +.. literalinclude:: /../../tests/yamlparser/fixtures/folders-job-name.yaml + +Folder Attribute Example: + +.. literalinclude:: /../../tests/yamlparser/fixtures/folders-attribute.yaml + + .. _ids: Item ID's diff --git a/jenkins_jobs/builder.py b/jenkins_jobs/builder.py index 238be9327..7626d461e 100644 --- a/jenkins_jobs/builder.py +++ b/jenkins_jobs/builder.py @@ -65,18 +65,43 @@ class JenkinsManager(object): self._view_list = None self._jjb_config = jjb_config + def _setup_output(self, output, item, config_xml=False): + output_dir = output + output_fn = os.path.join(output, item) + if '/' in item: + # in item folder + output_fn = os.path.join(output, os.path.normpath(item)) + output_dir = os.path.dirname(output_fn) + + # if in a folder, re-adding name to the directory here + if config_xml: + output_dir = os.path.join( + output_dir, os.path.basename(item)) + output_fn = os.path.join(output_dir, 'config.xml') + + if output_dir != output: + logger.info("Creating directory %s" % output_dir) + try: + os.makedirs(output_dir) + except OSError: + if not os.path.isdir(output_dir): + raise + + return output_fn + @property def jobs(self): if self._jobs is None: # populate jobs - self._jobs = self.jenkins.get_jobs() + self._jobs = self.jenkins.get_all_jobs() return self._jobs @property def job_list(self): if self._job_list is None: - self._job_list = set(job['name'] for job in self.jobs) + # python-jenkins uses 'fullname' for folder/name combination + self._job_list = set(job['fullname'] for job in self.jobs) return self._job_list def update_job(self, job_name, xml): @@ -154,17 +179,18 @@ class JenkinsManager(object): if keep is None: keep = [] for job in jobs: - if job['name'] not in keep: - if self.is_managed(job['name']): + # python-jenkins stores the folder and name as 'fullname' + if job['fullname'] not in keep: + if self.is_managed(job['fullname']): logger.info("Removing obsolete jenkins job {0}" - .format(job['name'])) - self.delete_job(job['name']) + .format(job['fullname'])) + self.delete_job(job['fullname']) deleted_jobs += 1 else: logger.info("Not deleting unmanaged jenkins job %s", - job['name']) + job['fullname']) else: - logger.debug("Keeping job %s", job['name']) + logger.debug("Keeping job %s", job['fullname']) return deleted_jobs def delete_jobs(self, jobs): @@ -231,17 +257,8 @@ class JenkinsManager(object): raise continue - if config_xml: - output_dir = os.path.join(output, job.name) - logger.info("Creating directory %s" % output_dir) - try: - os.makedirs(output_dir) - except OSError: - if not os.path.isdir(output_dir): - raise - output_fn = os.path.join(output_dir, 'config.xml') - else: - output_fn = os.path.join(output, job.name) + output_fn = self._setup_output(output, job.name, config_xml) + logger.debug("Writing XML to '{0}'".format(output_fn)) with io.open(output_fn, 'w', encoding='utf-8') as f: f.write(job.output().decode('utf-8')) @@ -383,17 +400,8 @@ class JenkinsManager(object): raise continue - if config_xml: - output_dir = os.path.join(output, view.name) - logger.info("Creating directory %s" % output_dir) - try: - os.makedirs(output_dir) - except OSError: - if not os.path.isdir(output_dir): - raise - output_fn = os.path.join(output_dir, 'config.xml') - else: - output_fn = os.path.join(output, view.name) + output_fn = self._setup_output(output, view.name, config_xml) + logger.debug("Writing XML to '{0}'".format(output_fn)) with io.open(output_fn, 'w', encoding='utf-8') as f: f.write(view.output().decode('utf-8')) diff --git a/jenkins_jobs/modules/general.py b/jenkins_jobs/modules/general.py index 658d3c14e..0bfca5375 100644 --- a/jenkins_jobs/modules/general.py +++ b/jenkins_jobs/modules/general.py @@ -56,6 +56,12 @@ Example: Path for a custom workspace. Defaults to Jenkins default configuration. + * **folder**: + The folder attribute provides an alternative to using '/' as + the job name to specify which Jenkins folder to upload the job to. + Requires the `CloudBees Folders Plugin. + `_ + * **child-workspace**: Path for a child custom workspace. Defaults to Jenkins default configuration. This parameter is only valid for matrix type jobs. @@ -103,7 +109,6 @@ Example: * **raw**: If present, this section should contain a single **xml** entry. This XML will be inserted at the top-level of the :ref:`Job` definition. - """ import logging diff --git a/jenkins_jobs/parser.py b/jenkins_jobs/parser.py index 5dee3cac3..1033b9fd3 100644 --- a/jenkins_jobs/parser.py +++ b/jenkins_jobs/parser.py @@ -226,6 +226,12 @@ class YamlParser(object): for macro in self.data.get(component_type, {}).values(): self._macro_registry.register(component_type, macro) + def _getfullname(self, data): + if 'folder' in data: + return "%s/%s" % (data['folder'], data['name']) + + return data['name'] + def expandYaml(self, registry, jobs_glob=None): changed = True while changed: @@ -240,15 +246,17 @@ class YamlParser(object): self._macro_registry.expand_macros(default) for job in self.data.get('job', {}).values(): self._macro_registry.expand_macros(job) + job = self._applyDefaults(job) + job['name'] = self._getfullname(job) if jobs_glob and not matches(job['name'], jobs_glob): logger.debug("Ignoring job {0}".format(job['name'])) continue logger.debug("Expanding job '{0}'".format(job['name'])) - job = self._applyDefaults(job) self._formatDescription(job) self.jobs.append(job) for view in self.data.get('view', {}).values(): + view['name'] = self._getfullname(view) logger.debug("Expanding view '{0}'".format(view['name'])) self._formatDescription(view) self.views.append(view) @@ -400,6 +408,7 @@ class YamlParser(object): "Failure formatting template '%s', containing '%s' with " "params '%s'", template_name, template, params) raise + expanded['name'] = self._getfullname(expanded) self._macro_registry.expand_macros(expanded, params) job_name = expanded.get('name') diff --git a/requirements.txt b/requirements.txt index f0e7ab425..52059b553 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,6 @@ six>=1.9.0 # MIT PyYAML>=3.10.0 # MIT pbr>=1.8 # Apache-2.0 stevedore>=1.17.1 # Apache-2.0 -python-jenkins>=0.4.8 +python-jenkins>=0.4.9 fasteners Jinja2 diff --git a/tests/base.py b/tests/base.py index becc211c5..06fedb28a 100644 --- a/tests/base.py +++ b/tests/base.py @@ -26,6 +26,7 @@ import re import xml.etree.ElementTree as XML import fixtures +import six from six.moves import StringIO import testtools from testtools.content import text_content @@ -64,11 +65,16 @@ def get_scenarios(fixtures_path, in_ext='yaml', out_ext='xml', - content of the fixture output file (aka expected) """ scenarios = [] - files = [] - for dirpath, dirs, fs in os.walk(fixtures_path): - files.extend([os.path.join(dirpath, f) for f in fs]) + files = {} + for dirpath, _, fs in os.walk(fixtures_path): + for fn in fs: + if fn in files: + files[fn].append(os.path.join(dirpath, fn)) + else: + files[fn] = [os.path.join(dirpath, fn)] - input_files = [f for f in files if re.match(r'.*\.{0}$'.format(in_ext), f)] + input_files = [files[f][0] for f in files if + re.match(r'.*\.{0}$'.format(in_ext), f)] for input_filename in input_files: if input_filename.endswith(plugins_info_ext): @@ -80,24 +86,27 @@ def get_scenarios(fixtures_path, in_ext='yaml', out_ext='xml', output_candidate = re.sub(r'\.{0}$'.format(in_ext), '.{0}'.format(out_ext), input_filename) # assume empty file if no output candidate found - if output_candidate not in files: - output_candidate = None + if os.path.basename(output_candidate) in files: + out_filenames = files[os.path.basename(output_candidate)] + else: + out_filenames = None plugins_info_candidate = re.sub(r'\.{0}$'.format(in_ext), '.{0}'.format(plugins_info_ext), input_filename) - if plugins_info_candidate not in files: + if os.path.basename(plugins_info_candidate) not in files: plugins_info_candidate = None conf_candidate = re.sub(r'\.yaml$|\.json$', '.conf', input_filename) - # If present, add the configuration file - if conf_candidate not in files: - conf_candidate = None + conf_filename = files.get(os.path.basename(conf_candidate), None) + + if conf_filename is not None: + conf_filename = conf_filename[0] scenarios.append((input_filename, { 'in_filename': input_filename, - 'out_filename': output_candidate, - 'conf_filename': conf_candidate, + 'out_filenames': out_filenames, + 'conf_filename': conf_filename, 'plugins_info_filename': plugins_info_candidate, })) @@ -117,12 +126,13 @@ class BaseTestCase(testtools.TestCase): def _read_utf8_content(self): # if None assume empty file - if self.out_filename is None: + if not self.out_filenames: return u"" # Read XML content, assuming it is unicode encoded - xml_content = u"%s" % io.open(self.out_filename, - 'r', encoding='utf-8').read() + xml_content = "" + for f in sorted(self.out_filenames): + xml_content += u"%s" % io.open(f, 'r', encoding='utf-8').read() return xml_content def _read_yaml_content(self, filename): @@ -195,6 +205,23 @@ class BaseScenariosTestCase(testscenarios.TestWithScenarios, BaseTestCase): # Generate the XML tree directly with modules/general pub.gen_xml(xml_project, yaml_content) + # check output file is under correct path + if 'name' in yaml_content: + prefix = os.path.dirname(self.in_filename) + # split using '/' since fullname uses URL path separator + expected_folders = [os.path.normpath( + os.path.join(prefix, + '/'.join(parser._getfullname(yaml_content). + split('/')[:-1])))] + actual_folders = [os.path.dirname(f) for f in self.out_filenames] + + self.assertEquals( + expected_folders, actual_folders, + "Output file under wrong path, was '%s', should be '%s'" % + (self.out_filenames[0], + os.path.join(expected_folders[0], + os.path.basename(self.out_filenames[0])))) + # Prettify generated XML pretty_xml = XmlJob(xml_project, 'fixturejob').output().decode('utf-8') @@ -226,6 +253,25 @@ class SingleJobTestCase(BaseScenariosTestCase): xml_jobs.sort(key=AlphanumSort) + # check reference files are under correct path for folders + prefix = os.path.dirname(self.in_filename) + # split using '/' since fullname uses URL path separator + expected_folders = list(set([ + os.path.normpath( + os.path.join(prefix, + '/'.join(job_data['name'].split('/')[:-1]))) + for job_data in job_data_list + ])) + actual_folders = [os.path.dirname(f) for f in self.out_filenames] + + six.assertCountEqual( + self, + expected_folders, actual_folders, + "Output file under wrong path, was '%s', should be '%s'" % + (self.out_filenames[0], + os.path.join(expected_folders[0], + os.path.basename(self.out_filenames[0])))) + # Prettify generated XML pretty_xml = u"\n".join(job.output().decode('utf-8') for job in xml_jobs) diff --git a/tests/cmd/subcommands/test_update.py b/tests/cmd/subcommands/test_update.py index 77dbc1d21..9606951f6 100644 --- a/tests/cmd/subcommands/test_update.py +++ b/tests/cmd/subcommands/test_update.py @@ -30,7 +30,7 @@ from tests.cmd.test_cmd import CmdTestsBase class UpdateTests(CmdTestsBase): @mock.patch('jenkins_jobs.builder.jenkins.Jenkins.job_exists') - @mock.patch('jenkins_jobs.builder.jenkins.Jenkins.get_jobs') + @mock.patch('jenkins_jobs.builder.jenkins.Jenkins.get_all_jobs') @mock.patch('jenkins_jobs.builder.jenkins.Jenkins.reconfig_job') def test_update_jobs(self, jenkins_reconfig_job, @@ -72,14 +72,14 @@ class UpdateTests(CmdTestsBase): six.text_type)) @mock.patch('jenkins_jobs.builder.jenkins.Jenkins.job_exists') - @mock.patch('jenkins_jobs.builder.jenkins.Jenkins.get_jobs') + @mock.patch('jenkins_jobs.builder.jenkins.Jenkins.get_all_jobs') @mock.patch('jenkins_jobs.builder.jenkins.Jenkins.reconfig_job') @mock.patch('jenkins_jobs.builder.jenkins.Jenkins.delete_job') def test_update_jobs_and_delete_old(self, jenkins_delete_job, jenkins_reconfig_job, - jenkins_get_jobs, - jenkins_job_exists, ): + jenkins_get_all_jobs, + jenkins_job_exists): """ Test update behaviour with --delete-old option @@ -99,8 +99,8 @@ class UpdateTests(CmdTestsBase): args = ['--conf', self.default_config_file, 'update', '--delete-old', path] - jenkins_get_jobs.return_value = [{'name': name} - for name in yaml_jobs + extra_jobs] + jenkins_get_all_jobs.return_value = [ + {'fullname': name} for name in yaml_jobs + extra_jobs] with mock.patch('jenkins_jobs.builder.JenkinsManager.is_managed', side_effect=(lambda name: name != 'unmanaged')): diff --git a/tests/general/fixtures/folders/folders001.xml b/tests/general/fixtures/folders/folders001.xml new file mode 100644 index 000000000..1505db519 --- /dev/null +++ b/tests/general/fixtures/folders/folders001.xml @@ -0,0 +1,9 @@ + + + + false + false + false + false + true + diff --git a/tests/general/fixtures/folders/test-nested-folder/folders002.xml b/tests/general/fixtures/folders/test-nested-folder/folders002.xml new file mode 100644 index 000000000..1505db519 --- /dev/null +++ b/tests/general/fixtures/folders/test-nested-folder/folders002.xml @@ -0,0 +1,9 @@ + + + + false + false + false + false + true + diff --git a/tests/general/fixtures/folders001.yaml b/tests/general/fixtures/folders001.yaml new file mode 100644 index 000000000..948ccc6d0 --- /dev/null +++ b/tests/general/fixtures/folders001.yaml @@ -0,0 +1,3 @@ +name: test-folder +project-type: freestyle +folder: folders diff --git a/tests/general/fixtures/folders002.yaml b/tests/general/fixtures/folders002.yaml new file mode 100644 index 000000000..b5626607b --- /dev/null +++ b/tests/general/fixtures/folders002.yaml @@ -0,0 +1,3 @@ +name: test-folder +project-type: freestyle +folder: folders/test-nested-folder diff --git a/tests/jenkins_manager/test_manager.py b/tests/jenkins_manager/test_manager.py index a1738329a..cc8e3d417 100644 --- a/tests/jenkins_manager/test_manager.py +++ b/tests/jenkins_manager/test_manager.py @@ -62,8 +62,8 @@ class TestCaseTestJenkinsManager(base.BaseTestCase): get_jobs=mock.DEFAULT, is_managed=mock.DEFAULT, delete_job=mock.DEFAULT) as patches: - patches['get_jobs'].return_value = [{'name': 'job1'}, - {'name': 'job2'}] + patches['get_jobs'].return_value = [{'fullname': 'job1'}, + {'fullname': 'job2'}] patches['is_managed'].side_effect = [True, True] self.builder.delete_old_managed() diff --git a/tests/yamlparser/fixtures/folders-attribute.yaml b/tests/yamlparser/fixtures/folders-attribute.yaml new file mode 100644 index 000000000..7346fa2f9 --- /dev/null +++ b/tests/yamlparser/fixtures/folders-attribute.yaml @@ -0,0 +1,12 @@ +- defaults: + name: team1 + folder: team1-jobs + +- job: + name: ruby-jobs/rspec + defaults: team1 + builders: + - shell: | + rvm use --create ruby-2.3.0@rspec + bundle install + bundle exec rspec diff --git a/tests/yamlparser/fixtures/folders-job-name.yaml b/tests/yamlparser/fixtures/folders-job-name.yaml new file mode 100644 index 000000000..d90323937 --- /dev/null +++ b/tests/yamlparser/fixtures/folders-job-name.yaml @@ -0,0 +1,5 @@ +- job: + name: python-jobs/tox-py27 + builders: + - shell: | + tox -e py27 diff --git a/tests/yamlparser/fixtures/folders/folders001.xml b/tests/yamlparser/fixtures/folders/folders001.xml new file mode 100644 index 000000000..1d2494f8a --- /dev/null +++ b/tests/yamlparser/fixtures/folders/folders001.xml @@ -0,0 +1,16 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + + + + diff --git a/tests/yamlparser/fixtures/folders001.xml b/tests/yamlparser/fixtures/folders001.xml new file mode 100644 index 000000000..0570e10a5 --- /dev/null +++ b/tests/yamlparser/fixtures/folders001.xml @@ -0,0 +1,15 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + + + diff --git a/tests/yamlparser/fixtures/folders001.yaml b/tests/yamlparser/fixtures/folders001.yaml new file mode 100644 index 000000000..20f6ed0cd --- /dev/null +++ b/tests/yamlparser/fixtures/folders001.yaml @@ -0,0 +1,14 @@ + +- job-template: + name: 'same-{text}-name' + folder: '{path}' + +- project: + name: test-same-job-name + path: folders + text: job + jobs: + - 'same-{text}-name' + +- job: + name: same-job-name diff --git a/tests/yamlparser/fixtures/python-jobs/folders-job-name.xml b/tests/yamlparser/fixtures/python-jobs/folders-job-name.xml new file mode 100644 index 000000000..8a3d28bd3 --- /dev/null +++ b/tests/yamlparser/fixtures/python-jobs/folders-job-name.xml @@ -0,0 +1,20 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + + tox -e py27 + + + + + + diff --git a/tests/yamlparser/fixtures/team1-jobs/ruby-jobs/folders-attribute.xml b/tests/yamlparser/fixtures/team1-jobs/ruby-jobs/folders-attribute.xml new file mode 100644 index 000000000..2c57eb750 --- /dev/null +++ b/tests/yamlparser/fixtures/team1-jobs/ruby-jobs/folders-attribute.xml @@ -0,0 +1,22 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + + rvm use --create ruby-2.3.0@rspec +bundle install +bundle exec rspec + + + + + +