From af9d984baa7f93ba8e846ff30a681d04117397e7 Mon Sep 17 00:00:00 2001 From: Darragh Bailey Date: Mon, 30 Nov 2015 13:20:15 +0100 Subject: [PATCH] Basic folder support Allows specifying a folder attribute for each job generated, which in turn is used when creating or uploading to place the job under the requested folder. The job name is expanded after defaults are applied, to support the attribute being defined within a set of defaults applied to a number of jobs. This in turn allows for multiple jobs with the same basename to exist, provided they are targeted at different folders. Does not support creating the folders if they do not exist. Change-Id: I8c2157c4c81087cc972a048d1b88d5f08ac65361 --- doc/source/definition.rst | 29 +++++++ jenkins_jobs/builder.py | 68 +++++++++-------- jenkins_jobs/modules/general.py | 7 +- jenkins_jobs/parser.py | 11 ++- requirements.txt | 2 +- tests/base.py | 76 +++++++++++++++---- tests/cmd/subcommands/test_update.py | 12 +-- tests/general/fixtures/folders/folders001.xml | 9 +++ .../folders/test-nested-folder/folders002.xml | 9 +++ tests/general/fixtures/folders001.yaml | 3 + tests/general/fixtures/folders002.yaml | 3 + tests/jenkins_manager/test_manager.py | 4 +- .../fixtures/folders-attribute.yaml | 12 +++ .../yamlparser/fixtures/folders-job-name.yaml | 5 ++ .../fixtures/folders/folders001.xml | 16 ++++ tests/yamlparser/fixtures/folders001.xml | 15 ++++ tests/yamlparser/fixtures/folders001.yaml | 14 ++++ .../fixtures/python-jobs/folders-job-name.xml | 20 +++++ .../ruby-jobs/folders-attribute.xml | 22 ++++++ 19 files changed, 281 insertions(+), 56 deletions(-) create mode 100644 tests/general/fixtures/folders/folders001.xml create mode 100644 tests/general/fixtures/folders/test-nested-folder/folders002.xml create mode 100644 tests/general/fixtures/folders001.yaml create mode 100644 tests/general/fixtures/folders002.yaml create mode 100644 tests/yamlparser/fixtures/folders-attribute.yaml create mode 100644 tests/yamlparser/fixtures/folders-job-name.yaml create mode 100644 tests/yamlparser/fixtures/folders/folders001.xml create mode 100644 tests/yamlparser/fixtures/folders001.xml create mode 100644 tests/yamlparser/fixtures/folders001.yaml create mode 100644 tests/yamlparser/fixtures/python-jobs/folders-job-name.xml create mode 100644 tests/yamlparser/fixtures/team1-jobs/ruby-jobs/folders-attribute.xml diff --git a/doc/source/definition.rst b/doc/source/definition.rst index 4cd8b0ea6..b9315b89e 100644 --- a/doc/source/definition.rst +++ b/doc/source/definition.rst @@ -329,6 +329,35 @@ always use ``{{`` to achieve a literal ``{``. A generic builder will need to consider the correct quoting based on its use of parameters. +.. _folders: + +Folders +^^^^^^^ + +Jenkins supports organising jobs, views, and slaves using a folder hierarchy. +This allows for easier separation of access as well credentials and resources +which can be assigned to only be available for a specific folder. + +JJB has two methods of supporting uploading jobs to a specific folder: + +* Name the job to contain the desired folder ``/my-job-name`` +* Use the ``folder`` attribute on a job definition, via a template, or through + `Defaults`_. + +Supporting both an attributed and use of it directly in job names allows for +teams to have all jobs using their defaults automatically use a top-level +folder, while still allowing for them to additionally nest jobs for their +own preferences. + +Job Name Example: + +.. literalinclude:: /../../tests/yamlparser/fixtures/folders-job-name.yaml + +Folder Attribute Example: + +.. literalinclude:: /../../tests/yamlparser/fixtures/folders-attribute.yaml + + .. _ids: Item ID's diff --git a/jenkins_jobs/builder.py b/jenkins_jobs/builder.py index 238be9327..7626d461e 100644 --- a/jenkins_jobs/builder.py +++ b/jenkins_jobs/builder.py @@ -65,18 +65,43 @@ class JenkinsManager(object): self._view_list = None self._jjb_config = jjb_config + def _setup_output(self, output, item, config_xml=False): + output_dir = output + output_fn = os.path.join(output, item) + if '/' in item: + # in item folder + output_fn = os.path.join(output, os.path.normpath(item)) + output_dir = os.path.dirname(output_fn) + + # if in a folder, re-adding name to the directory here + if config_xml: + output_dir = os.path.join( + output_dir, os.path.basename(item)) + output_fn = os.path.join(output_dir, 'config.xml') + + if output_dir != output: + logger.info("Creating directory %s" % output_dir) + try: + os.makedirs(output_dir) + except OSError: + if not os.path.isdir(output_dir): + raise + + return output_fn + @property def jobs(self): if self._jobs is None: # populate jobs - self._jobs = self.jenkins.get_jobs() + self._jobs = self.jenkins.get_all_jobs() return self._jobs @property def job_list(self): if self._job_list is None: - self._job_list = set(job['name'] for job in self.jobs) + # python-jenkins uses 'fullname' for folder/name combination + self._job_list = set(job['fullname'] for job in self.jobs) return self._job_list def update_job(self, job_name, xml): @@ -154,17 +179,18 @@ class JenkinsManager(object): if keep is None: keep = [] for job in jobs: - if job['name'] not in keep: - if self.is_managed(job['name']): + # python-jenkins stores the folder and name as 'fullname' + if job['fullname'] not in keep: + if self.is_managed(job['fullname']): logger.info("Removing obsolete jenkins job {0}" - .format(job['name'])) - self.delete_job(job['name']) + .format(job['fullname'])) + self.delete_job(job['fullname']) deleted_jobs += 1 else: logger.info("Not deleting unmanaged jenkins job %s", - job['name']) + job['fullname']) else: - logger.debug("Keeping job %s", job['name']) + logger.debug("Keeping job %s", job['fullname']) return deleted_jobs def delete_jobs(self, jobs): @@ -231,17 +257,8 @@ class JenkinsManager(object): raise continue - if config_xml: - output_dir = os.path.join(output, job.name) - logger.info("Creating directory %s" % output_dir) - try: - os.makedirs(output_dir) - except OSError: - if not os.path.isdir(output_dir): - raise - output_fn = os.path.join(output_dir, 'config.xml') - else: - output_fn = os.path.join(output, job.name) + output_fn = self._setup_output(output, job.name, config_xml) + logger.debug("Writing XML to '{0}'".format(output_fn)) with io.open(output_fn, 'w', encoding='utf-8') as f: f.write(job.output().decode('utf-8')) @@ -383,17 +400,8 @@ class JenkinsManager(object): raise continue - if config_xml: - output_dir = os.path.join(output, view.name) - logger.info("Creating directory %s" % output_dir) - try: - os.makedirs(output_dir) - except OSError: - if not os.path.isdir(output_dir): - raise - output_fn = os.path.join(output_dir, 'config.xml') - else: - output_fn = os.path.join(output, view.name) + output_fn = self._setup_output(output, view.name, config_xml) + logger.debug("Writing XML to '{0}'".format(output_fn)) with io.open(output_fn, 'w', encoding='utf-8') as f: f.write(view.output().decode('utf-8')) diff --git a/jenkins_jobs/modules/general.py b/jenkins_jobs/modules/general.py index 658d3c14e..0bfca5375 100644 --- a/jenkins_jobs/modules/general.py +++ b/jenkins_jobs/modules/general.py @@ -56,6 +56,12 @@ Example: Path for a custom workspace. Defaults to Jenkins default configuration. + * **folder**: + The folder attribute provides an alternative to using '/' as + the job name to specify which Jenkins folder to upload the job to. + Requires the `CloudBees Folders Plugin. + `_ + * **child-workspace**: Path for a child custom workspace. Defaults to Jenkins default configuration. This parameter is only valid for matrix type jobs. @@ -103,7 +109,6 @@ Example: * **raw**: If present, this section should contain a single **xml** entry. This XML will be inserted at the top-level of the :ref:`Job` definition. - """ import logging diff --git a/jenkins_jobs/parser.py b/jenkins_jobs/parser.py index 5dee3cac3..1033b9fd3 100644 --- a/jenkins_jobs/parser.py +++ b/jenkins_jobs/parser.py @@ -226,6 +226,12 @@ class YamlParser(object): for macro in self.data.get(component_type, {}).values(): self._macro_registry.register(component_type, macro) + def _getfullname(self, data): + if 'folder' in data: + return "%s/%s" % (data['folder'], data['name']) + + return data['name'] + def expandYaml(self, registry, jobs_glob=None): changed = True while changed: @@ -240,15 +246,17 @@ class YamlParser(object): self._macro_registry.expand_macros(default) for job in self.data.get('job', {}).values(): self._macro_registry.expand_macros(job) + job = self._applyDefaults(job) + job['name'] = self._getfullname(job) if jobs_glob and not matches(job['name'], jobs_glob): logger.debug("Ignoring job {0}".format(job['name'])) continue logger.debug("Expanding job '{0}'".format(job['name'])) - job = self._applyDefaults(job) self._formatDescription(job) self.jobs.append(job) for view in self.data.get('view', {}).values(): + view['name'] = self._getfullname(view) logger.debug("Expanding view '{0}'".format(view['name'])) self._formatDescription(view) self.views.append(view) @@ -400,6 +408,7 @@ class YamlParser(object): "Failure formatting template '%s', containing '%s' with " "params '%s'", template_name, template, params) raise + expanded['name'] = self._getfullname(expanded) self._macro_registry.expand_macros(expanded, params) job_name = expanded.get('name') diff --git a/requirements.txt b/requirements.txt index f0e7ab425..52059b553 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,6 @@ six>=1.9.0 # MIT PyYAML>=3.10.0 # MIT pbr>=1.8 # Apache-2.0 stevedore>=1.17.1 # Apache-2.0 -python-jenkins>=0.4.8 +python-jenkins>=0.4.9 fasteners Jinja2 diff --git a/tests/base.py b/tests/base.py index becc211c5..06fedb28a 100644 --- a/tests/base.py +++ b/tests/base.py @@ -26,6 +26,7 @@ import re import xml.etree.ElementTree as XML import fixtures +import six from six.moves import StringIO import testtools from testtools.content import text_content @@ -64,11 +65,16 @@ def get_scenarios(fixtures_path, in_ext='yaml', out_ext='xml', - content of the fixture output file (aka expected) """ scenarios = [] - files = [] - for dirpath, dirs, fs in os.walk(fixtures_path): - files.extend([os.path.join(dirpath, f) for f in fs]) + files = {} + for dirpath, _, fs in os.walk(fixtures_path): + for fn in fs: + if fn in files: + files[fn].append(os.path.join(dirpath, fn)) + else: + files[fn] = [os.path.join(dirpath, fn)] - input_files = [f for f in files if re.match(r'.*\.{0}$'.format(in_ext), f)] + input_files = [files[f][0] for f in files if + re.match(r'.*\.{0}$'.format(in_ext), f)] for input_filename in input_files: if input_filename.endswith(plugins_info_ext): @@ -80,24 +86,27 @@ def get_scenarios(fixtures_path, in_ext='yaml', out_ext='xml', output_candidate = re.sub(r'\.{0}$'.format(in_ext), '.{0}'.format(out_ext), input_filename) # assume empty file if no output candidate found - if output_candidate not in files: - output_candidate = None + if os.path.basename(output_candidate) in files: + out_filenames = files[os.path.basename(output_candidate)] + else: + out_filenames = None plugins_info_candidate = re.sub(r'\.{0}$'.format(in_ext), '.{0}'.format(plugins_info_ext), input_filename) - if plugins_info_candidate not in files: + if os.path.basename(plugins_info_candidate) not in files: plugins_info_candidate = None conf_candidate = re.sub(r'\.yaml$|\.json$', '.conf', input_filename) - # If present, add the configuration file - if conf_candidate not in files: - conf_candidate = None + conf_filename = files.get(os.path.basename(conf_candidate), None) + + if conf_filename is not None: + conf_filename = conf_filename[0] scenarios.append((input_filename, { 'in_filename': input_filename, - 'out_filename': output_candidate, - 'conf_filename': conf_candidate, + 'out_filenames': out_filenames, + 'conf_filename': conf_filename, 'plugins_info_filename': plugins_info_candidate, })) @@ -117,12 +126,13 @@ class BaseTestCase(testtools.TestCase): def _read_utf8_content(self): # if None assume empty file - if self.out_filename is None: + if not self.out_filenames: return u"" # Read XML content, assuming it is unicode encoded - xml_content = u"%s" % io.open(self.out_filename, - 'r', encoding='utf-8').read() + xml_content = "" + for f in sorted(self.out_filenames): + xml_content += u"%s" % io.open(f, 'r', encoding='utf-8').read() return xml_content def _read_yaml_content(self, filename): @@ -195,6 +205,23 @@ class BaseScenariosTestCase(testscenarios.TestWithScenarios, BaseTestCase): # Generate the XML tree directly with modules/general pub.gen_xml(xml_project, yaml_content) + # check output file is under correct path + if 'name' in yaml_content: + prefix = os.path.dirname(self.in_filename) + # split using '/' since fullname uses URL path separator + expected_folders = [os.path.normpath( + os.path.join(prefix, + '/'.join(parser._getfullname(yaml_content). + split('/')[:-1])))] + actual_folders = [os.path.dirname(f) for f in self.out_filenames] + + self.assertEquals( + expected_folders, actual_folders, + "Output file under wrong path, was '%s', should be '%s'" % + (self.out_filenames[0], + os.path.join(expected_folders[0], + os.path.basename(self.out_filenames[0])))) + # Prettify generated XML pretty_xml = XmlJob(xml_project, 'fixturejob').output().decode('utf-8') @@ -226,6 +253,25 @@ class SingleJobTestCase(BaseScenariosTestCase): xml_jobs.sort(key=AlphanumSort) + # check reference files are under correct path for folders + prefix = os.path.dirname(self.in_filename) + # split using '/' since fullname uses URL path separator + expected_folders = list(set([ + os.path.normpath( + os.path.join(prefix, + '/'.join(job_data['name'].split('/')[:-1]))) + for job_data in job_data_list + ])) + actual_folders = [os.path.dirname(f) for f in self.out_filenames] + + six.assertCountEqual( + self, + expected_folders, actual_folders, + "Output file under wrong path, was '%s', should be '%s'" % + (self.out_filenames[0], + os.path.join(expected_folders[0], + os.path.basename(self.out_filenames[0])))) + # Prettify generated XML pretty_xml = u"\n".join(job.output().decode('utf-8') for job in xml_jobs) diff --git a/tests/cmd/subcommands/test_update.py b/tests/cmd/subcommands/test_update.py index 77dbc1d21..9606951f6 100644 --- a/tests/cmd/subcommands/test_update.py +++ b/tests/cmd/subcommands/test_update.py @@ -30,7 +30,7 @@ from tests.cmd.test_cmd import CmdTestsBase class UpdateTests(CmdTestsBase): @mock.patch('jenkins_jobs.builder.jenkins.Jenkins.job_exists') - @mock.patch('jenkins_jobs.builder.jenkins.Jenkins.get_jobs') + @mock.patch('jenkins_jobs.builder.jenkins.Jenkins.get_all_jobs') @mock.patch('jenkins_jobs.builder.jenkins.Jenkins.reconfig_job') def test_update_jobs(self, jenkins_reconfig_job, @@ -72,14 +72,14 @@ class UpdateTests(CmdTestsBase): six.text_type)) @mock.patch('jenkins_jobs.builder.jenkins.Jenkins.job_exists') - @mock.patch('jenkins_jobs.builder.jenkins.Jenkins.get_jobs') + @mock.patch('jenkins_jobs.builder.jenkins.Jenkins.get_all_jobs') @mock.patch('jenkins_jobs.builder.jenkins.Jenkins.reconfig_job') @mock.patch('jenkins_jobs.builder.jenkins.Jenkins.delete_job') def test_update_jobs_and_delete_old(self, jenkins_delete_job, jenkins_reconfig_job, - jenkins_get_jobs, - jenkins_job_exists, ): + jenkins_get_all_jobs, + jenkins_job_exists): """ Test update behaviour with --delete-old option @@ -99,8 +99,8 @@ class UpdateTests(CmdTestsBase): args = ['--conf', self.default_config_file, 'update', '--delete-old', path] - jenkins_get_jobs.return_value = [{'name': name} - for name in yaml_jobs + extra_jobs] + jenkins_get_all_jobs.return_value = [ + {'fullname': name} for name in yaml_jobs + extra_jobs] with mock.patch('jenkins_jobs.builder.JenkinsManager.is_managed', side_effect=(lambda name: name != 'unmanaged')): diff --git a/tests/general/fixtures/folders/folders001.xml b/tests/general/fixtures/folders/folders001.xml new file mode 100644 index 000000000..1505db519 --- /dev/null +++ b/tests/general/fixtures/folders/folders001.xml @@ -0,0 +1,9 @@ + + + + false + false + false + false + true + diff --git a/tests/general/fixtures/folders/test-nested-folder/folders002.xml b/tests/general/fixtures/folders/test-nested-folder/folders002.xml new file mode 100644 index 000000000..1505db519 --- /dev/null +++ b/tests/general/fixtures/folders/test-nested-folder/folders002.xml @@ -0,0 +1,9 @@ + + + + false + false + false + false + true + diff --git a/tests/general/fixtures/folders001.yaml b/tests/general/fixtures/folders001.yaml new file mode 100644 index 000000000..948ccc6d0 --- /dev/null +++ b/tests/general/fixtures/folders001.yaml @@ -0,0 +1,3 @@ +name: test-folder +project-type: freestyle +folder: folders diff --git a/tests/general/fixtures/folders002.yaml b/tests/general/fixtures/folders002.yaml new file mode 100644 index 000000000..b5626607b --- /dev/null +++ b/tests/general/fixtures/folders002.yaml @@ -0,0 +1,3 @@ +name: test-folder +project-type: freestyle +folder: folders/test-nested-folder diff --git a/tests/jenkins_manager/test_manager.py b/tests/jenkins_manager/test_manager.py index a1738329a..cc8e3d417 100644 --- a/tests/jenkins_manager/test_manager.py +++ b/tests/jenkins_manager/test_manager.py @@ -62,8 +62,8 @@ class TestCaseTestJenkinsManager(base.BaseTestCase): get_jobs=mock.DEFAULT, is_managed=mock.DEFAULT, delete_job=mock.DEFAULT) as patches: - patches['get_jobs'].return_value = [{'name': 'job1'}, - {'name': 'job2'}] + patches['get_jobs'].return_value = [{'fullname': 'job1'}, + {'fullname': 'job2'}] patches['is_managed'].side_effect = [True, True] self.builder.delete_old_managed() diff --git a/tests/yamlparser/fixtures/folders-attribute.yaml b/tests/yamlparser/fixtures/folders-attribute.yaml new file mode 100644 index 000000000..7346fa2f9 --- /dev/null +++ b/tests/yamlparser/fixtures/folders-attribute.yaml @@ -0,0 +1,12 @@ +- defaults: + name: team1 + folder: team1-jobs + +- job: + name: ruby-jobs/rspec + defaults: team1 + builders: + - shell: | + rvm use --create ruby-2.3.0@rspec + bundle install + bundle exec rspec diff --git a/tests/yamlparser/fixtures/folders-job-name.yaml b/tests/yamlparser/fixtures/folders-job-name.yaml new file mode 100644 index 000000000..d90323937 --- /dev/null +++ b/tests/yamlparser/fixtures/folders-job-name.yaml @@ -0,0 +1,5 @@ +- job: + name: python-jobs/tox-py27 + builders: + - shell: | + tox -e py27 diff --git a/tests/yamlparser/fixtures/folders/folders001.xml b/tests/yamlparser/fixtures/folders/folders001.xml new file mode 100644 index 000000000..1d2494f8a --- /dev/null +++ b/tests/yamlparser/fixtures/folders/folders001.xml @@ -0,0 +1,16 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + + + + diff --git a/tests/yamlparser/fixtures/folders001.xml b/tests/yamlparser/fixtures/folders001.xml new file mode 100644 index 000000000..0570e10a5 --- /dev/null +++ b/tests/yamlparser/fixtures/folders001.xml @@ -0,0 +1,15 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + + + diff --git a/tests/yamlparser/fixtures/folders001.yaml b/tests/yamlparser/fixtures/folders001.yaml new file mode 100644 index 000000000..20f6ed0cd --- /dev/null +++ b/tests/yamlparser/fixtures/folders001.yaml @@ -0,0 +1,14 @@ + +- job-template: + name: 'same-{text}-name' + folder: '{path}' + +- project: + name: test-same-job-name + path: folders + text: job + jobs: + - 'same-{text}-name' + +- job: + name: same-job-name diff --git a/tests/yamlparser/fixtures/python-jobs/folders-job-name.xml b/tests/yamlparser/fixtures/python-jobs/folders-job-name.xml new file mode 100644 index 000000000..8a3d28bd3 --- /dev/null +++ b/tests/yamlparser/fixtures/python-jobs/folders-job-name.xml @@ -0,0 +1,20 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + + tox -e py27 + + + + + + diff --git a/tests/yamlparser/fixtures/team1-jobs/ruby-jobs/folders-attribute.xml b/tests/yamlparser/fixtures/team1-jobs/ruby-jobs/folders-attribute.xml new file mode 100644 index 000000000..2c57eb750 --- /dev/null +++ b/tests/yamlparser/fixtures/team1-jobs/ruby-jobs/folders-attribute.xml @@ -0,0 +1,22 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + + rvm use --create ruby-2.3.0@rspec +bundle install +bundle exec rspec + + + + + +