From 3e8c31fe94567007d7c63dac7678ce84a49acd7c Mon Sep 17 00:00:00 2001 From: Darragh Bailey Date: Fri, 19 Jun 2015 18:30:47 +0100 Subject: [PATCH] Take parameter order from yaml in trigger_parameterized_builds Support taking the order of the parameters to be passed through to a triggered parameterized build from the user defined yaml via config option. Use the existing order by default so as not to break current usage. The XML ordering of the parameters matters since when the same parameter is defined through multiple sources, Jenkins uses the ordering to determine which will be the one that that should win. ie if FOO=BAR is set as a build parameter of the current job and a properties file is generated during build that attempts to set FOO=BAZ; the XML ordering determines which value, BAZ or BAR, that the triggered job sees for the parameter FOO. Change-Id: Ia0dafad8ed3976e7b0b11feb6b3af844bbda9d4a --- doc/source/execution.rst | 14 ++ jenkins_jobs/cmd.py | 3 + jenkins_jobs/modules/publishers.py | 97 +++++++---- .../parameter-override-ordering.conf | 2 + .../parameter-override-ordering.xml | 39 +++++ .../parameter-override-ordering.yaml | 10 ++ .../parameter-override-ordering-001.conf | 2 + .../parameter-override-ordering-001.xml | 153 ++++++++++++++++++ .../parameter-override-ordering-001.yaml | 58 +++++++ .../parameter-override-ordering-002.conf | 2 + .../parameter-override-ordering-002.xml | 79 +++++++++ .../parameter-override-ordering-002.yaml | 39 +++++ 12 files changed, 465 insertions(+), 33 deletions(-) create mode 100644 tests/publishers/fixtures/trigger_parameterized_builds/parameter-override-ordering.conf create mode 100644 tests/publishers/fixtures/trigger_parameterized_builds/parameter-override-ordering.xml create mode 100644 tests/publishers/fixtures/trigger_parameterized_builds/parameter-override-ordering.yaml create mode 100644 tests/yamlparser/fixtures/trigger_parameterized_builds/parameter-override-ordering-001.conf create mode 100644 tests/yamlparser/fixtures/trigger_parameterized_builds/parameter-override-ordering-001.xml create mode 100644 tests/yamlparser/fixtures/trigger_parameterized_builds/parameter-override-ordering-001.yaml create mode 100644 tests/yamlparser/fixtures/trigger_parameterized_builds/parameter-override-ordering-002.conf create mode 100644 tests/yamlparser/fixtures/trigger_parameterized_builds/parameter-override-ordering-002.xml create mode 100644 tests/yamlparser/fixtures/trigger_parameterized_builds/parameter-override-ordering-002.yaml diff --git a/doc/source/execution.rst b/doc/source/execution.rst index 6946cf48f..356d6d1bb 100644 --- a/doc/source/execution.rst +++ b/doc/source/execution.rst @@ -114,6 +114,20 @@ stash section yaml part. +__future__ section +^^^^^^^^^^^^^^^^^^ + +This section is to control enabling of beta features or behaviour changes that +deviate from previously released behaviour in ways that may require effort to +convert existing JJB configs to adopt. This essentially will act as a method +to share these new behaviours while under active development so they can be +changed ahead of releases. + +**param_order_from_yaml** + Used to switch on using the order of the parameters are defined in yaml to + control the order of corresponding XML elements being written out. This is + intended as a global flag and can affect multiple modules. + Running ------- diff --git a/jenkins_jobs/cmd.py b/jenkins_jobs/cmd.py index 584e07b8d..f03df015d 100755 --- a/jenkins_jobs/cmd.py +++ b/jenkins_jobs/cmd.py @@ -50,6 +50,9 @@ query_plugins_info=True [hipchat] authtoken=dummy send-as=Jenkins + +[__future__] +param_order_from_yaml=False """ diff --git a/jenkins_jobs/modules/publishers.py b/jenkins_jobs/modules/publishers.py index 691da2342..4cb20e4d6 100644 --- a/jenkins_jobs/modules/publishers.py +++ b/jenkins_jobs/modules/publishers.py @@ -391,36 +391,67 @@ def trigger_parameterized_builds(parser, xml_parent, data): /../../tests/publishers/fixtures/trigger_parameterized_builds003.yaml :language: yaml """ + logger = logging.getLogger("%s:trigger-parameterized-builds" % __name__) pt_prefix = 'hudson.plugins.parameterizedtrigger.' tbuilder = XML.SubElement(xml_parent, pt_prefix + 'BuildTrigger') configs = XML.SubElement(tbuilder, 'configs') + + # original order + orig_order = [ + 'predefined-parameters', + 'git-revision', + 'property-file', + 'current-parameters', + 'node-parameters', + 'svn-revision', + 'restrict-matrix-project', + 'node-label-name', + 'node-label', + 'boolean-parameters', + ] + + try: + if parser.config.getboolean('__future__', + 'param_order_from_yaml'): + orig_order = None + except six.moves.configparser.NoSectionError: + pass + + if orig_order: + logger.warn( + "Using deprecated order for parameter sets in " + "triggered-parameterized-builds. This will be changed in a future " + "release to inherit the order from the user defined yaml. To " + "enable this behaviour immediately, set the config option " + "'__future__.param_order_from_yaml' to 'true' and change the " + "input job configuration to use the desired order") + for project_def in data: tconfig = XML.SubElement(configs, pt_prefix + 'BuildTriggerConfig') tconfigs = XML.SubElement(tconfig, 'configs') - if ('predefined-parameters' in project_def - or 'git-revision' in project_def - or 'property-file' in project_def - or 'current-parameters' in project_def - or 'node-parameters' in project_def - or 'svn-revision' in project_def - or 'restrict-matrix-project' in project_def - or 'node-label-name' in project_def - or 'node-label' in project_def - or 'boolean-parameters' in project_def): - if 'predefined-parameters' in project_def: + if orig_order: + parameters = orig_order + else: + parameters = project_def.keys() + + for param_type in parameters: + param_value = project_def.get(param_type) + if param_value is None: + continue + + if param_type == 'predefined-parameters': params = XML.SubElement(tconfigs, pt_prefix + 'PredefinedBuildParameters') properties = XML.SubElement(params, 'properties') - properties.text = project_def['predefined-parameters'] - - if 'git-revision' in project_def and project_def['git-revision']: + properties.text = param_value + elif param_type == 'git-revision' and param_value: params = XML.SubElement(tconfigs, 'hudson.plugins.git.' 'GitRevisionBuildParameters') XML.SubElement(params, 'combineQueuedCommits').text = str( project_def.get('combine-queued-commits', False)).lower() - if 'property-file' in project_def and project_def['property-file']: + elif param_type == 'property-file': params = XML.SubElement(tconfigs, pt_prefix + 'FileBuildParameters') properties = XML.SubElement(params, 'propertiesFile') @@ -442,50 +473,50 @@ def trigger_parameterized_builds(parser, xml_parent, data): XML.SubElement(params, "onlyExactRuns").text = ( str(project_def.get('only-exact-matrix-child-runs', False)).lower()) - if ('current-parameters' in project_def - and project_def['current-parameters']): + elif param_type == 'current-parameters' and param_value: XML.SubElement(tconfigs, pt_prefix + 'CurrentBuildParameters') - if ('node-parameters' in project_def - and project_def['node-parameters']): + elif param_type == 'node-parameters' and param_value: XML.SubElement(tconfigs, pt_prefix + 'NodeParameters') - if 'svn-revision' in project_def and project_def['svn-revision']: + elif param_type == 'svn-revision' and param_value: param = XML.SubElement(tconfigs, pt_prefix + 'SubversionRevisionBuildParameters') XML.SubElement(param, 'includeUpstreamParameters').text = str( project_def.get('include-upstream', False)).lower() - if ('restrict-matrix-project' in project_def - and project_def['restrict-matrix-project']): + elif param_type == 'restrict-matrix-project' and param_value: subset = XML.SubElement(tconfigs, pt_prefix + 'matrix.MatrixSubsetBuildParameters') XML.SubElement(subset, 'filter').text = \ project_def['restrict-matrix-project'] - if ('node-label-name' in project_def or - 'node-label' in project_def): - params = XML.SubElement(tconfigs, - 'org.jvnet.jenkins.plugins.' - 'nodelabelparameter.' - 'parameterizedtrigger.' - 'NodeLabelBuildParameter') + elif (param_type == 'node-label-name' or + param_type == 'node-label'): + tag_name = ('org.jvnet.jenkins.plugins.nodelabelparameter.' + 'parameterizedtrigger.NodeLabelBuildParameter') + if tconfigs.find(tag_name) is not None: + # already processed and can only have one + continue + params = XML.SubElement(tconfigs, tag_name) name = XML.SubElement(params, 'name') if 'node-label-name' in project_def: name.text = project_def['node-label-name'] label = XML.SubElement(params, 'nodeLabel') if 'node-label' in project_def: label.text = project_def['node-label'] - if ('boolean-parameters' in project_def - and project_def['boolean-parameters']): + elif param_type == 'boolean-parameters' and param_value: params = XML.SubElement(tconfigs, pt_prefix + 'BooleanParameters') config_tag = XML.SubElement(params, 'configs') param_tag_text = pt_prefix + 'BooleanParameterConfig' - params_list = project_def['boolean-parameters'] + params_list = param_value for name, value in params_list.items(): param_tag = XML.SubElement(config_tag, param_tag_text) XML.SubElement(param_tag, 'name').text = name XML.SubElement(param_tag, 'value').text = str( value or False).lower() - else: + + if not list(tconfigs): + # not child parameter tags added tconfigs.set('class', 'java.util.Collections$EmptyList') + projects = XML.SubElement(tconfig, 'projects') if isinstance(project_def['project'], list): diff --git a/tests/publishers/fixtures/trigger_parameterized_builds/parameter-override-ordering.conf b/tests/publishers/fixtures/trigger_parameterized_builds/parameter-override-ordering.conf new file mode 100644 index 000000000..8971c8054 --- /dev/null +++ b/tests/publishers/fixtures/trigger_parameterized_builds/parameter-override-ordering.conf @@ -0,0 +1,2 @@ +[__future__] +param_order_from_yaml = True diff --git a/tests/publishers/fixtures/trigger_parameterized_builds/parameter-override-ordering.xml b/tests/publishers/fixtures/trigger_parameterized_builds/parameter-override-ordering.xml new file mode 100644 index 000000000..9150e0ab7 --- /dev/null +++ b/tests/publishers/fixtures/trigger_parameterized_builds/parameter-override-ordering.xml @@ -0,0 +1,39 @@ + + + + + + + + + BUILD_NUM=${BUILD_NUMBER} + + + version.prop + false + + + + another_job + ALWAYS + false + + + + + + foo=bar + + + version.prop + false + + + yet_another_job + ALWAYS + false + + + + + diff --git a/tests/publishers/fixtures/trigger_parameterized_builds/parameter-override-ordering.yaml b/tests/publishers/fixtures/trigger_parameterized_builds/parameter-override-ordering.yaml new file mode 100644 index 000000000..5d8d5bda3 --- /dev/null +++ b/tests/publishers/fixtures/trigger_parameterized_builds/parameter-override-ordering.yaml @@ -0,0 +1,10 @@ +publishers: + - trigger-parameterized-builds: + - project: another_job + predefined-parameters: BUILD_NUM=${BUILD_NUMBER} + property-file: version.prop + current-parameters: true + - project: yet_another_job + current-parameters: true + predefined-parameters: foo=bar + property-file: version.prop diff --git a/tests/yamlparser/fixtures/trigger_parameterized_builds/parameter-override-ordering-001.conf b/tests/yamlparser/fixtures/trigger_parameterized_builds/parameter-override-ordering-001.conf new file mode 100644 index 000000000..8971c8054 --- /dev/null +++ b/tests/yamlparser/fixtures/trigger_parameterized_builds/parameter-override-ordering-001.conf @@ -0,0 +1,2 @@ +[__future__] +param_order_from_yaml = True diff --git a/tests/yamlparser/fixtures/trigger_parameterized_builds/parameter-override-ordering-001.xml b/tests/yamlparser/fixtures/trigger_parameterized_builds/parameter-override-ordering-001.xml new file mode 100644 index 000000000..eb59f2f0b --- /dev/null +++ b/tests/yamlparser/fixtures/trigger_parameterized_builds/parameter-override-ordering-001.xml @@ -0,0 +1,153 @@ + + + + testing macro trigger-parameterized-builds order v1.0<!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + + #!/usr/bin/env python +# +print("Doing something cool with python") + + + + + + + + + + BUILD_NUM=${BUILD_NUMBER} + + + default_version.prop + false + + + + default_job + ALWAYS + false + + + + + + + + + BUILD_NUM=${BUILD_NUMBER} + + + default_version.prop + false + + + + first_job + ALWAYS + false + + + + + + foo=bar + + + version.prop + false + + + second_job + ALWAYS + false + + + + + + + + + + + testing macro trigger-parameterized-builds order v1.2<!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + + #!/usr/bin/env python +# +print("Doing something cool with python") + + + + + + + + + + BUILD_NUM=${BUILD_NUMBER} + + + default_version.prop + false + + + + default_job + ALWAYS + false + + + + + + + + + BUILD_NUM=${BUILD_NUMBER} + + + + version.prop + false + + + 1.2_first_job + ALWAYS + false + + + + + + 1.2_version.prop + false + + + foo=bar + + + 1.2_second_job + ALWAYS + false + + + + + + diff --git a/tests/yamlparser/fixtures/trigger_parameterized_builds/parameter-override-ordering-001.yaml b/tests/yamlparser/fixtures/trigger_parameterized_builds/parameter-override-ordering-001.yaml new file mode 100644 index 000000000..423ad258a --- /dev/null +++ b/tests/yamlparser/fixtures/trigger_parameterized_builds/parameter-override-ordering-001.yaml @@ -0,0 +1,58 @@ +- wrapper: + name: timeout-wrapper + wrappers: + - timeout: + fail: true + elastic-percentage: 150 + elastic-default-timeout: 90 + type: elastic + +- publisher: + name: trigger_parameterized_1.0 + publishers: + - trigger-parameterized-builds: + - project: first_job + predefined-parameters: BUILD_NUM=${BUILD_NUMBER} + property-file: default_version.prop + current-parameters: true + - project: second_job + current-parameters: true + predefined-parameters: foo=bar + property-file: version.prop + +- publisher: + name: trigger_parameterized_1.2 + publishers: + - trigger-parameterized-builds: + - project: 1.2_first_job + predefined-parameters: BUILD_NUM=${BUILD_NUMBER} + current-parameters: true + property-file: version.prop + - project: 1.2_second_job + current-parameters: true + property-file: 1.2_version.prop + predefined-parameters: foo=bar + +- job-template: + name: 'trigger_parameterized_{version}' + description: testing macro trigger-parameterized-builds order v{version} + builders: + - shell: | + #!/usr/bin/env python + # + print("Doing something cool with python") + publishers: + - trigger-parameterized-builds: + - project: default_job + predefined-parameters: BUILD_NUM=${{BUILD_NUMBER}} + property-file: default_version.prop + current-parameters: true + - trigger_parameterized_{version} + +- project: + name: complete_trigger_paramterized + version: + - 1.0 + - 1.2 + jobs: + - 'trigger_parameterized_{version}' diff --git a/tests/yamlparser/fixtures/trigger_parameterized_builds/parameter-override-ordering-002.conf b/tests/yamlparser/fixtures/trigger_parameterized_builds/parameter-override-ordering-002.conf new file mode 100644 index 000000000..8971c8054 --- /dev/null +++ b/tests/yamlparser/fixtures/trigger_parameterized_builds/parameter-override-ordering-002.conf @@ -0,0 +1,2 @@ +[__future__] +param_order_from_yaml = True diff --git a/tests/yamlparser/fixtures/trigger_parameterized_builds/parameter-override-ordering-002.xml b/tests/yamlparser/fixtures/trigger_parameterized_builds/parameter-override-ordering-002.xml new file mode 100644 index 000000000..a6f5c4443 --- /dev/null +++ b/tests/yamlparser/fixtures/trigger_parameterized_builds/parameter-override-ordering-002.xml @@ -0,0 +1,79 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + JJB Test1 + false + false + false + true + + + + + + + + + + + + + + + false + + + false + + + + UNSTABLE_OR_BETTER + false + + + + + + + + + + + <!-- Managed by Jenkins Job Builder --> + false + JJB Test2 + false + false + false + true + + + + + + + + + + + false + + + + + + + false + + + + UNSTABLE_OR_BETTER + false + + + + + + diff --git a/tests/yamlparser/fixtures/trigger_parameterized_builds/parameter-override-ordering-002.yaml b/tests/yamlparser/fixtures/trigger_parameterized_builds/parameter-override-ordering-002.yaml new file mode 100644 index 000000000..fddb804ed --- /dev/null +++ b/tests/yamlparser/fixtures/trigger_parameterized_builds/parameter-override-ordering-002.yaml @@ -0,0 +1,39 @@ +- project: + name: bogus-jankerator + jobs: + - 'hydra_{name}_jjb-test1' + - 'hydra_{name}_jjb-test2' + +- job-template: + name: 'hydra_{name}_jjb-test1' + display-name: 'JJB Test1' + publishers: + - trigger-parameterized-builds: + - project: '{obj:tpb_projects}' + current-parameters: '{obj:tpb_current_parameters}' + predefined-parameters: '{obj:tpb_predefined_parameters}' + condition: '{obj:tpb_condition}' + property-file: '{obj:tpb_property_file}' + git-revision: '{obj:tpb_git_revision}' + +- job-template: + name: 'hydra_{name}_jjb-test2' + display-name: 'JJB Test2' + publishers: + - trigger-parameterized-builds: + - project: '{obj:tpb_projects}' + property-file: '{obj:tpb_property_file}' + predefined-parameters: '{obj:tpb_predefined_parameters}' + current-parameters: '{obj:tpb_current_parameters}' + condition: '{obj:tpb_condition}' + git-revision: '{obj:tpb_git_revision}' + +- defaults: + name: global + + tpb_predefined_parameters: '' + tpb_property_file: '' + tpb_current_parameters: True + tpb_condition: 'UNSTABLE_OR_BETTER' + tpb_git_revision: True + tpb_projects: ''