From 75d78b65409632c14577ba0cce74bfe2e6d3765a Mon Sep 17 00:00:00 2001 From: Vicky Chijwani Date: Wed, 30 May 2018 18:20:29 +0530 Subject: [PATCH] Add retain_anchors config option. If set to True, YAML anchors can be referenced across files, allowing jobs to be composed from bits of YAML defined in separate files. False by default. Story: 2000338 Task: 2547 Change-Id: I034ce3bce0030093cb8d4266dabbdb06d96306d6 --- doc/source/execution.rst | 9 +++++ jenkins_jobs/config.py | 8 ++++ jenkins_jobs/local_yaml.py | 5 ++- jenkins_jobs/parser.py | 6 ++- jenkins_jobs/utils.py | 3 ++ .../fixtures/custom_retain_anchors.yaml | 8 ++++ .../custom_retain_anchors_include001.yaml | 10 +++++ tests/localyaml/test_localyaml.py | 37 +++++++++++++++++++ 8 files changed, 82 insertions(+), 4 deletions(-) create mode 100644 tests/localyaml/fixtures/custom_retain_anchors.yaml create mode 100644 tests/localyaml/fixtures/custom_retain_anchors_include001.yaml diff --git a/doc/source/execution.rst b/doc/source/execution.rst index f8adc6837..7434be72b 100644 --- a/doc/source/execution.rst +++ b/doc/source/execution.rst @@ -57,6 +57,15 @@ job_builder section so user can be sure which instance was updated. User may click the link to go directly to that job. False by default. +**retain_anchors** + (Optional) If set to True, YAML anchors will be retained across files, + allowing jobs to be composed from bits of YAML defined in separate files. + Note this means that the order of processing files matters - `jenkins-jobs` + loads files in alphabetical order (all files in a dir are loaded before any + files in subdirs). For example, if your anchors are in a file named `foo.yml` + they will be accessible in `qux.yml` but not in `bar.yml`. They will also be + accessible in `mydir/bar.yml` and `mydir/qux.yml`. False by default. + jenkins section ^^^^^^^^^^^^^^^ diff --git a/jenkins_jobs/config.py b/jenkins_jobs/config.py index 7b0c2a1d8..ac15852d0 100644 --- a/jenkins_jobs/config.py +++ b/jenkins_jobs/config.py @@ -41,6 +41,7 @@ recursive=False exclude=.* allow_duplicates=False allow_empty_variables=False +retain_anchors=False # other named sections could be used in addition to the implicit [jenkins] # if you have multiple jenkins servers. @@ -300,6 +301,13 @@ class JJBConfig(object): config.has_option('job_builder', 'allow_empty_variables') and config.getboolean('job_builder', 'allow_empty_variables')) + # retain anchors across files? + retain_anchors = False + if config and config.has_option('job_builder', 'retain_anchors'): + retain_anchors = config.getboolean('job_builder', + 'retain_anchors') + self.yamlparser['retain_anchors'] = retain_anchors + def validate(self): # Inform the user as to what is likely to happen, as they may specify # a real jenkins instance in test mode to get the plugin info to check diff --git a/jenkins_jobs/local_yaml.py b/jenkins_jobs/local_yaml.py index 493dac079..a55587b97 100644 --- a/jenkins_jobs/local_yaml.py +++ b/jenkins_jobs/local_yaml.py @@ -564,8 +564,9 @@ class LazyLoader(CustomLoader): return self._cls.from_yaml(self._loader, node) -def load(stream, **kwargs): - LocalAnchorLoader.reset_anchors() +def load(stream, retain_anchors=False, **kwargs): + if not retain_anchors: + LocalAnchorLoader.reset_anchors() return yaml.load(stream, functools.partial(LocalLoader, **kwargs)) diff --git a/jenkins_jobs/parser.py b/jenkins_jobs/parser.py index 69f1115ad..8be0823a8 100644 --- a/jenkins_jobs/parser.py +++ b/jenkins_jobs/parser.py @@ -98,7 +98,7 @@ class YamlParser(object): for path in fn: if not hasattr(path, 'read') and os.path.isdir(path): files_to_process.extend([os.path.join(path, f) - for f in os.listdir(path) + for f in sorted(os.listdir(path)) if (f.endswith('.yml') or f.endswith('.yaml'))]) else: @@ -134,7 +134,9 @@ class YamlParser(object): def _parse_fp(self, fp): # wrap provided file streams to ensure correct encoding used - data = local_yaml.load(utils.wrap_stream(fp), search_path=self.path) + data = local_yaml.load(utils.wrap_stream(fp), + self.jjb_config.yamlparser['retain_anchors'], + search_path=self.path) if data: if not isinstance(data, list): raise JenkinsJobsException( diff --git a/jenkins_jobs/utils.py b/jenkins_jobs/utils.py index 385f87ba6..4ce8ad2cd 100644 --- a/jenkins_jobs/utils.py +++ b/jenkins_jobs/utils.py @@ -50,6 +50,9 @@ def recurse_path(root, excludes=None): relative = [e for e in excludes if os.path.sep in e and not os.path.isabs(e)] for root, dirs, files in os.walk(basepath, topdown=True): + # sort in-place to ensure dirnames are visited in alphabetical order + # a predictable order makes it easier to use the retain_anchors option + dirs.sort() dirs[:] = [ d for d in dirs if not any([fnmatch.fnmatch(d, pattern) for pattern in patterns]) diff --git a/tests/localyaml/fixtures/custom_retain_anchors.yaml b/tests/localyaml/fixtures/custom_retain_anchors.yaml new file mode 100644 index 000000000..aa3292352 --- /dev/null +++ b/tests/localyaml/fixtures/custom_retain_anchors.yaml @@ -0,0 +1,8 @@ +- project: + name: retain_anchors + jobs: + - retain_anchors + +- job-template: + name: retain_anchors + <<: *retain_anchors_defaults diff --git a/tests/localyaml/fixtures/custom_retain_anchors_include001.yaml b/tests/localyaml/fixtures/custom_retain_anchors_include001.yaml new file mode 100644 index 000000000..05242dd25 --- /dev/null +++ b/tests/localyaml/fixtures/custom_retain_anchors_include001.yaml @@ -0,0 +1,10 @@ +- retain_anchors_wrapper_defaults: &retain_anchors_wrapper_defaults + name: 'retain_anchors_wrapper_defaults' + wrappers: + - timeout: + timeout: 180 + fail: true + +- retain_anchors_defaults: &retain_anchors_defaults + name: 'retain_anchors_defaults' + <<: *retain_anchors_wrapper_defaults diff --git a/tests/localyaml/test_localyaml.py b/tests/localyaml/test_localyaml.py index 28460da6c..788bb3b26 100644 --- a/tests/localyaml/test_localyaml.py +++ b/tests/localyaml/test_localyaml.py @@ -15,6 +15,7 @@ # under the License. import os +import yaml from testtools import ExpectedException from yaml.composer import ComposerError @@ -80,3 +81,39 @@ class TestCaseLocalYamlIncludeAnchors(base.BaseTestCase): jjb_config.validate() j = YamlParser(jjb_config) j.load_files([os.path.join(self.fixtures_path, f) for f in files]) + + +class TestCaseLocalYamlRetainAnchors(base.BaseTestCase): + + fixtures_path = os.path.join(os.path.dirname(__file__), 'fixtures') + + def test_retain_anchors_default(self): + """ + Verify that anchors are NOT retained across files by default. + """ + + files = ["custom_retain_anchors_include001.yaml", + "custom_retain_anchors.yaml"] + + jjb_config = JJBConfig() + # use the default value for retain_anchors + jjb_config.validate() + j = YamlParser(jjb_config) + with ExpectedException(yaml.composer.ComposerError, + "found undefined alias.*"): + j.load_files([os.path.join(self.fixtures_path, f) for f in files]) + + def test_retain_anchors_enabled(self): + """ + Verify that anchors are retained across files if retain_anchors is + enabled in the config. + """ + + files = ["custom_retain_anchors_include001.yaml", + "custom_retain_anchors.yaml"] + + jjb_config = JJBConfig() + jjb_config.yamlparser['retain_anchors'] = True + jjb_config.validate() + j = YamlParser(jjb_config) + j.load_files([os.path.join(self.fixtures_path, f) for f in files])