Add retain_anchors config option.

If set to True, YAML anchors can be referenced across files, allowing jobs to be
composed from bits of YAML defined in separate files. False by default.

Story: 2000338
Task: 2547
Change-Id: I034ce3bce0030093cb8d4266dabbdb06d96306d6

doc/source/execution.rst

@@ -57,6 +57,15 @@ job_builder section
   so user can be sure which instance was updated. User may click the link to
   go directly to that job. False by default.
 
+**retain_anchors**
+  (Optional) If set to True, YAML anchors will be retained across files,
+  allowing jobs to be composed from bits of YAML defined in separate files.
+  Note this means that the order of processing files matters - `jenkins-jobs`
+  loads files in alphabetical order (all files in a dir are loaded before any
+  files in subdirs). For example, if your anchors are in a file named `foo.yml`
+  they will be accessible in `qux.yml` but not in `bar.yml`. They will also be
+  accessible in `mydir/bar.yml` and `mydir/qux.yml`. False by default.
+
 
 jenkins section
 ^^^^^^^^^^^^^^^

jenkins_jobs/config.py

@@ -41,6 +41,7 @@ recursive=False
 exclude=.*
 allow_duplicates=False
 allow_empty_variables=False
+retain_anchors=False
 
 # other named sections could be used in addition to the implicit [jenkins]
 # if you have multiple jenkins servers.
@@ -300,6 +301,13 @@ class JJBConfig(object):
             config.has_option('job_builder', 'allow_empty_variables') and
             config.getboolean('job_builder', 'allow_empty_variables'))
 
+        # retain anchors across files?
+        retain_anchors = False
+        if config and config.has_option('job_builder', 'retain_anchors'):
+            retain_anchors = config.getboolean('job_builder',
+                                               'retain_anchors')
+        self.yamlparser['retain_anchors'] = retain_anchors
+
     def validate(self):
         # Inform the user as to what is likely to happen, as they may specify
         # a real jenkins instance in test mode to get the plugin info to check

jenkins_jobs/local_yaml.py

@@ -564,8 +564,9 @@ class LazyLoader(CustomLoader):
         return self._cls.from_yaml(self._loader, node)
 
 
-def load(stream, **kwargs):
+def load(stream, retain_anchors=False, **kwargs):
-    LocalAnchorLoader.reset_anchors()
+    if not retain_anchors:
+        LocalAnchorLoader.reset_anchors()
     return yaml.load(stream, functools.partial(LocalLoader, **kwargs))
 
 

jenkins_jobs/parser.py

@@ -98,7 +98,7 @@ class YamlParser(object):
         for path in fn:
             if not hasattr(path, 'read') and os.path.isdir(path):
                 files_to_process.extend([os.path.join(path, f)
-                                         for f in os.listdir(path)
+                                         for f in sorted(os.listdir(path))
                                          if (f.endswith('.yml')
                                              or f.endswith('.yaml'))])
             else:
@@ -134,7 +134,9 @@ class YamlParser(object):
 
     def _parse_fp(self, fp):
         # wrap provided file streams to ensure correct encoding used
-        data = local_yaml.load(utils.wrap_stream(fp), search_path=self.path)
+        data = local_yaml.load(utils.wrap_stream(fp),
+                               self.jjb_config.yamlparser['retain_anchors'],
+                               search_path=self.path)
         if data:
             if not isinstance(data, list):
                 raise JenkinsJobsException(

jenkins_jobs/utils.py

@@ -50,6 +50,9 @@ def recurse_path(root, excludes=None):
     relative = [e for e in excludes if os.path.sep in e and
                 not os.path.isabs(e)]
     for root, dirs, files in os.walk(basepath, topdown=True):
+        # sort in-place to ensure dirnames are visited in alphabetical order
+        # a predictable order makes it easier to use the retain_anchors option
+        dirs.sort()
         dirs[:] = [
             d for d in dirs
             if not any([fnmatch.fnmatch(d, pattern) for pattern in patterns])

tests/localyaml/fixtures/custom_retain_anchors.yaml

@@ -0,0 +1,8 @@
+- project:
+    name: retain_anchors
+    jobs:
+      - retain_anchors
+
+- job-template:
+    name: retain_anchors
+    <<: *retain_anchors_defaults

tests/localyaml/fixtures/custom_retain_anchors_include001.yaml

@@ -0,0 +1,10 @@
+- retain_anchors_wrapper_defaults: &retain_anchors_wrapper_defaults
+    name: 'retain_anchors_wrapper_defaults'
+    wrappers:
+      - timeout:
+          timeout: 180
+          fail: true
+
+- retain_anchors_defaults: &retain_anchors_defaults
+    name: 'retain_anchors_defaults'
+    <<: *retain_anchors_wrapper_defaults

tests/localyaml/test_localyaml.py

@@ -15,6 +15,7 @@
 # under the License.
 
 import os
+import yaml
 
 from testtools import ExpectedException
 from yaml.composer import ComposerError
@@ -80,3 +81,39 @@ class TestCaseLocalYamlIncludeAnchors(base.BaseTestCase):
         jjb_config.validate()
         j = YamlParser(jjb_config)
         j.load_files([os.path.join(self.fixtures_path, f) for f in files])
+
+
+class TestCaseLocalYamlRetainAnchors(base.BaseTestCase):
+
+    fixtures_path = os.path.join(os.path.dirname(__file__), 'fixtures')
+
+    def test_retain_anchors_default(self):
+        """
+        Verify that anchors are NOT retained across files by default.
+        """
+
+        files = ["custom_retain_anchors_include001.yaml",
+                 "custom_retain_anchors.yaml"]
+
+        jjb_config = JJBConfig()
+        # use the default value for retain_anchors
+        jjb_config.validate()
+        j = YamlParser(jjb_config)
+        with ExpectedException(yaml.composer.ComposerError,
+                               "found undefined alias.*"):
+            j.load_files([os.path.join(self.fixtures_path, f) for f in files])
+
+    def test_retain_anchors_enabled(self):
+        """
+        Verify that anchors are retained across files if retain_anchors is
+        enabled in the config.
+        """
+
+        files = ["custom_retain_anchors_include001.yaml",
+                 "custom_retain_anchors.yaml"]
+
+        jjb_config = JJBConfig()
+        jjb_config.yamlparser['retain_anchors'] = True
+        jjb_config.validate()
+        j = YamlParser(jjb_config)
+        j.load_files([os.path.join(self.fixtures_path, f) for f in files])