Merge "Support non-top-level dirs as extra config path"

tests/fixtures/config/tenant-parser/extra.yaml

@@ -20,3 +20,6 @@
               extra-config-paths:
                 - extra.yaml
                 - extra.d/
+          - org/project6:
+              extra-config-paths:
+                - other/extra.d/

tests/fixtures/config/tenant-parser/git/org_project6/other/extra.d/extra.yaml

@@ -0,0 +1,9 @@
+- job:
+    name: project6-extra-dir
+    run: playbooks/common.yaml
+
+- project:
+    check:
+      jobs:
+        - project6-extra-dir
+

tests/fixtures/config/tenant-parser/git/org_project6/playbooks/common.yaml

@@ -0,0 +1,2 @@
+- hosts: all
+  tasks: []

tests/unit/test_configloader.py

@@ -891,6 +891,7 @@ class TestTenantExtra(TenantParserTestCase):
         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
         self.assertTrue('project2-extra-file' in tenant.layout.jobs)
         self.assertTrue('project2-extra-dir' in tenant.layout.jobs)
+        self.assertTrue('project6-extra-dir' in tenant.layout.jobs)
 
     def test_dynamic_extra(self):
         in_repo_conf = textwrap.dedent(
@@ -914,6 +915,30 @@ class TestTenantExtra(TenantParserTestCase):
             dict(name='project2-extra-file2', result='SUCCESS', changes='1,1'),
         ], ordered=False)
 
+    def test_dynamic_extra_dir(self):
+        in_repo_conf = textwrap.dedent(
+            """
+            - job:
+                name: project6-extra-dir2
+                parent: common-config-job
+            - project:
+                check:
+                  jobs:
+                    - project6-extra-dir
+                    - project6-extra-dir2
+            """)
+        file_dict = {
+            'other/extra.d/new/extra.yaml': in_repo_conf,
+        }
+        A = self.fake_gerrit.addFakeChange('org/project6', 'master', 'A',
+                                           files=file_dict)
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.assertHistory([
+            dict(name='project6-extra-dir', result='SUCCESS', changes='1,1'),
+            dict(name='project6-extra-dir2', result='SUCCESS', changes='1,1'),
+        ], ordered=False)
+
     def test_extra_reconfigure(self):
         in_repo_conf = textwrap.dedent(
             """

zuul/configloader.py

@@ -35,6 +35,8 @@ from zuul.lib.varnames import check_varnames
 from zuul.zk.config_cache import UnparsedConfigCache
 from zuul.zk.semaphore import SemaphoreHandler
 
+ZUUL_CONF_ROOT = ('zuul.yaml', 'zuul.d', '.zuul.yaml', '.zuul.d')
+
 
 # Several forms accept either a single item or a list, this makes
 # specifying that in the schema easy (and explicit).
@@ -1924,17 +1926,20 @@ class TenantParser(object):
         branch_cache = abide.getUnparsedBranchCache(
             source_context.project_canonical_name,
             source_context.branch)
-        for conf_root in (
-                ('zuul.yaml', 'zuul.d', '.zuul.yaml', '.zuul.d') +
-                tpc.extra_config_files + tpc.extra_config_dirs):
+        valid_dirs = ("zuul.d", ".zuul.d") + tpc.extra_config_dirs
+        for conf_root in (ZUUL_CONF_ROOT + tpc.extra_config_files
+                          + tpc.extra_config_dirs):
             for fn in sorted(files.keys()):
-                fn_root = fn.split('/')[0]
-                if fn_root != conf_root or not files.get(fn):
+                if not files.get(fn):
+                    continue
+                if not (fn == conf_root
+                        or (conf_root in valid_dirs
+                            and fn.startswith(f"{conf_root}/"))):
                     continue
                 # Don't load from more than one configuration in a
                 # project-branch (unless an "extra" file/dir).
-                if (conf_root not in tpc.extra_config_files and
-                    conf_root not in tpc.extra_config_dirs):
+                fn_root = fn.split('/')[0]
+                if (fn_root in ZUUL_CONF_ROOT):
                     if (loaded and loaded != conf_root):
                         self.log.warning("Multiple configuration files in %s",
                                          source_context)
@@ -2516,8 +2521,7 @@ class ConfigLoader(object):
 
                     # Don't load from more than one configuration in a
                     # project-branch (unless an "extra" file/dir).
-                    if (conf_root not in tpc.extra_config_files and
-                        conf_root not in tpc.extra_config_dirs):
+                    if (conf_root in ZUUL_CONF_ROOT):
                         if loaded and loaded != conf_root:
                             self.log.warning(
                                 "Configuration in %s ignored because "

zuul/merger/merger.py

@@ -652,7 +652,12 @@ class Repo(object):
                 ret[fn] = None
         if dirs:
             for dn in dirs:
-                if dn not in tree:
+                try:
+                    sub_tree = tree[dn]
+                except KeyError:
+                    continue
+
+                if sub_tree.type != "tree":
                     continue
 
                 # Some people like to keep playbooks, etc. grouped
@@ -660,7 +665,7 @@ class Repo(object):
                 # directories of any .zuul.ignore files and prune them
                 # from the config read.
                 to_ignore = []
-                for blob in tree[dn].traverse():
+                for blob in sub_tree.traverse():
                     if blob.path.endswith(".zuul.ignore"):
                         to_ignore.append(os.path.split(blob.path)[0])
 
@@ -670,7 +675,7 @@ class Repo(object):
                             return True
                     return False
 
-                for blob in tree[dn].traverse():
+                for blob in sub_tree.traverse():
                     if not _ignored(blob) and blob.path.endswith(".yaml"):
                         ret[blob.path] = blob.data_stream.read().decode(
                             'utf-8')