Fix error loading projects out of order

When a config altering change lands, we reload the tenant's
configuration using cached data except for the project-branch in
question.  Due to an error, we loaded all of the cached data before
the newly changed data.  Correct this to always load data in the
same sequence regardless of whether it was cached.

Change-Id: If8adc9f3e06474f9bcf8363ae1b64b7f42334a11

tests/fixtures/config/in-repo/git/org_project1/.zuul.yaml

@@ -0,0 +1,5 @@
+- project:
+    name: org/project1
+    tenant-one-gate:
+      jobs:
+        - project-test1

tests/unit/test_v3.py

@@ -486,6 +486,62 @@ class TestInRepoConfig(ZuulTestCase):
         self.assertIn('appears multiple times', A.messages[0],
                       "A should have a syntax error reported")
 
+    def test_multi_repo(self):
+        downstream_repo_conf = textwrap.dedent(
+            """
+            - project:
+                name: org/project1
+                tenant-one-gate:
+                  jobs:
+                    - project-test1
+
+            - job:
+                name: project1-test1
+                parent: project-test1
+            """)
+
+        file_dict = {'.zuul.yaml': downstream_repo_conf}
+        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A',
+                                           files=file_dict)
+        A.addApproval('code-review', 2)
+        self.fake_gerrit.addEvent(A.addApproval('approved', 1))
+        self.waitUntilSettled()
+
+        self.assertEqual(A.data['status'], 'MERGED')
+        self.fake_gerrit.addEvent(A.getChangeMergedEvent())
+        self.waitUntilSettled()
+
+        upstream_repo_conf = textwrap.dedent(
+            """
+            - job:
+                name: project-test1
+
+            - job:
+                name: project-test2
+
+            - project:
+                name: org/project
+                tenant-one-gate:
+                  jobs:
+                    - project-test1
+            """)
+
+        file_dict = {'.zuul.yaml': upstream_repo_conf}
+        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B',
+                                           files=file_dict)
+        B.addApproval('code-review', 2)
+        self.fake_gerrit.addEvent(B.addApproval('approved', 1))
+        self.waitUntilSettled()
+
+        self.assertEqual(B.data['status'], 'MERGED')
+        self.fake_gerrit.addEvent(B.getChangeMergedEvent())
+        self.waitUntilSettled()
+
+        tenant = self.sched.abide.tenants.get('tenant-one')
+        # Ensure the latest change is reflected in the config; if it
+        # isn't this will raise an exception.
+        tenant.layout.getJob('project-test2')
+
 
 class TestAnsible(AnsibleZuulTestCase):
     # A temporary class to hold new tests while others are disabled

zuul/configloader.py

@@ -1090,13 +1090,20 @@ class TenantParser(object):
         untrusted_projects_config = model.UnparsedTenantConfig()
         jobs = []
 
+        # In some cases, we can use cached data, but it's still
+        # important that we process that in the same order along with
+        # any jobs that we run.  This class is used to hold the cached
+        # data and is inserted in the ordered jobs list for later
+        # processing.
+        class CachedDataJob(object):
+            def __init__(self, config_project, project):
+                self.config_project = config_project
+                self.project = project
+
         for project in config_projects:
             # If we have cached data (this is a reconfiguration) use it.
             if cached and project.unparsed_config:
-                TenantParser.log.info(
-                    "Loading previously parsed configuration from %s" %
-                    (project,))
-                config_projects_config.extend(project.unparsed_config)
+                jobs.append(CachedDataJob(True, project))
                 continue
             # Otherwise, prepare an empty unparsed config object to
             # hold cached data later.
@@ -1115,10 +1122,7 @@ class TenantParser(object):
         for project in untrusted_projects:
             # If we have cached data (this is a reconfiguration) use it.
             if cached and project.unparsed_config:
-                TenantParser.log.info(
-                    "Loading previously parsed configuration from %s" %
-                    (project,))
-                untrusted_projects_config.extend(project.unparsed_config)
+                jobs.append(CachedDataJob(False, project))
                 continue
             # Otherwise, prepare an empty unparsed config object to
             # hold cached data later.
@@ -1146,6 +1150,17 @@ class TenantParser(object):
             # complete in the order they were executed which is the
             # same order they were defined in the main config file.
             # This is important for correct inheritance.
+            if isinstance(job, CachedDataJob):
+                TenantParser.log.info(
+                    "Loading previously parsed configuration from %s" %
+                    (job.project,))
+                if job.config_project:
+                    config_projects_config.extend(
+                        job.project.unparsed_config)
+                else:
+                    untrusted_projects_config.extend(
+                        job.project.unparsed_config)
+                continue
             TenantParser.log.debug("Waiting for cat job %s" % (job,))
             job.wait()
             TenantParser.log.debug("Cat job %s got files %s" %