Add support for 'repos' job configuration

If a job configuration gives a list of repos, add them to the list of
projects to update on the slave.

Test using a mock openstack dsvm job which should clone both nova and
keystone. Put this new mock job in the check pipeline rather than the
gate pipeline to keep the build history small, and assert that both the
launcher and the worker have cloned the project that did not trigger
the job.

Change-Id: I3ccf8713906d65cbd27929548499e81f798cea82

tests/base.py

@@ -759,7 +759,7 @@ class RecordingAnsibleJob(zuul.launcher.server.AnsibleJob):
         self.launcher_server.build_history.append(
             BuildHistory(name=build.name, result=result, changes=build.changes,
                          node=build.node, uuid=build.unique,
-                         parameters=build.parameters,
+                         parameters=build.parameters, jobdir=build.jobdir,
                          pipeline=build.parameters['ZUUL_PIPELINE'])
         )
         self.launcher_server.running_builds.remove(build)

tests/fixtures/config/openstack/git/project-config/playbooks/dsvm.yaml

@@ -0,0 +1,2 @@
+- hosts: all
+  tasks: []

tests/fixtures/config/openstack/git/project-config/zuul.yaml

@@ -71,12 +71,22 @@
         - python27
         - python35
 
+- job:
+    name: dsvm
+    parent: base
+    repos:
+      - openstack/keystone
+      - openstack/nova
+
 # Project definitions
 
 - project:
     name: openstack/nova
     templates:
       - python-jobs
+    check:
+      jobs:
+        - dsvm
     gate:
       queue: integrated
 
@@ -84,5 +94,8 @@
     name: openstack/keystone
     templates:
       - python-jobs
+    check:
+      jobs:
+        - dsvm
     gate:
       queue: integrated

tests/unit/test_openstack.py

@@ -14,6 +14,8 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import os
+
 from tests.base import AnsibleZuulTestCase
 
 
@@ -54,3 +56,45 @@ class TestOpenStack(AnsibleZuulTestCase):
                          "A should report start and success")
         self.assertEqual(self.getJobFromHistory('python27').node,
                          'ubuntu-trusty')
+
+    def test_dsvm_keystone_repo(self):
+        self.launch_server.keep_jobdir = True
+        A = self.fake_gerrit.addFakeChange('openstack/nova', 'master', 'A')
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        self.assertHistory([
+            dict(name='dsvm', result='SUCCESS', changes='1,1')])
+        build = self.getJobFromHistory('dsvm')
+
+        # Check that a change to nova triggered a keystone clone
+        launcher_git_dir = os.path.join(self.launcher_src_root,
+                                        'openstack', 'keystone', '.git')
+        self.assertTrue(os.path.exists(launcher_git_dir),
+                        msg='openstack/keystone should be cloned.')
+
+        jobdir_git_dir = os.path.join(build.jobdir.src_root,
+                                      'openstack', 'keystone', '.git')
+        self.assertTrue(os.path.exists(jobdir_git_dir),
+                        msg='openstack/keystone should be cloned.')
+
+    def test_dsvm_nova_repo(self):
+        self.launch_server.keep_jobdir = True
+        A = self.fake_gerrit.addFakeChange('openstack/keystone', 'master', 'A')
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        self.assertHistory([
+            dict(name='dsvm', result='SUCCESS', changes='1,1')])
+        build = self.getJobFromHistory('dsvm')
+
+        # Check that a change to keystone triggered a nova clone
+        launcher_git_dir = os.path.join(self.launcher_src_root,
+                                        'openstack', 'nova', '.git')
+        self.assertTrue(os.path.exists(launcher_git_dir),
+                        msg='openstack/nova should be cloned.')
+
+        jobdir_git_dir = os.path.join(build.jobdir.src_root,
+                                      'openstack', 'nova', '.git')
+        self.assertTrue(os.path.exists(jobdir_git_dir),
+                        msg='openstack/nova should be cloned.')

zuul/configloader.py

@@ -115,6 +115,7 @@ class JobParser(object):
                'run': str,
                '_source_context': model.SourceContext,
                'roles': to_list(role),
+               'repos': to_list(str),
                }
 
         return vs.Schema(job)
@@ -185,6 +186,11 @@ class JobParser(object):
                     ns.addNode(node)
             job.nodeset = ns
 
+        if 'repos' in conf:
+            # Accumulate repos in a set so that job inheritance
+            # is additive.
+            job.repos = job.repos.union(set(conf.get('repos', [])))
+
         tags = conf.get('tags')
         if tags:
             # Tags are merged via a union rather than a

zuul/launcher/client.py

@@ -348,6 +348,13 @@ class LaunchClient(object):
         params['nodes'] = nodes
         params['zuul'] = zuul_params
         projects = set()
+        if job.repos:
+            for repo in job.repos:
+                project = item.pipeline.source.getProject(repo)
+                params['projects'].append(
+                    dict(name=repo,
+                         url=item.pipeline.source.getGitUrl(project)))
+                projects.add(project)
         for item in all_items:
             if item.change.project not in projects:
                 params['projects'].append(

zuul/launcher/server.py

@@ -87,7 +87,7 @@ class JobDir(object):
         #     trusted.cfg
         #     untrusted.cfg
         #   work
-        #     git
+        #     src
         #     logs
         self.keep = keep
         self.root = tempfile.mkdtemp(dir=root)

zuul/model.py

@@ -699,6 +699,7 @@ class Job(object):
             attempts=3,
             final=False,
             roles=frozenset(),
+            repos=frozenset(),
         )
 
         # These are generally internal attributes which are not