From 2c1a62378af07dac675431388ba991e35438cb6d Mon Sep 17 00:00:00 2001 From: K Jonathan Harker Date: Tue, 21 Feb 2017 14:34:08 -0800 Subject: [PATCH] Add support for 'repos' job configuration If a job configuration gives a list of repos, add them to the list of projects to update on the slave. Test using a mock openstack dsvm job which should clone both nova and keystone. Put this new mock job in the check pipeline rather than the gate pipeline to keep the build history small, and assert that both the launcher and the worker have cloned the project that did not trigger the job. Change-Id: I3ccf8713906d65cbd27929548499e81f798cea82 --- tests/base.py | 2 +- .../git/project-config/playbooks/dsvm.yaml | 2 + .../openstack/git/project-config/zuul.yaml | 13 ++++++ tests/unit/test_openstack.py | 44 +++++++++++++++++++ zuul/configloader.py | 6 +++ zuul/launcher/client.py | 7 +++ zuul/launcher/server.py | 2 +- zuul/model.py | 1 + 8 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 tests/fixtures/config/openstack/git/project-config/playbooks/dsvm.yaml diff --git a/tests/base.py b/tests/base.py index 3ccc872d0f..b8f9d0c8a9 100755 --- a/tests/base.py +++ b/tests/base.py @@ -759,7 +759,7 @@ class RecordingAnsibleJob(zuul.launcher.server.AnsibleJob): self.launcher_server.build_history.append( BuildHistory(name=build.name, result=result, changes=build.changes, node=build.node, uuid=build.unique, - parameters=build.parameters, + parameters=build.parameters, jobdir=build.jobdir, pipeline=build.parameters['ZUUL_PIPELINE']) ) self.launcher_server.running_builds.remove(build) diff --git a/tests/fixtures/config/openstack/git/project-config/playbooks/dsvm.yaml b/tests/fixtures/config/openstack/git/project-config/playbooks/dsvm.yaml new file mode 100644 index 0000000000..f679dceaef --- /dev/null +++ b/tests/fixtures/config/openstack/git/project-config/playbooks/dsvm.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/openstack/git/project-config/zuul.yaml b/tests/fixtures/config/openstack/git/project-config/zuul.yaml index 9c2231a5b3..420d979468 100644 --- a/tests/fixtures/config/openstack/git/project-config/zuul.yaml +++ b/tests/fixtures/config/openstack/git/project-config/zuul.yaml @@ -71,12 +71,22 @@ - python27 - python35 +- job: + name: dsvm + parent: base + repos: + - openstack/keystone + - openstack/nova + # Project definitions - project: name: openstack/nova templates: - python-jobs + check: + jobs: + - dsvm gate: queue: integrated @@ -84,5 +94,8 @@ name: openstack/keystone templates: - python-jobs + check: + jobs: + - dsvm gate: queue: integrated diff --git a/tests/unit/test_openstack.py b/tests/unit/test_openstack.py index d0c7ab2165..670e578c79 100644 --- a/tests/unit/test_openstack.py +++ b/tests/unit/test_openstack.py @@ -14,6 +14,8 @@ # License for the specific language governing permissions and limitations # under the License. +import os + from tests.base import AnsibleZuulTestCase @@ -54,3 +56,45 @@ class TestOpenStack(AnsibleZuulTestCase): "A should report start and success") self.assertEqual(self.getJobFromHistory('python27').node, 'ubuntu-trusty') + + def test_dsvm_keystone_repo(self): + self.launch_server.keep_jobdir = True + A = self.fake_gerrit.addFakeChange('openstack/nova', 'master', 'A') + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + self.assertHistory([ + dict(name='dsvm', result='SUCCESS', changes='1,1')]) + build = self.getJobFromHistory('dsvm') + + # Check that a change to nova triggered a keystone clone + launcher_git_dir = os.path.join(self.launcher_src_root, + 'openstack', 'keystone', '.git') + self.assertTrue(os.path.exists(launcher_git_dir), + msg='openstack/keystone should be cloned.') + + jobdir_git_dir = os.path.join(build.jobdir.src_root, + 'openstack', 'keystone', '.git') + self.assertTrue(os.path.exists(jobdir_git_dir), + msg='openstack/keystone should be cloned.') + + def test_dsvm_nova_repo(self): + self.launch_server.keep_jobdir = True + A = self.fake_gerrit.addFakeChange('openstack/keystone', 'master', 'A') + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + self.assertHistory([ + dict(name='dsvm', result='SUCCESS', changes='1,1')]) + build = self.getJobFromHistory('dsvm') + + # Check that a change to keystone triggered a nova clone + launcher_git_dir = os.path.join(self.launcher_src_root, + 'openstack', 'nova', '.git') + self.assertTrue(os.path.exists(launcher_git_dir), + msg='openstack/nova should be cloned.') + + jobdir_git_dir = os.path.join(build.jobdir.src_root, + 'openstack', 'nova', '.git') + self.assertTrue(os.path.exists(jobdir_git_dir), + msg='openstack/nova should be cloned.') diff --git a/zuul/configloader.py b/zuul/configloader.py index e4fa62075c..3e94c37aac 100644 --- a/zuul/configloader.py +++ b/zuul/configloader.py @@ -115,6 +115,7 @@ class JobParser(object): 'run': str, '_source_context': model.SourceContext, 'roles': to_list(role), + 'repos': to_list(str), } return vs.Schema(job) @@ -185,6 +186,11 @@ class JobParser(object): ns.addNode(node) job.nodeset = ns + if 'repos' in conf: + # Accumulate repos in a set so that job inheritance + # is additive. + job.repos = job.repos.union(set(conf.get('repos', []))) + tags = conf.get('tags') if tags: # Tags are merged via a union rather than a diff --git a/zuul/launcher/client.py b/zuul/launcher/client.py index f36e2a7941..a22f82e409 100644 --- a/zuul/launcher/client.py +++ b/zuul/launcher/client.py @@ -348,6 +348,13 @@ class LaunchClient(object): params['nodes'] = nodes params['zuul'] = zuul_params projects = set() + if job.repos: + for repo in job.repos: + project = item.pipeline.source.getProject(repo) + params['projects'].append( + dict(name=repo, + url=item.pipeline.source.getGitUrl(project))) + projects.add(project) for item in all_items: if item.change.project not in projects: params['projects'].append( diff --git a/zuul/launcher/server.py b/zuul/launcher/server.py index 575d35226f..df24ff6c7b 100644 --- a/zuul/launcher/server.py +++ b/zuul/launcher/server.py @@ -87,7 +87,7 @@ class JobDir(object): # trusted.cfg # untrusted.cfg # work - # git + # src # logs self.keep = keep self.root = tempfile.mkdtemp(dir=root) diff --git a/zuul/model.py b/zuul/model.py index 10d0446e65..ac3a28613b 100644 --- a/zuul/model.py +++ b/zuul/model.py @@ -699,6 +699,7 @@ class Job(object): attempts=3, final=False, roles=frozenset(), + repos=frozenset(), ) # These are generally internal attributes which are not