From 74a974bf4e3e7faa78fda2af3604a2db5e6a1829 Mon Sep 17 00:00:00 2001 From: Paul Belanger Date: Tue, 14 Nov 2017 04:47:41 -0500 Subject: [PATCH] Allow run to be list of playbooks Like pre-run and post-run, allow a user to run a list of playbooks for a job. One example would be your job workflow would be to run multiple playbooks over using a site.yaml file with include_playbook commands. A second use case, more related to job design. With multiple playbooks support for job.run, the first playbook would be use deploy your server and the second playbook to validate the server was provisioned properly. Today, this can be done using a single run and post-run playbooks, however if post-run fails, zuul will return POST_FAILURE, not FAILURE. Not a large issue, but could be confusing to users when POST_FAILURE is returned. While it is possible a user could create a single site.yaml playbook, and use multiple include_playbook statements to get this functionality, there are downsides to this approach (mostly with the leaking of variables). Today, operators simply run ansible-playbook multiple times with the specific playbooks they only want. Story: 2002543 Task: 22101 Change-Id: I6268d9944e745cc07407ea7dd040fbfeb79dad4d Related-To: https://review.openstack.org/519596 Signed-off-by: Paul Belanger --- doc/source/user/config.rst | 6 +- .../notes/job-run-list-7036fbac9c146098.yaml | 4 ++ .../git/common-config/playbooks/bar.yaml | 5 ++ .../playbooks/first-fail-post.yaml | 11 ++++ .../common-config/playbooks/first-fail.yaml | 4 ++ .../git/common-config/playbooks/foo.yaml | 5 ++ .../common-config/playbooks/foobar-post.yaml | 33 ++++++++++ .../playbooks/multiple-parent-post.yaml | 11 ++++ .../playbooks/multiple-parent.yaml | 5 ++ .../ansible/git/common-config/zuul.yaml | 64 +++++++++++++++++++ .../config/ansible/git/org_project/.zuul.yaml | 4 ++ tests/unit/test_v3.py | 14 ++++ zuul/configloader.py | 10 +-- zuul/executor/server.py | 50 ++++++++------- 14 files changed, 196 insertions(+), 30 deletions(-) create mode 100644 releasenotes/notes/job-run-list-7036fbac9c146098.yaml create mode 100644 tests/fixtures/config/ansible/git/common-config/playbooks/bar.yaml create mode 100644 tests/fixtures/config/ansible/git/common-config/playbooks/first-fail-post.yaml create mode 100644 tests/fixtures/config/ansible/git/common-config/playbooks/first-fail.yaml create mode 100644 tests/fixtures/config/ansible/git/common-config/playbooks/foo.yaml create mode 100644 tests/fixtures/config/ansible/git/common-config/playbooks/foobar-post.yaml create mode 100644 tests/fixtures/config/ansible/git/common-config/playbooks/multiple-parent-post.yaml create mode 100644 tests/fixtures/config/ansible/git/common-config/playbooks/multiple-parent.yaml diff --git a/doc/source/user/config.rst b/doc/source/user/config.rst index 04de33e014..0bb1c46801 100644 --- a/doc/source/user/config.rst +++ b/doc/source/user/config.rst @@ -835,9 +835,9 @@ Here is an example of two job definitions: .. attr:: run - The name of the main playbook for this job. If it is not - supplied, the parent's playbook will be used (and likewise up - the inheritance chain). The full path within the repo is + The name of a playbook or list of playbooks for this job. If it + is not supplied, the parent's playbook will be used (and likewise + up the inheritance chain). The full path within the repo is required. Example: .. code-block:: yaml diff --git a/releasenotes/notes/job-run-list-7036fbac9c146098.yaml b/releasenotes/notes/job-run-list-7036fbac9c146098.yaml new file mode 100644 index 0000000000..0a4873615d --- /dev/null +++ b/releasenotes/notes/job-run-list-7036fbac9c146098.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + The :attr:`job.run` attribute now supports a single or list of playbooks. diff --git a/tests/fixtures/config/ansible/git/common-config/playbooks/bar.yaml b/tests/fixtures/config/ansible/git/common-config/playbooks/bar.yaml new file mode 100644 index 0000000000..c219e817e1 --- /dev/null +++ b/tests/fixtures/config/ansible/git/common-config/playbooks/bar.yaml @@ -0,0 +1,5 @@ +- hosts: bar + tasks: + - copy: + content: "bar456" + dest: "{{zuul.executor.log_root}}/bar.txt" diff --git a/tests/fixtures/config/ansible/git/common-config/playbooks/first-fail-post.yaml b/tests/fixtures/config/ansible/git/common-config/playbooks/first-fail-post.yaml new file mode 100644 index 0000000000..7e34c50458 --- /dev/null +++ b/tests/fixtures/config/ansible/git/common-config/playbooks/first-fail-post.yaml @@ -0,0 +1,11 @@ +- hosts: all + tasks: + - name: Register bar.txt file. + stat: + path: "{{zuul.executor.log_root}}/bar.txt" + register: bar_st + + - name: Assert bar.txt file. + assert: + that: + - not bar_st.stat.exists diff --git a/tests/fixtures/config/ansible/git/common-config/playbooks/first-fail.yaml b/tests/fixtures/config/ansible/git/common-config/playbooks/first-fail.yaml new file mode 100644 index 0000000000..3ff27a8101 --- /dev/null +++ b/tests/fixtures/config/ansible/git/common-config/playbooks/first-fail.yaml @@ -0,0 +1,4 @@ +- hosts: foo + tasks: + - fail: + msg: "Always fail" diff --git a/tests/fixtures/config/ansible/git/common-config/playbooks/foo.yaml b/tests/fixtures/config/ansible/git/common-config/playbooks/foo.yaml new file mode 100644 index 0000000000..86e22d6239 --- /dev/null +++ b/tests/fixtures/config/ansible/git/common-config/playbooks/foo.yaml @@ -0,0 +1,5 @@ +- hosts: foo + tasks: + - copy: + content: "foo123" + dest: "{{zuul.executor.log_root}}/foo.txt" diff --git a/tests/fixtures/config/ansible/git/common-config/playbooks/foobar-post.yaml b/tests/fixtures/config/ansible/git/common-config/playbooks/foobar-post.yaml new file mode 100644 index 0000000000..379e5d0842 --- /dev/null +++ b/tests/fixtures/config/ansible/git/common-config/playbooks/foobar-post.yaml @@ -0,0 +1,33 @@ +- hosts: all + tasks: + - name: Register parent.txt file. + stat: + path: "{{zuul.executor.log_root}}/parent.txt" + register: parent_st + + - name: Assert parent.txt does not exist. + assert: + that: + - not parent_st.stat.exists + + - name: Register foo.txt file. + stat: + path: "{{zuul.executor.log_root}}/foo.txt" + register: foo_st + + - name: Assert foo.txt exists. + assert: + that: + - foo_st.stat.exists + - foo_st.stat.isreg + + - name: Register bar.txt file. + stat: + path: "{{zuul.executor.log_root}}/bar.txt" + register: bar_st + + - name: Assert bar.txt exists. + assert: + that: + - bar_st.stat.exists + - bar_st.stat.isreg diff --git a/tests/fixtures/config/ansible/git/common-config/playbooks/multiple-parent-post.yaml b/tests/fixtures/config/ansible/git/common-config/playbooks/multiple-parent-post.yaml new file mode 100644 index 0000000000..8060fecd3c --- /dev/null +++ b/tests/fixtures/config/ansible/git/common-config/playbooks/multiple-parent-post.yaml @@ -0,0 +1,11 @@ +- hosts: all + tasks: + - name: Register parent.txt file. + stat: + path: "{{zuul.executor.log_root}}/parent.txt" + register: parent_st + + - name: Assert parent.txt exist. + assert: + that: + - parent_st.stat.exists diff --git a/tests/fixtures/config/ansible/git/common-config/playbooks/multiple-parent.yaml b/tests/fixtures/config/ansible/git/common-config/playbooks/multiple-parent.yaml new file mode 100644 index 0000000000..2d3d3fe621 --- /dev/null +++ b/tests/fixtures/config/ansible/git/common-config/playbooks/multiple-parent.yaml @@ -0,0 +1,5 @@ +- hosts: foo + tasks: + - copy: + content: "parent" + dest: "{{zuul.executor.log_root}}/parent.txt" diff --git a/tests/fixtures/config/ansible/git/common-config/zuul.yaml b/tests/fixtures/config/ansible/git/common-config/zuul.yaml index a7576d632d..eb252fdeb6 100644 --- a/tests/fixtures/config/ansible/git/common-config/zuul.yaml +++ b/tests/fixtures/config/ansible/git/common-config/zuul.yaml @@ -171,6 +171,70 @@ - secret: vartest_secret name: renamed_secret +- job: + name: multiple-parent + run: playbooks/multiple-parent.yaml + nodeset: + nodes: + - name: ubuntu-xenial + label: ubuntu-xenial + groups: + - name: foo + nodes: + - ubuntu-xenial + - name: bar + nodes: + - ubuntu-xenial + +- job: + name: multiple-child + parent: multiple-parent + run: + - playbooks/foo.yaml + - playbooks/bar.yaml + post-run: playbooks/foobar-post.yaml + +- job: + name: multiple-child-no-run + parent: multiple-parent + post-run: playbooks/multiple-parent-post.yaml + +- job: + name: multiple-run + run: + - playbooks/foo.yaml + - playbooks/bar.yaml + post-run: playbooks/foobar-post.yaml + nodeset: + nodes: + - name: ubuntu-xenial + label: ubuntu-xenial + groups: + - name: foo + nodes: + - ubuntu-xenial + - name: bar + nodes: + - ubuntu-xenial + +- job: + name: multiple-run-failure + run: + - playbooks/first-fail.yaml + - playbooks/bar.yaml + post-run: playbooks/first-fail-post.yaml + nodeset: + nodes: + - name: ubuntu-xenial + label: ubuntu-xenial + groups: + - name: foo + nodes: + - ubuntu-xenial + - name: bar + nodes: + - ubuntu-xenial + - job: parent: base-urls name: hello diff --git a/tests/fixtures/config/ansible/git/org_project/.zuul.yaml b/tests/fixtures/config/ansible/git/org_project/.zuul.yaml index a1d0d0281c..2a974c36d5 100644 --- a/tests/fixtures/config/ansible/git/org_project/.zuul.yaml +++ b/tests/fixtures/config/ansible/git/org_project/.zuul.yaml @@ -30,3 +30,7 @@ - post-timeout - hello-world - failpost + - multiple-child + - multiple-child-no-run + - multiple-run + - multiple-run-failure diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py index d6554a08a7..7337b8cf8a 100644 --- a/tests/unit/test_v3.py +++ b/tests/unit/test_v3.py @@ -2312,6 +2312,20 @@ class TestAnsible(AnsibleZuulTestCase): build_add_host = self.getJobFromHistory('add-host') with self.jobLog(build_add_host): self.assertEqual(build_add_host.result, 'SUCCESS') + build_multiple_child = self.getJobFromHistory('multiple-child') + with self.jobLog(build_multiple_child): + self.assertEqual(build_multiple_child.result, 'SUCCESS') + build_multiple_child_no_run = self.getJobFromHistory( + 'multiple-child-no-run') + with self.jobLog(build_multiple_child_no_run): + self.assertEqual(build_multiple_child_no_run.result, 'SUCCESS') + build_multiple_run = self.getJobFromHistory('multiple-run') + with self.jobLog(build_multiple_run): + self.assertEqual(build_multiple_run.result, 'SUCCESS') + build_multiple_run_failure = self.getJobFromHistory( + 'multiple-run-failure') + with self.jobLog(build_multiple_run_failure): + self.assertEqual(build_multiple_run_failure.result, 'FAILURE') build_python27 = self.getJobFromHistory('python27') with self.jobLog(build_python27): self.assertEqual(build_python27.result, 'SUCCESS') diff --git a/zuul/configloader.py b/zuul/configloader.py index 64318e2658..6965821bb0 100644 --- a/zuul/configloader.py +++ b/zuul/configloader.py @@ -564,7 +564,7 @@ class JobParser(object): 'attempts': int, 'pre-run': to_list(str), 'post-run': to_list(str), - 'run': str, + 'run': to_list(str), '_source_context': model.SourceContext, '_start_mark': ZuulMark, 'roles': to_list(role), @@ -717,10 +717,12 @@ class JobParser(object): post_run_name, job.roles, secrets) job.post_run = (post_run,) + job.post_run + if 'run' in conf: - run = model.PlaybookContext(job.source_context, conf['run'], - job.roles, secrets) - job.run = (run,) + for run_name in as_list(conf.get('run')): + run = model.PlaybookContext(job.source_context, run_name, + job.roles, secrets) + job.run = job.run + (run,) for k in self.simple_attributes: a = k.replace('-', '_') diff --git a/zuul/executor/server.py b/zuul/executor/server.py index 219c2342b6..a941c8992f 100644 --- a/zuul/executor/server.py +++ b/zuul/executor/server.py @@ -393,7 +393,6 @@ class JobDir(object): 'setup-inventory.yaml') self.logging_json = os.path.join(self.ansible_root, 'logging.json') self.playbooks = [] # The list of candidate playbooks - self.playbook = None # A pointer to the candidate we have chosen self.pre_playbooks = [] self.post_playbooks = [] self.job_output_file = os.path.join(self.log_root, 'job-output.txt') @@ -1126,26 +1125,30 @@ class AnsibleJob(object): self.cpu_times['children_system'])) if not pre_failed: - ansible_timeout = self.getAnsibleTimeout(time_started, job_timeout) - job_status, job_code = self.runAnsiblePlaybook( - self.jobdir.playbook, ansible_timeout, phase='run') - if job_status == self.RESULT_ABORTED: - return 'ABORTED' - elif job_status == self.RESULT_TIMED_OUT: - # Set the pre-failure flag so this doesn't get - # overridden by a post-failure. - pre_failed = True - result = 'TIMED_OUT' - elif job_status == self.RESULT_NORMAL: - success = (job_code == 0) - if success: - result = 'SUCCESS' + for index, playbook in enumerate(self.jobdir.playbooks): + ansible_timeout = self.getAnsibleTimeout( + time_started, job_timeout) + job_status, job_code = self.runAnsiblePlaybook( + playbook, ansible_timeout, phase='run', index=index) + if job_status == self.RESULT_ABORTED: + return 'ABORTED' + elif job_status == self.RESULT_TIMED_OUT: + # Set the pre-failure flag so this doesn't get + # overridden by a post-failure. + pre_failed = True + result = 'TIMED_OUT' + break + elif job_status == self.RESULT_NORMAL: + success = (job_code == 0) + if success: + result = 'SUCCESS' + else: + result = 'FAILURE' + break else: - result = 'FAILURE' - else: - # The result of the job is indeterminate. Zuul will - # run it again. - return None + # The result of the job is indeterminate. Zuul will + # run it again. + return None # check if we need to pause here result_data = self.getResultData() @@ -1343,14 +1346,15 @@ class AnsibleJob(object): jobdir_playbook = self.jobdir.addPrePlaybook() self.preparePlaybook(jobdir_playbook, playbook, args) + job_playbook = None for playbook in args['playbooks']: jobdir_playbook = self.jobdir.addPlaybook() self.preparePlaybook(jobdir_playbook, playbook, args) if jobdir_playbook.path is not None: - self.jobdir.playbook = jobdir_playbook - break + if job_playbook is None: + job_playbook = jobdir_playbook - if self.jobdir.playbook is None: + if job_playbook is None: raise ExecutorError("No playbook specified") for playbook in args['post_playbooks']: