Add test for running post playbooks after pre-playbooks fail

Also fix an error this uncovered.

Change-Id: I5bfbe6c64d42bc3e271e3dda2a9c633430238216

tests/fixtures/config/pre-playbook/git/common-config/playbooks/post.yaml

@@ -0,0 +1,5 @@
+- hosts: all
+  tasks:
+    - file:
+        path: "{{zuul._test.test_root}}/{{zuul.uuid}}.post.flag"
+        state: touch

tests/fixtures/config/pre-playbook/git/common-config/playbooks/pre.yaml

@@ -0,0 +1,8 @@
+- hosts: all
+  tasks:
+    - copy:
+        src: "{{zuul._test.test_root}}/{{zuul.uuid}}.flag"
+        dest: "{{zuul._test.test_root}}/{{zuul.uuid}}.failed"
+    - file:
+        path: "{{zuul._test.test_root}}/{{zuul.uuid}}.pre.flag"
+        state: touch

tests/fixtures/config/pre-playbook/git/common-config/playbooks/python27.yaml

@@ -0,0 +1,5 @@
+- hosts: all
+  tasks:
+    - file:
+        path: "{{zuul._test.test_root}}/{{zuul.uuid}}.main.flag"
+        state: touch

tests/fixtures/config/pre-playbook/git/common-config/zuul.yaml

@@ -0,0 +1,18 @@
+- pipeline:
+    name: check
+    manager: independent
+    allow-secrets: true
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        verified: 1
+    failure:
+      gerrit:
+        verified: -1
+
+- job:
+    name: python27
+    pre-run: playbooks/pre
+    post-run: playbooks/post

tests/fixtures/config/pre-playbook/git/org_project/.zuul.yaml

@@ -0,0 +1,5 @@
+- project:
+    name: org/project
+    check:
+      jobs:
+        - python27

tests/fixtures/config/pre-playbook/git/org_project/README

@@ -0,0 +1 @@
+test

tests/fixtures/config/pre-playbook/main.yaml

@@ -0,0 +1,9 @@
+- tenant:
+    name: tenant-one
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - org/project
+

tests/unit/test_v3.py

@@ -465,6 +465,31 @@ class TestAnsible(AnsibleZuulTestCase):
             self.assertEqual(f.read(), "test-username test-password")
 
 
+class TestPrePlaybooks(AnsibleZuulTestCase):
+    # A temporary class to hold new tests while others are disabled
+
+    tenant_config_file = 'config/pre-playbook/main.yaml'
+
+    def test_pre_playbook_fail(self):
+        # Test that we run the post playbooks (but not the actual
+        # playbook) when a pre-playbook fails.
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        build = self.getJobFromHistory('python27')
+        self.assertIsNone(build.result)
+        self.assertIn('RETRY_LIMIT', A.messages[0])
+        flag_path = os.path.join(self.test_root, build.uuid +
+                                 '.main.flag')
+        self.assertFalse(os.path.exists(flag_path))
+        pre_flag_path = os.path.join(self.test_root, build.uuid +
+                                     '.pre.flag')
+        self.assertFalse(os.path.exists(pre_flag_path))
+        post_flag_path = os.path.join(self.test_root, build.uuid +
+                                      '.post.flag')
+        self.assertTrue(os.path.exists(post_flag_path))
+
+
 class TestBrokenConfig(ZuulTestCase):
     # Test that we get an appropriate syntax error if we start with a
     # broken config.

zuul/executor/server.py

@@ -891,6 +891,7 @@ class AnsibleJob(object):
                 # These should really never fail, so return None and have
                 # zuul try again
                 pre_failed = True
+                success = False
                 break
 
         if not pre_failed: