Add test for running post playbooks after pre-playbooks fail

Also fix an error this uncovered.

Change-Id: I5bfbe6c64d42bc3e271e3dda2a9c633430238216
diff --git a/tests/fixtures/config/pre-playbook/git/common-config/playbooks/post.yaml b/tests/fixtures/config/pre-playbook/git/common-config/playbooks/post.yaml
new file mode 100644
index 0000000..2e512b1
--- /dev/null
+++ b/tests/fixtures/config/pre-playbook/git/common-config/playbooks/post.yaml
@@ -0,0 +1,5 @@
+- hosts: all
+  tasks:
+    - file:
+        path: "{{zuul._test.test_root}}/{{zuul.uuid}}.post.flag"
+        state: touch
diff --git a/tests/fixtures/config/pre-playbook/git/common-config/playbooks/pre.yaml b/tests/fixtures/config/pre-playbook/git/common-config/playbooks/pre.yaml
new file mode 100644
index 0000000..13c2208
--- /dev/null
+++ b/tests/fixtures/config/pre-playbook/git/common-config/playbooks/pre.yaml
@@ -0,0 +1,8 @@
+- hosts: all
+  tasks:
+    - copy:
+        src: "{{zuul._test.test_root}}/{{zuul.uuid}}.flag"
+        dest: "{{zuul._test.test_root}}/{{zuul.uuid}}.failed"
+    - file:
+        path: "{{zuul._test.test_root}}/{{zuul.uuid}}.pre.flag"
+        state: touch
diff --git a/tests/fixtures/config/pre-playbook/git/common-config/playbooks/python27.yaml b/tests/fixtures/config/pre-playbook/git/common-config/playbooks/python27.yaml
new file mode 100644
index 0000000..dbb64a5
--- /dev/null
+++ b/tests/fixtures/config/pre-playbook/git/common-config/playbooks/python27.yaml
@@ -0,0 +1,5 @@
+- hosts: all
+  tasks:
+    - file:
+        path: "{{zuul._test.test_root}}/{{zuul.uuid}}.main.flag"
+        state: touch
diff --git a/tests/fixtures/config/pre-playbook/git/common-config/zuul.yaml b/tests/fixtures/config/pre-playbook/git/common-config/zuul.yaml
new file mode 100644
index 0000000..3de0d6d
--- /dev/null
+++ b/tests/fixtures/config/pre-playbook/git/common-config/zuul.yaml
@@ -0,0 +1,18 @@
+- pipeline:
+    name: check
+    manager: independent
+    allow-secrets: true
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        verified: 1
+    failure:
+      gerrit:
+        verified: -1
+
+- job:
+    name: python27
+    pre-run: playbooks/pre
+    post-run: playbooks/post
diff --git a/tests/fixtures/config/pre-playbook/git/org_project/.zuul.yaml b/tests/fixtures/config/pre-playbook/git/org_project/.zuul.yaml
new file mode 100644
index 0000000..89a5674
--- /dev/null
+++ b/tests/fixtures/config/pre-playbook/git/org_project/.zuul.yaml
@@ -0,0 +1,5 @@
+- project:
+    name: org/project
+    check:
+      jobs:
+        - python27
diff --git a/tests/fixtures/config/pre-playbook/git/org_project/README b/tests/fixtures/config/pre-playbook/git/org_project/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/pre-playbook/git/org_project/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/pre-playbook/main.yaml b/tests/fixtures/config/pre-playbook/main.yaml
new file mode 100644
index 0000000..6033879
--- /dev/null
+++ b/tests/fixtures/config/pre-playbook/main.yaml
@@ -0,0 +1,9 @@
+- tenant:
+    name: tenant-one
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - org/project
+
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index 7d84b1f..282331e 100644
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -465,6 +465,31 @@
             self.assertEqual(f.read(), "test-username test-password")
 
 
+class TestPrePlaybooks(AnsibleZuulTestCase):
+    # A temporary class to hold new tests while others are disabled
+
+    tenant_config_file = 'config/pre-playbook/main.yaml'
+
+    def test_pre_playbook_fail(self):
+        # Test that we run the post playbooks (but not the actual
+        # playbook) when a pre-playbook fails.
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        build = self.getJobFromHistory('python27')
+        self.assertIsNone(build.result)
+        self.assertIn('RETRY_LIMIT', A.messages[0])
+        flag_path = os.path.join(self.test_root, build.uuid +
+                                 '.main.flag')
+        self.assertFalse(os.path.exists(flag_path))
+        pre_flag_path = os.path.join(self.test_root, build.uuid +
+                                     '.pre.flag')
+        self.assertFalse(os.path.exists(pre_flag_path))
+        post_flag_path = os.path.join(self.test_root, build.uuid +
+                                      '.post.flag')
+        self.assertTrue(os.path.exists(post_flag_path))
+
+
 class TestBrokenConfig(ZuulTestCase):
     # Test that we get an appropriate syntax error if we start with a
     # broken config.