Merge "Update test fixtures to use explicit run" into feature/zuulv3
diff --git a/tests/fixtures/config/ansible/git/common-config/zuul.yaml b/tests/fixtures/config/ansible/git/common-config/zuul.yaml
index 67d1c70..28bfce1 100644
--- a/tests/fixtures/config/ansible/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/ansible/git/common-config/zuul.yaml
@@ -79,10 +79,11 @@
     failure-url: https://failure.example.com/zuul-logs/{build.uuid}/
 
 - job:
-    parent: base-urls
     name: python27
-    pre-run: playbooks/pre
-    post-run: playbooks/post
+    parent: base-urls
+    run: playbooks/python27.yaml
+    pre-run: playbooks/pre.yaml
+    post-run: playbooks/post.yaml
     vars:
       flagpath: '{{zuul._test.test_root}}/{{zuul.build}}.flag'
     roles:
@@ -93,11 +94,13 @@
 - job:
     parent: python27
     name: timeout
+    run: playbooks/timeout.yaml
     timeout: 1
 
 - job:
     parent: python27
     name: check-vars
+    run: playbooks/check-vars.yaml
     nodeset:
       nodes:
         - name: ubuntu-xenial
@@ -113,6 +116,7 @@
 - job:
     parent: python27
     name: check-secret-names
+    run: playbooks/check-secret-names.yaml
     nodeset:
       nodes:
         - name: ubuntu-xenial
@@ -124,9 +128,11 @@
 - job:
     parent: base-urls
     name: hello
+    run: playbooks/hello-post.yaml
     post-run: playbooks/hello-post
 
 - job:
     parent: python27
     name: failpost
+    run: playbooks/post-broken.yaml
     post-run: playbooks/post-broken
diff --git a/tests/fixtures/config/ansible/git/org_project/.zuul.yaml b/tests/fixtures/config/ansible/git/org_project/.zuul.yaml
index e144325..447f6cd 100644
--- a/tests/fixtures/config/ansible/git/org_project/.zuul.yaml
+++ b/tests/fixtures/config/ansible/git/org_project/.zuul.yaml
@@ -1,10 +1,12 @@
 - job:
     parent: python27
     name: faillocal
+    run: playbooks/faillocal.yaml
 
 - job:
     parent: hello
     name: hello-world
+    run: playbooks/hello-world.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/base-jobs/git/org_project/.zuul.yaml b/tests/fixtures/config/base-jobs/git/org_project/.zuul.yaml
index 9844c14..a7e3bdb 100644
--- a/tests/fixtures/config/base-jobs/git/org_project/.zuul.yaml
+++ b/tests/fixtures/config/base-jobs/git/org_project/.zuul.yaml
@@ -1,9 +1,11 @@
 - job:
     name: my-job
+    run: playbooks/my-job.yaml
 
 - job:
     name: other-job
     parent: other-base
+    run: playbooks/other-job.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/data-return/git/common-config/zuul.yaml b/tests/fixtures/config/data-return/git/common-config/zuul.yaml
index 2d5c51f..97b6b28 100644
--- a/tests/fixtures/config/data-return/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/data-return/git/common-config/zuul.yaml
@@ -18,13 +18,16 @@
 
 - job:
     name: data-return
+    run: playbooks/data-return.yaml
 
 - job:
     name: data-return-relative
     success-url: docs/index.html
+    run: playbooks/data-return-relative.yaml
 
 - job:
     name: child
+    run: playbooks/child.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/dependency-graph/git/common-config/zuul.yaml b/tests/fixtures/config/dependency-graph/git/common-config/zuul.yaml
index 4179226..6a96b50 100644
--- a/tests/fixtures/config/dependency-graph/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/dependency-graph/git/common-config/zuul.yaml
@@ -25,24 +25,31 @@
 
 - job:
     name: A
+    run: playbooks/A.yaml
 
 - job:
     name: B
+    run: playbooks/B.yaml
 
 - job:
     name: C
+    run: playbooks/C.yaml
 
 - job:
     name: D
+    run: playbooks/D.yaml
 
 - job:
     name: E
+    run: playbooks/E.yaml
 
 - job:
     name: F
+    run: playbooks/F.yaml
 
 - job:
     name: G
+    run: playbooks/G.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/disk-accountant/git/common-config/zuul.yaml b/tests/fixtures/config/disk-accountant/git/common-config/zuul.yaml
index 893ea05..9ad8de5 100644
--- a/tests/fixtures/config/disk-accountant/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/disk-accountant/git/common-config/zuul.yaml
@@ -18,6 +18,7 @@
 
 - job:
     name: dd-big-empty-file
+    run: playbooks/dd-big-empty-file.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/duplicate-pipeline/git/common-config/zuul.yaml b/tests/fixtures/config/duplicate-pipeline/git/common-config/zuul.yaml
index 117e381..dbd63c5 100755
--- a/tests/fixtures/config/duplicate-pipeline/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/duplicate-pipeline/git/common-config/zuul.yaml
@@ -32,6 +32,7 @@
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/final/git/common-config/zuul.yaml b/tests/fixtures/config/final/git/common-config/zuul.yaml
index f08d66e..944626c 100644
--- a/tests/fixtures/config/final/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/final/git/common-config/zuul.yaml
@@ -20,9 +20,9 @@
     final: true
     vars:
       dont_override_this: dummy
+    run: playbooks/job-final.yaml
 
 - project:
     name: org/project
     check:
       jobs: []
-
diff --git a/tests/fixtures/config/git-driver/git/common-config/zuul.yaml b/tests/fixtures/config/git-driver/git/common-config/zuul.yaml
index 34d1136..784b5f2 100644
--- a/tests/fixtures/config/git-driver/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/git-driver/git/common-config/zuul.yaml
@@ -17,6 +17,7 @@
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/implicit-roles/git/org_norole-project/.zuul.yaml b/tests/fixtures/config/implicit-roles/git/org_norole-project/.zuul.yaml
index 74c8e8e..f66c616 100644
--- a/tests/fixtures/config/implicit-roles/git/org_norole-project/.zuul.yaml
+++ b/tests/fixtures/config/implicit-roles/git/org_norole-project/.zuul.yaml
@@ -1,11 +1,13 @@
 - job:
     name: implicit-role-fail
+    run: playbooks/implicit-role-fail.yaml
 
 - job:
     name: explicit-role-fail
     attempts: 1
     roles:
       - zuul: org/norole-project
+    run: playbooks/explicit-role-fail.yaml
 
 - project:
     name: org/norole-project
diff --git a/tests/fixtures/config/implicit-roles/git/org_role-project/.zuul.yaml b/tests/fixtures/config/implicit-roles/git/org_role-project/.zuul.yaml
index 42cae95..e6e902e 100644
--- a/tests/fixtures/config/implicit-roles/git/org_role-project/.zuul.yaml
+++ b/tests/fixtures/config/implicit-roles/git/org_role-project/.zuul.yaml
@@ -1,11 +1,13 @@
 - job:
     name: implicit-role-ok
+    run: playbooks/implicit-role-ok.yaml
 
 - job:
     name: explicit-role-ok
     roles:
       - zuul: org/role-project
         name: role-name
+    run: playbooks/explicit-role-ok.yaml
 
 - project:
     name: org/role-project
diff --git a/tests/fixtures/config/in-repo-join/git/common-config/zuul.yaml b/tests/fixtures/config/in-repo-join/git/common-config/zuul.yaml
index 561fc39..a8ee256 100644
--- a/tests/fixtures/config/in-repo-join/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/in-repo-join/git/common-config/zuul.yaml
@@ -38,6 +38,7 @@
 
 - job:
     name: common-config-test
+    run: playbooks/common-config-test.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/in-repo-join/git/org_project/.zuul.yaml b/tests/fixtures/config/in-repo-join/git/org_project/.zuul.yaml
index 280342c..3845b26 100644
--- a/tests/fixtures/config/in-repo-join/git/org_project/.zuul.yaml
+++ b/tests/fixtures/config/in-repo-join/git/org_project/.zuul.yaml
@@ -1,2 +1,3 @@
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
diff --git a/tests/fixtures/config/in-repo/git/common-config/zuul.yaml b/tests/fixtures/config/in-repo/git/common-config/zuul.yaml
index a97af51..c98651c 100644
--- a/tests/fixtures/config/in-repo/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/in-repo/git/common-config/zuul.yaml
@@ -62,10 +62,9 @@
         approval:
           - Code-Review: 2
             username: maintainer
-    require:
       github:
         review:
-          - username: '^(herp|derp)$'
+          - username: ^(herp|derp)$
             type: approved
     trigger: {}
 
@@ -75,9 +74,11 @@
 
 - job:
     name: common-config-test
+    run: playbooks/common-config-test.yaml
 
 - job:
     name: template-job
+    run: playbooks/template-job.yaml
 
 - project-template:
     name: common-config-template
diff --git a/tests/fixtures/config/in-repo/git/org_project/.zuul.yaml b/tests/fixtures/config/in-repo/git/org_project/.zuul.yaml
index e1c27bb..2c39a10 100644
--- a/tests/fixtures/config/in-repo/git/org_project/.zuul.yaml
+++ b/tests/fixtures/config/in-repo/git/org_project/.zuul.yaml
@@ -1,5 +1,6 @@
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/inventory/git/common-config/zuul.yaml b/tests/fixtures/config/inventory/git/common-config/zuul.yaml
index e5727a2..900abd6 100644
--- a/tests/fixtures/config/inventory/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/inventory/git/common-config/zuul.yaml
@@ -41,7 +41,9 @@
       nodes:
         - name: ubuntu-xenial
           label: ubuntu-xenial
+    run: playbooks/single-inventory.yaml
 
 - job:
     name: group-inventory
     nodeset: nodeset1
+    run: playbooks/group-inventory.yaml
diff --git a/tests/fixtures/config/job-output/git/common-config/zuul.yaml b/tests/fixtures/config/job-output/git/common-config/zuul.yaml
index f182d8d..4df0020 100644
--- a/tests/fixtures/config/job-output/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/job-output/git/common-config/zuul.yaml
@@ -19,6 +19,7 @@
 - job:
     parent: base
     name: job-output
+    run: playbooks/job-output.yaml
 
 - job:
     name: job-output-failure
diff --git a/tests/fixtures/config/merges/git/common-config/zuul.yaml b/tests/fixtures/config/merges/git/common-config/zuul.yaml
index 1ea5048..94dbca7 100644
--- a/tests/fixtures/config/merges/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/merges/git/common-config/zuul.yaml
@@ -38,13 +38,16 @@
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - job:
     name: project-merge
     hold-following-changes: true
+    run: playbooks/project-merge.yaml
 
 - project:
     name: org/project-merge
diff --git a/tests/fixtures/config/multi-driver/git/org_common-config/zuul.yaml b/tests/fixtures/config/multi-driver/git/org_common-config/zuul.yaml
index 7a5c190..94a9ecb 100644
--- a/tests/fixtures/config/multi-driver/git/org_common-config/zuul.yaml
+++ b/tests/fixtures/config/multi-driver/git/org_common-config/zuul.yaml
@@ -12,12 +12,12 @@
         - event: patchset-created
     success:
       github:
-        status: 'success'
+        status: success
       gerrit:
         Verified: 1
     failure:
       github:
-        status: 'failure'
+        status: failure
       gerrit:
         Verified: 1
     start:
@@ -32,9 +32,11 @@
 
 - job:
     name: project-gerrit
+    run: playbooks/project-gerrit.yaml
 
 - job:
     name: project1-github
+    run: playbooks/project1-github.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/multi-tenant-semaphore/git/tenant-one-config/zuul.yaml b/tests/fixtures/config/multi-tenant-semaphore/git/tenant-one-config/zuul.yaml
index 5e377e7..ca936b9 100644
--- a/tests/fixtures/config/multi-tenant-semaphore/git/tenant-one-config/zuul.yaml
+++ b/tests/fixtures/config/multi-tenant-semaphore/git/tenant-one-config/zuul.yaml
@@ -1,6 +1,7 @@
 - job:
     name: project1-test1
     semaphore: test-semaphore
+    run: playbooks/project1-test1.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/config/multi-tenant-semaphore/git/tenant-two-config/zuul.yaml b/tests/fixtures/config/multi-tenant-semaphore/git/tenant-two-config/zuul.yaml
index a310532..a2866b3 100644
--- a/tests/fixtures/config/multi-tenant-semaphore/git/tenant-two-config/zuul.yaml
+++ b/tests/fixtures/config/multi-tenant-semaphore/git/tenant-two-config/zuul.yaml
@@ -1,6 +1,7 @@
 - job:
     name: project2-test1
     semaphore: test-semaphore
+    run: playbooks/project2-test1.yaml
 
 - project:
     name: org/project2
diff --git a/tests/fixtures/config/multi-tenant/git/common-config/zuul.yaml b/tests/fixtures/config/multi-tenant/git/common-config/zuul.yaml
index 273469c..31f1e27 100644
--- a/tests/fixtures/config/multi-tenant/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/multi-tenant/git/common-config/zuul.yaml
@@ -21,3 +21,4 @@
       nodes:
         - name: controller
           label: ubuntu-trusty
+    run: playbooks/python27.yaml
diff --git a/tests/fixtures/config/multi-tenant/git/tenant-one-config/zuul.yaml b/tests/fixtures/config/multi-tenant/git/tenant-one-config/zuul.yaml
index 9a1b928..278c12c 100644
--- a/tests/fixtures/config/multi-tenant/git/tenant-one-config/zuul.yaml
+++ b/tests/fixtures/config/multi-tenant/git/tenant-one-config/zuul.yaml
@@ -27,6 +27,7 @@
 
 - job:
     name: project1-test1
+    run: playbooks/project1-test1.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/config/multi-tenant/git/tenant-two-config/zuul.yaml b/tests/fixtures/config/multi-tenant/git/tenant-two-config/zuul.yaml
index 9496a49..2b795b0 100644
--- a/tests/fixtures/config/multi-tenant/git/tenant-two-config/zuul.yaml
+++ b/tests/fixtures/config/multi-tenant/git/tenant-two-config/zuul.yaml
@@ -27,6 +27,7 @@
 
 - job:
     name: project2-test1
+    run: playbooks/project2-test1.yaml
 
 - project:
     name: org/project2
diff --git a/tests/fixtures/config/openstack/git/project-config/zuul.yaml b/tests/fixtures/config/openstack/git/project-config/zuul.yaml
index de6321d..93bdb11 100644
--- a/tests/fixtures/config/openstack/git/project-config/zuul.yaml
+++ b/tests/fixtures/config/openstack/git/project-config/zuul.yaml
@@ -41,10 +41,12 @@
       nodes:
         - name: controller
           label: ubuntu-xenial
+    run: playbooks/base.yaml
 
 - job:
     name: python27
     parent: base
+    run: playbooks/python27.yaml
 
 - job:
     name: python27
@@ -54,10 +56,12 @@
       nodes:
         - name: controller
           label: ubuntu-trusty
+    run: playbooks/python27.yaml
 
 - job:
     name: python35
     parent: base
+    run: playbooks/python35.yaml
 
 - project-template:
     name: python-jobs
@@ -72,6 +76,7 @@
     required-projects:
       - openstack/keystone
       - openstack/nova
+    run: playbooks/dsvm.yaml
 
 - project:
     name: openstack/nova
diff --git a/tests/fixtures/config/post-playbook/git/common-config/zuul.yaml b/tests/fixtures/config/post-playbook/git/common-config/zuul.yaml
index 92a5515..16d7dee 100644
--- a/tests/fixtures/config/post-playbook/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/post-playbook/git/common-config/zuul.yaml
@@ -22,3 +22,4 @@
     post-run: playbooks/post
     vars:
       waitpath: '{{zuul._test.test_root}}/{{zuul.build}}/test_wait'
+    run: playbooks/python27.yaml
diff --git a/tests/fixtures/config/pre-playbook/git/common-config/zuul.yaml b/tests/fixtures/config/pre-playbook/git/common-config/zuul.yaml
index 16d1966..7817745 100644
--- a/tests/fixtures/config/pre-playbook/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/pre-playbook/git/common-config/zuul.yaml
@@ -20,3 +20,4 @@
     name: python27
     pre-run: playbooks/pre
     post-run: playbooks/post
+    run: playbooks/python27.yaml
diff --git a/tests/fixtures/config/push-reqs/git/org_common-config/zuul.yaml b/tests/fixtures/config/push-reqs/git/org_common-config/zuul.yaml
index 63af1c9..e7d52d9 100644
--- a/tests/fixtures/config/push-reqs/git/org_common-config/zuul.yaml
+++ b/tests/fixtures/config/push-reqs/git/org_common-config/zuul.yaml
@@ -47,7 +47,7 @@
     manager: independent
     require:
       github:
-        status: 'zuul:check:success'
+        status: zuul:check:success
     trigger:
       github:
         - event: push
@@ -82,6 +82,7 @@
 
 - job:
     name: job1
+    run: playbooks/job1.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/config/requirements/email/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/email/git/common-config/zuul.yaml
index 90c9ac2..e41cfad 100644
--- a/tests/fixtures/config/requirements/email/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/requirements/email/git/common-config/zuul.yaml
@@ -36,9 +36,11 @@
 
 - job:
     name: project1-job
+    run: playbooks/project1-job.yaml
 
 - job:
     name: project2-job
+    run: playbooks/project2-job.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/config/requirements/newer-than/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/newer-than/git/common-config/zuul.yaml
index 5f266a4..96e04d9 100644
--- a/tests/fixtures/config/requirements/newer-than/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/requirements/newer-than/git/common-config/zuul.yaml
@@ -38,9 +38,11 @@
 
 - job:
     name: project1-job
+    run: playbooks/project1-job.yaml
 
 - job:
     name: project2-job
+    run: playbooks/project2-job.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/config/requirements/older-than/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/older-than/git/common-config/zuul.yaml
index 4287a94..182a036 100644
--- a/tests/fixtures/config/requirements/older-than/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/requirements/older-than/git/common-config/zuul.yaml
@@ -38,9 +38,11 @@
 
 - job:
     name: project1-job
+    run: playbooks/project1-job.yaml
 
 - job:
     name: project2-job
+    run: playbooks/project2-job.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/config/requirements/reject-username/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/reject-username/git/common-config/zuul.yaml
index aabfb6a..7cbba03 100644
--- a/tests/fixtures/config/requirements/reject-username/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/requirements/reject-username/git/common-config/zuul.yaml
@@ -36,9 +36,11 @@
 
 - job:
     name: project1-job
+    run: playbooks/project1-job.yaml
 
 - job:
     name: project2-job
+    run: playbooks/project2-job.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/config/requirements/reject/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/reject/git/common-config/zuul.yaml
index 2661eed..0ce66de 100644
--- a/tests/fixtures/config/requirements/reject/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/requirements/reject/git/common-config/zuul.yaml
@@ -52,9 +52,11 @@
 
 - job:
     name: project1-job
+    run: playbooks/project1-job.yaml
 
 - job:
     name: project2-job
+    run: playbooks/project2-job.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/config/requirements/state/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/state/git/common-config/zuul.yaml
index 715b89f..9f018c9 100644
--- a/tests/fixtures/config/requirements/state/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/requirements/state/git/common-config/zuul.yaml
@@ -55,6 +55,7 @@
 
 - job:
     name: project-job
+    run: playbooks/project-job.yaml
 
 - project:
     name: current-project
diff --git a/tests/fixtures/config/requirements/username/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/username/git/common-config/zuul.yaml
index 778ac16..05203ab 100644
--- a/tests/fixtures/config/requirements/username/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/requirements/username/git/common-config/zuul.yaml
@@ -36,9 +36,11 @@
 
 - job:
     name: project1-job
+    run: playbooks/project1-job.yaml
 
 - job:
     name: project2-job
+    run: playbooks/project2-job.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/config/requirements/vote1/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/vote1/git/common-config/zuul.yaml
index b5d7498..88f64e3 100644
--- a/tests/fixtures/config/requirements/vote1/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/requirements/vote1/git/common-config/zuul.yaml
@@ -38,9 +38,11 @@
 
 - job:
     name: project1-job
+    run: playbooks/project1-job.yaml
 
 - job:
     name: project2-job
+    run: playbooks/project2-job.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/config/requirements/vote2/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/vote2/git/common-config/zuul.yaml
index 3f41868..31b2329 100644
--- a/tests/fixtures/config/requirements/vote2/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/requirements/vote2/git/common-config/zuul.yaml
@@ -42,9 +42,11 @@
 
 - job:
     name: project1-job
+    run: playbooks/project1-job.yaml
 
 - job:
     name: project2-job
+    run: playbooks/project2-job.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/config/roles/git/common-config/zuul.yaml b/tests/fixtures/config/roles/git/common-config/zuul.yaml
index 7ae6263..ba34ad6 100644
--- a/tests/fixtures/config/roles/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/roles/git/common-config/zuul.yaml
@@ -38,6 +38,7 @@
 
 - job:
     name: common-config-test
+    run: playbooks/common-config-test.yaml
 
 - project:
     name: common-config
diff --git a/tests/fixtures/config/roles/git/org_project/.zuul.yaml b/tests/fixtures/config/roles/git/org_project/.zuul.yaml
index 35c2153..0986b82 100644
--- a/tests/fixtures/config/roles/git/org_project/.zuul.yaml
+++ b/tests/fixtures/config/roles/git/org_project/.zuul.yaml
@@ -2,6 +2,7 @@
     name: project-test
     roles:
       - zuul: bare-role
+    run: playbooks/project-test.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/secret-inheritance/git/common-config/zuul.yaml b/tests/fixtures/config/secret-inheritance/git/common-config/zuul.yaml
index d5fa2bc..ad16d4e 100644
--- a/tests/fixtures/config/secret-inheritance/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/secret-inheritance/git/common-config/zuul.yaml
@@ -38,15 +38,18 @@
 
 - job:
     name: trusted-secrets
+    run: playbooks/trusted-secrets.yaml
     secrets:
       - trusted-secret
 
 - job:
     name: trusted-secrets-trusted-child
+    run: playbooks/trusted-secrets-trusted-child.yaml
     parent: trusted-secrets
 
 - job:
     name: untrusted-secrets-trusted-child
+    run: playbooks/untrusted-secrets-trusted-child.yaml
     parent: untrusted-secrets
     
 - project:
diff --git a/tests/fixtures/config/secret-inheritance/git/org_project/.zuul.yaml b/tests/fixtures/config/secret-inheritance/git/org_project/.zuul.yaml
index 5eeced2..b384669 100644
--- a/tests/fixtures/config/secret-inheritance/git/org_project/.zuul.yaml
+++ b/tests/fixtures/config/secret-inheritance/git/org_project/.zuul.yaml
@@ -1,14 +1,17 @@
 - job:
     name: untrusted-secrets
+    run: playbooks/untrusted-secrets.yaml
     secrets:
       - untrusted-secret
 
 - job:
     name: trusted-secrets-untrusted-child
+    run: playbooks/trusted-secrets-untrusted-child.yaml
     parent: trusted-secrets
 
 - job:
     name: untrusted-secrets-untrusted-child
+    run: playbooks/untrusted-secrets-untrusted-child.yaml
     parent: untrusted-secrets
 
 - project:
diff --git a/tests/fixtures/config/secret-leaks/git/common-config/zuul.yaml b/tests/fixtures/config/secret-leaks/git/common-config/zuul.yaml
index 4ab198f..9321df8 100644
--- a/tests/fixtures/config/secret-leaks/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/secret-leaks/git/common-config/zuul.yaml
@@ -55,12 +55,14 @@
 - job:
     parent: base
     name: secret-file
+    run: playbooks/secret-file.yaml
     secrets:
       - test_secret
 
 - job:
     parent: base
     name: secret-file-fail
+    run: playbooks/secret-file-fail.yaml
     secrets:
       - test_secret
 
diff --git a/tests/fixtures/config/semaphore/git/common-config/zuul.yaml b/tests/fixtures/config/semaphore/git/common-config/zuul.yaml
index c8bd322..52a0e7d 100644
--- a/tests/fixtures/config/semaphore/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/semaphore/git/common-config/zuul.yaml
@@ -25,22 +25,27 @@
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: semaphore-one-test1
     semaphore: test-semaphore
+    run: playbooks/semaphore-one-test1.yaml
 
 - job:
     name: semaphore-one-test2
     semaphore: test-semaphore
+    run: playbooks/semaphore-one-test2.yaml
 
 - job:
     name: semaphore-two-test1
     semaphore: test-semaphore-two
+    run: playbooks/semaphore-two-test1.yaml
 
 - job:
     name: semaphore-two-test2
     semaphore: test-semaphore-two
+    run: playbooks/semaphore-two-test2.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/shadow/git/local-config/zuul.yaml b/tests/fixtures/config/shadow/git/local-config/zuul.yaml
index 87f46b7..8935d8a 100644
--- a/tests/fixtures/config/shadow/git/local-config/zuul.yaml
+++ b/tests/fixtures/config/shadow/git/local-config/zuul.yaml
@@ -14,9 +14,11 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: test2
+    run: playbooks/test2.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/shadow/git/stdlib/.zuul.yaml b/tests/fixtures/config/shadow/git/stdlib/.zuul.yaml
index 6a6f9c9..5132653 100644
--- a/tests/fixtures/config/shadow/git/stdlib/.zuul.yaml
+++ b/tests/fixtures/config/shadow/git/stdlib/.zuul.yaml
@@ -1,10 +1,13 @@
 - job:
     name: base
+    run: playbooks/base.yaml
 
 - job:
     name: test1
     parent: base
+    run: playbooks/test1.yaml
 
 - job:
     name: test2
     parent: base
+    run: playbooks/test2.yaml
diff --git a/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml b/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
index 2160ef9..b2f15f9 100644
--- a/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
@@ -52,6 +52,7 @@
       nodes:
         - name: controller
           label: label1
+    run: playbooks/project-merge.yaml
 
 - job:
     name: project-test1
@@ -60,6 +61,7 @@
       nodes:
         - name: controller
           label: label1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test1
@@ -68,6 +70,7 @@
       nodes:
         - name: controller
           label: label2
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-post
@@ -75,6 +78,7 @@
       nodes:
         - name: static
           label: ubuntu-xenial
+    run: playbooks/project-post.yaml
 
 - job:
     name: project-test2
@@ -82,6 +86,7 @@
       nodes:
         - name: controller
           label: label1
+    run: playbooks/project-test2.yaml
 
 - job:
     name: project1-project2-integration
@@ -89,11 +94,13 @@
       nodes:
         - name: controller
           label: label1
+    run: playbooks/project1-project2-integration.yaml
 
 - job:
     name: project-testfile
     files:
       - .*-requires
+    run: playbooks/project-testfile.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/split-config/git/common-config/zuul.d/jobs.yaml b/tests/fixtures/config/split-config/git/common-config/zuul.d/jobs.yaml
index 9d15599..12e1c24 100644
--- a/tests/fixtures/config/split-config/git/common-config/zuul.d/jobs.yaml
+++ b/tests/fixtures/config/split-config/git/common-config/zuul.d/jobs.yaml
@@ -4,3 +4,4 @@
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
diff --git a/tests/fixtures/config/split-config/git/org_project1/.zuul.d/jobs.yaml b/tests/fixtures/config/split-config/git/org_project1/.zuul.d/jobs.yaml
index 33d74f3..20cd16a 100644
--- a/tests/fixtures/config/split-config/git/org_project1/.zuul.d/jobs.yaml
+++ b/tests/fixtures/config/split-config/git/org_project1/.zuul.d/jobs.yaml
@@ -1,2 +1,3 @@
 - job:
     name: project1-project2-integration
+    run: playbooks/project1-project2-integration.yaml
diff --git a/tests/fixtures/config/sql-driver/git/common-config/zuul.yaml b/tests/fixtures/config/sql-driver/git/common-config/zuul.yaml
index 8fce9e7..82c85c7 100644
--- a/tests/fixtures/config/sql-driver/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/sql-driver/git/common-config/zuul.yaml
@@ -7,12 +7,12 @@
     success:
       gerrit:
         Verified: 1
-      resultsdb:
+      resultsdb: null
     failure:
       gerrit:
         Verified: -1
-      resultsdb:
-      resultsdb_failures:
+      resultsdb: null
+      resultsdb_failures: null
 
 - job:
     name: base
@@ -20,15 +20,19 @@
 
 - job:
     name: project-merge
+    run: playbooks/project-merge.yaml
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - job:
     name: project-test3
+    run: playbooks/project-test3.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/streamer/git/common-config/zuul.yaml b/tests/fixtures/config/streamer/git/common-config/zuul.yaml
index f9925fe..8e67bfb 100644
--- a/tests/fixtures/config/streamer/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/streamer/git/common-config/zuul.yaml
@@ -19,3 +19,4 @@
     name: python27
     vars:
       waitpath: '{{zuul._test.test_root}}/{{zuul.build}}/test_wait'
+    run: playbooks/python27.yaml
diff --git a/tests/fixtures/config/success-url/git/common-config/zuul.yaml b/tests/fixtures/config/success-url/git/common-config/zuul.yaml
index 8929240..b9f4bff 100644
--- a/tests/fixtures/config/success-url/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/success-url/git/common-config/zuul.yaml
@@ -23,10 +23,12 @@
 - job:
     name: docs-draft-test
     success-url: http://docs-draft.example.org/{change.number:.2}/{change.number}/{change.patchset}/{pipeline.name}/{job.name}/{build.uuid:.7}/publish-docs/
+    run: playbooks/docs-draft-test.yaml
 
 - job:
     name: docs-draft-test2
     success-url: http://docs-draft.example.org/{NOPE}/{build.parameters[BAD]}/publish-docs/
+    run: playbooks/docs-draft-test2.yaml
 
 - project:
     name: org/docs
diff --git a/tests/fixtures/config/templated-project/git/common-config/zuul.d/jobs.yaml b/tests/fixtures/config/templated-project/git/common-config/zuul.d/jobs.yaml
index f9de1ad..7ad791d 100644
--- a/tests/fixtures/config/templated-project/git/common-config/zuul.d/jobs.yaml
+++ b/tests/fixtures/config/templated-project/git/common-config/zuul.d/jobs.yaml
@@ -4,18 +4,24 @@
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - job:
     name: layered-project-test3
+    run: playbooks/layered-project-test3.yaml
 
 - job:
     name: layered-project-test4
+    run: playbooks/layered-project-test4.yaml
 
 - job:
     name: layered-project-foo-test5
+    run: playbooks/layered-project-foo-test5.yaml
 
 - job:
     name: project-test6
+    run: playbooks/project-test6.yaml
diff --git a/tests/fixtures/config/unprotected-branches/git/org_project1/zuul.yaml b/tests/fixtures/config/unprotected-branches/git/org_project1/zuul.yaml
index 31abadf..1deffb3 100644
--- a/tests/fixtures/config/unprotected-branches/git/org_project1/zuul.yaml
+++ b/tests/fixtures/config/unprotected-branches/git/org_project1/zuul.yaml
@@ -1,5 +1,6 @@
 - job:
     name: project-test
+    run: playbooks/project-test.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/config/zuul-connections-multiple-gerrits/git/common-config/zuul.yaml b/tests/fixtures/config/zuul-connections-multiple-gerrits/git/common-config/zuul.yaml
index d70a384..6a0865e 100644
--- a/tests/fixtures/config/zuul-connections-multiple-gerrits/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/zuul-connections-multiple-gerrits/git/common-config/zuul.yaml
@@ -49,9 +49,11 @@
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - project:
     name: review.example.com/org/project1
@@ -65,7 +67,6 @@
       jobs:
         - project-test2
 
-
 - project:
     name: review.example.com/org/project2
     common_check:
diff --git a/tests/fixtures/config/zuul-connections-same-gerrit/git/common-config/zuul.yaml b/tests/fixtures/config/zuul-connections-same-gerrit/git/common-config/zuul.yaml
index eb65279..14a9b11 100644
--- a/tests/fixtures/config/zuul-connections-same-gerrit/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/zuul-connections-same-gerrit/git/common-config/zuul.yaml
@@ -17,9 +17,11 @@
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/zuultrigger/parent-change-enqueued/git/common-config/zuul.yaml b/tests/fixtures/config/zuultrigger/parent-change-enqueued/git/common-config/zuul.yaml
index 3dd8324..3fcc43c 100644
--- a/tests/fixtures/config/zuultrigger/parent-change-enqueued/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/zuultrigger/parent-change-enqueued/git/common-config/zuul.yaml
@@ -51,9 +51,11 @@
 
 - job:
     name: project-check
+    run: playbooks/project-check.yaml
 
 - job:
     name: project-gate
+    run: playbooks/project-gate.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/zuultrigger/project-change-merged/git/common-config/zuul.yaml b/tests/fixtures/config/zuultrigger/project-change-merged/git/common-config/zuul.yaml
index a5c5a1c..045e0a9 100644
--- a/tests/fixtures/config/zuultrigger/project-change-merged/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/zuultrigger/project-change-merged/git/common-config/zuul.yaml
@@ -49,9 +49,11 @@
 
 - job:
     name: project-check
+    run: playbooks/project-check.yaml
 
 - job:
     name: project-gate
+    run: playbooks/project-gate.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/autohold.yaml b/tests/fixtures/layouts/autohold.yaml
index 578f886..32b6822 100644
--- a/tests/fixtures/layouts/autohold.yaml
+++ b/tests/fixtures/layouts/autohold.yaml
@@ -14,6 +14,7 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-test2
@@ -21,6 +22,7 @@
       nodes:
         - name: controller
           label: label1
+    run: playbooks/project-test2.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/basic-github.yaml b/tests/fixtures/layouts/basic-github.yaml
index d7b323a..217e874 100644
--- a/tests/fixtures/layouts/basic-github.yaml
+++ b/tests/fixtures/layouts/basic-github.yaml
@@ -8,10 +8,10 @@
             - opened
             - changed
             - reopened
-          branch: '^master$'
+          branch: ^master$
         - event: pull_request
           action: comment
-          comment: 'test me'
+          comment: test me
     success:
       github: {}
     failure:
@@ -20,12 +20,15 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/crd-github.yaml b/tests/fixtures/layouts/crd-github.yaml
index 9696226..6ef881f 100644
--- a/tests/fixtures/layouts/crd-github.yaml
+++ b/tests/fixtures/layouts/crd-github.yaml
@@ -30,24 +30,31 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project1-test
+    run: playbooks/project1-test.yaml
 
 - job:
     name: project2-test
+    run: playbooks/project2-test.yaml
 
 - job:
     name: project3-test
+    run: playbooks/project3-test.yaml
 
 - job:
     name: project4-test
+    run: playbooks/project4-test.yaml
 
 - job:
     name: project5-test
+    run: playbooks/project5-test.yaml
 
 - job:
     name: project6-test
+    run: playbooks/project6-test.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/layouts/delayed-repo-init.yaml b/tests/fixtures/layouts/delayed-repo-init.yaml
index c89e2fa..0c9a152 100644
--- a/tests/fixtures/layouts/delayed-repo-init.yaml
+++ b/tests/fixtures/layouts/delayed-repo-init.yaml
@@ -43,18 +43,23 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-merge
+    run: playbooks/project-merge.yaml
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - job:
     name: project-post
+    run: playbooks/project-post.yaml
 
 - project:
     name: org/new-project
diff --git a/tests/fixtures/layouts/dependent-github.yaml b/tests/fixtures/layouts/dependent-github.yaml
index eb74163..6ad6bd2 100644
--- a/tests/fixtures/layouts/dependent-github.yaml
+++ b/tests/fixtures/layouts/dependent-github.yaml
@@ -6,29 +6,33 @@
       github:
         - event: pull_request
           action: labeled
-          label: 'merge'
+          label: merge
     success:
       github:
         merge: true
-        unlabel: 'merge'
+        unlabel: merge
     failure:
       github:
-        unlabel: 'merge'
+        unlabel: merge
 
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - job:
     name: project-merge
     failure-message: Unable to merge change
     hold-following-changes: true
+    run: playbooks/project-merge.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/dequeue-github.yaml b/tests/fixtures/layouts/dequeue-github.yaml
index ae61cd5..72d8145 100644
--- a/tests/fixtures/layouts/dequeue-github.yaml
+++ b/tests/fixtures/layouts/dequeue-github.yaml
@@ -11,9 +11,11 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: one-job-project-merge
+    run: playbooks/one-job-project-merge.yaml
 
 - project:
     name: org/one-job-project
diff --git a/tests/fixtures/layouts/disable_at.yaml b/tests/fixtures/layouts/disable_at.yaml
index 8c24c1b..a090d11 100644
--- a/tests/fixtures/layouts/disable_at.yaml
+++ b/tests/fixtures/layouts/disable_at.yaml
@@ -18,6 +18,7 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-test1
@@ -25,6 +26,7 @@
       nodes:
         - name: controller
           label: label1
+    run: playbooks/project-test1.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/dont-ignore-ref-deletes.yaml b/tests/fixtures/layouts/dont-ignore-ref-deletes.yaml
index bb98b57..9a9e592 100644
--- a/tests/fixtures/layouts/dont-ignore-ref-deletes.yaml
+++ b/tests/fixtures/layouts/dont-ignore-ref-deletes.yaml
@@ -10,6 +10,7 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-post
@@ -17,6 +18,7 @@
       nodes:
         - name: static
           label: ubuntu-xenial
+    run: playbooks/project-post.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/files-github.yaml b/tests/fixtures/layouts/files-github.yaml
index ec35259..ed053f9 100644
--- a/tests/fixtures/layouts/files-github.yaml
+++ b/tests/fixtures/layouts/files-github.yaml
@@ -9,11 +9,13 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-test1
     files:
-      - '.*-requires'
+      - .*-requires
+    run: playbooks/project-test1.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/footer-message.yaml b/tests/fixtures/layouts/footer-message.yaml
index 4ee25f6..746d384 100644
--- a/tests/fixtures/layouts/footer-message.yaml
+++ b/tests/fixtures/layouts/footer-message.yaml
@@ -28,10 +28,11 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-test1
-#    success-url: http://logs.exxxample.com/{change.number}/{change.patchset}/{pipeline.name}/{job.name}
+    run: playbooks/project-test1.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/idle.yaml b/tests/fixtures/layouts/idle.yaml
index 4cc07ae..4f3efe4 100644
--- a/tests/fixtures/layouts/idle.yaml
+++ b/tests/fixtures/layouts/idle.yaml
@@ -8,6 +8,7 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-bitrot
@@ -15,10 +16,10 @@
       nodes:
         - name: static
           label: ubuntu-xenial
+    run: playbooks/project-bitrot.yaml
 
 - project:
     name: org/project
     periodic:
       jobs:
         - project-bitrot
-
diff --git a/tests/fixtures/layouts/ignore-dependencies.yaml b/tests/fixtures/layouts/ignore-dependencies.yaml
index 89a82b3..b869dab 100644
--- a/tests/fixtures/layouts/ignore-dependencies.yaml
+++ b/tests/fixtures/layouts/ignore-dependencies.yaml
@@ -15,27 +15,35 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project1-merge
+    run: playbooks/project1-merge.yaml
 
 - job:
     name: project1-test1
+    run: playbooks/project1-test1.yaml
 
 - job:
     name: project1-test2
+    run: playbooks/project1-test2.yaml
 
 - job:
     name: project2-merge
+    run: playbooks/project2-merge.yaml
 
 - job:
     name: project2-test1
+    run: playbooks/project2-test1.yaml
 
 - job:
     name: project2-test2
+    run: playbooks/project2-test2.yaml
 
 - job:
     name: project1-project2-integration
+    run: playbooks/project1-project2-integration.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/layouts/inheritance.yaml b/tests/fixtures/layouts/inheritance.yaml
index 3fe7fd4..35f1402 100644
--- a/tests/fixtures/layouts/inheritance.yaml
+++ b/tests/fixtures/layouts/inheritance.yaml
@@ -14,23 +14,28 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-test-irrelevant-starts-empty
+    run: playbooks/project-test-irrelevant-starts-empty.yaml
 
 - job:
     name: project-test-irrelevant-starts-full
     irrelevant-files:
       - ^README$
       - ^ignoreme$
+    run: playbooks/project-test-irrelevant-starts-full.yaml
 
 - job:
     name: project-test-nomatch-starts-empty
+    run: playbooks/project-test-nomatch-starts-empty.yaml
 
 - job:
     name: project-test-nomatch-starts-full
     irrelevant-files:
       - ^README$
+    run: playbooks/project-test-nomatch-starts-full.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/irrelevant-files.yaml b/tests/fixtures/layouts/irrelevant-files.yaml
index 97f58e7..80be9af 100644
--- a/tests/fixtures/layouts/irrelevant-files.yaml
+++ b/tests/fixtures/layouts/irrelevant-files.yaml
@@ -14,9 +14,11 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-test-irrelevant-files
+    run: playbooks/project-test-irrelevant-files.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/job-variants.yaml b/tests/fixtures/layouts/job-variants.yaml
index de99e8a..356034f 100644
--- a/tests/fixtures/layouts/job-variants.yaml
+++ b/tests/fixtures/layouts/job-variants.yaml
@@ -20,6 +20,7 @@
       nodes:
         - name: controller
           label: base
+    run: playbooks/base.yaml
 
 - job:
     name: python27
@@ -33,6 +34,7 @@
       nodes:
         - name: controller
           label: new
+    run: playbooks/python27.yaml
 
 - job:
     name: python27
@@ -53,6 +55,7 @@
       - stable/essex
     pre-run: py27-essex-pre
     post-run: py27-essex-post
+    run: playbooks/python27.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/job-vars.yaml b/tests/fixtures/layouts/job-vars.yaml
index 22fc5c2..e46f084 100644
--- a/tests/fixtures/layouts/job-vars.yaml
+++ b/tests/fixtures/layouts/job-vars.yaml
@@ -14,6 +14,7 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: parentjob
@@ -24,6 +25,7 @@
       override: 0
       child1override: 0
       parent: 0
+    run: playbooks/parentjob.yaml
 
 - job:
     name: child1
@@ -34,6 +36,7 @@
       override: 1
       child1override: 1
       child1: 1
+    run: playbooks/child1.yaml
 
 - job:
     name: child2
@@ -43,10 +46,12 @@
     vars:
       override: 2
       child2: 2
+    run: playbooks/child2.yaml
 
 - job:
     name: child3
     parent: parentjob
+    run: playbooks/child3.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/labeling-github.yaml b/tests/fixtures/layouts/labeling-github.yaml
index 2441a9c..fbbf068 100644
--- a/tests/fixtures/layouts/labeling-github.yaml
+++ b/tests/fixtures/layouts/labeling-github.yaml
@@ -7,24 +7,26 @@
         - event: pull_request
           action: labeled
           label:
-            - 'test'
+            - test
         - event: pull_request
           action: unlabeled
           label:
-            - 'do not test'
+            - do not test
     success:
       github:
         label:
-          - 'tests passed'
+          - tests passed
         unlabel:
-          - 'test'
+          - test
 
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-labels
+    run: playbooks/project-labels.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/live-reconfiguration-add-job.yaml b/tests/fixtures/layouts/live-reconfiguration-add-job.yaml
index 57d2a5f..c871f1d 100644
--- a/tests/fixtures/layouts/live-reconfiguration-add-job.yaml
+++ b/tests/fixtures/layouts/live-reconfiguration-add-job.yaml
@@ -22,40 +22,46 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-merge
     hold-following-changes: true
+    run: playbooks/project-merge.yaml
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - job:
     name: project-test3
+    run: playbooks/project-test3.yaml
 
 - job:
     name: project-testfile
     files:
-      - '.*-requires'
+      - .*-requires
+    run: playbooks/project-testfile.yaml
 
 - project:
     name: org/project
     merge-mode: cherry-pick
     gate:
       jobs:
-      - project-merge
-      - project-test1:
-          dependencies:
-            - project-merge
-      - project-test2:
-          dependencies:
-            - project-merge
-      - project-test3:
-          dependencies:
-            - project-merge
-      - project-testfile:
-          dependencies:
-            - project-merge
+        - project-merge
+        - project-test1:
+            dependencies:
+              - project-merge
+        - project-test2:
+            dependencies:
+              - project-merge
+        - project-test3:
+            dependencies:
+              - project-merge
+        - project-testfile:
+            dependencies:
+              - project-merge
diff --git a/tests/fixtures/layouts/live-reconfiguration-del-project.yaml b/tests/fixtures/layouts/live-reconfiguration-del-project.yaml
index b149af0..259de84 100644
--- a/tests/fixtures/layouts/live-reconfiguration-del-project.yaml
+++ b/tests/fixtures/layouts/live-reconfiguration-del-project.yaml
@@ -14,19 +14,24 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-merge
     hold-following-changes: true
+    run: playbooks/project-merge.yaml
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - job:
     name: project-testfile
+    run: playbooks/project-testfile.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/live-reconfiguration-failed-job.yaml b/tests/fixtures/layouts/live-reconfiguration-failed-job.yaml
index c4719f4..e5cc651 100644
--- a/tests/fixtures/layouts/live-reconfiguration-failed-job.yaml
+++ b/tests/fixtures/layouts/live-reconfiguration-failed-job.yaml
@@ -14,26 +14,30 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-merge
     hold-following-changes: true
+    run: playbooks/project-merge.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - job:
     name: project-testfile
+    run: playbooks/project-testfile.yaml
 
 - project:
     name: org/project
     merge-mode: cherry-pick
     check:
       jobs:
-      - project-merge
-      - project-test2:
-          dependencies:
-            - project-merge
-      - project-testfile:
-          dependencies:
-            - project-merge
+        - project-merge
+        - project-test2:
+            dependencies:
+              - project-merge
+        - project-testfile:
+            dependencies:
+              - project-merge
diff --git a/tests/fixtures/layouts/live-reconfiguration-shared-queue.yaml b/tests/fixtures/layouts/live-reconfiguration-shared-queue.yaml
index e363b4c..49a1c2c 100644
--- a/tests/fixtures/layouts/live-reconfiguration-shared-queue.yaml
+++ b/tests/fixtures/layouts/live-reconfiguration-shared-queue.yaml
@@ -35,19 +35,24 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-merge
     hold-following-changes: true
+    run: playbooks/project-merge.yaml
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - job:
     name: project1-project2-integration
+    run: playbooks/project1-project2-integration.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/layouts/matcher-test.yaml b/tests/fixtures/layouts/matcher-test.yaml
index b511a2f..3239b52 100644
--- a/tests/fixtures/layouts/matcher-test.yaml
+++ b/tests/fixtures/layouts/matcher-test.yaml
@@ -35,6 +35,7 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-test1
@@ -42,6 +43,7 @@
       nodes:
         - name: controller
           label: label1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: ignore-branch
@@ -50,6 +52,7 @@
       nodes:
         - name: controller
           label: label2
+    run: playbooks/ignore-branch.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/merge-failure.yaml b/tests/fixtures/layouts/merge-failure.yaml
index 7c5121c..3828a06 100644
--- a/tests/fixtures/layouts/merge-failure.yaml
+++ b/tests/fixtures/layouts/merge-failure.yaml
@@ -23,7 +23,7 @@
     name: gate
     manager: dependent
     failure-message: Build failed.  For information on how to proceed, see http://wiki.example.org/Test_Failures
-    merge-failure-message: "The merge failed! For more information..."
+    merge-failure-message: The merge failed! For more information...
     trigger:
       gerrit:
         - event: comment-added
@@ -49,16 +49,20 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-merge
     hold-following-changes: true
+    run: playbooks/project-merge.yaml
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/merging-github.yaml b/tests/fixtures/layouts/merging-github.yaml
index c9673b9..20df0d4 100644
--- a/tests/fixtures/layouts/merging-github.yaml
+++ b/tests/fixtures/layouts/merging-github.yaml
@@ -2,12 +2,12 @@
     name: merge
     description: Pipeline for merging the pull request
     manager: independent
-    merge-failure-message: 'Merge failed'
+    merge-failure-message: Merge failed
     trigger:
       github:
         - event: pull_request
           action: comment
-          comment: 'merge me'
+          comment: merge me
     success:
       github:
         merge: true
@@ -16,6 +16,7 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/multiple-templates.yaml b/tests/fixtures/layouts/multiple-templates.yaml
index 7272cad..ece8396 100644
--- a/tests/fixtures/layouts/multiple-templates.yaml
+++ b/tests/fixtures/layouts/multiple-templates.yaml
@@ -14,9 +14,11 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: py27
+    run: playbooks/py27.yaml
 
 - project-template:
     name: python-jobs
diff --git a/tests/fixtures/layouts/no-jobs-project.yaml b/tests/fixtures/layouts/no-jobs-project.yaml
index 8f965e2..e23f36c 100644
--- a/tests/fixtures/layouts/no-jobs-project.yaml
+++ b/tests/fixtures/layouts/no-jobs-project.yaml
@@ -14,11 +14,13 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-testfile
     files:
       - .*-requires
+    run: playbooks/project-testfile.yaml
 
 - project:
     name: org/no-jobs-project
diff --git a/tests/fixtures/layouts/no-jobs.yaml b/tests/fixtures/layouts/no-jobs.yaml
index 301b27a..7d483ec 100644
--- a/tests/fixtures/layouts/no-jobs.yaml
+++ b/tests/fixtures/layouts/no-jobs.yaml
@@ -35,9 +35,11 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: gate-noop
+    run: playbooks/gate-noop.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/no-timer.yaml b/tests/fixtures/layouts/no-timer.yaml
index 7aaa1ed..67a3244 100644
--- a/tests/fixtures/layouts/no-timer.yaml
+++ b/tests/fixtures/layouts/no-timer.yaml
@@ -23,9 +23,11 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-bitrot
@@ -33,6 +35,7 @@
       nodes:
         - name: static
           label: ubuntu-xenial
+    run: playbooks/project-bitrot.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/nonvoting-job.yaml b/tests/fixtures/layouts/nonvoting-job.yaml
index 6a912bf..5b8e9be 100644
--- a/tests/fixtures/layouts/nonvoting-job.yaml
+++ b/tests/fixtures/layouts/nonvoting-job.yaml
@@ -22,17 +22,21 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: nonvoting-project-merge
     hold-following-changes: true
+    run: playbooks/nonvoting-project-merge.yaml
 
 - job:
     name: nonvoting-project-test1
+    run: playbooks/nonvoting-project-test1.yaml
 
 - job:
     name: nonvoting-project-test2
     voting: false
+    run: playbooks/nonvoting-project-test2.yaml
 
 - project:
     name: org/nonvoting-project
diff --git a/tests/fixtures/layouts/nonvoting-pipeline.yaml b/tests/fixtures/layouts/nonvoting-pipeline.yaml
index d8468dd..afe0528 100644
--- a/tests/fixtures/layouts/nonvoting-pipeline.yaml
+++ b/tests/fixtures/layouts/nonvoting-pipeline.yaml
@@ -12,13 +12,16 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-merge
     hold-following-changes: true
+    run: playbooks/project-merge.yaml
 
 - job:
     name: experimental-project-test
+    run: playbooks/experimental-project-test.yaml
 
 - project:
     name: org/experimental-project
diff --git a/tests/fixtures/layouts/one-job-project.yaml b/tests/fixtures/layouts/one-job-project.yaml
index 4b682d3..d5346f6 100644
--- a/tests/fixtures/layouts/one-job-project.yaml
+++ b/tests/fixtures/layouts/one-job-project.yaml
@@ -43,13 +43,16 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: one-job-project-merge
     hold-following-changes: true
+    run: playbooks/one-job-project-merge.yaml
 
 - job:
     name: one-job-project-post
+    run: playbooks/one-job-project-post.yaml
 
 - project:
     name: org/one-job-project
diff --git a/tests/fixtures/layouts/parent-matchers.yaml b/tests/fixtures/layouts/parent-matchers.yaml
index d53741e..2080215 100644
--- a/tests/fixtures/layouts/parent-matchers.yaml
+++ b/tests/fixtures/layouts/parent-matchers.yaml
@@ -14,18 +14,22 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: parent-job
     files: foo.txt
+    run: playbooks/parent-job.yaml
 
 - job:
     name: parent-job
     files: bar.txt
-    
+    run: playbooks/parent-job.yaml
+
 - job:
     name: child-job
     parent: parent-job
+    run: playbooks/child-job.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/push-tag-github.yaml b/tests/fixtures/layouts/push-tag-github.yaml
index 5805127..d689201 100644
--- a/tests/fixtures/layouts/push-tag-github.yaml
+++ b/tests/fixtures/layouts/push-tag-github.yaml
@@ -4,7 +4,7 @@
     trigger:
       github:
         - event: push
-          ref: '^refs/heads/master$'
+          ref: ^refs/heads/master$
 
 - pipeline:
     name: tag
@@ -17,12 +17,15 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-post
+    run: playbooks/project-post.yaml
 
 - job:
     name: project-tag
+    run: playbooks/project-tag.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/rate-limit.yaml b/tests/fixtures/layouts/rate-limit.yaml
index 1f32dbf..b432d51 100644
--- a/tests/fixtures/layouts/rate-limit.yaml
+++ b/tests/fixtures/layouts/rate-limit.yaml
@@ -27,15 +27,19 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-merge
+    run: playbooks/project-merge.yaml
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/repo-checkout-four-project.yaml b/tests/fixtures/layouts/repo-checkout-four-project.yaml
index 17303f5..11212e8 100644
--- a/tests/fixtures/layouts/repo-checkout-four-project.yaml
+++ b/tests/fixtures/layouts/repo-checkout-four-project.yaml
@@ -35,6 +35,7 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: integration
@@ -43,6 +44,7 @@
       - org/project2
       - org/project3
       - org/project4
+    run: playbooks/integration.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/layouts/repo-checkout-no-timer-override.yaml b/tests/fixtures/layouts/repo-checkout-no-timer-override.yaml
index 4680869..ca49292 100644
--- a/tests/fixtures/layouts/repo-checkout-no-timer-override.yaml
+++ b/tests/fixtures/layouts/repo-checkout-no-timer-override.yaml
@@ -10,6 +10,7 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: integration
@@ -17,6 +18,7 @@
     override-branch: stable/havana
     required-projects:
       - org/project1
+    run: playbooks/integration.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/layouts/repo-checkout-no-timer.yaml b/tests/fixtures/layouts/repo-checkout-no-timer.yaml
index ed20bb1..6b88801 100644
--- a/tests/fixtures/layouts/repo-checkout-no-timer.yaml
+++ b/tests/fixtures/layouts/repo-checkout-no-timer.yaml
@@ -10,12 +10,14 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: integration
     override-branch: stable/havana
     required-projects:
       - org/project1
+    run: playbooks/integration.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/layouts/repo-checkout-post.yaml b/tests/fixtures/layouts/repo-checkout-post.yaml
index 191569c..2e702bc 100644
--- a/tests/fixtures/layouts/repo-checkout-post.yaml
+++ b/tests/fixtures/layouts/repo-checkout-post.yaml
@@ -9,12 +9,14 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: integration
     required-projects:
       - org/project1
       - org/project2
+    run: playbooks/integration.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/layouts/repo-checkout-six-project.yaml b/tests/fixtures/layouts/repo-checkout-six-project.yaml
index 9a81eae..6079612 100644
--- a/tests/fixtures/layouts/repo-checkout-six-project.yaml
+++ b/tests/fixtures/layouts/repo-checkout-six-project.yaml
@@ -35,6 +35,7 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: integration
@@ -46,6 +47,7 @@
         override-branch: master
       - org/project5
       - org/project6
+    run: playbooks/integration.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/layouts/repo-checkout-timer-override.yaml b/tests/fixtures/layouts/repo-checkout-timer-override.yaml
index 99fc4f5..af5bd3c 100644
--- a/tests/fixtures/layouts/repo-checkout-timer-override.yaml
+++ b/tests/fixtures/layouts/repo-checkout-timer-override.yaml
@@ -8,6 +8,7 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: integration
@@ -15,6 +16,7 @@
     override-branch: stable/havana
     required-projects:
       - org/project1
+    run: playbooks/integration.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/layouts/repo-checkout-timer.yaml b/tests/fixtures/layouts/repo-checkout-timer.yaml
index e707732..739c066 100644
--- a/tests/fixtures/layouts/repo-checkout-timer.yaml
+++ b/tests/fixtures/layouts/repo-checkout-timer.yaml
@@ -8,11 +8,13 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: integration
     required-projects:
       - org/project1
+    run: playbooks/integration.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/layouts/repo-checkout-two-project.yaml b/tests/fixtures/layouts/repo-checkout-two-project.yaml
index 7910ae7..64c6ee9 100644
--- a/tests/fixtures/layouts/repo-checkout-two-project.yaml
+++ b/tests/fixtures/layouts/repo-checkout-two-project.yaml
@@ -35,12 +35,14 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: integration
     required-projects:
       - org/project1
       - org/project2
+    run: playbooks/integration.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/layouts/repo-deleted.yaml b/tests/fixtures/layouts/repo-deleted.yaml
index 3a7f6b3..2ee8ebd 100644
--- a/tests/fixtures/layouts/repo-deleted.yaml
+++ b/tests/fixtures/layouts/repo-deleted.yaml
@@ -35,10 +35,12 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-merge
     hold-following-changes: true
+    run: playbooks/project-merge.yaml
 
 - job:
     name: project-test1
@@ -46,6 +48,7 @@
       nodes:
         - name: controller
           label: label1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test1
@@ -54,9 +57,11 @@
       nodes:
         - name: controller
           label: label2
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - project:
     name: org/delete-project
diff --git a/tests/fixtures/layouts/reporting-github.yaml b/tests/fixtures/layouts/reporting-github.yaml
index 159f205..c909cf4 100644
--- a/tests/fixtures/layouts/reporting-github.yaml
+++ b/tests/fixtures/layouts/reporting-github.yaml
@@ -8,11 +8,11 @@
           action: opened
     start:
       github:
-        status: 'pending'
+        status: pending
         comment: false
     success:
       github:
-        status: 'success'
+        status: success
 
 - pipeline:
     name: reporting
@@ -22,13 +22,13 @@
       github:
         - event: pull_request
           action: comment
-          comment: 'reporting check'
+          comment: reporting check
     start:
       github: {}
     success:
       github:
         comment: false
-        status: 'success'
+        status: success
         status-url: http://logs.example.com/{tenant.name}/{pipeline.name}/{change.project}/{change.number}/{buildset.uuid}/
     failure:
       github:
@@ -42,14 +42,14 @@
       github:
         - event: pull_request
           action: comment
-          comment: 'long pipeline'
+          comment: long pipeline
     start:
       github:
-        status: 'pending'
+        status: pending
     success:
       github:
         comment: false
-        status: 'success'
+        status: success
         status-url: http://logs.example.com/{tenant.name}/{pipeline.name}/{change.project}/{change.number}/{buildset.uuid}/
     failure:
       github:
@@ -67,23 +67,25 @@
     start:
       github:
         comment: true
-        status: 'pending'
+        status: pending
     success:
       github:
         comment: true
-        status: 'success'
+        status: success
         merge: true
     failure:
       github:
         comment: true
-        status: 'failure'
+        status: failure
 
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/reporting-multiple-github.yaml b/tests/fixtures/layouts/reporting-multiple-github.yaml
index 0126ec5..67a237a 100644
--- a/tests/fixtures/layouts/reporting-multiple-github.yaml
+++ b/tests/fixtures/layouts/reporting-multiple-github.yaml
@@ -11,26 +11,29 @@
           action: opened
     start:
       github:
-        status: 'pending'
+        status: pending
         comment: false
       github_ent:
-        status: 'pending'
+        status: pending
         comment: false
     success:
       github:
-        status: 'success'
+        status: success
       github_ent:
-        status: 'success'
+        status: success
 
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project1-test1
+    run: playbooks/project1-test1.yaml
 
 - job:
     name: project2-test2
+    run: playbooks/project2-test2.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/layouts/requirements-github.yaml b/tests/fixtures/layouts/requirements-github.yaml
index f2ecd16..92bd9cb 100644
--- a/tests/fixtures/layouts/requirements-github.yaml
+++ b/tests/fixtures/layouts/requirements-github.yaml
@@ -3,12 +3,12 @@
     manager: independent
     require:
       github:
-        status: "zuul:check:success"
+        status: zuul:check:success
     trigger:
       github:
         - event: pull_request
           action: comment
-          comment: 'test me'
+          comment: test me
     success:
       github:
         comment: true
@@ -20,8 +20,8 @@
       github:
         - event: pull_request
           action: comment
-          comment: 'trigger me'
-          require-status: "zuul:check:success"
+          comment: trigger me
+          require-status: zuul:check:success
     success:
       github:
         comment: true
@@ -33,13 +33,13 @@
       github:
         - event: pull_request
           action: status
-          status: 'zuul:check:success'
+          status: zuul:check:success
     success:
       github:
-        status: 'success'
+        status: success
     failure:
       github:
-        status: 'failure'
+        status: failure
 
 - pipeline:
     name: reviewusername
@@ -47,13 +47,13 @@
     require:
       github:
         review:
-          - username: '^(herp|derp)$'
+          - username: ^(herp|derp)$
             type: approved
     trigger:
       github:
         - event: pull_request
           action: comment
-          comment: 'test me'
+          comment: test me
     success:
       github:
         comment: true
@@ -75,7 +75,7 @@
       github:
         - event: pull_request
           action: comment
-          comment: 'test me'
+          comment: test me
     success:
       github:
         comment: true
@@ -86,7 +86,7 @@
     require:
       github:
         review:
-          - username: 'derp'
+          - username: derp
             type: approved
             permission: write
     reject:
@@ -98,7 +98,7 @@
       github:
         - event: pull_request
           action: comment
-          comment: 'test me'
+          comment: test me
     success:
       github:
         comment: true
@@ -116,7 +116,7 @@
       github:
         - event: pull_request
           action: comment
-          comment: 'test me'
+          comment: test me
     success:
       github:
         comment: true
@@ -134,7 +134,7 @@
       github:
         - event: pull_request
           action: comment
-          comment: 'test me'
+          comment: test me
     success:
       github:
         comment: true
@@ -149,7 +149,7 @@
       github:
         - event: pull_request
           action: comment
-          comment: 'test me'
+          comment: test me
     success:
       github:
         comment: true
@@ -178,7 +178,7 @@
       github:
         - event: pull_request
           action: comment
-          comment: 'test me'
+          comment: test me
     success:
       github:
         comment: true
@@ -186,36 +186,47 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project1-pipeline
+    run: playbooks/project1-pipeline.yaml
 
 - job:
     name: project2-trigger
+    run: playbooks/project2-trigger.yaml
 
 - job:
     name: project3-reviewusername
+    run: playbooks/project3-reviewusername.yaml
 
 - job:
     name: project4-reviewreq
+    run: playbooks/project4-reviewreq.yaml
 
 - job:
     name: project5-reviewuserstate
+    run: playbooks/project5-reviewuserstate.yaml
 
 - job:
     name: project6-newerthan
+    run: playbooks/project6-newerthan.yaml
 
 - job:
     name: project7-olderthan
+    run: playbooks/project7-olderthan.yaml
 
 - job:
     name: project8-requireopen
+    run: playbooks/project8-requireopen.yaml
 
 - job:
     name: project9-requirecurrent
+    run: playbooks/project9-requirecurrent.yaml
 
 - job:
     name: project10-label
+    run: playbooks/project10-label.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/layouts/reviews-github.yaml b/tests/fixtures/layouts/reviews-github.yaml
index f186fbe..abc3d99 100644
--- a/tests/fixtures/layouts/reviews-github.yaml
+++ b/tests/fixtures/layouts/reviews-github.yaml
@@ -5,18 +5,20 @@
       github:
         - event: pull_request_review
           action: submitted
-          state: 'approve'
+          state: approve
     success:
       github:
         label:
-          - 'tests passed'
+          - tests passed
 
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-reviews
+    run: playbooks/project-reviews.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/smtp.yaml b/tests/fixtures/layouts/smtp.yaml
index 0654448..77391a0 100644
--- a/tests/fixtures/layouts/smtp.yaml
+++ b/tests/fixtures/layouts/smtp.yaml
@@ -41,10 +41,12 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-merge
     hold-following-changes: true
+    run: playbooks/project-merge.yaml
 
 - job:
     name: project-test1
@@ -52,6 +54,7 @@
       nodes:
         - name: controller
           label: label1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test1
@@ -60,9 +63,11 @@
       nodes:
         - name: controller
           label: label2
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/tags.yaml b/tests/fixtures/layouts/tags.yaml
index f86f5ab..2fda2db 100644
--- a/tests/fixtures/layouts/tags.yaml
+++ b/tests/fixtures/layouts/tags.yaml
@@ -14,11 +14,13 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: merge
     tags:
       - merge
+    run: playbooks/merge.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/layouts/three-projects.yaml b/tests/fixtures/layouts/three-projects.yaml
index 51cd406..33e81ac 100644
--- a/tests/fixtures/layouts/three-projects.yaml
+++ b/tests/fixtures/layouts/three-projects.yaml
@@ -35,19 +35,24 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-merge
     hold-following-changes: true
+    run: playbooks/project-merge.yaml
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - job:
     name: project1-project2-integration
+    run: playbooks/project1-project2-integration.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/layouts/timer-smtp.yaml b/tests/fixtures/layouts/timer-smtp.yaml
index a27b183..d9e4282 100644
--- a/tests/fixtures/layouts/timer-smtp.yaml
+++ b/tests/fixtures/layouts/timer-smtp.yaml
@@ -13,14 +13,17 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-bitrot-stable-old
     success-url: http://logs.example.com/{job.name}/{build.number}
+    run: playbooks/project-bitrot-stable-old.yaml
 
 - job:
     name: project-bitrot-stable-older
     success-url: http://logs.example.com/{job.name}/{build.number}
+    run: playbooks/project-bitrot-stable-older.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/timer.yaml b/tests/fixtures/layouts/timer.yaml
index 8c0cc2b..e9e9a17 100644
--- a/tests/fixtures/layouts/timer.yaml
+++ b/tests/fixtures/layouts/timer.yaml
@@ -21,12 +21,15 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - job:
     name: project-bitrot
@@ -34,6 +37,7 @@
       nodes:
         - name: static
           label: ubuntu-xenial
+    run: playbooks/project-bitrot.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/untrusted-secrets.yaml b/tests/fixtures/layouts/untrusted-secrets.yaml
index b90d3d7..337587a 100644
--- a/tests/fixtures/layouts/untrusted-secrets.yaml
+++ b/tests/fixtures/layouts/untrusted-secrets.yaml
@@ -14,10 +14,12 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project1-test
     post-review: true
+    run: playbooks/project1-test.yaml
 
 - project:
     name: org/project1
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index 1b5f9dd..e30a43d 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -2460,7 +2460,7 @@
         self.assertEqual([x['path'] for x in p['post_playbooks']],
                          ['py27-post-a', 'py27-post-b', 'base-post'])
         self.assertEqual([x['path'] for x in p['playbooks']],
-                         ['playbooks/python27', 'playbooks/base'])
+                         ['playbooks/python27.yaml'])
 
         p = self.history[1].parameters
         self.assertEqual(p['timeout'], 50)
@@ -2484,7 +2484,7 @@
                          ['py27-essex-post', 'py27-post-a', 'py27-post-b',
                           'base-post'])
         self.assertEqual([x['path'] for x in p['playbooks']],
-                         ['playbooks/python27', 'playbooks/base'])
+                         ['playbooks/python27.yaml'])
 
     def test_queue_names(self):
         "Test shared change queue names"
@@ -5879,6 +5879,7 @@
 
             - job:
                 name: project-test2
+                run: playbooks/project-test2.yaml
                 semaphore: test-semaphore
 
             - project:
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index b9ae04b..80e6ccc 100755
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -128,6 +128,7 @@
             - job:
                 name: project-test
                 parent: job-final
+                run: playbooks/project-test.yaml
 
             - project:
                 name: org/project
@@ -310,6 +311,7 @@
 
             - job:
                 name: project-test2
+                run: playbooks/project-test2.yaml
 
             - project:
                 name: org/project
@@ -460,9 +462,11 @@
             """
             - job:
                 name: project-test1
+                run: playbooks/project-test1.yaml
 
             - job:
                 name: project-test2
+                run: playbooks/project-test2.yaml
 
             - project:
                 name: org/project
@@ -493,9 +497,11 @@
             """
             - job:
                 name: project-test1
+                run: playbooks/project-test1.yaml
 
             - job:
                 name: project-test2
+                run: playbooks/project-test2.yaml
 
             - project:
                 name: org/project
@@ -536,6 +542,7 @@
 
             - job:
                 name: project-test2
+                run: playbooks/project-test2.yaml
 
             - project:
                 name: org/project
@@ -608,6 +615,7 @@
 
             - job:
                 name: project-test2
+                run: playbooks/project-test2.yaml
 
             - project:
                 name: org/project
@@ -1075,6 +1083,7 @@
             """
             - job:
                 name: project-test1
+                run: playbooks/project-test1.yaml
             - project-template:
                 name: some-jobs
                 tenant-one-gate:
@@ -1133,6 +1142,7 @@
             """
             - job:
                 name: project-test1
+                run: playbooks/project-test1.yaml
 
             - project:
                 name: org/project1
@@ -1187,6 +1197,7 @@
             """
             - job:
                 name: project-test1
+                run: playbooks/project-test1.yaml
 
             - job:
                 name: project-test2
@@ -1303,9 +1314,11 @@
             """
             - job:
                 name: project-test1
+                run: playbooks/project-test1.yaml
 
             - job:
                 name: project-test2
+                run: playbooks/project-test2.yaml
 
             - project:
                 name: org/project
@@ -1349,6 +1362,7 @@
             """
             - job:
                 name: project-test1
+                run: playbooks/project-test1.yaml
 
             - project:
                 name: org/project
@@ -1482,13 +1496,14 @@
             """
             - job:
                 name: %s
+                run: playbooks/%s.yaml
 
             - project:
                 name: org/plugin-project
                 check:
                   jobs:
                     - %s
-            """ % (job_name, job_name))
+            """ % (job_name, job_name, job_name))
 
         file_dict = {'.zuul.yaml': conf}
         A = self.fake_gerrit.addFakeChange('org/plugin-project', 'master', 'A',
@@ -1690,6 +1705,7 @@
             - job:
                 name: project-test
                 parent: parent
+                run: playbooks/project-test.yaml
                 roles:
                   - zuul: org/project
 
@@ -1726,6 +1742,7 @@
             """
             - job:
                 name: project-test
+                run: playbooks/project-test.yaml
                 roles:
                   - zuul: common-config
 
@@ -2014,7 +2031,7 @@
                   'username': 'test-username'}
         self.assertEqual(
             self._getSecrets('trusted-secrets', 'playbooks'),
-            [{'trusted-secret': secret}, {}])
+            [{'trusted-secret': secret}])
         self.assertEqual(
             self._getSecrets('trusted-secrets', 'pre_playbooks'), [])
         self.assertEqual(
@@ -2022,8 +2039,7 @@
 
         self.assertEqual(
             self._getSecrets('trusted-secrets-trusted-child',
-                             'playbooks'),
-            [{}, {'trusted-secret': secret}, {}])
+                             'playbooks'), [{}])
         self.assertEqual(
             self._getSecrets('trusted-secrets-trusted-child',
                              'pre_playbooks'), [])
@@ -2033,8 +2049,7 @@
 
         self.assertEqual(
             self._getSecrets('trusted-secrets-untrusted-child',
-                             'playbooks'),
-            [{}, {'trusted-secret': secret}, {}])
+                             'playbooks'), [{}])
         self.assertEqual(
             self._getSecrets('trusted-secrets-untrusted-child',
                              'pre_playbooks'), [])
@@ -2048,7 +2063,7 @@
                   'username': 'test-username'}
         self.assertEqual(
             self._getSecrets('untrusted-secrets', 'playbooks'),
-            [{'untrusted-secret': secret}, {}])
+            [{'untrusted-secret': secret}])
         self.assertEqual(
             self._getSecrets('untrusted-secrets', 'pre_playbooks'), [])
         self.assertEqual(
@@ -2056,8 +2071,7 @@
 
         self.assertEqual(
             self._getSecrets('untrusted-secrets-trusted-child',
-                             'playbooks'),
-            [{}, {'untrusted-secret': secret}, {}])
+                             'playbooks'), [{}])
         self.assertEqual(
             self._getSecrets('untrusted-secrets-trusted-child',
                              'pre_playbooks'), [])
@@ -2067,8 +2081,7 @@
 
         self.assertEqual(
             self._getSecrets('untrusted-secrets-untrusted-child',
-                             'playbooks'),
-            [{}, {'untrusted-secret': secret}, {}])
+                             'playbooks'), [{}])
         self.assertEqual(
             self._getSecrets('untrusted-secrets-untrusted-child',
                              'pre_playbooks'), [])