Merge "Increase github delay to 10 seconds" into feature/zuulv3
diff --git a/.zuul.yaml b/.zuul.yaml
index ede4391..fccd3f3 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -17,22 +17,22 @@
     name: zuul-stream-functional
     parent: multinode
     nodeset: zuul-functional
-    pre-run: playbooks/zuul-stream/pre
-    run: playbooks/zuul-stream/functional
+    pre-run: playbooks/zuul-stream/pre.yaml
+    run: playbooks/zuul-stream/functional.yaml
     post-run:
-      - playbooks/zuul-stream/post
-      - playbooks/zuul-stream/post-ara
+      - playbooks/zuul-stream/post.yaml
+      - playbooks/zuul-stream/post-ara.yaml
     required-projects:
       - openstack/ara
     files:
-      - "zuul/ansible/callback/.*"
-      - "playbooks/zuul-stream/.*"
+      - zuul/ansible/callback/.*
+      - playbooks/zuul-stream/.*
 
 - job:
     name: zuul-migrate
     parent: unittests
-    run: playbooks/zuul-migrate/run
-    post-run: playbooks/zuul-migrate/post
+    run: playbooks/zuul-migrate/run.yaml
+    post-run: playbooks/zuul-migrate/post.yaml
     # We're adding zuul to the required-projects so that we can also trigger
     # this from project-config changes
     required-projects:
diff --git a/doc/source/user/config.rst b/doc/source/user/config.rst
index 80c9136..0019932 100644
--- a/doc/source/user/config.rst
+++ b/doc/source/user/config.rst
@@ -781,42 +781,55 @@
 
    .. attr:: pre-run
 
-      The name of a playbook or list of playbooks without file
-      extension to run before the main body of a job.  The full path
-      to the playbook in the repo where the job is defined is
-      expected.
+      The name of a playbook or list of playbooks to run before the
+      main body of a job.  The full path to the playbook in the repo
+      where the job is defined is expected.
 
       When a job inherits from a parent, the child's pre-run playbooks
       are run after the parent's.  See :ref:`job` for more
       information.
 
+      .. warning::
+
+         If the path as specified does not exist, Zuul will try
+         appending the extensions ``.yaml`` and ``.yml``.  This
+         behavior is deprecated and will be removed in the future all
+         playbook paths should include the file extension.
+
    .. attr:: post-run
 
-      The name of a playbook or list of playbooks without file
-      extension to run after the main body of a job.  The full path to
-      the playbook in the repo where the job is defined is expected.
+      The name of a playbook or list of playbooks to run after the
+      main body of a job.  The full path to the playbook in the repo
+      where the job is defined is expected.
 
       When a job inherits from a parent, the child's post-run
       playbooks are run before the parent's.  See :ref:`job` for more
       information.
 
+      .. warning::
+
+         If the path as specified does not exist, Zuul will try
+         appending the extensions ``.yaml`` and ``.yml``.  This
+         behavior is deprecated and will be removed in the future all
+         playbook paths should include the file extension.
+
    .. attr:: run
 
-      The name of the main playbook for this job.  This parameter is
-      not normally necessary, as it defaults to a playbook with the
-      same name as the job inside of the ``playbooks/`` directory
-      (e.g., the ``foo`` job would default to ``playbooks/foo``.
-      However, if a playbook with a different name is needed, it can
-      be specified here.  The file extension is not required, but the
-      full path within the repo is.  When a child inherits from a
-      parent, a playbook with the name of the child job is implicitly
-      searched first, before falling back on the playbook used by the
-      parent job (unless the child job specifies a ``run`` attribute,
-      in which case that value is used).  Example:
+      The name of the main playbook for this job.  If it is not
+      supplied, the parent's playbook will be used (and likewise up
+      the inheritance chain).  The full path within the repo is
+      required.  Example:
 
       .. code-block:: yaml
 
-         run: playbooks/<name of the job>
+         run: playbooks/job-playbook.yaml
+
+      .. warning::
+
+         If the path as specified does not exist, Zuul will try
+         appending the extensions ``.yaml`` and ``.yml``.  This
+         behavior is deprecated and will be removed in the future all
+         playbook paths should include the file extension.
 
    .. attr:: roles
 
diff --git a/playbooks/zuul-stream/fixtures/test-stream.yaml b/playbooks/zuul-stream/fixtures/test-stream.yaml
index fd28757..c4946e8 100644
--- a/playbooks/zuul-stream/fixtures/test-stream.yaml
+++ b/playbooks/zuul-stream/fixtures/test-stream.yaml
@@ -46,3 +46,6 @@
       args:
         chdir: /itemloop/somewhere/that/does/not/exist
       failed_when: false
+
+    - name: Print binary data
+      command: echo -e '\x80abc'
diff --git a/playbooks/zuul-stream/functional.yaml b/playbooks/zuul-stream/functional.yaml
index 6b67b05..779a102 100644
--- a/playbooks/zuul-stream/functional.yaml
+++ b/playbooks/zuul-stream/functional.yaml
@@ -58,3 +58,8 @@
       shell: |
         egrep "^.+\| node1 \| OSError.+\/failure-itemloop\/" job-output.txt
         egrep "^.+\| node2 \| OSError.+\/failure-itemloop\/" job-output.txt
+
+    - name: Validate output - binary data
+      shell: |
+        egrep "^.*\| node1 \| \\\\x80abc" job-output.txt
+        egrep "^.*\| node2 \| \\\\x80abc" job-output.txt
diff --git a/tests/base.py b/tests/base.py
index 2da4d47..2a2f164 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -2230,8 +2230,6 @@
                                      files={'README': ''},
                                      branch='master', tag='init')
             if 'job' in item:
-                jobname = item['job']['name']
-                files['playbooks/%s.yaml' % jobname] = ''
                 if 'run' in item['job']:
                     files['%s.yaml' % item['job']['run']] = ''
                 for fn in zuul.configloader.as_list(
diff --git a/tests/fixtures/config/ansible/git/common-config/zuul.yaml b/tests/fixtures/config/ansible/git/common-config/zuul.yaml
index 67d1c70..28bfce1 100644
--- a/tests/fixtures/config/ansible/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/ansible/git/common-config/zuul.yaml
@@ -79,10 +79,11 @@
     failure-url: https://failure.example.com/zuul-logs/{build.uuid}/
 
 - job:
-    parent: base-urls
     name: python27
-    pre-run: playbooks/pre
-    post-run: playbooks/post
+    parent: base-urls
+    run: playbooks/python27.yaml
+    pre-run: playbooks/pre.yaml
+    post-run: playbooks/post.yaml
     vars:
       flagpath: '{{zuul._test.test_root}}/{{zuul.build}}.flag'
     roles:
@@ -93,11 +94,13 @@
 - job:
     parent: python27
     name: timeout
+    run: playbooks/timeout.yaml
     timeout: 1
 
 - job:
     parent: python27
     name: check-vars
+    run: playbooks/check-vars.yaml
     nodeset:
       nodes:
         - name: ubuntu-xenial
@@ -113,6 +116,7 @@
 - job:
     parent: python27
     name: check-secret-names
+    run: playbooks/check-secret-names.yaml
     nodeset:
       nodes:
         - name: ubuntu-xenial
@@ -124,9 +128,11 @@
 - job:
     parent: base-urls
     name: hello
+    run: playbooks/hello-post.yaml
     post-run: playbooks/hello-post
 
 - job:
     parent: python27
     name: failpost
+    run: playbooks/post-broken.yaml
     post-run: playbooks/post-broken
diff --git a/tests/fixtures/config/ansible/git/org_project/.zuul.yaml b/tests/fixtures/config/ansible/git/org_project/.zuul.yaml
index e144325..447f6cd 100644
--- a/tests/fixtures/config/ansible/git/org_project/.zuul.yaml
+++ b/tests/fixtures/config/ansible/git/org_project/.zuul.yaml
@@ -1,10 +1,12 @@
 - job:
     parent: python27
     name: faillocal
+    run: playbooks/faillocal.yaml
 
 - job:
     parent: hello
     name: hello-world
+    run: playbooks/hello-world.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/base-jobs/git/org_project/.zuul.yaml b/tests/fixtures/config/base-jobs/git/org_project/.zuul.yaml
index 9844c14..a7e3bdb 100644
--- a/tests/fixtures/config/base-jobs/git/org_project/.zuul.yaml
+++ b/tests/fixtures/config/base-jobs/git/org_project/.zuul.yaml
@@ -1,9 +1,11 @@
 - job:
     name: my-job
+    run: playbooks/my-job.yaml
 
 - job:
     name: other-job
     parent: other-base
+    run: playbooks/other-job.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/central-jobs/git/central-jobs/README b/tests/fixtures/config/central-jobs/git/central-jobs/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/central-jobs/git/central-jobs/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/central-jobs/git/central-jobs/playbooks/central-job.yaml b/tests/fixtures/config/central-jobs/git/central-jobs/playbooks/central-job.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/central-jobs/git/central-jobs/playbooks/central-job.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+  tasks: []
diff --git a/tests/fixtures/config/central-jobs/git/central-jobs/zuul.yaml b/tests/fixtures/config/central-jobs/git/central-jobs/zuul.yaml
new file mode 100644
index 0000000..2bf782e
--- /dev/null
+++ b/tests/fixtures/config/central-jobs/git/central-jobs/zuul.yaml
@@ -0,0 +1,9 @@
+- job:
+    name: central-job
+    run: playbooks/central-job.yaml
+
+- project-template:
+    name: central-jobs
+    check:
+      jobs:
+        - central-job
diff --git a/tests/fixtures/config/central-jobs/git/common-config/zuul.yaml b/tests/fixtures/config/central-jobs/git/common-config/zuul.yaml
new file mode 100644
index 0000000..c31af45
--- /dev/null
+++ b/tests/fixtures/config/central-jobs/git/common-config/zuul.yaml
@@ -0,0 +1,52 @@
+- pipeline:
+    name: check
+    manager: independent
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        Verified: 1
+    failure:
+      gerrit:
+        Verified: -1
+
+- pipeline:
+    name: gate
+    manager: dependent
+    trigger:
+      gerrit:
+        - event: comment-added
+          approval:
+            - Approved: 1
+    success:
+      gerrit:
+        Verified: 2
+        submit: true
+    failure:
+      gerrit:
+        Verified: -2
+    start:
+      gerrit:
+        Verified: 0
+    precedence: high
+
+- job:
+    name: base
+    parent: null
+
+- project:
+    name: common-config
+    check:
+      jobs: []
+    gate:
+      jobs:
+        - noop
+
+- project:
+    name: org/project
+    check:
+      jobs: []
+    gate:
+      jobs:
+        - noop
diff --git a/tests/fixtures/config/central-jobs/git/org_project/README b/tests/fixtures/config/central-jobs/git/org_project/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/central-jobs/git/org_project/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/central-jobs/main.yaml b/tests/fixtures/config/central-jobs/main.yaml
new file mode 100644
index 0000000..08f4d5d
--- /dev/null
+++ b/tests/fixtures/config/central-jobs/main.yaml
@@ -0,0 +1,9 @@
+- tenant:
+    name: tenant-one
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - central-jobs
+          - org/project
diff --git a/tests/fixtures/config/data-return/git/common-config/zuul.yaml b/tests/fixtures/config/data-return/git/common-config/zuul.yaml
index 2d5c51f..97b6b28 100644
--- a/tests/fixtures/config/data-return/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/data-return/git/common-config/zuul.yaml
@@ -18,13 +18,16 @@
 
 - job:
     name: data-return
+    run: playbooks/data-return.yaml
 
 - job:
     name: data-return-relative
     success-url: docs/index.html
+    run: playbooks/data-return-relative.yaml
 
 - job:
     name: child
+    run: playbooks/child.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/dependency-graph/git/common-config/zuul.yaml b/tests/fixtures/config/dependency-graph/git/common-config/zuul.yaml
index 4179226..6a96b50 100644
--- a/tests/fixtures/config/dependency-graph/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/dependency-graph/git/common-config/zuul.yaml
@@ -25,24 +25,31 @@
 
 - job:
     name: A
+    run: playbooks/A.yaml
 
 - job:
     name: B
+    run: playbooks/B.yaml
 
 - job:
     name: C
+    run: playbooks/C.yaml
 
 - job:
     name: D
+    run: playbooks/D.yaml
 
 - job:
     name: E
+    run: playbooks/E.yaml
 
 - job:
     name: F
+    run: playbooks/F.yaml
 
 - job:
     name: G
+    run: playbooks/G.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/disk-accountant/git/common-config/zuul.yaml b/tests/fixtures/config/disk-accountant/git/common-config/zuul.yaml
index 893ea05..9ad8de5 100644
--- a/tests/fixtures/config/disk-accountant/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/disk-accountant/git/common-config/zuul.yaml
@@ -18,6 +18,7 @@
 
 - job:
     name: dd-big-empty-file
+    run: playbooks/dd-big-empty-file.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/duplicate-pipeline/git/common-config/zuul.yaml b/tests/fixtures/config/duplicate-pipeline/git/common-config/zuul.yaml
index 117e381..dbd63c5 100755
--- a/tests/fixtures/config/duplicate-pipeline/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/duplicate-pipeline/git/common-config/zuul.yaml
@@ -32,6 +32,7 @@
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/final/git/common-config/zuul.yaml b/tests/fixtures/config/final/git/common-config/zuul.yaml
index f08d66e..944626c 100644
--- a/tests/fixtures/config/final/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/final/git/common-config/zuul.yaml
@@ -20,9 +20,9 @@
     final: true
     vars:
       dont_override_this: dummy
+    run: playbooks/job-final.yaml
 
 - project:
     name: org/project
     check:
       jobs: []
-
diff --git a/tests/fixtures/config/git-driver/git/common-config/zuul.yaml b/tests/fixtures/config/git-driver/git/common-config/zuul.yaml
index 34d1136..784b5f2 100644
--- a/tests/fixtures/config/git-driver/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/git-driver/git/common-config/zuul.yaml
@@ -17,6 +17,7 @@
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/implicit-roles/git/org_norole-project/.zuul.yaml b/tests/fixtures/config/implicit-roles/git/org_norole-project/.zuul.yaml
index 74c8e8e..f66c616 100644
--- a/tests/fixtures/config/implicit-roles/git/org_norole-project/.zuul.yaml
+++ b/tests/fixtures/config/implicit-roles/git/org_norole-project/.zuul.yaml
@@ -1,11 +1,13 @@
 - job:
     name: implicit-role-fail
+    run: playbooks/implicit-role-fail.yaml
 
 - job:
     name: explicit-role-fail
     attempts: 1
     roles:
       - zuul: org/norole-project
+    run: playbooks/explicit-role-fail.yaml
 
 - project:
     name: org/norole-project
diff --git a/tests/fixtures/config/implicit-roles/git/org_role-project/.zuul.yaml b/tests/fixtures/config/implicit-roles/git/org_role-project/.zuul.yaml
index 42cae95..e6e902e 100644
--- a/tests/fixtures/config/implicit-roles/git/org_role-project/.zuul.yaml
+++ b/tests/fixtures/config/implicit-roles/git/org_role-project/.zuul.yaml
@@ -1,11 +1,13 @@
 - job:
     name: implicit-role-ok
+    run: playbooks/implicit-role-ok.yaml
 
 - job:
     name: explicit-role-ok
     roles:
       - zuul: org/role-project
         name: role-name
+    run: playbooks/explicit-role-ok.yaml
 
 - project:
     name: org/role-project
diff --git a/tests/fixtures/config/in-repo-join/git/common-config/zuul.yaml b/tests/fixtures/config/in-repo-join/git/common-config/zuul.yaml
index 561fc39..a8ee256 100644
--- a/tests/fixtures/config/in-repo-join/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/in-repo-join/git/common-config/zuul.yaml
@@ -38,6 +38,7 @@
 
 - job:
     name: common-config-test
+    run: playbooks/common-config-test.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/in-repo-join/git/org_project/.zuul.yaml b/tests/fixtures/config/in-repo-join/git/org_project/.zuul.yaml
index 280342c..3845b26 100644
--- a/tests/fixtures/config/in-repo-join/git/org_project/.zuul.yaml
+++ b/tests/fixtures/config/in-repo-join/git/org_project/.zuul.yaml
@@ -1,2 +1,3 @@
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
diff --git a/tests/fixtures/config/in-repo/git/common-config/zuul.yaml b/tests/fixtures/config/in-repo/git/common-config/zuul.yaml
index a97af51..c98651c 100644
--- a/tests/fixtures/config/in-repo/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/in-repo/git/common-config/zuul.yaml
@@ -62,10 +62,9 @@
         approval:
           - Code-Review: 2
             username: maintainer
-    require:
       github:
         review:
-          - username: '^(herp|derp)$'
+          - username: ^(herp|derp)$
             type: approved
     trigger: {}
 
@@ -75,9 +74,11 @@
 
 - job:
     name: common-config-test
+    run: playbooks/common-config-test.yaml
 
 - job:
     name: template-job
+    run: playbooks/template-job.yaml
 
 - project-template:
     name: common-config-template
diff --git a/tests/fixtures/config/in-repo/git/org_project/.zuul.yaml b/tests/fixtures/config/in-repo/git/org_project/.zuul.yaml
index e1c27bb..2c39a10 100644
--- a/tests/fixtures/config/in-repo/git/org_project/.zuul.yaml
+++ b/tests/fixtures/config/in-repo/git/org_project/.zuul.yaml
@@ -1,5 +1,6 @@
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/inventory/git/common-config/zuul.yaml b/tests/fixtures/config/inventory/git/common-config/zuul.yaml
index e5727a2..900abd6 100644
--- a/tests/fixtures/config/inventory/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/inventory/git/common-config/zuul.yaml
@@ -41,7 +41,9 @@
       nodes:
         - name: ubuntu-xenial
           label: ubuntu-xenial
+    run: playbooks/single-inventory.yaml
 
 - job:
     name: group-inventory
     nodeset: nodeset1
+    run: playbooks/group-inventory.yaml
diff --git a/tests/fixtures/config/job-output/git/common-config/zuul.yaml b/tests/fixtures/config/job-output/git/common-config/zuul.yaml
index f182d8d..4df0020 100644
--- a/tests/fixtures/config/job-output/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/job-output/git/common-config/zuul.yaml
@@ -19,6 +19,7 @@
 - job:
     parent: base
     name: job-output
+    run: playbooks/job-output.yaml
 
 - job:
     name: job-output-failure
diff --git a/tests/fixtures/config/merges/git/common-config/zuul.yaml b/tests/fixtures/config/merges/git/common-config/zuul.yaml
index 1ea5048..94dbca7 100644
--- a/tests/fixtures/config/merges/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/merges/git/common-config/zuul.yaml
@@ -38,13 +38,16 @@
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - job:
     name: project-merge
     hold-following-changes: true
+    run: playbooks/project-merge.yaml
 
 - project:
     name: org/project-merge
diff --git a/tests/fixtures/config/multi-driver/git/org_common-config/zuul.yaml b/tests/fixtures/config/multi-driver/git/org_common-config/zuul.yaml
index 7a5c190..94a9ecb 100644
--- a/tests/fixtures/config/multi-driver/git/org_common-config/zuul.yaml
+++ b/tests/fixtures/config/multi-driver/git/org_common-config/zuul.yaml
@@ -12,12 +12,12 @@
         - event: patchset-created
     success:
       github:
-        status: 'success'
+        status: success
       gerrit:
         Verified: 1
     failure:
       github:
-        status: 'failure'
+        status: failure
       gerrit:
         Verified: 1
     start:
@@ -32,9 +32,11 @@
 
 - job:
     name: project-gerrit
+    run: playbooks/project-gerrit.yaml
 
 - job:
     name: project1-github
+    run: playbooks/project1-github.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/multi-tenant-semaphore/git/tenant-one-config/zuul.yaml b/tests/fixtures/config/multi-tenant-semaphore/git/tenant-one-config/zuul.yaml
index 5e377e7..ca936b9 100644
--- a/tests/fixtures/config/multi-tenant-semaphore/git/tenant-one-config/zuul.yaml
+++ b/tests/fixtures/config/multi-tenant-semaphore/git/tenant-one-config/zuul.yaml
@@ -1,6 +1,7 @@
 - job:
     name: project1-test1
     semaphore: test-semaphore
+    run: playbooks/project1-test1.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/config/multi-tenant-semaphore/git/tenant-two-config/zuul.yaml b/tests/fixtures/config/multi-tenant-semaphore/git/tenant-two-config/zuul.yaml
index a310532..a2866b3 100644
--- a/tests/fixtures/config/multi-tenant-semaphore/git/tenant-two-config/zuul.yaml
+++ b/tests/fixtures/config/multi-tenant-semaphore/git/tenant-two-config/zuul.yaml
@@ -1,6 +1,7 @@
 - job:
     name: project2-test1
     semaphore: test-semaphore
+    run: playbooks/project2-test1.yaml
 
 - project:
     name: org/project2
diff --git a/tests/fixtures/config/multi-tenant/git/common-config/zuul.yaml b/tests/fixtures/config/multi-tenant/git/common-config/zuul.yaml
index 273469c..31f1e27 100644
--- a/tests/fixtures/config/multi-tenant/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/multi-tenant/git/common-config/zuul.yaml
@@ -21,3 +21,4 @@
       nodes:
         - name: controller
           label: ubuntu-trusty
+    run: playbooks/python27.yaml
diff --git a/tests/fixtures/config/multi-tenant/git/tenant-one-config/zuul.yaml b/tests/fixtures/config/multi-tenant/git/tenant-one-config/zuul.yaml
index 9a1b928..278c12c 100644
--- a/tests/fixtures/config/multi-tenant/git/tenant-one-config/zuul.yaml
+++ b/tests/fixtures/config/multi-tenant/git/tenant-one-config/zuul.yaml
@@ -27,6 +27,7 @@
 
 - job:
     name: project1-test1
+    run: playbooks/project1-test1.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/config/multi-tenant/git/tenant-two-config/zuul.yaml b/tests/fixtures/config/multi-tenant/git/tenant-two-config/zuul.yaml
index 9496a49..2b795b0 100644
--- a/tests/fixtures/config/multi-tenant/git/tenant-two-config/zuul.yaml
+++ b/tests/fixtures/config/multi-tenant/git/tenant-two-config/zuul.yaml
@@ -27,6 +27,7 @@
 
 - job:
     name: project2-test1
+    run: playbooks/project2-test1.yaml
 
 - project:
     name: org/project2
diff --git a/tests/fixtures/config/openstack/git/project-config/zuul.yaml b/tests/fixtures/config/openstack/git/project-config/zuul.yaml
index de6321d..93bdb11 100644
--- a/tests/fixtures/config/openstack/git/project-config/zuul.yaml
+++ b/tests/fixtures/config/openstack/git/project-config/zuul.yaml
@@ -41,10 +41,12 @@
       nodes:
         - name: controller
           label: ubuntu-xenial
+    run: playbooks/base.yaml
 
 - job:
     name: python27
     parent: base
+    run: playbooks/python27.yaml
 
 - job:
     name: python27
@@ -54,10 +56,12 @@
       nodes:
         - name: controller
           label: ubuntu-trusty
+    run: playbooks/python27.yaml
 
 - job:
     name: python35
     parent: base
+    run: playbooks/python35.yaml
 
 - project-template:
     name: python-jobs
@@ -72,6 +76,7 @@
     required-projects:
       - openstack/keystone
       - openstack/nova
+    run: playbooks/dsvm.yaml
 
 - project:
     name: openstack/nova
diff --git a/tests/fixtures/config/post-playbook/git/common-config/zuul.yaml b/tests/fixtures/config/post-playbook/git/common-config/zuul.yaml
index 92a5515..16d7dee 100644
--- a/tests/fixtures/config/post-playbook/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/post-playbook/git/common-config/zuul.yaml
@@ -22,3 +22,4 @@
     post-run: playbooks/post
     vars:
       waitpath: '{{zuul._test.test_root}}/{{zuul.build}}/test_wait'
+    run: playbooks/python27.yaml
diff --git a/tests/fixtures/config/pre-playbook/git/common-config/zuul.yaml b/tests/fixtures/config/pre-playbook/git/common-config/zuul.yaml
index 16d1966..7817745 100644
--- a/tests/fixtures/config/pre-playbook/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/pre-playbook/git/common-config/zuul.yaml
@@ -20,3 +20,4 @@
     name: python27
     pre-run: playbooks/pre
     post-run: playbooks/post
+    run: playbooks/python27.yaml
diff --git a/tests/fixtures/config/push-reqs/git/org_common-config/zuul.yaml b/tests/fixtures/config/push-reqs/git/org_common-config/zuul.yaml
index 63af1c9..e7d52d9 100644
--- a/tests/fixtures/config/push-reqs/git/org_common-config/zuul.yaml
+++ b/tests/fixtures/config/push-reqs/git/org_common-config/zuul.yaml
@@ -47,7 +47,7 @@
     manager: independent
     require:
       github:
-        status: 'zuul:check:success'
+        status: zuul:check:success
     trigger:
       github:
         - event: push
@@ -82,6 +82,7 @@
 
 - job:
     name: job1
+    run: playbooks/job1.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/config/requirements/email/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/email/git/common-config/zuul.yaml
index 90c9ac2..e41cfad 100644
--- a/tests/fixtures/config/requirements/email/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/requirements/email/git/common-config/zuul.yaml
@@ -36,9 +36,11 @@
 
 - job:
     name: project1-job
+    run: playbooks/project1-job.yaml
 
 - job:
     name: project2-job
+    run: playbooks/project2-job.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/config/requirements/newer-than/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/newer-than/git/common-config/zuul.yaml
index 5f266a4..96e04d9 100644
--- a/tests/fixtures/config/requirements/newer-than/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/requirements/newer-than/git/common-config/zuul.yaml
@@ -38,9 +38,11 @@
 
 - job:
     name: project1-job
+    run: playbooks/project1-job.yaml
 
 - job:
     name: project2-job
+    run: playbooks/project2-job.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/config/requirements/older-than/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/older-than/git/common-config/zuul.yaml
index 4287a94..182a036 100644
--- a/tests/fixtures/config/requirements/older-than/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/requirements/older-than/git/common-config/zuul.yaml
@@ -38,9 +38,11 @@
 
 - job:
     name: project1-job
+    run: playbooks/project1-job.yaml
 
 - job:
     name: project2-job
+    run: playbooks/project2-job.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/config/requirements/reject-username/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/reject-username/git/common-config/zuul.yaml
index aabfb6a..7cbba03 100644
--- a/tests/fixtures/config/requirements/reject-username/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/requirements/reject-username/git/common-config/zuul.yaml
@@ -36,9 +36,11 @@
 
 - job:
     name: project1-job
+    run: playbooks/project1-job.yaml
 
 - job:
     name: project2-job
+    run: playbooks/project2-job.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/config/requirements/reject/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/reject/git/common-config/zuul.yaml
index 2661eed..0ce66de 100644
--- a/tests/fixtures/config/requirements/reject/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/requirements/reject/git/common-config/zuul.yaml
@@ -52,9 +52,11 @@
 
 - job:
     name: project1-job
+    run: playbooks/project1-job.yaml
 
 - job:
     name: project2-job
+    run: playbooks/project2-job.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/config/requirements/state/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/state/git/common-config/zuul.yaml
index 715b89f..9f018c9 100644
--- a/tests/fixtures/config/requirements/state/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/requirements/state/git/common-config/zuul.yaml
@@ -55,6 +55,7 @@
 
 - job:
     name: project-job
+    run: playbooks/project-job.yaml
 
 - project:
     name: current-project
diff --git a/tests/fixtures/config/requirements/username/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/username/git/common-config/zuul.yaml
index 778ac16..05203ab 100644
--- a/tests/fixtures/config/requirements/username/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/requirements/username/git/common-config/zuul.yaml
@@ -36,9 +36,11 @@
 
 - job:
     name: project1-job
+    run: playbooks/project1-job.yaml
 
 - job:
     name: project2-job
+    run: playbooks/project2-job.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/config/requirements/vote1/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/vote1/git/common-config/zuul.yaml
index b5d7498..88f64e3 100644
--- a/tests/fixtures/config/requirements/vote1/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/requirements/vote1/git/common-config/zuul.yaml
@@ -38,9 +38,11 @@
 
 - job:
     name: project1-job
+    run: playbooks/project1-job.yaml
 
 - job:
     name: project2-job
+    run: playbooks/project2-job.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/config/requirements/vote2/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/vote2/git/common-config/zuul.yaml
index 3f41868..31b2329 100644
--- a/tests/fixtures/config/requirements/vote2/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/requirements/vote2/git/common-config/zuul.yaml
@@ -42,9 +42,11 @@
 
 - job:
     name: project1-job
+    run: playbooks/project1-job.yaml
 
 - job:
     name: project2-job
+    run: playbooks/project2-job.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/config/roles/git/common-config/zuul.yaml b/tests/fixtures/config/roles/git/common-config/zuul.yaml
index 7ae6263..ba34ad6 100644
--- a/tests/fixtures/config/roles/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/roles/git/common-config/zuul.yaml
@@ -38,6 +38,7 @@
 
 - job:
     name: common-config-test
+    run: playbooks/common-config-test.yaml
 
 - project:
     name: common-config
diff --git a/tests/fixtures/config/roles/git/org_project/.zuul.yaml b/tests/fixtures/config/roles/git/org_project/.zuul.yaml
index 35c2153..0986b82 100644
--- a/tests/fixtures/config/roles/git/org_project/.zuul.yaml
+++ b/tests/fixtures/config/roles/git/org_project/.zuul.yaml
@@ -2,6 +2,7 @@
     name: project-test
     roles:
       - zuul: bare-role
+    run: playbooks/project-test.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/secret-inheritance/git/common-config/zuul.yaml b/tests/fixtures/config/secret-inheritance/git/common-config/zuul.yaml
index d5fa2bc..ad16d4e 100644
--- a/tests/fixtures/config/secret-inheritance/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/secret-inheritance/git/common-config/zuul.yaml
@@ -38,15 +38,18 @@
 
 - job:
     name: trusted-secrets
+    run: playbooks/trusted-secrets.yaml
     secrets:
       - trusted-secret
 
 - job:
     name: trusted-secrets-trusted-child
+    run: playbooks/trusted-secrets-trusted-child.yaml
     parent: trusted-secrets
 
 - job:
     name: untrusted-secrets-trusted-child
+    run: playbooks/untrusted-secrets-trusted-child.yaml
     parent: untrusted-secrets
     
 - project:
diff --git a/tests/fixtures/config/secret-inheritance/git/org_project/.zuul.yaml b/tests/fixtures/config/secret-inheritance/git/org_project/.zuul.yaml
index 5eeced2..b384669 100644
--- a/tests/fixtures/config/secret-inheritance/git/org_project/.zuul.yaml
+++ b/tests/fixtures/config/secret-inheritance/git/org_project/.zuul.yaml
@@ -1,14 +1,17 @@
 - job:
     name: untrusted-secrets
+    run: playbooks/untrusted-secrets.yaml
     secrets:
       - untrusted-secret
 
 - job:
     name: trusted-secrets-untrusted-child
+    run: playbooks/trusted-secrets-untrusted-child.yaml
     parent: trusted-secrets
 
 - job:
     name: untrusted-secrets-untrusted-child
+    run: playbooks/untrusted-secrets-untrusted-child.yaml
     parent: untrusted-secrets
 
 - project:
diff --git a/tests/fixtures/config/secret-leaks/git/common-config/zuul.yaml b/tests/fixtures/config/secret-leaks/git/common-config/zuul.yaml
index 4ab198f..9321df8 100644
--- a/tests/fixtures/config/secret-leaks/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/secret-leaks/git/common-config/zuul.yaml
@@ -55,12 +55,14 @@
 - job:
     parent: base
     name: secret-file
+    run: playbooks/secret-file.yaml
     secrets:
       - test_secret
 
 - job:
     parent: base
     name: secret-file-fail
+    run: playbooks/secret-file-fail.yaml
     secrets:
       - test_secret
 
diff --git a/tests/fixtures/config/semaphore/git/common-config/zuul.yaml b/tests/fixtures/config/semaphore/git/common-config/zuul.yaml
index c8bd322..52a0e7d 100644
--- a/tests/fixtures/config/semaphore/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/semaphore/git/common-config/zuul.yaml
@@ -25,22 +25,27 @@
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: semaphore-one-test1
     semaphore: test-semaphore
+    run: playbooks/semaphore-one-test1.yaml
 
 - job:
     name: semaphore-one-test2
     semaphore: test-semaphore
+    run: playbooks/semaphore-one-test2.yaml
 
 - job:
     name: semaphore-two-test1
     semaphore: test-semaphore-two
+    run: playbooks/semaphore-two-test1.yaml
 
 - job:
     name: semaphore-two-test2
     semaphore: test-semaphore-two
+    run: playbooks/semaphore-two-test2.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/shadow/git/local-config/zuul.yaml b/tests/fixtures/config/shadow/git/local-config/zuul.yaml
index 87f46b7..8935d8a 100644
--- a/tests/fixtures/config/shadow/git/local-config/zuul.yaml
+++ b/tests/fixtures/config/shadow/git/local-config/zuul.yaml
@@ -14,9 +14,11 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: test2
+    run: playbooks/test2.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/shadow/git/stdlib/.zuul.yaml b/tests/fixtures/config/shadow/git/stdlib/.zuul.yaml
index 6a6f9c9..5132653 100644
--- a/tests/fixtures/config/shadow/git/stdlib/.zuul.yaml
+++ b/tests/fixtures/config/shadow/git/stdlib/.zuul.yaml
@@ -1,10 +1,13 @@
 - job:
     name: base
+    run: playbooks/base.yaml
 
 - job:
     name: test1
     parent: base
+    run: playbooks/test1.yaml
 
 - job:
     name: test2
     parent: base
+    run: playbooks/test2.yaml
diff --git a/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml b/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
index 2160ef9..b2f15f9 100644
--- a/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
@@ -52,6 +52,7 @@
       nodes:
         - name: controller
           label: label1
+    run: playbooks/project-merge.yaml
 
 - job:
     name: project-test1
@@ -60,6 +61,7 @@
       nodes:
         - name: controller
           label: label1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test1
@@ -68,6 +70,7 @@
       nodes:
         - name: controller
           label: label2
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-post
@@ -75,6 +78,7 @@
       nodes:
         - name: static
           label: ubuntu-xenial
+    run: playbooks/project-post.yaml
 
 - job:
     name: project-test2
@@ -82,6 +86,7 @@
       nodes:
         - name: controller
           label: label1
+    run: playbooks/project-test2.yaml
 
 - job:
     name: project1-project2-integration
@@ -89,11 +94,13 @@
       nodes:
         - name: controller
           label: label1
+    run: playbooks/project1-project2-integration.yaml
 
 - job:
     name: project-testfile
     files:
       - .*-requires
+    run: playbooks/project-testfile.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/split-config/git/common-config/zuul.d/jobs.yaml b/tests/fixtures/config/split-config/git/common-config/zuul.d/jobs.yaml
index 9d15599..12e1c24 100644
--- a/tests/fixtures/config/split-config/git/common-config/zuul.d/jobs.yaml
+++ b/tests/fixtures/config/split-config/git/common-config/zuul.d/jobs.yaml
@@ -4,3 +4,4 @@
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
diff --git a/tests/fixtures/config/split-config/git/org_project1/.zuul.d/jobs.yaml b/tests/fixtures/config/split-config/git/org_project1/.zuul.d/jobs.yaml
index 33d74f3..20cd16a 100644
--- a/tests/fixtures/config/split-config/git/org_project1/.zuul.d/jobs.yaml
+++ b/tests/fixtures/config/split-config/git/org_project1/.zuul.d/jobs.yaml
@@ -1,2 +1,3 @@
 - job:
     name: project1-project2-integration
+    run: playbooks/project1-project2-integration.yaml
diff --git a/tests/fixtures/config/sql-driver/git/common-config/zuul.yaml b/tests/fixtures/config/sql-driver/git/common-config/zuul.yaml
index 8fce9e7..82c85c7 100644
--- a/tests/fixtures/config/sql-driver/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/sql-driver/git/common-config/zuul.yaml
@@ -7,12 +7,12 @@
     success:
       gerrit:
         Verified: 1
-      resultsdb:
+      resultsdb: null
     failure:
       gerrit:
         Verified: -1
-      resultsdb:
-      resultsdb_failures:
+      resultsdb: null
+      resultsdb_failures: null
 
 - job:
     name: base
@@ -20,15 +20,19 @@
 
 - job:
     name: project-merge
+    run: playbooks/project-merge.yaml
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - job:
     name: project-test3
+    run: playbooks/project-test3.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/streamer/git/common-config/zuul.yaml b/tests/fixtures/config/streamer/git/common-config/zuul.yaml
index f9925fe..8e67bfb 100644
--- a/tests/fixtures/config/streamer/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/streamer/git/common-config/zuul.yaml
@@ -19,3 +19,4 @@
     name: python27
     vars:
       waitpath: '{{zuul._test.test_root}}/{{zuul.build}}/test_wait'
+    run: playbooks/python27.yaml
diff --git a/tests/fixtures/config/success-url/git/common-config/zuul.yaml b/tests/fixtures/config/success-url/git/common-config/zuul.yaml
index 8929240..b9f4bff 100644
--- a/tests/fixtures/config/success-url/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/success-url/git/common-config/zuul.yaml
@@ -23,10 +23,12 @@
 - job:
     name: docs-draft-test
     success-url: http://docs-draft.example.org/{change.number:.2}/{change.number}/{change.patchset}/{pipeline.name}/{job.name}/{build.uuid:.7}/publish-docs/
+    run: playbooks/docs-draft-test.yaml
 
 - job:
     name: docs-draft-test2
     success-url: http://docs-draft.example.org/{NOPE}/{build.parameters[BAD]}/publish-docs/
+    run: playbooks/docs-draft-test2.yaml
 
 - project:
     name: org/docs
diff --git a/tests/fixtures/config/templated-project/git/common-config/zuul.d/jobs.yaml b/tests/fixtures/config/templated-project/git/common-config/zuul.d/jobs.yaml
index f9de1ad..7ad791d 100644
--- a/tests/fixtures/config/templated-project/git/common-config/zuul.d/jobs.yaml
+++ b/tests/fixtures/config/templated-project/git/common-config/zuul.d/jobs.yaml
@@ -4,18 +4,24 @@
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - job:
     name: layered-project-test3
+    run: playbooks/layered-project-test3.yaml
 
 - job:
     name: layered-project-test4
+    run: playbooks/layered-project-test4.yaml
 
 - job:
     name: layered-project-foo-test5
+    run: playbooks/layered-project-foo-test5.yaml
 
 - job:
     name: project-test6
+    run: playbooks/project-test6.yaml
diff --git a/tests/fixtures/config/unprotected-branches/git/org_project1/zuul.yaml b/tests/fixtures/config/unprotected-branches/git/org_project1/zuul.yaml
index 31abadf..1deffb3 100644
--- a/tests/fixtures/config/unprotected-branches/git/org_project1/zuul.yaml
+++ b/tests/fixtures/config/unprotected-branches/git/org_project1/zuul.yaml
@@ -1,5 +1,6 @@
 - job:
     name: project-test
+    run: playbooks/project-test.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/config/zuul-connections-multiple-gerrits/git/common-config/zuul.yaml b/tests/fixtures/config/zuul-connections-multiple-gerrits/git/common-config/zuul.yaml
index d70a384..6a0865e 100644
--- a/tests/fixtures/config/zuul-connections-multiple-gerrits/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/zuul-connections-multiple-gerrits/git/common-config/zuul.yaml
@@ -49,9 +49,11 @@
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - project:
     name: review.example.com/org/project1
@@ -65,7 +67,6 @@
       jobs:
         - project-test2
 
-
 - project:
     name: review.example.com/org/project2
     common_check:
diff --git a/tests/fixtures/config/zuul-connections-same-gerrit/git/common-config/zuul.yaml b/tests/fixtures/config/zuul-connections-same-gerrit/git/common-config/zuul.yaml
index eb65279..14a9b11 100644
--- a/tests/fixtures/config/zuul-connections-same-gerrit/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/zuul-connections-same-gerrit/git/common-config/zuul.yaml
@@ -17,9 +17,11 @@
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/zuultrigger/parent-change-enqueued/git/common-config/zuul.yaml b/tests/fixtures/config/zuultrigger/parent-change-enqueued/git/common-config/zuul.yaml
index 3dd8324..3fcc43c 100644
--- a/tests/fixtures/config/zuultrigger/parent-change-enqueued/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/zuultrigger/parent-change-enqueued/git/common-config/zuul.yaml
@@ -51,9 +51,11 @@
 
 - job:
     name: project-check
+    run: playbooks/project-check.yaml
 
 - job:
     name: project-gate
+    run: playbooks/project-gate.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/config/zuultrigger/project-change-merged/git/common-config/zuul.yaml b/tests/fixtures/config/zuultrigger/project-change-merged/git/common-config/zuul.yaml
index a5c5a1c..045e0a9 100644
--- a/tests/fixtures/config/zuultrigger/project-change-merged/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/zuultrigger/project-change-merged/git/common-config/zuul.yaml
@@ -49,9 +49,11 @@
 
 - job:
     name: project-check
+    run: playbooks/project-check.yaml
 
 - job:
     name: project-gate
+    run: playbooks/project-gate.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/autohold.yaml b/tests/fixtures/layouts/autohold.yaml
index 578f886..32b6822 100644
--- a/tests/fixtures/layouts/autohold.yaml
+++ b/tests/fixtures/layouts/autohold.yaml
@@ -14,6 +14,7 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-test2
@@ -21,6 +22,7 @@
       nodes:
         - name: controller
           label: label1
+    run: playbooks/project-test2.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/basic-github.yaml b/tests/fixtures/layouts/basic-github.yaml
index d7b323a..217e874 100644
--- a/tests/fixtures/layouts/basic-github.yaml
+++ b/tests/fixtures/layouts/basic-github.yaml
@@ -8,10 +8,10 @@
             - opened
             - changed
             - reopened
-          branch: '^master$'
+          branch: ^master$
         - event: pull_request
           action: comment
-          comment: 'test me'
+          comment: test me
     success:
       github: {}
     failure:
@@ -20,12 +20,15 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/crd-github.yaml b/tests/fixtures/layouts/crd-github.yaml
index 9696226..6ef881f 100644
--- a/tests/fixtures/layouts/crd-github.yaml
+++ b/tests/fixtures/layouts/crd-github.yaml
@@ -30,24 +30,31 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project1-test
+    run: playbooks/project1-test.yaml
 
 - job:
     name: project2-test
+    run: playbooks/project2-test.yaml
 
 - job:
     name: project3-test
+    run: playbooks/project3-test.yaml
 
 - job:
     name: project4-test
+    run: playbooks/project4-test.yaml
 
 - job:
     name: project5-test
+    run: playbooks/project5-test.yaml
 
 - job:
     name: project6-test
+    run: playbooks/project6-test.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/layouts/delayed-repo-init.yaml b/tests/fixtures/layouts/delayed-repo-init.yaml
index c89e2fa..0c9a152 100644
--- a/tests/fixtures/layouts/delayed-repo-init.yaml
+++ b/tests/fixtures/layouts/delayed-repo-init.yaml
@@ -43,18 +43,23 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-merge
+    run: playbooks/project-merge.yaml
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - job:
     name: project-post
+    run: playbooks/project-post.yaml
 
 - project:
     name: org/new-project
diff --git a/tests/fixtures/layouts/dependent-github.yaml b/tests/fixtures/layouts/dependent-github.yaml
index eb74163..6ad6bd2 100644
--- a/tests/fixtures/layouts/dependent-github.yaml
+++ b/tests/fixtures/layouts/dependent-github.yaml
@@ -6,29 +6,33 @@
       github:
         - event: pull_request
           action: labeled
-          label: 'merge'
+          label: merge
     success:
       github:
         merge: true
-        unlabel: 'merge'
+        unlabel: merge
     failure:
       github:
-        unlabel: 'merge'
+        unlabel: merge
 
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - job:
     name: project-merge
     failure-message: Unable to merge change
     hold-following-changes: true
+    run: playbooks/project-merge.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/dequeue-github.yaml b/tests/fixtures/layouts/dequeue-github.yaml
index ae61cd5..72d8145 100644
--- a/tests/fixtures/layouts/dequeue-github.yaml
+++ b/tests/fixtures/layouts/dequeue-github.yaml
@@ -11,9 +11,11 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: one-job-project-merge
+    run: playbooks/one-job-project-merge.yaml
 
 - project:
     name: org/one-job-project
diff --git a/tests/fixtures/layouts/disable_at.yaml b/tests/fixtures/layouts/disable_at.yaml
index 8c24c1b..a090d11 100644
--- a/tests/fixtures/layouts/disable_at.yaml
+++ b/tests/fixtures/layouts/disable_at.yaml
@@ -18,6 +18,7 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-test1
@@ -25,6 +26,7 @@
       nodes:
         - name: controller
           label: label1
+    run: playbooks/project-test1.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/dont-ignore-ref-deletes.yaml b/tests/fixtures/layouts/dont-ignore-ref-deletes.yaml
index bb98b57..9a9e592 100644
--- a/tests/fixtures/layouts/dont-ignore-ref-deletes.yaml
+++ b/tests/fixtures/layouts/dont-ignore-ref-deletes.yaml
@@ -10,6 +10,7 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-post
@@ -17,6 +18,7 @@
       nodes:
         - name: static
           label: ubuntu-xenial
+    run: playbooks/project-post.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/files-github.yaml b/tests/fixtures/layouts/files-github.yaml
index ec35259..ed053f9 100644
--- a/tests/fixtures/layouts/files-github.yaml
+++ b/tests/fixtures/layouts/files-github.yaml
@@ -9,11 +9,13 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-test1
     files:
-      - '.*-requires'
+      - .*-requires
+    run: playbooks/project-test1.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/footer-message.yaml b/tests/fixtures/layouts/footer-message.yaml
index 4ee25f6..746d384 100644
--- a/tests/fixtures/layouts/footer-message.yaml
+++ b/tests/fixtures/layouts/footer-message.yaml
@@ -28,10 +28,11 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-test1
-#    success-url: http://logs.exxxample.com/{change.number}/{change.patchset}/{pipeline.name}/{job.name}
+    run: playbooks/project-test1.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/idle.yaml b/tests/fixtures/layouts/idle.yaml
index 4cc07ae..4f3efe4 100644
--- a/tests/fixtures/layouts/idle.yaml
+++ b/tests/fixtures/layouts/idle.yaml
@@ -8,6 +8,7 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-bitrot
@@ -15,10 +16,10 @@
       nodes:
         - name: static
           label: ubuntu-xenial
+    run: playbooks/project-bitrot.yaml
 
 - project:
     name: org/project
     periodic:
       jobs:
         - project-bitrot
-
diff --git a/tests/fixtures/layouts/ignore-dependencies.yaml b/tests/fixtures/layouts/ignore-dependencies.yaml
index 89a82b3..b869dab 100644
--- a/tests/fixtures/layouts/ignore-dependencies.yaml
+++ b/tests/fixtures/layouts/ignore-dependencies.yaml
@@ -15,27 +15,35 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project1-merge
+    run: playbooks/project1-merge.yaml
 
 - job:
     name: project1-test1
+    run: playbooks/project1-test1.yaml
 
 - job:
     name: project1-test2
+    run: playbooks/project1-test2.yaml
 
 - job:
     name: project2-merge
+    run: playbooks/project2-merge.yaml
 
 - job:
     name: project2-test1
+    run: playbooks/project2-test1.yaml
 
 - job:
     name: project2-test2
+    run: playbooks/project2-test2.yaml
 
 - job:
     name: project1-project2-integration
+    run: playbooks/project1-project2-integration.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/layouts/inheritance.yaml b/tests/fixtures/layouts/inheritance.yaml
index 3fe7fd4..35f1402 100644
--- a/tests/fixtures/layouts/inheritance.yaml
+++ b/tests/fixtures/layouts/inheritance.yaml
@@ -14,23 +14,28 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-test-irrelevant-starts-empty
+    run: playbooks/project-test-irrelevant-starts-empty.yaml
 
 - job:
     name: project-test-irrelevant-starts-full
     irrelevant-files:
       - ^README$
       - ^ignoreme$
+    run: playbooks/project-test-irrelevant-starts-full.yaml
 
 - job:
     name: project-test-nomatch-starts-empty
+    run: playbooks/project-test-nomatch-starts-empty.yaml
 
 - job:
     name: project-test-nomatch-starts-full
     irrelevant-files:
       - ^README$
+    run: playbooks/project-test-nomatch-starts-full.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/irrelevant-files.yaml b/tests/fixtures/layouts/irrelevant-files.yaml
index 97f58e7..80be9af 100644
--- a/tests/fixtures/layouts/irrelevant-files.yaml
+++ b/tests/fixtures/layouts/irrelevant-files.yaml
@@ -14,9 +14,11 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-test-irrelevant-files
+    run: playbooks/project-test-irrelevant-files.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/job-variants.yaml b/tests/fixtures/layouts/job-variants.yaml
index de99e8a..356034f 100644
--- a/tests/fixtures/layouts/job-variants.yaml
+++ b/tests/fixtures/layouts/job-variants.yaml
@@ -20,6 +20,7 @@
       nodes:
         - name: controller
           label: base
+    run: playbooks/base.yaml
 
 - job:
     name: python27
@@ -33,6 +34,7 @@
       nodes:
         - name: controller
           label: new
+    run: playbooks/python27.yaml
 
 - job:
     name: python27
@@ -53,6 +55,7 @@
       - stable/essex
     pre-run: py27-essex-pre
     post-run: py27-essex-post
+    run: playbooks/python27.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/job-vars.yaml b/tests/fixtures/layouts/job-vars.yaml
index 22fc5c2..e46f084 100644
--- a/tests/fixtures/layouts/job-vars.yaml
+++ b/tests/fixtures/layouts/job-vars.yaml
@@ -14,6 +14,7 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: parentjob
@@ -24,6 +25,7 @@
       override: 0
       child1override: 0
       parent: 0
+    run: playbooks/parentjob.yaml
 
 - job:
     name: child1
@@ -34,6 +36,7 @@
       override: 1
       child1override: 1
       child1: 1
+    run: playbooks/child1.yaml
 
 - job:
     name: child2
@@ -43,10 +46,12 @@
     vars:
       override: 2
       child2: 2
+    run: playbooks/child2.yaml
 
 - job:
     name: child3
     parent: parentjob
+    run: playbooks/child3.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/labeling-github.yaml b/tests/fixtures/layouts/labeling-github.yaml
index 2441a9c..fbbf068 100644
--- a/tests/fixtures/layouts/labeling-github.yaml
+++ b/tests/fixtures/layouts/labeling-github.yaml
@@ -7,24 +7,26 @@
         - event: pull_request
           action: labeled
           label:
-            - 'test'
+            - test
         - event: pull_request
           action: unlabeled
           label:
-            - 'do not test'
+            - do not test
     success:
       github:
         label:
-          - 'tests passed'
+          - tests passed
         unlabel:
-          - 'test'
+          - test
 
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-labels
+    run: playbooks/project-labels.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/live-reconfiguration-add-job.yaml b/tests/fixtures/layouts/live-reconfiguration-add-job.yaml
index 57d2a5f..c871f1d 100644
--- a/tests/fixtures/layouts/live-reconfiguration-add-job.yaml
+++ b/tests/fixtures/layouts/live-reconfiguration-add-job.yaml
@@ -22,40 +22,46 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-merge
     hold-following-changes: true
+    run: playbooks/project-merge.yaml
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - job:
     name: project-test3
+    run: playbooks/project-test3.yaml
 
 - job:
     name: project-testfile
     files:
-      - '.*-requires'
+      - .*-requires
+    run: playbooks/project-testfile.yaml
 
 - project:
     name: org/project
     merge-mode: cherry-pick
     gate:
       jobs:
-      - project-merge
-      - project-test1:
-          dependencies:
-            - project-merge
-      - project-test2:
-          dependencies:
-            - project-merge
-      - project-test3:
-          dependencies:
-            - project-merge
-      - project-testfile:
-          dependencies:
-            - project-merge
+        - project-merge
+        - project-test1:
+            dependencies:
+              - project-merge
+        - project-test2:
+            dependencies:
+              - project-merge
+        - project-test3:
+            dependencies:
+              - project-merge
+        - project-testfile:
+            dependencies:
+              - project-merge
diff --git a/tests/fixtures/layouts/live-reconfiguration-del-project.yaml b/tests/fixtures/layouts/live-reconfiguration-del-project.yaml
index b149af0..259de84 100644
--- a/tests/fixtures/layouts/live-reconfiguration-del-project.yaml
+++ b/tests/fixtures/layouts/live-reconfiguration-del-project.yaml
@@ -14,19 +14,24 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-merge
     hold-following-changes: true
+    run: playbooks/project-merge.yaml
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - job:
     name: project-testfile
+    run: playbooks/project-testfile.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/live-reconfiguration-failed-job.yaml b/tests/fixtures/layouts/live-reconfiguration-failed-job.yaml
index c4719f4..e5cc651 100644
--- a/tests/fixtures/layouts/live-reconfiguration-failed-job.yaml
+++ b/tests/fixtures/layouts/live-reconfiguration-failed-job.yaml
@@ -14,26 +14,30 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-merge
     hold-following-changes: true
+    run: playbooks/project-merge.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - job:
     name: project-testfile
+    run: playbooks/project-testfile.yaml
 
 - project:
     name: org/project
     merge-mode: cherry-pick
     check:
       jobs:
-      - project-merge
-      - project-test2:
-          dependencies:
-            - project-merge
-      - project-testfile:
-          dependencies:
-            - project-merge
+        - project-merge
+        - project-test2:
+            dependencies:
+              - project-merge
+        - project-testfile:
+            dependencies:
+              - project-merge
diff --git a/tests/fixtures/layouts/live-reconfiguration-shared-queue.yaml b/tests/fixtures/layouts/live-reconfiguration-shared-queue.yaml
index e363b4c..49a1c2c 100644
--- a/tests/fixtures/layouts/live-reconfiguration-shared-queue.yaml
+++ b/tests/fixtures/layouts/live-reconfiguration-shared-queue.yaml
@@ -35,19 +35,24 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-merge
     hold-following-changes: true
+    run: playbooks/project-merge.yaml
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - job:
     name: project1-project2-integration
+    run: playbooks/project1-project2-integration.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/layouts/matcher-test.yaml b/tests/fixtures/layouts/matcher-test.yaml
index b511a2f..3239b52 100644
--- a/tests/fixtures/layouts/matcher-test.yaml
+++ b/tests/fixtures/layouts/matcher-test.yaml
@@ -35,6 +35,7 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-test1
@@ -42,6 +43,7 @@
       nodes:
         - name: controller
           label: label1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: ignore-branch
@@ -50,6 +52,7 @@
       nodes:
         - name: controller
           label: label2
+    run: playbooks/ignore-branch.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/merge-failure.yaml b/tests/fixtures/layouts/merge-failure.yaml
index 7c5121c..3828a06 100644
--- a/tests/fixtures/layouts/merge-failure.yaml
+++ b/tests/fixtures/layouts/merge-failure.yaml
@@ -23,7 +23,7 @@
     name: gate
     manager: dependent
     failure-message: Build failed.  For information on how to proceed, see http://wiki.example.org/Test_Failures
-    merge-failure-message: "The merge failed! For more information..."
+    merge-failure-message: The merge failed! For more information...
     trigger:
       gerrit:
         - event: comment-added
@@ -49,16 +49,20 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-merge
     hold-following-changes: true
+    run: playbooks/project-merge.yaml
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/merging-github.yaml b/tests/fixtures/layouts/merging-github.yaml
index c9673b9..20df0d4 100644
--- a/tests/fixtures/layouts/merging-github.yaml
+++ b/tests/fixtures/layouts/merging-github.yaml
@@ -2,12 +2,12 @@
     name: merge
     description: Pipeline for merging the pull request
     manager: independent
-    merge-failure-message: 'Merge failed'
+    merge-failure-message: Merge failed
     trigger:
       github:
         - event: pull_request
           action: comment
-          comment: 'merge me'
+          comment: merge me
     success:
       github:
         merge: true
@@ -16,6 +16,7 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/multiple-templates.yaml b/tests/fixtures/layouts/multiple-templates.yaml
index 7272cad..ece8396 100644
--- a/tests/fixtures/layouts/multiple-templates.yaml
+++ b/tests/fixtures/layouts/multiple-templates.yaml
@@ -14,9 +14,11 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: py27
+    run: playbooks/py27.yaml
 
 - project-template:
     name: python-jobs
diff --git a/tests/fixtures/layouts/no-jobs-project.yaml b/tests/fixtures/layouts/no-jobs-project.yaml
index 8f965e2..e23f36c 100644
--- a/tests/fixtures/layouts/no-jobs-project.yaml
+++ b/tests/fixtures/layouts/no-jobs-project.yaml
@@ -14,11 +14,13 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-testfile
     files:
       - .*-requires
+    run: playbooks/project-testfile.yaml
 
 - project:
     name: org/no-jobs-project
diff --git a/tests/fixtures/layouts/no-jobs.yaml b/tests/fixtures/layouts/no-jobs.yaml
index 301b27a..7d483ec 100644
--- a/tests/fixtures/layouts/no-jobs.yaml
+++ b/tests/fixtures/layouts/no-jobs.yaml
@@ -35,9 +35,11 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: gate-noop
+    run: playbooks/gate-noop.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/no-run.yaml b/tests/fixtures/layouts/no-run.yaml
new file mode 100644
index 0000000..bccee9c
--- /dev/null
+++ b/tests/fixtures/layouts/no-run.yaml
@@ -0,0 +1,22 @@
+- pipeline:
+    name: check
+    manager: independent
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        Verified: 1
+    failure:
+      gerrit:
+        Verified: -1
+
+- job:
+    name: base
+    parent: null
+
+- project:
+    name: org/project
+    check:
+      jobs:
+        - base
diff --git a/tests/fixtures/layouts/no-timer.yaml b/tests/fixtures/layouts/no-timer.yaml
index 7aaa1ed..67a3244 100644
--- a/tests/fixtures/layouts/no-timer.yaml
+++ b/tests/fixtures/layouts/no-timer.yaml
@@ -23,9 +23,11 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-bitrot
@@ -33,6 +35,7 @@
       nodes:
         - name: static
           label: ubuntu-xenial
+    run: playbooks/project-bitrot.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/nonvoting-job.yaml b/tests/fixtures/layouts/nonvoting-job.yaml
index 6a912bf..5b8e9be 100644
--- a/tests/fixtures/layouts/nonvoting-job.yaml
+++ b/tests/fixtures/layouts/nonvoting-job.yaml
@@ -22,17 +22,21 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: nonvoting-project-merge
     hold-following-changes: true
+    run: playbooks/nonvoting-project-merge.yaml
 
 - job:
     name: nonvoting-project-test1
+    run: playbooks/nonvoting-project-test1.yaml
 
 - job:
     name: nonvoting-project-test2
     voting: false
+    run: playbooks/nonvoting-project-test2.yaml
 
 - project:
     name: org/nonvoting-project
diff --git a/tests/fixtures/layouts/nonvoting-pipeline.yaml b/tests/fixtures/layouts/nonvoting-pipeline.yaml
index d8468dd..afe0528 100644
--- a/tests/fixtures/layouts/nonvoting-pipeline.yaml
+++ b/tests/fixtures/layouts/nonvoting-pipeline.yaml
@@ -12,13 +12,16 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-merge
     hold-following-changes: true
+    run: playbooks/project-merge.yaml
 
 - job:
     name: experimental-project-test
+    run: playbooks/experimental-project-test.yaml
 
 - project:
     name: org/experimental-project
diff --git a/tests/fixtures/layouts/one-job-project.yaml b/tests/fixtures/layouts/one-job-project.yaml
index 4b682d3..d5346f6 100644
--- a/tests/fixtures/layouts/one-job-project.yaml
+++ b/tests/fixtures/layouts/one-job-project.yaml
@@ -43,13 +43,16 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: one-job-project-merge
     hold-following-changes: true
+    run: playbooks/one-job-project-merge.yaml
 
 - job:
     name: one-job-project-post
+    run: playbooks/one-job-project-post.yaml
 
 - project:
     name: org/one-job-project
diff --git a/tests/fixtures/layouts/parent-matchers.yaml b/tests/fixtures/layouts/parent-matchers.yaml
index d53741e..2080215 100644
--- a/tests/fixtures/layouts/parent-matchers.yaml
+++ b/tests/fixtures/layouts/parent-matchers.yaml
@@ -14,18 +14,22 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: parent-job
     files: foo.txt
+    run: playbooks/parent-job.yaml
 
 - job:
     name: parent-job
     files: bar.txt
-    
+    run: playbooks/parent-job.yaml
+
 - job:
     name: child-job
     parent: parent-job
+    run: playbooks/child-job.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/push-tag-github.yaml b/tests/fixtures/layouts/push-tag-github.yaml
index 5805127..d689201 100644
--- a/tests/fixtures/layouts/push-tag-github.yaml
+++ b/tests/fixtures/layouts/push-tag-github.yaml
@@ -4,7 +4,7 @@
     trigger:
       github:
         - event: push
-          ref: '^refs/heads/master$'
+          ref: ^refs/heads/master$
 
 - pipeline:
     name: tag
@@ -17,12 +17,15 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-post
+    run: playbooks/project-post.yaml
 
 - job:
     name: project-tag
+    run: playbooks/project-tag.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/rate-limit.yaml b/tests/fixtures/layouts/rate-limit.yaml
index 1f32dbf..b432d51 100644
--- a/tests/fixtures/layouts/rate-limit.yaml
+++ b/tests/fixtures/layouts/rate-limit.yaml
@@ -27,15 +27,19 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-merge
+    run: playbooks/project-merge.yaml
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/repo-checkout-four-project.yaml b/tests/fixtures/layouts/repo-checkout-four-project.yaml
index 17303f5..11212e8 100644
--- a/tests/fixtures/layouts/repo-checkout-four-project.yaml
+++ b/tests/fixtures/layouts/repo-checkout-four-project.yaml
@@ -35,6 +35,7 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: integration
@@ -43,6 +44,7 @@
       - org/project2
       - org/project3
       - org/project4
+    run: playbooks/integration.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/layouts/repo-checkout-no-timer-override.yaml b/tests/fixtures/layouts/repo-checkout-no-timer-override.yaml
index 4680869..ca49292 100644
--- a/tests/fixtures/layouts/repo-checkout-no-timer-override.yaml
+++ b/tests/fixtures/layouts/repo-checkout-no-timer-override.yaml
@@ -10,6 +10,7 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: integration
@@ -17,6 +18,7 @@
     override-branch: stable/havana
     required-projects:
       - org/project1
+    run: playbooks/integration.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/layouts/repo-checkout-no-timer.yaml b/tests/fixtures/layouts/repo-checkout-no-timer.yaml
index ed20bb1..6b88801 100644
--- a/tests/fixtures/layouts/repo-checkout-no-timer.yaml
+++ b/tests/fixtures/layouts/repo-checkout-no-timer.yaml
@@ -10,12 +10,14 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: integration
     override-branch: stable/havana
     required-projects:
       - org/project1
+    run: playbooks/integration.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/layouts/repo-checkout-post.yaml b/tests/fixtures/layouts/repo-checkout-post.yaml
index 191569c..2e702bc 100644
--- a/tests/fixtures/layouts/repo-checkout-post.yaml
+++ b/tests/fixtures/layouts/repo-checkout-post.yaml
@@ -9,12 +9,14 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: integration
     required-projects:
       - org/project1
       - org/project2
+    run: playbooks/integration.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/layouts/repo-checkout-six-project.yaml b/tests/fixtures/layouts/repo-checkout-six-project.yaml
index 9a81eae..6079612 100644
--- a/tests/fixtures/layouts/repo-checkout-six-project.yaml
+++ b/tests/fixtures/layouts/repo-checkout-six-project.yaml
@@ -35,6 +35,7 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: integration
@@ -46,6 +47,7 @@
         override-branch: master
       - org/project5
       - org/project6
+    run: playbooks/integration.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/layouts/repo-checkout-timer-override.yaml b/tests/fixtures/layouts/repo-checkout-timer-override.yaml
index 99fc4f5..af5bd3c 100644
--- a/tests/fixtures/layouts/repo-checkout-timer-override.yaml
+++ b/tests/fixtures/layouts/repo-checkout-timer-override.yaml
@@ -8,6 +8,7 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: integration
@@ -15,6 +16,7 @@
     override-branch: stable/havana
     required-projects:
       - org/project1
+    run: playbooks/integration.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/layouts/repo-checkout-timer.yaml b/tests/fixtures/layouts/repo-checkout-timer.yaml
index e707732..739c066 100644
--- a/tests/fixtures/layouts/repo-checkout-timer.yaml
+++ b/tests/fixtures/layouts/repo-checkout-timer.yaml
@@ -8,11 +8,13 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: integration
     required-projects:
       - org/project1
+    run: playbooks/integration.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/layouts/repo-checkout-two-project.yaml b/tests/fixtures/layouts/repo-checkout-two-project.yaml
index 7910ae7..64c6ee9 100644
--- a/tests/fixtures/layouts/repo-checkout-two-project.yaml
+++ b/tests/fixtures/layouts/repo-checkout-two-project.yaml
@@ -35,12 +35,14 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: integration
     required-projects:
       - org/project1
       - org/project2
+    run: playbooks/integration.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/layouts/repo-deleted.yaml b/tests/fixtures/layouts/repo-deleted.yaml
index 3a7f6b3..2ee8ebd 100644
--- a/tests/fixtures/layouts/repo-deleted.yaml
+++ b/tests/fixtures/layouts/repo-deleted.yaml
@@ -35,10 +35,12 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-merge
     hold-following-changes: true
+    run: playbooks/project-merge.yaml
 
 - job:
     name: project-test1
@@ -46,6 +48,7 @@
       nodes:
         - name: controller
           label: label1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test1
@@ -54,9 +57,11 @@
       nodes:
         - name: controller
           label: label2
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - project:
     name: org/delete-project
diff --git a/tests/fixtures/layouts/reporting-github.yaml b/tests/fixtures/layouts/reporting-github.yaml
index 159f205..c909cf4 100644
--- a/tests/fixtures/layouts/reporting-github.yaml
+++ b/tests/fixtures/layouts/reporting-github.yaml
@@ -8,11 +8,11 @@
           action: opened
     start:
       github:
-        status: 'pending'
+        status: pending
         comment: false
     success:
       github:
-        status: 'success'
+        status: success
 
 - pipeline:
     name: reporting
@@ -22,13 +22,13 @@
       github:
         - event: pull_request
           action: comment
-          comment: 'reporting check'
+          comment: reporting check
     start:
       github: {}
     success:
       github:
         comment: false
-        status: 'success'
+        status: success
         status-url: http://logs.example.com/{tenant.name}/{pipeline.name}/{change.project}/{change.number}/{buildset.uuid}/
     failure:
       github:
@@ -42,14 +42,14 @@
       github:
         - event: pull_request
           action: comment
-          comment: 'long pipeline'
+          comment: long pipeline
     start:
       github:
-        status: 'pending'
+        status: pending
     success:
       github:
         comment: false
-        status: 'success'
+        status: success
         status-url: http://logs.example.com/{tenant.name}/{pipeline.name}/{change.project}/{change.number}/{buildset.uuid}/
     failure:
       github:
@@ -67,23 +67,25 @@
     start:
       github:
         comment: true
-        status: 'pending'
+        status: pending
     success:
       github:
         comment: true
-        status: 'success'
+        status: success
         merge: true
     failure:
       github:
         comment: true
-        status: 'failure'
+        status: failure
 
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/reporting-multiple-github.yaml b/tests/fixtures/layouts/reporting-multiple-github.yaml
index 0126ec5..67a237a 100644
--- a/tests/fixtures/layouts/reporting-multiple-github.yaml
+++ b/tests/fixtures/layouts/reporting-multiple-github.yaml
@@ -11,26 +11,29 @@
           action: opened
     start:
       github:
-        status: 'pending'
+        status: pending
         comment: false
       github_ent:
-        status: 'pending'
+        status: pending
         comment: false
     success:
       github:
-        status: 'success'
+        status: success
       github_ent:
-        status: 'success'
+        status: success
 
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project1-test1
+    run: playbooks/project1-test1.yaml
 
 - job:
     name: project2-test2
+    run: playbooks/project2-test2.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/layouts/requirements-github.yaml b/tests/fixtures/layouts/requirements-github.yaml
index f2ecd16..92bd9cb 100644
--- a/tests/fixtures/layouts/requirements-github.yaml
+++ b/tests/fixtures/layouts/requirements-github.yaml
@@ -3,12 +3,12 @@
     manager: independent
     require:
       github:
-        status: "zuul:check:success"
+        status: zuul:check:success
     trigger:
       github:
         - event: pull_request
           action: comment
-          comment: 'test me'
+          comment: test me
     success:
       github:
         comment: true
@@ -20,8 +20,8 @@
       github:
         - event: pull_request
           action: comment
-          comment: 'trigger me'
-          require-status: "zuul:check:success"
+          comment: trigger me
+          require-status: zuul:check:success
     success:
       github:
         comment: true
@@ -33,13 +33,13 @@
       github:
         - event: pull_request
           action: status
-          status: 'zuul:check:success'
+          status: zuul:check:success
     success:
       github:
-        status: 'success'
+        status: success
     failure:
       github:
-        status: 'failure'
+        status: failure
 
 - pipeline:
     name: reviewusername
@@ -47,13 +47,13 @@
     require:
       github:
         review:
-          - username: '^(herp|derp)$'
+          - username: ^(herp|derp)$
             type: approved
     trigger:
       github:
         - event: pull_request
           action: comment
-          comment: 'test me'
+          comment: test me
     success:
       github:
         comment: true
@@ -75,7 +75,7 @@
       github:
         - event: pull_request
           action: comment
-          comment: 'test me'
+          comment: test me
     success:
       github:
         comment: true
@@ -86,7 +86,7 @@
     require:
       github:
         review:
-          - username: 'derp'
+          - username: derp
             type: approved
             permission: write
     reject:
@@ -98,7 +98,7 @@
       github:
         - event: pull_request
           action: comment
-          comment: 'test me'
+          comment: test me
     success:
       github:
         comment: true
@@ -116,7 +116,7 @@
       github:
         - event: pull_request
           action: comment
-          comment: 'test me'
+          comment: test me
     success:
       github:
         comment: true
@@ -134,7 +134,7 @@
       github:
         - event: pull_request
           action: comment
-          comment: 'test me'
+          comment: test me
     success:
       github:
         comment: true
@@ -149,7 +149,7 @@
       github:
         - event: pull_request
           action: comment
-          comment: 'test me'
+          comment: test me
     success:
       github:
         comment: true
@@ -178,7 +178,7 @@
       github:
         - event: pull_request
           action: comment
-          comment: 'test me'
+          comment: test me
     success:
       github:
         comment: true
@@ -186,36 +186,47 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project1-pipeline
+    run: playbooks/project1-pipeline.yaml
 
 - job:
     name: project2-trigger
+    run: playbooks/project2-trigger.yaml
 
 - job:
     name: project3-reviewusername
+    run: playbooks/project3-reviewusername.yaml
 
 - job:
     name: project4-reviewreq
+    run: playbooks/project4-reviewreq.yaml
 
 - job:
     name: project5-reviewuserstate
+    run: playbooks/project5-reviewuserstate.yaml
 
 - job:
     name: project6-newerthan
+    run: playbooks/project6-newerthan.yaml
 
 - job:
     name: project7-olderthan
+    run: playbooks/project7-olderthan.yaml
 
 - job:
     name: project8-requireopen
+    run: playbooks/project8-requireopen.yaml
 
 - job:
     name: project9-requirecurrent
+    run: playbooks/project9-requirecurrent.yaml
 
 - job:
     name: project10-label
+    run: playbooks/project10-label.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/layouts/reviews-github.yaml b/tests/fixtures/layouts/reviews-github.yaml
index f186fbe..abc3d99 100644
--- a/tests/fixtures/layouts/reviews-github.yaml
+++ b/tests/fixtures/layouts/reviews-github.yaml
@@ -5,18 +5,20 @@
       github:
         - event: pull_request_review
           action: submitted
-          state: 'approve'
+          state: approve
     success:
       github:
         label:
-          - 'tests passed'
+          - tests passed
 
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-reviews
+    run: playbooks/project-reviews.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/smtp.yaml b/tests/fixtures/layouts/smtp.yaml
index 0654448..77391a0 100644
--- a/tests/fixtures/layouts/smtp.yaml
+++ b/tests/fixtures/layouts/smtp.yaml
@@ -41,10 +41,12 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-merge
     hold-following-changes: true
+    run: playbooks/project-merge.yaml
 
 - job:
     name: project-test1
@@ -52,6 +54,7 @@
       nodes:
         - name: controller
           label: label1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test1
@@ -60,9 +63,11 @@
       nodes:
         - name: controller
           label: label2
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/tags.yaml b/tests/fixtures/layouts/tags.yaml
index f86f5ab..2fda2db 100644
--- a/tests/fixtures/layouts/tags.yaml
+++ b/tests/fixtures/layouts/tags.yaml
@@ -14,11 +14,13 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: merge
     tags:
       - merge
+    run: playbooks/merge.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/layouts/three-projects.yaml b/tests/fixtures/layouts/three-projects.yaml
index 51cd406..33e81ac 100644
--- a/tests/fixtures/layouts/three-projects.yaml
+++ b/tests/fixtures/layouts/three-projects.yaml
@@ -35,19 +35,24 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-merge
     hold-following-changes: true
+    run: playbooks/project-merge.yaml
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - job:
     name: project1-project2-integration
+    run: playbooks/project1-project2-integration.yaml
 
 - project:
     name: org/project1
diff --git a/tests/fixtures/layouts/timer-smtp.yaml b/tests/fixtures/layouts/timer-smtp.yaml
index a27b183..d9e4282 100644
--- a/tests/fixtures/layouts/timer-smtp.yaml
+++ b/tests/fixtures/layouts/timer-smtp.yaml
@@ -13,14 +13,17 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-bitrot-stable-old
     success-url: http://logs.example.com/{job.name}/{build.number}
+    run: playbooks/project-bitrot-stable-old.yaml
 
 - job:
     name: project-bitrot-stable-older
     success-url: http://logs.example.com/{job.name}/{build.number}
+    run: playbooks/project-bitrot-stable-older.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/timer.yaml b/tests/fixtures/layouts/timer.yaml
index 8c0cc2b..e9e9a17 100644
--- a/tests/fixtures/layouts/timer.yaml
+++ b/tests/fixtures/layouts/timer.yaml
@@ -21,12 +21,15 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project-test1
+    run: playbooks/project-test1.yaml
 
 - job:
     name: project-test2
+    run: playbooks/project-test2.yaml
 
 - job:
     name: project-bitrot
@@ -34,6 +37,7 @@
       nodes:
         - name: static
           label: ubuntu-xenial
+    run: playbooks/project-bitrot.yaml
 
 - project:
     name: org/project
diff --git a/tests/fixtures/layouts/untrusted-secrets.yaml b/tests/fixtures/layouts/untrusted-secrets.yaml
index b90d3d7..337587a 100644
--- a/tests/fixtures/layouts/untrusted-secrets.yaml
+++ b/tests/fixtures/layouts/untrusted-secrets.yaml
@@ -14,10 +14,12 @@
 - job:
     name: base
     parent: null
+    run: playbooks/base.yaml
 
 - job:
     name: project1-test
     post-review: true
+    run: playbooks/project1-test.yaml
 
 - project:
     name: org/project1
diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py
index 11f4eeb..784fcb3 100644
--- a/tests/unit/test_model.py
+++ b/tests/unit/test_model.py
@@ -194,7 +194,8 @@
             'name': 'project',
             'gate': {
                 'jobs': [
-                    {'python27': {'timeout': 70}}
+                    {'python27': {'timeout': 70,
+                                  'run': 'playbooks/python27.yaml'}}
                 ]
             }
         }])
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index 1b5f9dd..b0df2b2 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -2460,7 +2460,7 @@
         self.assertEqual([x['path'] for x in p['post_playbooks']],
                          ['py27-post-a', 'py27-post-b', 'base-post'])
         self.assertEqual([x['path'] for x in p['playbooks']],
-                         ['playbooks/python27', 'playbooks/base'])
+                         ['playbooks/python27.yaml'])
 
         p = self.history[1].parameters
         self.assertEqual(p['timeout'], 50)
@@ -2484,7 +2484,16 @@
                          ['py27-essex-post', 'py27-post-a', 'py27-post-b',
                           'base-post'])
         self.assertEqual([x['path'] for x in p['playbooks']],
-                         ['playbooks/python27', 'playbooks/base'])
+                         ['playbooks/python27.yaml'])
+
+    @simple_layout("layouts/no-run.yaml")
+    def test_job_without_run(self):
+        "Test that a job without a run playbook errors"
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        self.assertIn('Job base does not specify a run playbook',
+                      A.messages[-1])
 
     def test_queue_names(self):
         "Test shared change queue names"
@@ -5879,6 +5888,7 @@
 
             - job:
                 name: project-test2
+                run: playbooks/project-test2.yaml
                 semaphore: test-semaphore
 
             - project:
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index b9ae04b..c04604d 100755
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -128,6 +128,7 @@
             - job:
                 name: project-test
                 parent: job-final
+                run: playbooks/project-test.yaml
 
             - project:
                 name: org/project
@@ -250,6 +251,94 @@
         self.waitUntilSettled()
 
 
+class TestCentralJobs(ZuulTestCase):
+    tenant_config_file = 'config/central-jobs/main.yaml'
+
+    def setUp(self):
+        super(TestCentralJobs, self).setUp()
+        self.create_branch('org/project', 'stable')
+        self.fake_gerrit.addEvent(
+            self.fake_gerrit.getFakeBranchCreatedEvent(
+                'org/project', 'stable'))
+        self.waitUntilSettled()
+
+    def _updateConfig(self, config, branch):
+        file_dict = {'.zuul.yaml': config}
+        C = self.fake_gerrit.addFakeChange('org/project', branch, 'C',
+                                           files=file_dict)
+        C.addApproval('Code-Review', 2)
+        self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
+        self.waitUntilSettled()
+        self.fake_gerrit.addEvent(C.getChangeMergedEvent())
+        self.waitUntilSettled()
+
+    def _test_central_job_on_branch(self, branch, other_branch):
+        # Test that a job defined on a branchless repo only runs on
+        # the branch applied
+        config = textwrap.dedent(
+            """
+            - project:
+                name: org/project
+                check:
+                  jobs:
+                    - central-job
+            """)
+        self._updateConfig(config, branch)
+
+        A = self.fake_gerrit.addFakeChange('org/project', branch, 'A')
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        self.assertHistory([
+            dict(name='central-job', result='SUCCESS', changes='2,1')])
+
+        # No jobs should run for this change.
+        B = self.fake_gerrit.addFakeChange('org/project', other_branch, 'B')
+        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        self.assertHistory([
+            dict(name='central-job', result='SUCCESS', changes='2,1')])
+
+    def test_central_job_on_stable(self):
+        self._test_central_job_on_branch('master', 'stable')
+
+    def test_central_job_on_master(self):
+        self._test_central_job_on_branch('stable', 'master')
+
+    def _test_central_template_on_branch(self, branch, other_branch):
+        # Test that a project-template defined on a branchless repo
+        # only runs on the branch applied
+        config = textwrap.dedent(
+            """
+            - project:
+                name: org/project
+                templates: ['central-jobs']
+            """)
+        self._updateConfig(config, branch)
+
+        A = self.fake_gerrit.addFakeChange('org/project', branch, 'A')
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        self.assertHistory([
+            dict(name='central-job', result='SUCCESS', changes='2,1')])
+
+        # No jobs should run for this change.
+        B = self.fake_gerrit.addFakeChange('org/project', other_branch, 'B')
+        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        self.assertHistory([
+            dict(name='central-job', result='SUCCESS', changes='2,1')])
+
+    def test_central_template_on_stable(self):
+        self._test_central_template_on_branch('master', 'stable')
+
+    def test_central_template_on_master(self):
+        self._test_central_template_on_branch('stable', 'master')
+
+
 class TestInRepoConfig(ZuulTestCase):
     # A temporary class to hold new tests while others are disabled
 
@@ -310,6 +399,7 @@
 
             - job:
                 name: project-test2
+                run: playbooks/project-test2.yaml
 
             - project:
                 name: org/project
@@ -355,6 +445,8 @@
             dict(name='project-test2', result='SUCCESS', changes='2,1')])
 
     def test_dynamic_template(self):
+        # Tests that a project can't update a template in another
+        # project.
         in_repo_conf = textwrap.dedent(
             """
             - job:
@@ -376,8 +468,12 @@
                                            files=file_dict)
         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
         self.waitUntilSettled()
-        self.assertHistory([
-            dict(name='template-job', result='SUCCESS', changes='1,1')])
+
+        self.assertEqual(A.patchsets[0]['approvals'][0]['value'], "-1")
+        self.assertIn('Project template common-config-template '
+                      'is already defined',
+                      A.messages[0],
+                      "A should have failed the check pipeline")
 
     def test_dynamic_config_non_existing_job(self):
         """Test that requesting a non existent job fails"""
@@ -460,9 +556,11 @@
             """
             - job:
                 name: project-test1
+                run: playbooks/project-test1.yaml
 
             - job:
                 name: project-test2
+                run: playbooks/project-test2.yaml
 
             - project:
                 name: org/project
@@ -493,9 +591,11 @@
             """
             - job:
                 name: project-test1
+                run: playbooks/project-test1.yaml
 
             - job:
                 name: project-test2
+                run: playbooks/project-test2.yaml
 
             - project:
                 name: org/project
@@ -536,6 +636,7 @@
 
             - job:
                 name: project-test2
+                run: playbooks/project-test2.yaml
 
             - project:
                 name: org/project
@@ -608,6 +709,7 @@
 
             - job:
                 name: project-test2
+                run: playbooks/project-test2.yaml
 
             - project:
                 name: org/project
@@ -1075,6 +1177,7 @@
             """
             - job:
                 name: project-test1
+                run: playbooks/project-test1.yaml
             - project-template:
                 name: some-jobs
                 tenant-one-gate:
@@ -1133,6 +1236,7 @@
             """
             - job:
                 name: project-test1
+                run: playbooks/project-test1.yaml
 
             - project:
                 name: org/project1
@@ -1187,6 +1291,7 @@
             """
             - job:
                 name: project-test1
+                run: playbooks/project-test1.yaml
 
             - job:
                 name: project-test2
@@ -1303,9 +1408,11 @@
             """
             - job:
                 name: project-test1
+                run: playbooks/project-test1.yaml
 
             - job:
                 name: project-test2
+                run: playbooks/project-test2.yaml
 
             - project:
                 name: org/project
@@ -1349,6 +1456,7 @@
             """
             - job:
                 name: project-test1
+                run: playbooks/project-test1.yaml
 
             - project:
                 name: org/project
@@ -1482,13 +1590,14 @@
             """
             - job:
                 name: %s
+                run: playbooks/%s.yaml
 
             - project:
                 name: org/plugin-project
                 check:
                   jobs:
                     - %s
-            """ % (job_name, job_name))
+            """ % (job_name, job_name, job_name))
 
         file_dict = {'.zuul.yaml': conf}
         A = self.fake_gerrit.addFakeChange('org/plugin-project', 'master', 'A',
@@ -1690,6 +1799,7 @@
             - job:
                 name: project-test
                 parent: parent
+                run: playbooks/project-test.yaml
                 roles:
                   - zuul: org/project
 
@@ -1726,6 +1836,7 @@
             """
             - job:
                 name: project-test
+                run: playbooks/project-test.yaml
                 roles:
                   - zuul: common-config
 
@@ -2014,7 +2125,7 @@
                   'username': 'test-username'}
         self.assertEqual(
             self._getSecrets('trusted-secrets', 'playbooks'),
-            [{'trusted-secret': secret}, {}])
+            [{'trusted-secret': secret}])
         self.assertEqual(
             self._getSecrets('trusted-secrets', 'pre_playbooks'), [])
         self.assertEqual(
@@ -2022,8 +2133,7 @@
 
         self.assertEqual(
             self._getSecrets('trusted-secrets-trusted-child',
-                             'playbooks'),
-            [{}, {'trusted-secret': secret}, {}])
+                             'playbooks'), [{}])
         self.assertEqual(
             self._getSecrets('trusted-secrets-trusted-child',
                              'pre_playbooks'), [])
@@ -2033,8 +2143,7 @@
 
         self.assertEqual(
             self._getSecrets('trusted-secrets-untrusted-child',
-                             'playbooks'),
-            [{}, {'trusted-secret': secret}, {}])
+                             'playbooks'), [{}])
         self.assertEqual(
             self._getSecrets('trusted-secrets-untrusted-child',
                              'pre_playbooks'), [])
@@ -2048,7 +2157,7 @@
                   'username': 'test-username'}
         self.assertEqual(
             self._getSecrets('untrusted-secrets', 'playbooks'),
-            [{'untrusted-secret': secret}, {}])
+            [{'untrusted-secret': secret}])
         self.assertEqual(
             self._getSecrets('untrusted-secrets', 'pre_playbooks'), [])
         self.assertEqual(
@@ -2056,8 +2165,7 @@
 
         self.assertEqual(
             self._getSecrets('untrusted-secrets-trusted-child',
-                             'playbooks'),
-            [{}, {'untrusted-secret': secret}, {}])
+                             'playbooks'), [{}])
         self.assertEqual(
             self._getSecrets('untrusted-secrets-trusted-child',
                              'pre_playbooks'), [])
@@ -2067,8 +2175,7 @@
 
         self.assertEqual(
             self._getSecrets('untrusted-secrets-untrusted-child',
-                             'playbooks'),
-            [{}, {'untrusted-secret': secret}, {}])
+                             'playbooks'), [{}])
         self.assertEqual(
             self._getSecrets('untrusted-secrets-untrusted-child',
                              'pre_playbooks'), [])
diff --git a/zuul/ansible/callback/zuul_stream.py b/zuul/ansible/callback/zuul_stream.py
index 8ba3b86..8845e9b 100644
--- a/zuul/ansible/callback/zuul_stream.py
+++ b/zuul/ansible/callback/zuul_stream.py
@@ -128,22 +128,29 @@
                 continue
             msg = "%s\n" % log_id
             s.send(msg.encode("utf-8"))
-            buff = s.recv(4096).decode("utf-8")
+            buff = s.recv(4096)
             buffering = True
             while buffering:
-                if "\n" in buff:
-                    (line, buff) = buff.split("\n", 1)
-                    done = self._log_streamline(host, line)
+                if b'\n' in buff:
+                    (line, buff) = buff.split(b'\n', 1)
+                    # We can potentially get binary data here. In order to
+                    # being able to handle that use the backslashreplace
+                    # error handling method. This decodes unknown utf-8
+                    # code points to escape sequences which exactly represent
+                    # the correct data without throwing a decoding exception.
+                    done = self._log_streamline(
+                        host, line.decode("utf-8", "backslashreplace"))
                     if done:
                         return
                 else:
-                    more = s.recv(4096).decode("utf-8")
+                    more = s.recv(4096)
                     if not more:
                         buffering = False
                     else:
                         buff += more
             if buff:
-                self._log_streamline(host, line)
+                self._log_streamline(
+                    host, line.decode("utf-8", "backslashreplace"))
 
     def _log_streamline(self, host, line):
         if "[Zuul] Task exit code" in line:
diff --git a/zuul/configloader.py b/zuul/configloader.py
index 2cb23d9..6737c7b 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -477,12 +477,7 @@
     ]
 
     @staticmethod
-    def _getImpliedBranches(tenant, job, project_pipeline):
-        # If this is a project pipeline, don't create implied branch
-        # matchers -- that's handled in ProjectParser.
-        if project_pipeline:
-            return None
-
+    def _getImpliedBranches(tenant, job):
         # If the user has set a pragma directive for this, use the
         # value (if unset, the value is None).
         if job.source_context.implied_branch_matchers is True:
@@ -612,12 +607,6 @@
             run = model.PlaybookContext(job.source_context, conf['run'],
                                         job.roles, secrets)
             job.run = (run,)
-        else:
-            if not project_pipeline:
-                run_name = os.path.join('playbooks', job.name)
-                run = model.PlaybookContext(job.source_context, run_name,
-                                            job.roles, secrets)
-                job.implied_run = (run,)
 
         for k in JobParser.simple_attributes:
             a = k.replace('-', '_')
@@ -679,8 +668,7 @@
 
         branches = None
         if ('branches' not in conf):
-            branches = JobParser._getImpliedBranches(
-                tenant, job, project_pipeline)
+            branches = JobParser._getImpliedBranches(tenant, job)
         if (not branches) and ('branches' in conf):
             branches = as_list(conf['branches'])
         if branches:
@@ -751,8 +739,8 @@
         if validate:
             with configuration_exceptions('project-template', conf):
                 self.schema(conf)
-        project_template = model.ProjectConfig(conf['name'])
         source_context = conf['_source_context']
+        project_template = model.ProjectConfig(conf['name'], source_context)
         start_mark = conf['_start_mark']
         for pipeline in self.layout.pipelines.values():
             conf_pipeline = conf.get(pipeline.name)
@@ -1130,7 +1118,7 @@
 
     @staticmethod
     def fromYaml(base, project_key_dir, connections, scheduler, merger, conf,
-                 cached):
+                 old_tenant):
         TenantParser.getSchema(connections)(conf)
         tenant = model.Tenant(conf['name'])
         if conf.get('max-nodes-per-job') is not None:
@@ -1147,16 +1135,20 @@
         # tpcs is TenantProjectConfigs
         config_tpcs, untrusted_tpcs = \
             TenantParser._loadTenantProjects(
-                tenant, project_key_dir, connections, conf)
+                project_key_dir, connections, conf)
         for tpc in config_tpcs:
             tenant.addConfigProject(tpc)
         for tpc in untrusted_tpcs:
             tenant.addUntrustedProject(tpc)
 
         for tpc in config_tpcs + untrusted_tpcs:
-            TenantParser._getProjectBranches(tenant, tpc)
+            TenantParser._getProjectBranches(tenant, tpc, old_tenant)
             TenantParser._resolveShadowProjects(tenant, tpc)
 
+        if old_tenant:
+            cached = True
+        else:
+            cached = False
         tenant.config_projects_config, tenant.untrusted_projects_config = \
             TenantParser._loadTenantInRepoLayouts(merger, connections,
                                                   tenant.config_projects,
@@ -1178,9 +1170,16 @@
         tpc.shadow_projects = frozenset(shadow_projects)
 
     @staticmethod
-    def _getProjectBranches(tenant, tpc):
-        branches = sorted(tpc.project.source.getProjectBranches(
-            tpc.project, tenant))
+    def _getProjectBranches(tenant, tpc, old_tenant):
+        # If we're performing a tenant reconfiguration, we will have
+        # an old_tenant object, however, we may be doing so because of
+        # a branch creation event, so if we don't have any cached
+        # data, query the branches again as well.
+        if old_tenant and tpc.project.unparsed_config:
+            branches = old_tenant.getProjectBranches(tpc.project)[:]
+        else:
+            branches = sorted(tpc.project.source.getProjectBranches(
+                tpc.project, tenant))
         if 'master' in branches:
             branches.remove('master')
             branches = ['master'] + branches
@@ -1236,7 +1235,7 @@
                 encryption.deserialize_rsa_keypair(f.read())
 
     @staticmethod
-    def _getProject(tenant, source, conf, current_include):
+    def _getProject(source, conf, current_include):
         if isinstance(conf, str):
             # Return a project object whether conf is a dict or a str
             project = source.getProject(conf)
@@ -1268,13 +1267,13 @@
         return tenant_project_config
 
     @staticmethod
-    def _getProjects(tenant, source, conf, current_include):
+    def _getProjects(source, conf, current_include):
         # Return a project object whether conf is a dict or a str
         projects = []
         if isinstance(conf, str):
             # A simple project name string
             projects.append(TenantParser._getProject(
-                tenant, source, conf, current_include))
+                source, conf, current_include))
         elif len(conf.keys()) > 1 and 'projects' in conf:
             # This is a project group
             if 'include' in conf:
@@ -1286,18 +1285,18 @@
                 current_include = current_include - exclude
             for project in conf['projects']:
                 sub_projects = TenantParser._getProjects(
-                    tenant, source, project, current_include)
+                    source, project, current_include)
                 projects.extend(sub_projects)
         elif len(conf.keys()) == 1:
             # A project with overrides
             projects.append(TenantParser._getProject(
-                tenant, source, conf, current_include))
+                source, conf, current_include))
         else:
             raise Exception("Unable to parse project %s", conf)
         return projects
 
     @staticmethod
-    def _loadTenantProjects(tenant, project_key_dir, connections, conf_tenant):
+    def _loadTenantProjects(project_key_dir, connections, conf_tenant):
         config_projects = []
         untrusted_projects = []
 
@@ -1310,7 +1309,7 @@
             current_include = default_include
             for conf_repo in conf_source.get('config-projects', []):
                 # tpcs = TenantProjectConfigs
-                tpcs = TenantParser._getProjects(tenant, source, conf_repo,
+                tpcs = TenantParser._getProjects(source, conf_repo,
                                                  current_include)
                 for tpc in tpcs:
                     TenantParser._loadProjectKeys(
@@ -1319,7 +1318,7 @@
 
             current_include = frozenset(default_include - set(['pipeline']))
             for conf_repo in conf_source.get('untrusted-projects', []):
-                tpcs = TenantParser._getProjects(tenant, source, conf_repo,
+                tpcs = TenantParser._getProjects(source, conf_repo,
                                                  current_include)
                 for tpc in tpcs:
                     TenantParser._loadProjectKeys(
@@ -1568,8 +1567,9 @@
             classes = TenantParser._getLoadClasses(tenant, config_template)
             if 'project-template' not in classes:
                 continue
-            layout.addProjectTemplate(project_template_parser.fromYaml(
-                config_template))
+            with configuration_exceptions('project-template', config_template):
+                layout.addProjectTemplate(project_template_parser.fromYaml(
+                    config_template))
 
         project_parser = ProjectParser(tenant, layout, project_template_parser)
         for config_projects in data.projects.values():
@@ -1635,7 +1635,7 @@
             # When performing a full reload, do not use cached data.
             tenant = TenantParser.fromYaml(
                 base, project_key_dir, connections, scheduler, merger,
-                conf_tenant, cached=False)
+                conf_tenant, old_tenant=None)
             abide.tenants[tenant.name] = tenant
         return abide
 
@@ -1650,7 +1650,7 @@
         # When reloading a tenant only, use cached data if available.
         new_tenant = TenantParser.fromYaml(
             base, project_key_dir, connections, scheduler, merger,
-            tenant.unparsed_config, cached=True)
+            tenant.unparsed_config, old_tenant=tenant)
         new_abide.tenants[tenant.name] = new_tenant
         return new_abide
 
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index 82e7bf4..2951043 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -935,7 +935,7 @@
                     "Ansible plugin dir %s found adjacent to playbook %s in "
                     "non-trusted repo." % (entry, path))
 
-    def findPlaybook(self, path, required=False, trusted=False):
+    def findPlaybook(self, path, trusted=False):
         for ext in ['', '.yaml', '.yml']:
             fn = path + ext
             if os.path.exists(fn):
@@ -943,35 +943,30 @@
                     playbook_dir = os.path.dirname(os.path.abspath(fn))
                     self._blockPluginDirs(playbook_dir)
                 return fn
-        if required:
-            raise ExecutorError("Unable to find playbook %s" % path)
-        return None
+        raise ExecutorError("Unable to find playbook %s" % path)
 
     def preparePlaybooks(self, args):
         self.writeAnsibleConfig(self.jobdir.setup_playbook)
 
         for playbook in args['pre_playbooks']:
             jobdir_playbook = self.jobdir.addPrePlaybook()
-            self.preparePlaybook(jobdir_playbook, playbook,
-                                 args, required=True)
+            self.preparePlaybook(jobdir_playbook, playbook, args)
 
         for playbook in args['playbooks']:
             jobdir_playbook = self.jobdir.addPlaybook()
-            self.preparePlaybook(jobdir_playbook, playbook,
-                                 args, required=False)
+            self.preparePlaybook(jobdir_playbook, playbook, args)
             if jobdir_playbook.path is not None:
                 self.jobdir.playbook = jobdir_playbook
                 break
 
         if self.jobdir.playbook is None:
-            raise ExecutorError("No valid playbook found")
+            raise ExecutorError("No playbook specified")
 
         for playbook in args['post_playbooks']:
             jobdir_playbook = self.jobdir.addPostPlaybook()
-            self.preparePlaybook(jobdir_playbook, playbook,
-                                 args, required=True)
+            self.preparePlaybook(jobdir_playbook, playbook, args)
 
-    def preparePlaybook(self, jobdir_playbook, playbook, args, required):
+    def preparePlaybook(self, jobdir_playbook, playbook, args):
         self.log.debug("Prepare playbook repo for %s" %
                        (playbook['project'],))
         # Check out the playbook repo if needed and set the path to
@@ -1006,7 +1001,6 @@
 
         jobdir_playbook.path = self.findPlaybook(
             path,
-            required=required,
             trusted=playbook['trusted'])
 
         # If this playbook doesn't exist, don't bother preparing
@@ -1696,7 +1690,7 @@
 
     def unregister_work(self):
         self.accepting_work = False
-        self.executor_worker.unregisterFunction("executor:execute")
+        self.executor_worker.unRegisterFunction("executor:execute")
 
     def stop(self):
         self.log.debug("Stopping")
diff --git a/zuul/model.py b/zuul/model.py
index 30d4b26..cf63f64 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -856,7 +856,6 @@
             source_line=None,
             inheritance_path=(),
             parent_data=None,
-            implied_run=(),
         )
 
         self.inheritable_attributes = {}
@@ -914,10 +913,6 @@
     def setBase(self):
         self.inheritance_path = self.inheritance_path + (repr(self),)
 
-    def setRun(self):
-        if not self.run:
-            self.run = self.implied_run
-
     def addRoles(self, roles):
         newroles = []
         # Start with a copy of the existing roles, but if any of them
@@ -1036,11 +1031,6 @@
         if other._get('roles') is not None:
             self.addRoles(other.roles)
 
-        # We only want to update implied run for inheritance, not
-        # variance.
-        if self.name != other.name:
-            other_implied_run = self.freezePlaybooks(other.implied_run)
-            self.implied_run = other_implied_run + self.implied_run
         if other._get('run') is not None:
             other_run = self.freezePlaybooks(other.run)
             self.run = other_run
@@ -1107,7 +1097,8 @@
                 if not job.branch_matcher and implied_branch:
                     job = job.copy()
                     job.setBranchMatcher([implied_branch])
-                joblist.append(job)
+                if job not in joblist:
+                    joblist.append(job)
 
 
 class JobGraph(object):
@@ -2222,8 +2213,11 @@
 
 class ProjectConfig(object):
     # Represents a project cofiguration
-    def __init__(self, name):
+    def __init__(self, name, source_context=None):
         self.name = name
+        # If this is a template, it will have a source_context, but
+        # not if it is a project definition.
+        self.source_context = source_context
         self.merge_mode = None
         # The default branch for the project (usually master).
         self.default_branch = None
@@ -2423,6 +2417,7 @@
         # inherit from the reference definition.
         noop = Job('noop')
         noop.parent = noop.BASE_JOB_MARKER
+        noop.run = 'noop.yaml'
         self.jobs = {'noop': [noop]}
         self.nodesets = {}
         self.secrets = {}
@@ -2491,12 +2486,18 @@
         self.pipelines[pipeline.name] = pipeline
 
     def addProjectTemplate(self, project_template):
-        if project_template.name in self.project_templates:
-            # TODO(jeblair): issue a warning to the logs on loading
-            # the config, and an error when this hits in a proposed
-            # change.
-            return
-        self.project_templates[project_template.name] = project_template
+        template = self.project_templates.get(project_template.name)
+        if template:
+            if (project_template.source_context.project !=
+                template.source_context.project):
+                raise Exception("Project template %s is already defined" %
+                                (project_template.name,))
+            for pipeline in project_template.pipelines:
+                template.pipelines[pipeline].job_list.\
+                    inheritFrom(project_template.pipelines[pipeline].job_list,
+                                None)
+        else:
+            self.project_templates[project_template.name] = project_template
 
     def addProjectConfig(self, project_config):
         self.project_configs[project_config.name] = project_config
@@ -2551,11 +2552,7 @@
                 else:
                     frozen_job.applyVariant(variant)
                     frozen_job.name = variant.name
-            # Set the implied run based on the last top-level job
-            # definition, before we start applying project-pipeline
-            # variants.
             frozen_job.name = jobname
-            frozen_job.setRun()
             # Whether the change matches any of the project pipeline
             # variants
             matched = False
@@ -2575,6 +2572,9 @@
                 raise Exception("Pre-review pipeline %s does not allow "
                                 "post-review job %s" % (
                                     pipeline.name, frozen_job.name))
+            if not frozen_job.run:
+                raise Exception("Job %s does not specify a run playbook" % (
+                    frozen_job.name,))
             job_graph.addJob(frozen_job)
 
     def createJobGraph(self, item):