diff --git a/tests/base.py b/tests/base.py
index 2c478ad..98f4c7f 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -16,6 +16,7 @@
 # under the License.
 
 import configparser
+from contextlib import contextmanager
 import datetime
 import gc
 import hashlib
@@ -2769,6 +2770,24 @@
     """ZuulTestCase but with an actual ansible executor running"""
     run_ansible = True
 
+    @contextmanager
+    def jobLog(self, build):
+        """Print job logs on assertion errors
+
+        This method is a context manager which, if it encounters an
+        ecxeption, adds the build log to the debug output.
+
+        :arg Build build: The build that's being asserted.
+        """
+        try:
+            yield
+        except Exception:
+            path = os.path.join(self.test_root, build.uuid,
+                                'work', 'logs', 'job-output.txt')
+            with open(path) as f:
+                self.log.debug(f.read())
+            raise
+
 
 class SSLZuulTestCase(ZuulTestCase):
     """ZuulTestCase but using SSL when possible"""
diff --git a/tests/fixtures/config/ansible/git/common-config/playbooks/check-vars.yaml b/tests/fixtures/config/ansible/git/common-config/playbooks/check-vars.yaml
index ce392a4..3f62c4c 100644
--- a/tests/fixtures/config/ansible/git/common-config/playbooks/check-vars.yaml
+++ b/tests/fixtures/config/ansible/git/common-config/playbooks/check-vars.yaml
@@ -20,3 +20,13 @@
           - zuul.project.name == 'org/project'
           - zuul.project.canonical_hostname == 'review.example.com'
           - zuul.project.canonical_name == 'review.example.com/org/project'
+
+    - debug:
+        msg: "vartest secret {{ vartest_secret }}"
+
+    - name: Assert variable precedence.
+      assert:
+        that:
+          - vartest_job == 'vartest_job'
+          - vartest_secret.value == 'vartest_secret'
+          - vartest_site == 'vartest_site'
\ No newline at end of file
diff --git a/tests/fixtures/config/ansible/git/common-config/zuul.yaml b/tests/fixtures/config/ansible/git/common-config/zuul.yaml
index 1a1b22f..f050c6f 100644
--- a/tests/fixtures/config/ansible/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/ansible/git/common-config/zuul.yaml
@@ -48,6 +48,27 @@
         Z3QSO1NjbBxWnaHKZYT7nkrJm8AMCgZU0ZArFLpaufKCeiK5ECSsDxic4FIsY1OkWT42qEUfL0Wd
         +150AKGNZpPJnnP3QYY4W/MWcKH/zdO400+zWN52WevbSqZy90tqKDJrBkMl1ydqbuw1E4ZHvIs=
 
+# This is used by the check-vars job to evaluate variable precedence.
+# The name of this secret conflicts with a site variable.
+- secret:
+    name: vartest_site
+    data:
+      value: vartest_secret
+
+# This is used by the check-vars job to evaluate variable precedence.
+# The name of this secret conflicts with a job variable.
+- secret:
+    name: vartest_job
+    data:
+      value: vartest_secret
+
+# This is used by the check-vars job to evaluate variable precedence.
+# The name of this secret should not conflict.
+- secret:
+    name: vartest_secret
+    data:
+      value: vartest_secret
+
 - job:
     name: base-urls
     success-url: https://success.example.com/zuul-logs/{build.uuid}/
@@ -77,6 +98,14 @@
     nodes:
       - name: ubuntu-xenial
         label: ubuntu-xenial
+    vars:
+      vartest_job: vartest_job
+      vartest_secret: vartest_job
+      vartest_site: vartest_job
+    auth:
+      secrets:
+        - vartest_site
+        - vartest_secret
 
 - job:
     parent: base-urls
diff --git a/tests/fixtures/config/ansible/variables.yaml b/tests/fixtures/config/ansible/variables.yaml
new file mode 100644
index 0000000..692eb7f
--- /dev/null
+++ b/tests/fixtures/config/ansible/variables.yaml
@@ -0,0 +1 @@
+vartest_site: vartest_site
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index fb80660..ad0b2cc 100644
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -595,54 +595,70 @@
     tenant_config_file = 'config/ansible/main.yaml'
 
     def test_playbook(self):
+        # Keep the jobdir around so we can inspect contents if an
+        # assert fails.
+        self.executor_server.keep_jobdir = True
+        # Output extra ansible info so we might see errors.
+        self.executor_server.verbose = True
+        # Add a site variables file, used by check-vars
+        path = os.path.join(FIXTURE_DIR, 'config', 'ansible',
+                            'variables.yaml')
+        self.config.set('executor', 'variables', path)
         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
         self.waitUntilSettled()
         build_timeout = self.getJobFromHistory('timeout')
-        self.assertEqual(build_timeout.result, 'TIMED_OUT')
+        with self.jobLog(build_timeout):
+            self.assertEqual(build_timeout.result, 'TIMED_OUT')
         build_faillocal = self.getJobFromHistory('faillocal')
-        self.assertEqual(build_faillocal.result, 'FAILURE')
+        with self.jobLog(build_faillocal):
+            self.assertEqual(build_faillocal.result, 'FAILURE')
         build_failpost = self.getJobFromHistory('failpost')
-        self.assertEqual(build_failpost.result, 'POST_FAILURE')
+        with self.jobLog(build_failpost):
+            self.assertEqual(build_failpost.result, 'POST_FAILURE')
         build_check_vars = self.getJobFromHistory('check-vars')
-        self.assertEqual(build_check_vars.result, 'SUCCESS')
+        with self.jobLog(build_check_vars):
+            self.assertEqual(build_check_vars.result, 'SUCCESS')
         build_hello = self.getJobFromHistory('hello-world')
-        self.assertEqual(build_hello.result, 'SUCCESS')
+        with self.jobLog(build_hello):
+            self.assertEqual(build_hello.result, 'SUCCESS')
         build_python27 = self.getJobFromHistory('python27')
-        self.assertEqual(build_python27.result, 'SUCCESS')
-        flag_path = os.path.join(self.test_root, build_python27.uuid + '.flag')
-        self.assertTrue(os.path.exists(flag_path))
-        copied_path = os.path.join(self.test_root, build_python27.uuid +
-                                   '.copied')
-        self.assertTrue(os.path.exists(copied_path))
-        failed_path = os.path.join(self.test_root, build_python27.uuid +
-                                   '.failed')
-        self.assertFalse(os.path.exists(failed_path))
-        pre_flag_path = os.path.join(self.test_root, build_python27.uuid +
-                                     '.pre.flag')
-        self.assertTrue(os.path.exists(pre_flag_path))
-        post_flag_path = os.path.join(self.test_root, build_python27.uuid +
-                                      '.post.flag')
-        self.assertTrue(os.path.exists(post_flag_path))
-        bare_role_flag_path = os.path.join(self.test_root,
-                                           build_python27.uuid +
-                                           '.bare-role.flag')
-        self.assertTrue(os.path.exists(bare_role_flag_path))
+        with self.jobLog(build_python27):
+            self.assertEqual(build_python27.result, 'SUCCESS')
+            flag_path = os.path.join(self.test_root,
+                                     build_python27.uuid + '.flag')
+            self.assertTrue(os.path.exists(flag_path))
+            copied_path = os.path.join(self.test_root, build_python27.uuid +
+                                       '.copied')
+            self.assertTrue(os.path.exists(copied_path))
+            failed_path = os.path.join(self.test_root, build_python27.uuid +
+                                       '.failed')
+            self.assertFalse(os.path.exists(failed_path))
+            pre_flag_path = os.path.join(self.test_root, build_python27.uuid +
+                                         '.pre.flag')
+            self.assertTrue(os.path.exists(pre_flag_path))
+            post_flag_path = os.path.join(self.test_root, build_python27.uuid +
+                                          '.post.flag')
+            self.assertTrue(os.path.exists(post_flag_path))
+            bare_role_flag_path = os.path.join(self.test_root,
+                                               build_python27.uuid +
+                                               '.bare-role.flag')
+            self.assertTrue(os.path.exists(bare_role_flag_path))
+            secrets_path = os.path.join(self.test_root,
+                                        build_python27.uuid + '.secrets')
+            with open(secrets_path) as f:
+                self.assertEqual(f.read(), "test-username test-password")
 
-        secrets_path = os.path.join(self.test_root,
-                                    build_python27.uuid + '.secrets')
-        with open(secrets_path) as f:
-            self.assertEqual(f.read(), "test-username test-password")
-
-        msg = A.messages[0]
-        success = "{} https://success.example.com/zuul-logs/{}"
-        fail = "{} https://failure.example.com/zuul-logs/{}"
-        self.assertIn(success.format("python27", build_python27.uuid), msg)
-        self.assertIn(fail.format("faillocal", build_faillocal.uuid), msg)
-        self.assertIn(success.format("check-vars", build_check_vars.uuid), msg)
-        self.assertIn(success.format("hello-world", build_hello.uuid), msg)
-        self.assertIn(fail.format("timeout", build_timeout.uuid), msg)
-        self.assertIn(fail.format("failpost", build_failpost.uuid), msg)
+            msg = A.messages[0]
+            success = "{} https://success.example.com/zuul-logs/{}"
+            fail = "{} https://failure.example.com/zuul-logs/{}"
+            self.assertIn(success.format("python27", build_python27.uuid), msg)
+            self.assertIn(fail.format("faillocal", build_faillocal.uuid), msg)
+            self.assertIn(success.format("check-vars",
+                                         build_check_vars.uuid), msg)
+            self.assertIn(success.format("hello-world", build_hello.uuid), msg)
+            self.assertIn(fail.format("timeout", build_timeout.uuid), msg)
+            self.assertIn(fail.format("failpost", build_failpost.uuid), msg)
 
 
 class TestPrePlaybooks(AnsibleZuulTestCase):