diff --git a/playbooks/zuul-stream/fixtures/test-stream-failure.yaml b/playbooks/zuul-stream/fixtures/test-stream-failure.yaml
new file mode 100644
index 0000000..272431a
--- /dev/null
+++ b/playbooks/zuul-stream/fixtures/test-stream-failure.yaml
@@ -0,0 +1,21 @@
+- name: Run some commands to show that logging works on failed tasks too
+  hosts: node
+  tasks:
+
+    - block:
+
+      - name: Run a shell task with an ansible python exception
+        command: echo foo
+        args:
+          chdir: /somewhere/that/does/not/exist
+
+    - always:
+
+      - name: Loop with items on an ansible python exception
+        command: "echo {{ item }}"
+        with_items:
+          - item1
+          - item2
+          - item3
+        args:
+          chdir: /somewhere/that/does/not/exist
diff --git a/playbooks/zuul-stream/fixtures/test-stream.yaml b/playbooks/zuul-stream/fixtures/test-stream.yaml
index 065e332..5bffc16 100644
--- a/playbooks/zuul-stream/fixtures/test-stream.yaml
+++ b/playbooks/zuul-stream/fixtures/test-stream.yaml
@@ -19,3 +19,19 @@
         - item1
         - item2
         - item3
+
+    - name: Run a shell task with an ansible python exception
+      command: echo foo
+      args:
+        chdir: /somewhere/that/does/not/exist
+      failed_when: false
+
+    - name: Loop with items on an ansible python exception
+      command: "echo {{ item }}"
+      with_items:
+        - item1
+        - item2
+        - item3
+      args:
+        chdir: /somewhere/that/does/not/exist
+      failed_when: false