diff --git a/playbooks/zuul-stream/fixtures/test-stream-failure.yaml b/playbooks/zuul-stream/fixtures/test-stream-failure.yaml
new file mode 100644
index 0000000..272431a
--- /dev/null
+++ b/playbooks/zuul-stream/fixtures/test-stream-failure.yaml
@@ -0,0 +1,21 @@
+- name: Run some commands to show that logging works on failed tasks too
+  hosts: node
+  tasks:
+
+    - block:
+
+      - name: Run a shell task with an ansible python exception
+        command: echo foo
+        args:
+          chdir: /somewhere/that/does/not/exist
+
+    - always:
+
+      - name: Loop with items on an ansible python exception
+        command: "echo {{ item }}"
+        with_items:
+          - item1
+          - item2
+          - item3
+        args:
+          chdir: /somewhere/that/does/not/exist
diff --git a/playbooks/zuul-stream/fixtures/test-stream.yaml b/playbooks/zuul-stream/fixtures/test-stream.yaml
index 065e332..5bffc16 100644
--- a/playbooks/zuul-stream/fixtures/test-stream.yaml
+++ b/playbooks/zuul-stream/fixtures/test-stream.yaml
@@ -19,3 +19,19 @@
         - item1
         - item2
         - item3
+
+    - name: Run a shell task with an ansible python exception
+      command: echo foo
+      args:
+        chdir: /somewhere/that/does/not/exist
+      failed_when: false
+
+    - name: Loop with items on an ansible python exception
+      command: "echo {{ item }}"
+      with_items:
+        - item1
+        - item2
+        - item3
+      args:
+        chdir: /somewhere/that/does/not/exist
+      failed_when: false
diff --git a/playbooks/zuul-stream/functional.yaml b/playbooks/zuul-stream/functional.yaml
index 727598c..0c7c617 100644
--- a/playbooks/zuul-stream/functional.yaml
+++ b/playbooks/zuul-stream/functional.yaml
@@ -7,6 +7,14 @@
         ZUUL_JOB_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json"
         ARA_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json"
 
+    - name: Run ansible playbook that should fail
+      command: ansible-playbook -vvv src/git.openstack.org/openstack-infra/zuul/playbooks/zuul-stream/fixtures/test-stream-failure.yaml
+      register: failed_results
+      failed_when: "failed_results.rc != 2"
+      environment:
+        ZUUL_JOB_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json"
+        ARA_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json"
+
     - name: Generate ARA html
       command: ara generate html ara-output