Handle logging ansible python errors properly

If an exception is thrown from ansible, we don't log it properly.

Add a playbook to the functional test of logging that doesn't mask
failures (it lets the tasks report as failed) So that we cover all four
possibilities for ansible module failure exception:

* failed task
* failed task in item
* failed task that reports as success
* failed task in item that reports as success

Change-Id: I9e355350f5ec34251f5030eaaa3f4bb7b4438a6e
diff --git a/playbooks/zuul-stream/fixtures/test-stream-failure.yaml b/playbooks/zuul-stream/fixtures/test-stream-failure.yaml
new file mode 100644
index 0000000..272431a
--- /dev/null
+++ b/playbooks/zuul-stream/fixtures/test-stream-failure.yaml
@@ -0,0 +1,21 @@
+- name: Run some commands to show that logging works on failed tasks too
+  hosts: node
+  tasks:
+
+    - block:
+
+      - name: Run a shell task with an ansible python exception
+        command: echo foo
+        args:
+          chdir: /somewhere/that/does/not/exist
+
+    - always:
+
+      - name: Loop with items on an ansible python exception
+        command: "echo {{ item }}"
+        with_items:
+          - item1
+          - item2
+          - item3
+        args:
+          chdir: /somewhere/that/does/not/exist
diff --git a/playbooks/zuul-stream/fixtures/test-stream.yaml b/playbooks/zuul-stream/fixtures/test-stream.yaml
index 065e332..5bffc16 100644
--- a/playbooks/zuul-stream/fixtures/test-stream.yaml
+++ b/playbooks/zuul-stream/fixtures/test-stream.yaml
@@ -19,3 +19,19 @@
         - item1
         - item2
         - item3
+
+    - name: Run a shell task with an ansible python exception
+      command: echo foo
+      args:
+        chdir: /somewhere/that/does/not/exist
+      failed_when: false
+
+    - name: Loop with items on an ansible python exception
+      command: "echo {{ item }}"
+      with_items:
+        - item1
+        - item2
+        - item3
+      args:
+        chdir: /somewhere/that/does/not/exist
+      failed_when: false
diff --git a/playbooks/zuul-stream/functional.yaml b/playbooks/zuul-stream/functional.yaml
index 727598c..0c7c617 100644
--- a/playbooks/zuul-stream/functional.yaml
+++ b/playbooks/zuul-stream/functional.yaml
@@ -7,6 +7,14 @@
         ZUUL_JOB_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json"
         ARA_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json"
 
+    - name: Run ansible playbook that should fail
+      command: ansible-playbook -vvv src/git.openstack.org/openstack-infra/zuul/playbooks/zuul-stream/fixtures/test-stream-failure.yaml
+      register: failed_results
+      failed_when: "failed_results.rc != 2"
+      environment:
+        ZUUL_JOB_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json"
+        ARA_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json"
+
     - name: Generate ARA html
       command: ara generate html ara-output