Use three separate jobs for code coverage diffing
Reusable roles and playbooks are quite important upstream (I have
zuul-jobs inclusion in mind). Besides that, in Zuul's Ansible a playbook
is supposed to come from the same repo as the one where the job
definition is placed, and one can only override a job's `run` playbook.
This makes it rather annoying to modify, say, `tox-coverage` definition.
My previous approach with a job which performed two things (build the
old source *and* then generate coverage report) also would not scale
that well if the build+test phase is long.
Switch to three jobs:
- build the current, proposed state, and gather coverage info,
- build the past state, gather coverage info,
- download two artifacts from previous runs, reproduce the source trees,
and generate the final coverage diff report.
See-also: https://review.gerrithub.io/c/Telecominfraproject/oopt-zuul-jobs/+/489964
Change-Id: I6eb61669c6f28440a55ba08569a4f2683513a9fa
diff --git a/playbooks/coverage-diff/checkout-previous.yaml b/playbooks/coverage-diff/checkout-previous.yaml
new file mode 100644
index 0000000..330adce
--- /dev/null
+++ b/playbooks/coverage-diff/checkout-previous.yaml
@@ -0,0 +1,3 @@
+- hosts: all
+ roles:
+ - git-use-previous-commit
diff --git a/playbooks/coverage-diff/pre.yaml b/playbooks/coverage-diff/pre.yaml
deleted file mode 100644
index 190ac26..0000000
--- a/playbooks/coverage-diff/pre.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-- hosts: all
- roles:
- - ensure-pycobertura
- - git-use-previous-commit
- - download-next-coverage
diff --git a/playbooks/coverage-diff/prepare.yaml b/playbooks/coverage-diff/prepare.yaml
new file mode 100644
index 0000000..94e9198
--- /dev/null
+++ b/playbooks/coverage-diff/prepare.yaml
@@ -0,0 +1,12 @@
+- hosts: all
+ roles:
+ - ensure-pycobertura
+ - prepare-previous-src
+ - name: download-coverage
+ vars:
+ job_name: '{{ coverage_job_name_previous }}'
+ coverage_destination: '{{ ansible_user_dir }}/coverage-previous.xml'
+ - name: download-coverage
+ vars:
+ job_name: '{{ coverage_job_name_current }}'
+ coverage_destination: '{{ ansible_user_dir }}/coverage-current.xml'
diff --git a/playbooks/coverage-diff/upload-diff.yaml b/playbooks/coverage-diff/upload-diff.yaml
new file mode 100644
index 0000000..a71d814
--- /dev/null
+++ b/playbooks/coverage-diff/upload-diff.yaml
@@ -0,0 +1,3 @@
+- hosts: all
+ roles:
+ - fetch-coverage-diff
diff --git a/playbooks/coverage-diff/upload.yaml b/playbooks/coverage-diff/upload-one-report.yaml
similarity index 100%
rename from playbooks/coverage-diff/upload.yaml
rename to playbooks/coverage-diff/upload-one-report.yaml
diff --git a/roles/download-next-coverage/tasks/download.yaml b/roles/download-coverage/tasks/download.yaml
similarity index 81%
rename from roles/download-next-coverage/tasks/download.yaml
rename to roles/download-coverage/tasks/download.yaml
index f5b2d45..576aeb7 100644
--- a/roles/download-next-coverage/tasks/download.yaml
+++ b/roles/download-coverage/tasks/download.yaml
@@ -5,4 +5,4 @@
- name: Download one artifact
get_url:
url: "{{ artifact.url }}"
- dest: "{{ new_source_prefix }}/coverage.xml"
+ dest: "{{ coverage_destination }}"
diff --git a/roles/download-coverage/tasks/main.yaml b/roles/download-coverage/tasks/main.yaml
new file mode 100644
index 0000000..8ae4695
--- /dev/null
+++ b/roles/download-coverage/tasks/main.yaml
@@ -0,0 +1,7 @@
+- name: Retrieve coverage from parent jobs
+ include_tasks: download.yaml
+ loop: "{{ zuul.artifacts | json_query(artifact_selector) }}"
+ vars:
+ artifact_selector: "[([?metadata.type == 'cobertura_xml' && project == '{{ zuul.project.name }}' && job == '{{ job_name }}'])[-1]]"
+ loop_control:
+ loop_var: artifact
diff --git a/roles/download-next-coverage/tasks/main.yaml b/roles/download-next-coverage/tasks/main.yaml
deleted file mode 100644
index ae7cbfd..0000000
--- a/roles/download-next-coverage/tasks/main.yaml
+++ /dev/null
@@ -1,17 +0,0 @@
-- name: Place for new version of sources
- file:
- path: "{{ new_source_prefix }}"
- state: directory
-
-- name: Prepare copy of the new version of sources
- command: git worktree add {{ new_source_prefix }}/{{ zuul.project.short_name }} {{ zuul.branch }}
- args:
- chdir: "{{ zuul.project.src_dir }}"
-
-- name: Retrieve coverage from parent jobs
- include_tasks: download.yaml
- loop: "{{ zuul.artifacts | json_query(artifact_seletcor) }}"
- vars:
- artifact_seletcor: "[([?metadata.type == 'cobertura_xml' && project == '{{ zuul.project.name }}'])[-1]]"
- loop_control:
- loop_var: artifact
diff --git a/roles/ensure-pycobertura/tasks/main.yaml b/roles/ensure-pycobertura/tasks/main.yaml
index 473d543..ab309bd 100644
--- a/roles/ensure-pycobertura/tasks/main.yaml
+++ b/roles/ensure-pycobertura/tasks/main.yaml
@@ -1,4 +1,4 @@
- name: Ensure pycobertura is available
pip:
name: pycobertura
- extra_args: --user
+ become: yes
diff --git a/roles/fetch-coverage-diff/tasks/main.yaml b/roles/fetch-coverage-diff/tasks/main.yaml
new file mode 100644
index 0000000..0e7ce70
--- /dev/null
+++ b/roles/fetch-coverage-diff/tasks/main.yaml
@@ -0,0 +1,34 @@
+- name: Check for coverage-diff.html
+ stat:
+ path: "{{ ansible_user_dir }}/zuul-output/logs/coverage-diff.html"
+ get_checksum: false
+ get_mime: false
+ get_md5: false
+ register: coverage_diff_html_stat
+
+- name: Register current coverage report as an artifact
+ when: coverage_diff_html_stat.stat.exists
+ zuul_return:
+ data:
+ zuul:
+ artifacts:
+ - name: "Full coverage report (current)"
+ url: "coverage-current.html"
+
+- name: Register previous coverage report as an artifact
+ when: coverage_diff_html_stat.stat.exists
+ zuul_return:
+ data:
+ zuul:
+ artifacts:
+ - name: "Full coverage report (previous)"
+ url: "coverage-previous.html"
+
+- name: Register coverage report diff as an artifact
+ when: coverage_diff_html_stat.stat.exists
+ zuul_return:
+ data:
+ zuul:
+ artifacts:
+ - name: "Coverage report: difference in coverage"
+ url: "coverage-diff.html"
diff --git a/roles/fetch-coverage-xml/defaults/main.yaml b/roles/fetch-coverage-xml/defaults/main.yaml
new file mode 100644
index 0000000..2f1e36d
--- /dev/null
+++ b/roles/fetch-coverage-xml/defaults/main.yaml
@@ -0,0 +1 @@
+coverage_artifact_name: "Coverage data"
diff --git a/roles/fetch-coverage-xml/tasks/main.yaml b/roles/fetch-coverage-xml/tasks/main.yaml
index c247e49..5316a35 100644
--- a/roles/fetch-coverage-xml/tasks/main.yaml
+++ b/roles/fetch-coverage-xml/tasks/main.yaml
@@ -25,7 +25,7 @@
data:
zuul:
artifacts:
- - name: "Coverage data"
+ - name: "{{ coverage_artifact_name }}"
metadata:
type: "cobertura_xml"
url: "coverage.xml"
diff --git a/roles/git-use-previous-commit/tasks/main.yaml b/roles/git-use-previous-commit/tasks/main.yaml
index cb7458e..db63a97 100644
--- a/roles/git-use-previous-commit/tasks/main.yaml
+++ b/roles/git-use-previous-commit/tasks/main.yaml
@@ -1,7 +1,8 @@
- name: Prepare git submodules
shell: |
- echo "{{ item.key }}"
- git checkout -b "zuul_origin_{{ zuul.branch }}" "origin/{{ zuul.branch }}"
+ PREVIOUS_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+ echo "{{ item.key }}: ${PREVIOUS_BRANCH}"
+ git checkout -b zuul_origin_${PREVIOUS_BRANCH} origin/${PREVIOUS_BRANCH}
args:
chdir: "{{ ansible_user_dir }}/{{ item.value.src_dir }}"
loop: "{{ query('dict', zuul.projects) }}"
diff --git a/roles/prepare-previous-src/tasks/main.yaml b/roles/prepare-previous-src/tasks/main.yaml
new file mode 100644
index 0000000..250d40d
--- /dev/null
+++ b/roles/prepare-previous-src/tasks/main.yaml
@@ -0,0 +1,9 @@
+- name: Place for previous version of sources
+ file:
+ path: "{{ (previous_source_prefix + '/' + zuul.project.canonical_hostname + '/' + zuul.project.name) | dirname }}"
+ state: directory
+
+- name: Prepare copy of the previous version of sources
+ shell:
+ cmd: git worktree add {{ previous_source_prefix }}/{{ zuul.project.canonical_hostname }}/{{ zuul.project.name }} origin/$(git rev-parse --abbrev-ref HEAD)
+ chdir: "{{ zuul.project.src_dir }}"
diff --git a/roles/pycobertura-diff/tasks/main.yaml b/roles/pycobertura-diff/tasks/main.yaml
index 418b273..471c22e 100644
--- a/roles/pycobertura-diff/tasks/main.yaml
+++ b/roles/pycobertura-diff/tasks/main.yaml
@@ -1,45 +1,18 @@
- name: Generate pycobertura report
shell: |
set -ex
- PATH=${PATH}:~/.local/bin
- pycobertura show --format html --output "{{ ansible_user_dir }}/zuul-output/logs/coverage.html" \
- --source "{{ new_source_prefix }}/{{ zuul.project.short_name }}" "{{ new_source_prefix }}/coverage.xml"
+
+ pycobertura show --format html --output "{{ ansible_user_dir }}/zuul-output/logs/coverage-previous.html" \
+ --source "{{ previous_source_prefix }}/{{ zuul.project.canonical_hostname }}/{{ zuul.project.name}}" \
+ "{{ ansible_user_dir }}/coverage-previous.xml"
+
+ pycobertura show --format html --output "{{ ansible_user_dir }}/zuul-output/logs/coverage-current.html" \
+ --source "{{ zuul.project.src_dir }}" \
+ "{{ ansible_user_dir }}/coverage-current.xml"
+
pycobertura diff --format html --output "{{ ansible_user_dir }}/zuul-output/logs/coverage-diff.html" \
- --source1 "{{ zuul.project.src_dir }}" "{{ coverage_xml_dir }}/coverage.xml" \
- --source2 "{{ new_source_prefix }}/{{ zuul.project.short_name }}" "{{ new_source_prefix }}/coverage.xml"
- sed -e 's;^ </body>$; <p>Show <a href="coverage.html">final coverage</a> only</p>\n </body>;' \
- -i "{{ ansible_user_dir }}/zuul-output/logs/coverage-diff.html"
+ --source1 "{{ previous_source_prefix }}/{{ zuul.project.canonical_hostname }}/{{ zuul.project.name}}" "{{ ansible_user_dir }}/coverage-previous.xml" \
+ --source2 "{{ zuul.project.src_dir }}" "{{ ansible_user_dir }}/coverage-current.xml"
-- name: Check for coverage-diff.html
- stat:
- path: "{{ ansible_user_dir }}/zuul-output/logs/coverage-diff.html"
- get_checksum: false
- get_mime: false
- get_md5: false
- register: coverage_diff_html_stat
-
-- name: Check for coverage.html
- stat:
- path: "{{ ansible_user_dir }}/zuul-output/logs/coverage.html"
- get_checksum: false
- get_mime: false
- get_md5: false
- register: coverage_html_stat
-
-- name: Register coverage reports as artifacts
- when: coverage_html_stat.stat.exists
- zuul_return:
- data:
- zuul:
- artifacts:
- - name: "Full coverage report"
- url: "coverage.html"
-
-- name: Register coverage reports as artifacts
- when: coverage_diff_html_stat.stat.exists
- zuul_return:
- data:
- zuul:
- artifacts:
- - name: "Change of code coverage report"
- url: "coverage-diff.html"
+ sed -e 's;^ </body>$; <p>Show <a href="coverage-current.html">previous</a> or <a href="coverage-current.html">final coverage</a> only</p>\n </body>;' \
+ -i "{{ ansible_user_dir }}/zuul-output/logs/coverage-diff.html" || true
diff --git a/zuul.yaml b/zuul.yaml
index 3671559..ff32b3b 100644
--- a/zuul.yaml
+++ b/zuul.yaml
@@ -33,27 +33,53 @@
- job:
name: f29-gcc-cover
description: |
- Build on Fedora 29 with GCC.
+ Build on Fedora 29 with GCC and code coverage.
parent: f29-gcc
- post-run: playbooks/coverage-diff/upload.yaml
+ post-run: playbooks/coverage-diff/upload-one-report.yaml
+ provides: coverage-f29-gcc-new
vars:
coverage_xml_dir: "{{ ansible_user_dir }}/build"
- job:
name: f29-gcc-cover-diff
+ parent: noop # TODO: remove this job once all users are adopted
+
+- job:
+ name: f29-gcc-cover-previous
description: |
- Compare difference in coverage as introduced by this change.
- parent: f29-gcc
- pre-run:
- - playbooks/coverage-diff/pre.yaml
- run:
- - playbooks/run-test-command/run.yaml
- - playbooks/coverage-diff/generate.yaml
- success-url: coverage-diff.html
- failure-url: coverage-diff.html
+ Build previous version of sources on Fedora 29 with GCC and code coverage.
+ parent: f29-gcc-cover
+ pre-run: playbooks/coverage-diff/checkout-previous.yaml
+ provides: coverage-f29-gcc-old
+ voting: false
vars:
coverage_xml_dir: "{{ ansible_user_dir }}/build"
- new_source_prefix: "{{ ansible_user_dir }}/new"
+
+- job:
+ name: cpp-coverage-diff
+ description: |
+ Report how coverage changed compared to the previous state of the repo.
+ nodeset:
+ nodes:
+ - name: w
+ label: f29
+ timeout: 120
+ pre-run: playbooks/coverage-diff/prepare.yaml
+ run: playbooks/coverage-diff/generate.yaml
+ post-run: playbooks/coverage-diff/upload-diff.yaml
+ success-url: coverage-diff.html
+ failure-url: coverage-diff.html
+ requires:
+ - coverage-f29-gcc-old
+ - coverage-f29-gcc-new
+ dependencies:
+ - f29-gcc-cover
+ - f29-gcc-cover-previous
+ vars:
+ previous_source_prefix: "{{ ansible_user_dir }}/previous-src"
+ coverage_xml_dir: "{{ zuul.project.src_dir }}/cover"
+ coverage_job_name_previous: f29-gcc-cover-previous
+ coverage_job_name_current: f29-gcc-cover
- job:
name: f29-gcc-asan