Uploading artifacts to Swift
There's a new role, the `upload-artifacts-swift`. It is a pretty
bare-bones role which should serve as a (mostly) drop-in replacement of
the add-fileserver + publish-artifacts-to-fileserver combo. It still
requires "something" to ensure that the artifacts are available at the
executor, and it will not register these artifacts, nor return them to
Zuul. That all should be done separately (check the rest of our CI setup
for details, especially publish-artifacts-base and
publish-artifacts-tenant in ci/project-config.git), this one is only a
building block.
The rest happens in the updated role `download-artifacts-from-check`. It
is no longer intended to run within the post playbook; it should now be
defined in a parent job's `run`.
Also, I'm moving everything to the executor so that we don't require a
dummy build node. This necessitated a fix to the executor's bubblewrap
configuration for accessing the TLS certificates and preferences and
what not.
Change-Id: I4e4bb26e5976127ca8c504ed3e6d1f8b9ede9393
diff --git a/playbooks/download-artifacts-from-check/run.yaml b/playbooks/download-artifacts-from-check/run.yaml
deleted file mode 100644
index 920dce2..0000000
--- a/playbooks/download-artifacts-from-check/run.yaml
+++ /dev/null
@@ -1,3 +0,0 @@
-- hosts: all
- roles:
- - download-artifacts-from-check
diff --git a/roles/download-artifacts-from-check/defaults/main.yaml b/roles/download-artifacts-from-check/defaults/main.yaml
deleted file mode 100644
index 3e6da11..0000000
--- a/roles/download-artifacts-from-check/defaults/main.yaml
+++ /dev/null
@@ -1 +0,0 @@
-zuul_output_dir: "{{ ansible_user_dir }}/zuul-output"
diff --git a/roles/download-artifacts-from-check/tasks/main.yaml b/roles/download-artifacts-from-check/tasks/main.yaml
index 75d490c..48ef576 100644
--- a/roles/download-artifacts-from-check/tasks/main.yaml
+++ b/roles/download-artifacts-from-check/tasks/main.yaml
@@ -2,17 +2,25 @@
msg: "This job only works in the 'promote' pipeline."
when: zuul.pipeline != 'promote'
+- name: Ensure artifacts directory exists
+ delegate_to: localhost
+ file:
+ path: "{{ zuul.executor.work_root }}/artifacts"
+ state: directory
+
- name: Retrieve artifacts built within the check pipeline
shell: |
set -ex
- curl "https://zuul.gerrit.cesnet.cz/api/tenant/{{ zuul.tenant }}/builds?pipeline=check&change={{ zuul.change }}&patchset={{ zuul.patchset }}" --output {{ zuul_output_dir }}/logs/check_jobs_for_change.json
- ALL_JOBS=$(jq -r '.[].job_name' < {{ zuul_output_dir }}/logs/check_jobs_for_change.json | sort | uniq)
+ CHECKS_JSON={{ zuul.executor.work_root }}/check_jobs_for_change.json
+ curl "{{ zuul_root_url }}/api/tenant/{{ zuul.tenant }}/builds?pipeline=check&change={{ zuul.change }}&patchset={{ zuul.patchset }}" --output ${CHECKS_JSON}
+ ALL_JOBS=$(jq -r '.[].job_name' < ${CHECKS_JSON} | sort | uniq)
for JOB_NAME in ${ALL_JOBS}; do
- ARTIFACTS_URL=$(jq -r ". | map(select(.job_name == \"${JOB_NAME}\"))[0].artifacts[].url" < {{ zuul_output_dir }}/logs/check_jobs_for_change.json)
- mkdir {{ zuul_output_dir }}/artifacts/${JOB_NAME}
- pushd {{ zuul_output_dir }}/artifacts/${JOB_NAME}
+ ARTIFACTS_URL=$(jq -r ". | map(select(.job_name == \"${JOB_NAME}\"))[0].artifacts[].url" < ${CHECKS_JSON})
+ mkdir {{ zuul.executor.work_root }}/artifacts/${JOB_NAME}
+ pushd {{ zuul.executor.work_root }}/artifacts/${JOB_NAME}
for ONE_URL in ${ARTIFACTS_URL}; do
curl -O "${ONE_URL}"
done
popd
done
+ rm -f ${CHECKS_JSON}
diff --git a/roles/fetch-artifacts-for-publish/defaults/main.yaml b/roles/fetch-artifacts-for-publish/defaults/main.yaml
deleted file mode 100644
index 3e6da11..0000000
--- a/roles/fetch-artifacts-for-publish/defaults/main.yaml
+++ /dev/null
@@ -1 +0,0 @@
-zuul_output_dir: "{{ ansible_user_dir }}/zuul-output"
diff --git a/roles/fetch-artifacts-for-publish/tasks/main.yaml b/roles/fetch-artifacts-for-publish/tasks/main.yaml
deleted file mode 100644
index 5283d65..0000000
--- a/roles/fetch-artifacts-for-publish/tasks/main.yaml
+++ /dev/null
@@ -1,26 +0,0 @@
-- fail:
- msg: "This role only works in the 'promote' pipeline."
- when: zuul.pipeline != 'promote'
-
-- name: Ensure artifacts directory exists
- delegate_to: localhost
- file:
- path: "{{ zuul.executor.work_root }}/artifacts"
- state: directory
-
-- name: Collect artifacts
- synchronize:
- dest: "{{ zuul.executor.work_root }}/artifacts/"
- mode: pull
- src: "{{ zuul_output_dir }}/artifacts/"
- verify_host: true
-
-- name: "Purge artifacts on the build node so that they are not uploaded the promote job's log directory"
- file:
- path: "{{ zuul_output_dir }}/artifacts/"
- state: absent
-
-- name: "Re-create an empty artifacts directory on the build node so that `fetch-output` doesn't complain"
- file:
- path: "{{ zuul_output_dir }}/artifacts/"
- state: directory
diff --git a/roles/upload-artifacts-swift/tasks/main.yaml b/roles/upload-artifacts-swift/tasks/main.yaml
new file mode 100644
index 0000000..ed0110b
--- /dev/null
+++ b/roles/upload-artifacts-swift/tasks/main.yaml
@@ -0,0 +1,25 @@
+- fail:
+ msg: "This job only works in the 'promote' pipeline."
+ when: zuul.pipeline != 'promote'
+
+- name: find artifact name
+ find:
+ paths: '{{ zuul.executor.work_root }}/artifacts'
+ file_type: 'file'
+ recurse: true
+ register: found_artifacts
+
+- name: some debugging
+ debug:
+ msg: 'Will create artifact: {{ zuul_artifacts_prefix }}/{{ item.path | dirname | relpath(zuul.executor.work_root + "/artifacts") }}/{{ item.path | basename }}'
+ loop: '{{ found_artifacts.files }}'
+
+- name: Upload artifacts to Swift object storage
+ os_object:
+ cloud: '{{ zuul_artifacts_cloud_config }}'
+ container: '{{ zuul_artifacts_container }}'
+ container_access: 'public'
+ filename: '{{ item.path }}'
+ name: '{{ zuul_artifacts_prefix }}/{{ item.path | dirname | relpath(zuul.executor.work_root + "/artifacts") }}/{{ item.path | basename }}'
+ state: present
+ loop: '{{ found_artifacts.files }}'
diff --git a/zuul.yaml b/zuul.yaml
index 297d140..090d816 100644
--- a/zuul.yaml
+++ b/zuul.yaml
@@ -12,21 +12,14 @@
timeout: 900
- job:
- name: publish-artifacts-from-check
+ name: promote-artifacts
parent: publish-artifacts-tenant
description: |
Retrieve all artifacts built by the last matching run in the check
pipeline, and promote them into a permanent location in the log server.
This is intended to be run within a promote pipeline.
- run: playbooks/download-artifacts-from-check/run.yaml
-
-- job:
- name: promote-artifacts
- parent: publish-artifacts-from-check
nodeset:
- nodes:
- - name: f29
- label: f29
+ nodes: []
- job:
name: f29-gcc