Merge "Replace ssh-keyscan with host keys from node" into feature/zuulv3
diff --git a/.zuul.yaml b/.zuul.yaml
index ed66422..98b880d 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -1,10 +1,12 @@
- job:
name: base
- pre-run: base-pre
- post-run: base-post
+ pre-run: base/pre
+ post-run: base/post
success-url: http://zuulv3-dev.openstack.org/logs/{build.uuid}/
failure-url: http://zuulv3-dev.openstack.org/logs/{build.uuid}/
timeout: 1800
+ vars:
+ zuul_workspace_root: /home/zuul
nodes:
- name: ubuntu-xenial
image: ubuntu-xenial
@@ -12,26 +14,42 @@
- job:
name: tox
parent: base
- pre-run: tox-pre
- post-run: tox-post
+ pre-run: tox/pre
+ post-run: tox/post
- job:
name: tox-cover
parent: tox
+ run: tox/cover
voting: false
- job:
+ name: tox-docs
+ parent: tox
+ run: tox/docs
+
+- job:
name: tox-linters
parent: tox
+ run: tox/docs
- job:
name: tox-py27
parent: tox
+ run: tox/py27
+
+- job:
+ name: tox-tarball
+ parent: tox
+ run: tox/tarball
+ post-run: tox/tarball-post
- project:
name: openstack-infra/zuul
check:
jobs:
+ - tox-docs
- tox-cover
- tox-linters
- tox-py27
+ - tox-tarball
diff --git a/README.rst b/README.rst
index 84b9b7a..932edbf 100644
--- a/README.rst
+++ b/README.rst
@@ -118,7 +118,7 @@
Construct a test to fully simulate the series of events you want to
see, then run it in the foreground. For example::
- .tox/py27/bin/python -m testtools.run tests.test_scheduler.TestScheduler.test_jobs_executed
+ .tox/py27/bin/python -m testtools.run tests.unit.test_scheduler.TestScheduler.test_jobs_executed
See TESTING.rst for more information.
diff --git a/doc/source/zuul.rst b/doc/source/zuul.rst
index 8b325ba..e4ce737 100644
--- a/doc/source/zuul.rst
+++ b/doc/source/zuul.rst
@@ -771,8 +771,11 @@
executed are listed. If a job is entered as a dictionary key, then
jobs contained within that key are only executed if the key job
succeeds. In the above example, project-unittest, project-pep8, and
-project-pyflakes are only executed if project-merge succeeds. This
-can help avoid running unnecessary jobs.
+project-pyflakes are only executed if project-merge succeeds.
+Furthermore, project-finaltest is executed only if project-unittest,
+project-pep8 and project-pyflakes all succeed. This can help avoid
+running unnecessary jobs while maximizing parallelism. It is also
+useful when distributing results between jobs.
The special job named ``noop`` is internal to Zuul and will always
return ``SUCCESS`` immediately. This can be useful if you require
diff --git a/etc/zuul.conf-sample b/etc/zuul.conf-sample
index 7207c73..bf19895 100644
--- a/etc/zuul.conf-sample
+++ b/etc/zuul.conf-sample
@@ -18,15 +18,6 @@
;git_user_name=zuul
zuul_url=http://zuul.example.com/p
-[swift]
-authurl=https://identity.api.example.org/v2.0/
-user=username
-key=password
-
-default_container=logs
-region_name=EXP
-logserver_prefix=http://logs.example.org/server.app/
-
[webapp]
listen_address=0.0.0.0
port=8001
diff --git a/playbooks/base-post.yaml b/playbooks/base/post.yaml
similarity index 100%
rename from playbooks/base-post.yaml
rename to playbooks/base/post.yaml
diff --git a/playbooks/base-pre.yaml b/playbooks/base/pre.yaml
similarity index 100%
rename from playbooks/base-pre.yaml
rename to playbooks/base/pre.yaml
diff --git a/playbooks/base/roles b/playbooks/base/roles
new file mode 120000
index 0000000..7b9ade8
--- /dev/null
+++ b/playbooks/base/roles
@@ -0,0 +1 @@
+../roles/
\ No newline at end of file
diff --git a/playbooks/roles/extra-test-setup/tasks/main.yaml b/playbooks/roles/extra-test-setup/tasks/main.yaml
new file mode 100644
index 0000000..da4259e
--- /dev/null
+++ b/playbooks/roles/extra-test-setup/tasks/main.yaml
@@ -0,0 +1,13 @@
+---
+- name: Check if projects tools/test-setup.sh exists.
+ stat:
+ path: "{{ zuul_workspace_root }}/src/{{ zuul.project }}/tools/test-setup.sh"
+ register: p
+
+- name: Run tools/test-setup.sh.
+ shell: tools/test-setup.sh
+ args:
+ chdir: "{{ zuul_workspace_root }}/src/{{ zuul.project }}"
+ when:
+ - p.stat.exists
+ - p.stat.executable
diff --git a/playbooks/roles/prepare-workspace/defaults/main.yaml b/playbooks/roles/prepare-workspace/defaults/main.yaml
deleted file mode 100644
index 9127ad8..0000000
--- a/playbooks/roles/prepare-workspace/defaults/main.yaml
+++ /dev/null
@@ -1,3 +0,0 @@
----
-# tasks/main.yaml
-prepare_workspace_root: /home/zuul/workspace
diff --git a/playbooks/roles/prepare-workspace/tasks/main.yaml b/playbooks/roles/prepare-workspace/tasks/main.yaml
index 26d0670..c5952c7 100644
--- a/playbooks/roles/prepare-workspace/tasks/main.yaml
+++ b/playbooks/roles/prepare-workspace/tasks/main.yaml
@@ -10,15 +10,12 @@
- name: Create workspace directory.
file:
- path: "{{ prepare_workspace_root }}"
+ path: "{{ zuul_workspace_root }}"
owner: zuul
group: zuul
state: directory
- name: Synchronize src repos to workspace directory.
synchronize:
- dest: "{{ prepare_workspace_root }}"
+ dest: "{{ zuul_workspace_root }}"
src: "{{ zuul.executor.src_root }}"
-
-- name: Run configure_mirror.sh
- shell: /opt/nodepool-scripts/configure_mirror.sh
diff --git a/playbooks/roles/revoke-sudo/tasks/main.yaml b/playbooks/roles/revoke-sudo/tasks/main.yaml
new file mode 100644
index 0000000..1c18187
--- /dev/null
+++ b/playbooks/roles/revoke-sudo/tasks/main.yaml
@@ -0,0 +1,8 @@
+- name: Remove sudo access for zuul user.
+ become: yes
+ file:
+ path: /etc/sudoers.d/zuul-sudo
+ state: absent
+
+- name: Prove that general sudo access is actually revoked.
+ shell: ! sudo -n true
diff --git a/playbooks/roles/run-bindep/tasks/main.yaml b/playbooks/roles/run-bindep/tasks/main.yaml
index 7717c86..5a9d33e 100644
--- a/playbooks/roles/run-bindep/tasks/main.yaml
+++ b/playbooks/roles/run-bindep/tasks/main.yaml
@@ -2,4 +2,4 @@
- name: Run install-distro-packages.sh
shell: /usr/local/jenkins/slave_scripts/install-distro-packages.sh
args:
- chdir: "/home/zuul/workspace/src/{{ zuul.project }}"
+ chdir: "{{ zuul_workspace_root }}/src/{{ zuul.project }}"
diff --git a/playbooks/roles/run-cover/defaults/main.yaml b/playbooks/roles/run-cover/defaults/main.yaml
new file mode 100644
index 0000000..2e32efe
--- /dev/null
+++ b/playbooks/roles/run-cover/defaults/main.yaml
@@ -0,0 +1,2 @@
+---
+run_cover_envlist: cover
diff --git a/playbooks/roles/run-cover/tasks/main.yaml b/playbooks/roles/run-cover/tasks/main.yaml
new file mode 100644
index 0000000..caed13c
--- /dev/null
+++ b/playbooks/roles/run-cover/tasks/main.yaml
@@ -0,0 +1,4 @@
+- name: Execute run-cover.sh.
+ shell: "/usr/local/jenkins/slave_scripts/run-cover.sh {{ run_cover_envlist }}"
+ args:
+ chdir: "{{ zuul_workspace_root }}/src/{{ zuul.project }}"
diff --git a/playbooks/roles/run-docs/defaults/main.yaml b/playbooks/roles/run-docs/defaults/main.yaml
new file mode 100644
index 0000000..5855a3d
--- /dev/null
+++ b/playbooks/roles/run-docs/defaults/main.yaml
@@ -0,0 +1,2 @@
+---
+run_docs_envlist: venv
diff --git a/playbooks/roles/run-docs/tasks/main.yaml b/playbooks/roles/run-docs/tasks/main.yaml
new file mode 100644
index 0000000..2250593
--- /dev/null
+++ b/playbooks/roles/run-docs/tasks/main.yaml
@@ -0,0 +1,4 @@
+- name: Execute run-docs.sh.
+ shell: "/usr/local/jenkins/slave_scripts/run-docs.sh {{ run_docs_envlist }}"
+ args:
+ chdir: "{{ zuul_workspace_root }}/src/{{ zuul.project }}"
diff --git a/playbooks/roles/run-tarball/defaults/main.yaml b/playbooks/roles/run-tarball/defaults/main.yaml
new file mode 100644
index 0000000..072828a
--- /dev/null
+++ b/playbooks/roles/run-tarball/defaults/main.yaml
@@ -0,0 +1,2 @@
+---
+run_tarball_envlist: venv
diff --git a/playbooks/roles/run-tarball/tasks/main.yaml b/playbooks/roles/run-tarball/tasks/main.yaml
new file mode 100644
index 0000000..e21c4c8
--- /dev/null
+++ b/playbooks/roles/run-tarball/tasks/main.yaml
@@ -0,0 +1,4 @@
+- name: Execute run-tarball.sh.
+ shell: "/usr/local/jenkins/slave_scripts/run-tarball.sh {{ run_tarball_envlist }}"
+ args:
+ chdir: "{{ zuul_workspace_root }}/src/{{ zuul.project }}"
diff --git a/playbooks/roles/run-tox/tasks/main.yaml b/playbooks/roles/run-tox/tasks/main.yaml
index 1053690..29a4cc4 100644
--- a/playbooks/roles/run-tox/tasks/main.yaml
+++ b/playbooks/roles/run-tox/tasks/main.yaml
@@ -1,4 +1,4 @@
- name: Run tox
shell: "/usr/local/jenkins/slave_scripts/run-tox.sh {{ run_tox_envlist }}"
args:
- chdir: "/home/zuul/workspace/src/{{ zuul.project }}"
+ chdir: "{{ zuul_workspace_root }}/src/{{ zuul.project }}"
diff --git a/playbooks/roles/run-wheel/defaults/main.yaml b/playbooks/roles/run-wheel/defaults/main.yaml
new file mode 100644
index 0000000..8645d33
--- /dev/null
+++ b/playbooks/roles/run-wheel/defaults/main.yaml
@@ -0,0 +1,2 @@
+---
+run_wheel_envlist: venv
diff --git a/playbooks/roles/run-wheel/tasks/main.yaml b/playbooks/roles/run-wheel/tasks/main.yaml
new file mode 100644
index 0000000..f5aaf54
--- /dev/null
+++ b/playbooks/roles/run-wheel/tasks/main.yaml
@@ -0,0 +1,4 @@
+- name: Execute run-wheel.sh.
+ shell: "/usr/local/jenkins/slave_scripts/run-wheel.sh {{ run_wheel_envlist }}"
+ args:
+ chdir: "{{ zuul_workspace_root }}/src/{{ zuul.project }}"
diff --git a/playbooks/tox-cover.yaml b/playbooks/tox-cover.yaml
deleted file mode 100644
index 942aa23..0000000
--- a/playbooks/tox-cover.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-- hosts: all
- vars:
- run_tox_envlist: cover
- roles:
- - run-tox
diff --git a/playbooks/tox/cover.yaml b/playbooks/tox/cover.yaml
new file mode 100644
index 0000000..642eb4e
--- /dev/null
+++ b/playbooks/tox/cover.yaml
@@ -0,0 +1,5 @@
+- hosts: all
+ roles:
+ - extra-test-setup
+ - revoke-sudo
+ - run-cover
diff --git a/playbooks/tox/docs.yaml b/playbooks/tox/docs.yaml
new file mode 100644
index 0000000..028e1c5
--- /dev/null
+++ b/playbooks/tox/docs.yaml
@@ -0,0 +1,4 @@
+- hosts: all
+ roles:
+ - revoke-sudo
+ - run-docs
diff --git a/playbooks/tox-linters.yaml b/playbooks/tox/linters.yaml
similarity index 79%
rename from playbooks/tox-linters.yaml
rename to playbooks/tox/linters.yaml
index 9da2e8a..d1e7f13 100644
--- a/playbooks/tox-linters.yaml
+++ b/playbooks/tox/linters.yaml
@@ -2,4 +2,5 @@
vars:
run_tox_envlist: pep8
roles:
+ - revoke-sudo
- run-tox
diff --git a/playbooks/tox-post.yaml b/playbooks/tox/post.yaml
similarity index 86%
rename from playbooks/tox-post.yaml
rename to playbooks/tox/post.yaml
index 697b727..3b035f8 100644
--- a/playbooks/tox-post.yaml
+++ b/playbooks/tox/post.yaml
@@ -3,7 +3,7 @@
- name: Find tox directories to synchrionize.
find:
file_type: directory
- paths: "/home/zuul/workspace/src/{{ zuul.project }}/.tox"
+ paths: "{{ zuul_workspace_root }}/src/{{ zuul.project }}/.tox"
# NOTE(pabelanger): The .tox/log folder is empty, ignore it.
patterns: ^(?!log).*$
use_regex: yes
diff --git a/playbooks/tox-pre.yaml b/playbooks/tox/pre.yaml
similarity index 100%
rename from playbooks/tox-pre.yaml
rename to playbooks/tox/pre.yaml
diff --git a/playbooks/tox-py27.yaml b/playbooks/tox/py27.yaml
similarity index 62%
rename from playbooks/tox-py27.yaml
rename to playbooks/tox/py27.yaml
index 13756b5..fd45f27 100644
--- a/playbooks/tox-py27.yaml
+++ b/playbooks/tox/py27.yaml
@@ -2,4 +2,6 @@
vars:
run_tox_envlist: py27
roles:
+ - extra-test-setup
+ - revoke-sudo
- run-tox
diff --git a/playbooks/tox/roles b/playbooks/tox/roles
new file mode 120000
index 0000000..7b9ade8
--- /dev/null
+++ b/playbooks/tox/roles
@@ -0,0 +1 @@
+../roles/
\ No newline at end of file
diff --git a/playbooks/tox/tarball-post.yaml b/playbooks/tox/tarball-post.yaml
new file mode 100644
index 0000000..fb41707
--- /dev/null
+++ b/playbooks/tox/tarball-post.yaml
@@ -0,0 +1,10 @@
+- hosts: all
+ tasks:
+ - name: Collect tarball artifacts.
+ synchronize:
+ dest: "{{ zuul.executor.src_root }}/tarballs"
+ mode: pull
+ src: "{{ zuul_workspace_root }}/src/{{ zuul.project }}/dist/{{ item }}"
+ with_items:
+ - "*.tar.gz"
+ - "*.whl"
diff --git a/playbooks/tox/tarball.yaml b/playbooks/tox/tarball.yaml
new file mode 100644
index 0000000..4d5a8f6
--- /dev/null
+++ b/playbooks/tox/tarball.yaml
@@ -0,0 +1,5 @@
+- hosts: all
+ roles:
+ - revoke-sudo
+ - run-tarball
+ - run-wheel
diff --git a/tests/base.py b/tests/base.py
index b72927f..9a6fb69 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -53,7 +53,6 @@
import zuul.driver.gerrit.gerritsource as gerritsource
import zuul.driver.gerrit.gerritconnection as gerritconnection
-import zuul.connection.sql
import zuul.scheduler
import zuul.webapp
import zuul.rpclistener
@@ -1923,8 +1922,8 @@
class ZuulDBTestCase(ZuulTestCase):
- def setup_config(self, config_file='zuul-connections-same-gerrit.conf'):
- super(ZuulDBTestCase, self).setup_config(config_file)
+ def setup_config(self):
+ super(ZuulDBTestCase, self).setup_config()
for section_name in self.config.sections():
con_match = re.match(r'^connection ([\'\"]?)(.*)(\1)$',
section_name, re.I)
diff --git a/tests/fixtures/config/ansible/git/common-config/zuul.yaml b/tests/fixtures/config/ansible/git/common-config/zuul.yaml
index aa70054..50f353d 100644
--- a/tests/fixtures/config/ansible/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/ansible/git/common-config/zuul.yaml
@@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: patchset-created
@@ -17,8 +16,7 @@
name: gate
manager: dependent
success-message: Build succeeded (gate).
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: comment-added
@@ -41,7 +39,7 @@
pre-run: pre
post-run: post
vars:
- flagpath: "{{zuul._test.test_root}}/{{zuul.uuid}}.flag"
+ flagpath: '{{zuul._test.test_root}}/{{zuul.uuid}}.flag'
roles:
- zuul: bare-role
diff --git a/tests/fixtures/config/ansible/git/org_project/.zuul.yaml b/tests/fixtures/config/ansible/git/org_project/.zuul.yaml
index b38f88e..24ba019 100644
--- a/tests/fixtures/config/ansible/git/org_project/.zuul.yaml
+++ b/tests/fixtures/config/ansible/git/org_project/.zuul.yaml
@@ -4,7 +4,6 @@
- project:
name: org/project
-
check:
jobs:
- python27
diff --git a/tests/fixtures/config/dependency-graph/git/common-config/playbooks/A.yaml b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/A.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/A.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/dependency-graph/git/common-config/playbooks/B.yaml b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/B.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/B.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/dependency-graph/git/common-config/playbooks/C.yaml b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/C.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/C.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/dependency-graph/git/common-config/playbooks/D.yaml b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/D.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/D.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/dependency-graph/git/common-config/playbooks/E.yaml b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/E.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/E.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/dependency-graph/git/common-config/playbooks/F.yaml b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/F.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/F.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/dependency-graph/git/common-config/playbooks/G.yaml b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/G.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/G.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/dependency-graph/git/common-config/playbooks/project-test1.yaml b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/project-test1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/dependency-graph/git/common-config/playbooks/project-test1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/dependency-graph/git/common-config/zuul.yaml b/tests/fixtures/config/dependency-graph/git/common-config/zuul.yaml
new file mode 100644
index 0000000..60f3651
--- /dev/null
+++ b/tests/fixtures/config/dependency-graph/git/common-config/zuul.yaml
@@ -0,0 +1,73 @@
+- pipeline:
+ name: gate
+ manager: dependent
+ success-message: Build succeeded (gate).
+ source: gerrit
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - approved: 1
+ success:
+ gerrit:
+ verified: 2
+ submit: true
+ failure:
+ gerrit:
+ verified: -2
+ start:
+ gerrit:
+ verified: 0
+ precedence: high
+
+- job:
+ name: A
+
+- job:
+ name: B
+
+- job:
+ name: C
+
+- job:
+ name: D
+
+- job:
+ name: E
+
+- job:
+ name: F
+
+- job:
+ name: G
+
+- project:
+ name: org/project
+ gate:
+ jobs:
+ # Job dependencies, starting with A
+ # A
+ # / \
+ # B C
+ # / \ / \
+ # D F E
+ # |
+ # G
+ # This is intentionally not listed in the natural order to
+ # ensure that we can reference dependencies before they are
+ # defined.
+ - E:
+ dependencies: C
+ - A
+ - B:
+ dependencies: A
+ - C:
+ dependencies: A
+ - F:
+ dependencies:
+ - B
+ - C
+ - D:
+ dependencies: B
+ - G:
+ dependencies: F
diff --git a/tests/fixtures/config/dependency-graph/git/org_project/README b/tests/fixtures/config/dependency-graph/git/org_project/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/dependency-graph/git/org_project/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/dependency-graph/main.yaml b/tests/fixtures/config/dependency-graph/main.yaml
new file mode 100644
index 0000000..d9868fa
--- /dev/null
+++ b/tests/fixtures/config/dependency-graph/main.yaml
@@ -0,0 +1,8 @@
+- tenant:
+ name: tenant-one
+ source:
+ gerrit:
+ config-repos:
+ - common-config
+ project-repos:
+ - org/project
diff --git a/tests/fixtures/config/duplicate-pipeline/git/common-config/zuul.yaml b/tests/fixtures/config/duplicate-pipeline/git/common-config/zuul.yaml
index bc88b06..5005108 100755
--- a/tests/fixtures/config/duplicate-pipeline/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/duplicate-pipeline/git/common-config/zuul.yaml
@@ -2,8 +2,7 @@
name: dup1
manager: independent
success-message: Build succeeded (dup1).
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: change-restored
@@ -18,8 +17,7 @@
name: dup2
manager: independent
success-message: Build succeeded (dup2).
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: change-restored
@@ -39,7 +37,6 @@
queue: integrated
jobs:
- project-test1
-
dup2:
queue: integrated
jobs:
diff --git a/tests/fixtures/config/in-repo/git/common-config/zuul.yaml b/tests/fixtures/config/in-repo/git/common-config/zuul.yaml
index d8b7200..55169ce 100644
--- a/tests/fixtures/config/in-repo/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/in-repo/git/common-config/zuul.yaml
@@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: patchset-created
@@ -17,8 +16,7 @@
name: tenant-one-gate
manager: dependent
success-message: Build succeeded (tenant-one-gate).
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: comment-added
diff --git a/tests/fixtures/config/merges/git/common-config/zuul.yaml b/tests/fixtures/config/merges/git/common-config/zuul.yaml
index bb91f3a..ab4e24c 100644
--- a/tests/fixtures/config/merges/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/merges/git/common-config/zuul.yaml
@@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: patchset-created
@@ -17,8 +16,7 @@
name: gate
manager: dependent
success-message: Build succeeded (gate).
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: comment-added
@@ -37,16 +35,13 @@
precedence: high
- job:
- name:
- project-test1
+ name: project-test1
- job:
- name:
- project-test2
+ name: project-test2
- job:
- name:
- project-merge
+ name: project-merge
hold-following-changes: true
- project:
@@ -75,6 +70,6 @@
merge-mode: cherry-pick
gate:
jobs:
- - project-merge:
- jobs:
- - project-test1
+ - project-merge
+ - project-test1:
+ dependencies: project-merge
diff --git a/tests/fixtures/config/multi-tenant/git/common-config/zuul.yaml b/tests/fixtures/config/multi-tenant/git/common-config/zuul.yaml
index 08117d6..004f2df 100644
--- a/tests/fixtures/config/multi-tenant/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/multi-tenant/git/common-config/zuul.yaml
@@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: patchset-created
@@ -14,8 +13,7 @@
verified: -1
- job:
- name:
- python27
+ name: python27
nodes:
- name: controller
image: ubuntu-trusty
diff --git a/tests/fixtures/config/multi-tenant/git/tenant-one-config/zuul.yaml b/tests/fixtures/config/multi-tenant/git/tenant-one-config/zuul.yaml
index 4a653f6..5769cf5 100644
--- a/tests/fixtures/config/multi-tenant/git/tenant-one-config/zuul.yaml
+++ b/tests/fixtures/config/multi-tenant/git/tenant-one-config/zuul.yaml
@@ -2,8 +2,7 @@
name: tenant-one-gate
manager: dependent
success-message: Build succeeded (tenant-one-gate).
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: comment-added
@@ -28,8 +27,7 @@
image: controller-image
- job:
- name:
- project1-test1
+ name: project1-test1
- project:
name: org/project1
diff --git a/tests/fixtures/config/multi-tenant/git/tenant-two-config/zuul.yaml b/tests/fixtures/config/multi-tenant/git/tenant-two-config/zuul.yaml
index 7c79720..19782ce 100644
--- a/tests/fixtures/config/multi-tenant/git/tenant-two-config/zuul.yaml
+++ b/tests/fixtures/config/multi-tenant/git/tenant-two-config/zuul.yaml
@@ -2,8 +2,7 @@
name: tenant-two-gate
manager: dependent
success-message: Build succeeded (tenant-two-gate).
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: comment-added
@@ -28,8 +27,7 @@
image: controller-image
- job:
- name:
- project2-test1
+ name: project2-test1
- project:
name: org/project2
diff --git a/tests/fixtures/config/one-job-project/git/common-config/zuul.yaml b/tests/fixtures/config/one-job-project/git/common-config/zuul.yaml
index 148ba42..4579062 100644
--- a/tests/fixtures/config/one-job-project/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/one-job-project/git/common-config/zuul.yaml
@@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: patchset-created
@@ -17,8 +16,7 @@
name: gate
manager: dependent
success-message: Build succeeded (gate).
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: comment-added
@@ -39,8 +37,7 @@
- pipeline:
name: post
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: ref-updated
diff --git a/tests/fixtures/config/openstack/git/project-config/zuul.yaml b/tests/fixtures/config/openstack/git/project-config/zuul.yaml
index 420d979..760adb8 100644
--- a/tests/fixtures/config/openstack/git/project-config/zuul.yaml
+++ b/tests/fixtures/config/openstack/git/project-config/zuul.yaml
@@ -1,11 +1,8 @@
-# Pipeline definitions
-
- pipeline:
name: check
manager: independent
success-message: Build succeeded (check).
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: patchset-created
@@ -20,8 +17,7 @@
name: gate
manager: dependent
success-message: Build succeeded (gate).
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: comment-added
@@ -39,8 +35,6 @@
verified: 0
precedence: high
-# Job definitions
-
- job:
name: base
timeout: 30
@@ -78,8 +72,6 @@
- openstack/keystone
- openstack/nova
-# Project definitions
-
- project:
name: openstack/nova
templates:
diff --git a/tests/fixtures/config/requirements/email/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/email/git/common-config/zuul.yaml
index 09e0cc6..78d2a18 100644
--- a/tests/fixtures/config/requirements/email/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/requirements/email/git/common-config/zuul.yaml
@@ -1,8 +1,7 @@
- pipeline:
name: pipeline
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: comment-added
@@ -19,8 +18,7 @@
- pipeline:
name: trigger
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: comment-added
diff --git a/tests/fixtures/config/requirements/newer-than/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/newer-than/git/common-config/zuul.yaml
index cd76afd..1e84e18 100644
--- a/tests/fixtures/config/requirements/newer-than/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/requirements/newer-than/git/common-config/zuul.yaml
@@ -1,8 +1,7 @@
- pipeline:
name: pipeline
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: comment-added
@@ -20,8 +19,7 @@
- pipeline:
name: trigger
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: comment-added
diff --git a/tests/fixtures/config/requirements/older-than/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/older-than/git/common-config/zuul.yaml
index 8dca5e6..efbd79a 100644
--- a/tests/fixtures/config/requirements/older-than/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/requirements/older-than/git/common-config/zuul.yaml
@@ -1,8 +1,7 @@
- pipeline:
name: pipeline
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: comment-added
@@ -20,8 +19,7 @@
- pipeline:
name: trigger
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: comment-added
diff --git a/tests/fixtures/config/requirements/reject-username/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/reject-username/git/common-config/zuul.yaml
index 92c7de2..7212944 100644
--- a/tests/fixtures/config/requirements/reject-username/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/requirements/reject-username/git/common-config/zuul.yaml
@@ -1,11 +1,10 @@
- pipeline:
name: pipeline
manager: independent
- source:
- gerrit
+ source: gerrit
reject:
approval:
- - username: 'jenkins'
+ - username: jenkins
trigger:
gerrit:
- event: comment-added
@@ -19,13 +18,12 @@
- pipeline:
name: trigger
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: comment-added
reject-approval:
- - username: 'jenkins'
+ - username: jenkins
success:
gerrit:
verified: 1
diff --git a/tests/fixtures/config/requirements/reject/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/reject/git/common-config/zuul.yaml
index 12a2538..9f5b125 100644
--- a/tests/fixtures/config/requirements/reject/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/requirements/reject/git/common-config/zuul.yaml
@@ -1,15 +1,18 @@
- pipeline:
name: pipeline
manager: independent
- source:
- gerrit
+ source: gerrit
require:
approval:
- username: jenkins
- verified: [1, 2]
+ verified:
+ - 1
+ - 2
reject:
approval:
- - verified: [-1, -2]
+ - verified:
+ - -1
+ - -2
trigger:
gerrit:
- event: comment-added
@@ -23,16 +26,19 @@
- pipeline:
name: trigger
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: comment-added
require-approval:
- username: jenkins
- verified: [1, 2]
+ verified:
+ - 1
+ - 2
reject-approval:
- - verified: [-1, -2]
+ - verified:
+ - -1
+ - -2
success:
gerrit:
verified: 1
diff --git a/tests/fixtures/config/requirements/state/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/state/git/common-config/zuul.yaml
index 9491bff..01ceb46 100644
--- a/tests/fixtures/config/requirements/state/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/requirements/state/git/common-config/zuul.yaml
@@ -1,10 +1,9 @@
- pipeline:
name: current-check
manager: independent
- source:
- gerrit
+ source: gerrit
require:
- current-patchset: True
+ current-patchset: true
trigger:
gerrit:
- event: patchset-created
@@ -19,10 +18,9 @@
- pipeline:
name: open-check
manager: independent
- source:
- gerrit
+ source: gerrit
require:
- open: True
+ open: true
trigger:
gerrit:
- event: patchset-created
@@ -37,8 +35,7 @@
- pipeline:
name: status-check
manager: independent
- source:
- gerrit
+ source: gerrit
require:
status: NEW
trigger:
diff --git a/tests/fixtures/config/requirements/username/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/username/git/common-config/zuul.yaml
index ca2ff97..9789e71 100644
--- a/tests/fixtures/config/requirements/username/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/requirements/username/git/common-config/zuul.yaml
@@ -1,8 +1,7 @@
- pipeline:
name: pipeline
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: comment-added
@@ -19,8 +18,7 @@
- pipeline:
name: trigger
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: comment-added
diff --git a/tests/fixtures/config/requirements/vote1/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/vote1/git/common-config/zuul.yaml
index 00afe79..7989363 100644
--- a/tests/fixtures/config/requirements/vote1/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/requirements/vote1/git/common-config/zuul.yaml
@@ -1,4 +1,3 @@
-
- pipeline:
name: pipeline
manager: independent
@@ -6,8 +5,7 @@
approval:
- username: jenkins
verified: 1
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: comment-added
@@ -21,8 +19,7 @@
- pipeline:
name: trigger
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: comment-added
diff --git a/tests/fixtures/config/requirements/vote2/git/common-config/zuul.yaml b/tests/fixtures/config/requirements/vote2/git/common-config/zuul.yaml
index 73db7a7..9348afb 100644
--- a/tests/fixtures/config/requirements/vote2/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/requirements/vote2/git/common-config/zuul.yaml
@@ -4,9 +4,10 @@
require:
approval:
- username: jenkins
- verified: [1, 2]
- source:
- gerrit
+ verified:
+ - 1
+ - 2
+ source: gerrit
trigger:
gerrit:
- event: comment-added
@@ -20,14 +21,15 @@
- pipeline:
name: trigger
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: comment-added
require-approval:
- username: jenkins
- verified: [1, 2]
+ verified:
+ - 1
+ - 2
success:
gerrit:
verified: 1
diff --git a/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml b/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
index b91bf6f..47c173d 100644
--- a/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
@@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: patchset-created
@@ -17,8 +16,7 @@
name: gate
manager: dependent
success-message: Build succeeded (gate).
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: comment-added
@@ -39,8 +37,7 @@
- pipeline:
name: post
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: ref-updated
@@ -49,8 +46,7 @@
- pipeline:
name: experimental
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: patchset-created
@@ -107,23 +103,26 @@
- job:
name: project-testfile
files:
- - '.*-requires'
+ - .*-requires
- project:
name: org/project
check:
jobs:
- - project-merge:
- jobs:
- - project-test1
- - project-test2
+ - project-merge
+ - project-test1:
+ dependencies: project-merge
+ - project-test2:
+ dependencies: project-merge
gate:
jobs:
- - project-merge:
- jobs:
- - project-test1
- - project-test2
- - project-testfile
+ - project-merge
+ - project-test1:
+ dependencies: project-merge
+ - project-test2:
+ dependencies: project-merge
+ - project-testfile:
+ dependencies: project-merge
post:
jobs:
- project-post
@@ -132,48 +131,58 @@
name: org/project1
check:
jobs:
- - project-merge:
- jobs:
- - project-test1
- - project-test2
- - project1-project2-integration
+ - project-merge
+ - project-test1:
+ dependencies: project-merge
+ - project-test2:
+ dependencies: project-merge
+ - project1-project2-integration:
+ dependencies: project-merge
gate:
queue: integrated
jobs:
- - project-merge:
- jobs:
- - project-test1
- - project-test2
- - project1-project2-integration
+ - project-merge
+ - project-test1:
+ dependencies: project-merge
+ - project-test2:
+ dependencies: project-merge
+ - project1-project2-integration:
+ dependencies: project-merge
- project:
name: org/project2
gate:
queue: integrated
jobs:
- - project-merge:
- jobs:
- - project-test1
- - project-test2
- - project1-project2-integration
+ - project-merge
+ - project-test1:
+ dependencies: project-merge
+ - project-test2:
+ dependencies: project-merge
+ - project1-project2-integration:
+ dependencies: project-merge
- project:
name: org/project3
check:
jobs:
- - project-merge:
- jobs:
- - project-test1
- - project-test2
- - project1-project2-integration
+ - project-merge
+ - project-test1:
+ dependencies: project-merge
+ - project-test2:
+ dependencies: project-merge
+ - project1-project2-integration:
+ dependencies: project-merge
gate:
queue: integrated
jobs:
- - project-merge:
- jobs:
- - project-test1
- - project-test2
- - project1-project2-integration
+ - project-merge
+ - project-test1:
+ dependencies: project-merge
+ - project-test2:
+ dependencies: project-merge
+ - project1-project2-integration:
+ dependencies: project-merge
post:
jobs:
- project-post
@@ -182,9 +191,9 @@
name: org/experimental-project
experimental:
jobs:
- - project-merge:
- jobs:
- - experimental-project-test
+ - project-merge
+ - experimental-project-test:
+ dependencies: project-merge
- project:
name: org/noop-project
@@ -199,16 +208,18 @@
name: org/nonvoting-project
check:
jobs:
- - nonvoting-project-merge:
- jobs:
- - nonvoting-project-test1
- - nonvoting-project-test2
+ - nonvoting-project-merge
+ - nonvoting-project-test1:
+ dependencies: nonvoting-project-merge
+ - nonvoting-project-test2:
+ dependencies: nonvoting-project-merge
gate:
jobs:
- - nonvoting-project-merge:
- jobs:
- - nonvoting-project-test1
- - nonvoting-project-test2
+ - nonvoting-project-merge
+ - nonvoting-project-test1:
+ dependencies: nonvoting-project-merge
+ - nonvoting-project-test2:
+ dependencies: nonvoting-project-merge
- project:
name: org/no-jobs-project
diff --git a/tests/fixtures/config/single-tenant/git/layout-disabled-at/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-disabled-at/zuul.yaml
index 4cf6f16..bdc19ac 100644
--- a/tests/fixtures/config/single-tenant/git/layout-disabled-at/zuul.yaml
+++ b/tests/fixtures/config/single-tenant/git/layout-disabled-at/zuul.yaml
@@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: patchset-created
diff --git a/tests/fixtures/config/single-tenant/git/layout-dont-ignore-ref-deletes/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-dont-ignore-ref-deletes/zuul.yaml
index 30e574a..334d9ac 100644
--- a/tests/fixtures/config/single-tenant/git/layout-dont-ignore-ref-deletes/zuul.yaml
+++ b/tests/fixtures/config/single-tenant/git/layout-dont-ignore-ref-deletes/zuul.yaml
@@ -1,13 +1,12 @@
- pipeline:
name: post
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: ref-updated
ref: ^(?!refs/).*$
- ignore-deletes: False
+ ignore-deletes: false
- job:
name: project-post
@@ -20,4 +19,3 @@
post:
jobs:
- project-post
-
diff --git a/tests/fixtures/config/single-tenant/git/layout-footer-message/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-footer-message/zuul.yaml
index 0c04070..c698378 100644
--- a/tests/fixtures/config/single-tenant/git/layout-footer-message/zuul.yaml
+++ b/tests/fixtures/config/single-tenant/git/layout-footer-message/zuul.yaml
@@ -2,8 +2,7 @@
name: gate
manager: dependent
success-message: Build succeeded (gate).
- source:
- gerrit
+ source: gerrit
failure-message: Build failed. For information on how to proceed, see http://wiki.example.org/Test_Failures
footer-message: For CI problems and help debugging, contact ci@example.org
trigger:
@@ -35,4 +34,3 @@
gate:
jobs:
- project-test1
-
diff --git a/tests/fixtures/config/single-tenant/git/layout-idle/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-idle/zuul.yaml
index f71f3e4..d1fa04b 100644
--- a/tests/fixtures/config/single-tenant/git/layout-idle/zuul.yaml
+++ b/tests/fixtures/config/single-tenant/git/layout-idle/zuul.yaml
@@ -1,8 +1,7 @@
- pipeline:
name: periodic
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
timer:
- time: '* * * * * */1'
diff --git a/tests/fixtures/config/single-tenant/git/layout-inheritance/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-inheritance/zuul.yaml
index 3070af0..ab8c9a5 100644
--- a/tests/fixtures/config/single-tenant/git/layout-inheritance/zuul.yaml
+++ b/tests/fixtures/config/single-tenant/git/layout-inheritance/zuul.yaml
@@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: patchset-created
@@ -13,7 +12,6 @@
gerrit:
verified: -1
-
- job:
name: project-test-irrelevant-starts-empty
diff --git a/tests/fixtures/config/single-tenant/git/layout-irrelevant-files/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-irrelevant-files/zuul.yaml
index f243bcc..5d72fc0 100644
--- a/tests/fixtures/config/single-tenant/git/layout-irrelevant-files/zuul.yaml
+++ b/tests/fixtures/config/single-tenant/git/layout-irrelevant-files/zuul.yaml
@@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: patchset-created
@@ -13,7 +12,6 @@
gerrit:
verified: -1
-
- job:
name: project-test-irrelevant-files
diff --git a/tests/fixtures/config/single-tenant/git/layout-mutex-reconfiguration/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-mutex-reconfiguration/zuul.yaml
index 12f1747..0e332e4 100644
--- a/tests/fixtures/config/single-tenant/git/layout-mutex-reconfiguration/zuul.yaml
+++ b/tests/fixtures/config/single-tenant/git/layout-mutex-reconfiguration/zuul.yaml
@@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: patchset-created
diff --git a/tests/fixtures/config/single-tenant/git/layout-mutex/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-mutex/zuul.yaml
index e91903a..bb92b7a 100644
--- a/tests/fixtures/config/single-tenant/git/layout-mutex/zuul.yaml
+++ b/tests/fixtures/config/single-tenant/git/layout-mutex/zuul.yaml
@@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: patchset-created
diff --git a/tests/fixtures/config/single-tenant/git/layout-no-timer/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-no-timer/zuul.yaml
index f754e37..ab919a4 100644
--- a/tests/fixtures/config/single-tenant/git/layout-no-timer/zuul.yaml
+++ b/tests/fixtures/config/single-tenant/git/layout-no-timer/zuul.yaml
@@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: patchset-created
@@ -18,8 +17,7 @@
manager: independent
# Trigger is required, set it to one that is a noop
# during tests that check the timer trigger.
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: ref-updated
diff --git a/tests/fixtures/config/single-tenant/git/layout-repo-deleted/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-repo-deleted/zuul.yaml
index 2bffc3e..5851d75 100644
--- a/tests/fixtures/config/single-tenant/git/layout-repo-deleted/zuul.yaml
+++ b/tests/fixtures/config/single-tenant/git/layout-repo-deleted/zuul.yaml
@@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: patchset-created
@@ -17,8 +16,7 @@
name: gate
manager: dependent
success-message: Build succeeded (gate).
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: comment-added
@@ -60,13 +58,15 @@
name: org/delete-project
check:
jobs:
- - project-merge:
- jobs:
- - project-test1
- - project-test2
+ - project-merge
+ - project-test1:
+ dependencies: project-merge
+ - project-test2:
+ dependencies: project-merge
gate:
jobs:
- - project-merge:
- jobs:
- - project-test1
- - project-test2
+ - project-merge
+ - project-test1:
+ dependencies: project-merge
+ - project-test2:
+ dependencies: project-merge
diff --git a/tests/fixtures/config/single-tenant/git/layout-smtp/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-smtp/zuul.yaml
index 9effb1f..be90d48 100644
--- a/tests/fixtures/config/single-tenant/git/layout-smtp/zuul.yaml
+++ b/tests/fixtures/config/single-tenant/git/layout-smtp/zuul.yaml
@@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: patchset-created
@@ -23,8 +22,7 @@
name: gate
manager: dependent
success-message: Build succeeded (gate).
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: comment-added
@@ -69,13 +67,15 @@
name: org/project
check:
jobs:
- - project-merge:
- jobs:
- - project-test1
- - project-test2
+ - project-merge
+ - project-test1:
+ dependencies: project-merge
+ - project-test2:
+ dependencies: project-merge
gate:
jobs:
- - project-merge:
- jobs:
- - project-test1
- - project-test2
+ - project-merge
+ - project-test1:
+ dependencies: project-merge
+ - project-test2:
+ dependencies: project-merge
diff --git a/tests/fixtures/config/single-tenant/git/layout-tags/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-tags/zuul.yaml
index c921c90..07f0657 100644
--- a/tests/fixtures/config/single-tenant/git/layout-tags/zuul.yaml
+++ b/tests/fixtures/config/single-tenant/git/layout-tags/zuul.yaml
@@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: patchset-created
@@ -34,19 +33,23 @@
check:
jobs:
- merge:
- jobs:
- - test1
- - test2
- - integration
tags:
- extratag
+ - test1:
+ dependencies: merge
+ - test2:
+ dependencies: merge
+ - integration:
+ dependencies: merge
- project:
name: org/project2
check:
jobs:
- - merge:
- jobs:
- - test1
- - test2
- - integration
+ - merge
+ - test1:
+ dependencies: merge
+ - test2:
+ dependencies: merge
+ - integration:
+ dependencies: merge
diff --git a/tests/fixtures/config/single-tenant/git/layout-timer-smtp/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-timer-smtp/zuul.yaml
index 4a14107..2a2eca5 100644
--- a/tests/fixtures/config/single-tenant/git/layout-timer-smtp/zuul.yaml
+++ b/tests/fixtures/config/single-tenant/git/layout-timer-smtp/zuul.yaml
@@ -1,8 +1,7 @@
- pipeline:
name: periodic
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
timer:
- time: '* * * * * */1'
@@ -10,7 +9,7 @@
smtp:
to: alternative_me@example.com
from: zuul_from@example.com
- subject: 'Periodic check for {change.project} succeeded'
+ subject: Periodic check for {change.project} succeeded
- job:
name: project-bitrot-stable-old
diff --git a/tests/fixtures/config/single-tenant/git/layout-timer/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-timer/zuul.yaml
index f69a91d..8072644 100644
--- a/tests/fixtures/config/single-tenant/git/layout-timer/zuul.yaml
+++ b/tests/fixtures/config/single-tenant/git/layout-timer/zuul.yaml
@@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: patchset-created
@@ -16,8 +15,7 @@
- pipeline:
name: periodic
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
timer:
- time: '* * * * * */1'
diff --git a/tests/fixtures/config/sql-driver/git/common-config/playbooks/project-merge.yaml b/tests/fixtures/config/sql-driver/git/common-config/playbooks/project-merge.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/sql-driver/git/common-config/playbooks/project-merge.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/sql-driver/git/common-config/playbooks/project-test1.yaml b/tests/fixtures/config/sql-driver/git/common-config/playbooks/project-test1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/sql-driver/git/common-config/playbooks/project-test1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/sql-driver/git/common-config/playbooks/project-test2.yaml b/tests/fixtures/config/sql-driver/git/common-config/playbooks/project-test2.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/sql-driver/git/common-config/playbooks/project-test2.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/sql-driver/git/common-config/zuul.yaml b/tests/fixtures/config/sql-driver/git/common-config/zuul.yaml
new file mode 100644
index 0000000..36c7602
--- /dev/null
+++ b/tests/fixtures/config/sql-driver/git/common-config/zuul.yaml
@@ -0,0 +1,38 @@
+- pipeline:
+ name: check
+ manager: independent
+ source: gerrit
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ resultsdb:
+ score: 1
+ failure:
+ gerrit:
+ verified: -1
+ resultsdb:
+ score: -1
+ resultsdb_failures:
+ score: -1
+
+- job:
+ name: project-merge
+
+- job:
+ name: project-test1
+
+- job:
+ name: project-test2
+
+- project:
+ name: org/project
+ check:
+ jobs:
+ - project-merge
+ - project-test1:
+ dependencies: project-merge
+ - project-test2:
+ dependencies: project-merge
diff --git a/tests/fixtures/config/sql-driver/git/org_project/README b/tests/fixtures/config/sql-driver/git/org_project/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/sql-driver/git/org_project/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/sql-driver/main.yaml b/tests/fixtures/config/sql-driver/main.yaml
new file mode 100644
index 0000000..d9868fa
--- /dev/null
+++ b/tests/fixtures/config/sql-driver/main.yaml
@@ -0,0 +1,8 @@
+- tenant:
+ name: tenant-one
+ source:
+ gerrit:
+ config-repos:
+ - common-config
+ project-repos:
+ - org/project
diff --git a/tests/fixtures/config/success-url/git/common-config/zuul.yaml b/tests/fixtures/config/success-url/git/common-config/zuul.yaml
index 7edb340..f2d5251 100644
--- a/tests/fixtures/config/success-url/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/success-url/git/common-config/zuul.yaml
@@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: patchset-created
@@ -18,7 +17,6 @@
gerrit:
verified: -1
-
- job:
name: docs-draft-test
success-url: http://docs-draft.example.org/{build.parameters[LOG_PATH]}/publish-docs/
diff --git a/tests/fixtures/config/templated-project/git/common-config/zuul.yaml b/tests/fixtures/config/templated-project/git/common-config/zuul.yaml
index 22a2d6d..8d2c8a0 100644
--- a/tests/fixtures/config/templated-project/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/templated-project/git/common-config/zuul.yaml
@@ -1,8 +1,7 @@
- pipeline:
name: check
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: patchset-created
@@ -17,8 +16,7 @@
name: gate
manager: dependent
success-message: Build succeeded (gate).
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: comment-added
@@ -39,8 +37,7 @@
- pipeline:
name: post
manager: independent
- source:
- gerrit
+ source: gerrit
trigger:
gerrit:
- event: ref-updated
@@ -56,15 +53,15 @@
- project-template:
name: test-three-and-four
check:
- jobs:
- - layered-project-test3
- - layered-project-test4
+ jobs:
+ - layered-project-test3
+ - layered-project-test4
- project-template:
name: test-five
check:
jobs:
- - layered-project-foo-test5
+ - layered-project-foo-test5
- job:
name: project-test1
diff --git a/tests/fixtures/layout-sql-reporter.yaml b/tests/fixtures/layout-sql-reporter.yaml
deleted file mode 100644
index c79a432..0000000
--- a/tests/fixtures/layout-sql-reporter.yaml
+++ /dev/null
@@ -1,27 +0,0 @@
-pipelines:
- - name: check
- manager: IndependentPipelineManager
- source:
- review_gerrit
- trigger:
- review_gerrit:
- - event: patchset-created
- success:
- review_gerrit:
- verified: 1
- resultsdb:
- score: 1
- failure:
- review_gerrit:
- verified: -1
- resultsdb:
- score: -1
- resultsdb_failures:
- score: -1
-
-projects:
- - name: org/project
- check:
- - project-merge:
- - project-test1
- - project-test2
diff --git a/tests/fixtures/zuul-connections-bad-sql.conf b/tests/fixtures/zuul-connections-bad-sql.conf
deleted file mode 100644
index 2d1e804..0000000
--- a/tests/fixtures/zuul-connections-bad-sql.conf
+++ /dev/null
@@ -1,40 +0,0 @@
-[gearman]
-server=127.0.0.1
-
-[zuul]
-layout_config=layout-connections-multiple-voters.yaml
-url_pattern=http://logs.example.com/{change.number}/{change.patchset}/{pipeline.name}/{job.name}/{build.number}
-job_name_in_report=true
-
-[merger]
-git_dir=/tmp/zuul-test/git
-git_user_email=zuul@example.com
-git_user_name=zuul
-zuul_url=http://zuul.example.com/p
-
-[connection review_gerrit]
-driver=gerrit
-server=review.example.com
-user=jenkins
-sshkey=none
-
-[connection alt_voting_gerrit]
-driver=gerrit
-server=alt_review.example.com
-user=civoter
-sshkey=none
-
-[connection outgoing_smtp]
-driver=smtp
-server=localhost
-port=25
-default_from=zuul@example.com
-default_to=you@example.com
-
-[connection resultsdb]
-driver=sql
-dburi=mysql+pymysql://bad:creds@host/db
-
-[connection resultsdb_failures]
-driver=sql
-dburi=mysql+pymysql://bad:creds@host/db
diff --git a/tests/fixtures/zuul-connections-same-gerrit.conf b/tests/fixtures/zuul-connections-same-gerrit.conf
index 69f5239..6156df4 100644
--- a/tests/fixtures/zuul-connections-same-gerrit.conf
+++ b/tests/fixtures/zuul-connections-same-gerrit.conf
@@ -33,12 +33,3 @@
port=25
default_from=zuul@example.com
default_to=you@example.com
-
-# TODOv3(jeblair): commented out until sqlalchemy conenction ported to
-# v3 driver syntax
-#[connection resultsdb] driver=sql
-#dburi=$MYSQL_FIXTURE_DBURI$
-
-#[connection resultsdb_failures]
-#driver=sql
-#dburi=$MYSQL_FIXTURE_DBURI$
diff --git a/tests/fixtures/zuul-git-driver.conf b/tests/fixtures/zuul-git-driver.conf
index 936c530..0a4e230 100644
--- a/tests/fixtures/zuul-git-driver.conf
+++ b/tests/fixtures/zuul-git-driver.conf
@@ -15,16 +15,6 @@
[executor]
git_dir=/tmp/zuul-test/executor-git
-[swift]
-authurl=https://identity.api.example.org/v2.0/
-user=username
-key=password
-tenant_name=" "
-
-default_container=logs
-region_name=EXP
-logserver_prefix=http://logs.example.org/server.app/
-
[connection gerrit]
driver=gerrit
server=review.example.com
diff --git a/tests/fixtures/zuul-sql-driver-bad.conf b/tests/fixtures/zuul-sql-driver-bad.conf
new file mode 100644
index 0000000..d91e2f6
--- /dev/null
+++ b/tests/fixtures/zuul-sql-driver-bad.conf
@@ -0,0 +1,30 @@
+[gearman]
+server=127.0.0.1
+
+[zuul]
+tenant_config=main.yaml
+url_pattern=http://logs.example.com/{change.number}/{change.patchset}/{pipeline.name}/{job.name}/{build.number}
+job_name_in_report=true
+
+[merger]
+git_dir=/tmp/zuul-test/merger-git
+git_user_email=zuul@example.com
+git_user_name=zuul
+zuul_url=http://zuul.example.com/p
+
+[executor]
+git_dir=/tmp/zuul-test/executor-git
+
+[connection gerrit]
+driver=gerrit
+server=review.example.com
+user=jenkins
+sshkey=fake_id_rsa1
+
+[connection resultsdb]
+driver=sql
+dburi=mysql+pymysql://bad:creds@host/db
+
+[connection resultsdb_failures]
+driver=sql
+dburi=mysql+pymysql://bad:creds@host/db
diff --git a/tests/fixtures/zuul-sql-driver.conf b/tests/fixtures/zuul-sql-driver.conf
new file mode 100644
index 0000000..42ab1ba
--- /dev/null
+++ b/tests/fixtures/zuul-sql-driver.conf
@@ -0,0 +1,30 @@
+[gearman]
+server=127.0.0.1
+
+[zuul]
+tenant_config=main.yaml
+url_pattern=http://logs.example.com/{change.number}/{change.patchset}/{pipeline.name}/{job.name}/{build.number}
+job_name_in_report=true
+
+[merger]
+git_dir=/tmp/zuul-test/merger-git
+git_user_email=zuul@example.com
+git_user_name=zuul
+zuul_url=http://zuul.example.com/p
+
+[executor]
+git_dir=/tmp/zuul-test/executor-git
+
+[connection gerrit]
+driver=gerrit
+server=review.example.com
+user=jenkins
+sshkey=fake_id_rsa1
+
+[connection resultsdb]
+driver=sql
+dburi=$MYSQL_FIXTURE_DBURI$
+
+[connection resultsdb_failures]
+driver=sql
+dburi=$MYSQL_FIXTURE_DBURI$
diff --git a/tests/fixtures/zuul.conf b/tests/fixtures/zuul.conf
index 516ce08..ce29310 100644
--- a/tests/fixtures/zuul.conf
+++ b/tests/fixtures/zuul.conf
@@ -15,16 +15,6 @@
[executor]
git_dir=/tmp/zuul-test/executor-git
-[swift]
-authurl=https://identity.api.example.org/v2.0/
-user=username
-key=password
-tenant_name=" "
-
-default_container=logs
-region_name=EXP
-logserver_prefix=http://logs.example.org/server.app/
-
[connection gerrit]
driver=gerrit
server=review.example.com
diff --git a/tests/nodepool/test_nodepool_integration.py b/tests/nodepool/test_nodepool_integration.py
index 67968a3..2c9a9b3 100644
--- a/tests/nodepool/test_nodepool_integration.py
+++ b/tests/nodepool/test_nodepool_integration.py
@@ -12,7 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
-
+import socket
import time
from unittest import skip
@@ -30,9 +30,9 @@
def setUp(self):
super(BaseTestCase, self).setUp()
- self.zk_config = zuul.zk.ZooKeeperConnectionConfig('localhost')
self.zk = zuul.zk.ZooKeeper()
- self.zk.connect([self.zk_config])
+ self.zk.connect('localhost:2181')
+ self.hostname = socket.gethostname()
self.provisioned_requests = []
# This class implements the scheduler methods zuul.nodepool
diff --git a/tests/unit/test_connection.py b/tests/unit/test_connection.py
index 22cf331..ee9a0b0 100644
--- a/tests/unit/test_connection.py
+++ b/tests/unit/test_connection.py
@@ -13,7 +13,6 @@
# under the License.
import sqlalchemy as sa
-from unittest import skip
from tests.base import ZuulTestCase, ZuulDBTestCase
@@ -57,45 +56,44 @@
self.assertEqual(B.patchsets[-1]['approvals'][0]['by']['username'],
'civoter')
- def _test_sql_tables_created(self, metadata_table=None):
+
+class TestSQLConnection(ZuulDBTestCase):
+ config_file = 'zuul-sql-driver.conf'
+ tenant_config_file = 'config/sql-driver/main.yaml'
+
+ def test_sql_tables_created(self, metadata_table=None):
"Test the tables for storing results are created properly"
buildset_table = 'zuul_buildset'
build_table = 'zuul_build'
insp = sa.engine.reflection.Inspector(
- self.connections['resultsdb'].engine)
+ self.connections.connections['resultsdb'].engine)
self.assertEqual(9, len(insp.get_columns(buildset_table)))
self.assertEqual(10, len(insp.get_columns(build_table)))
- @skip("Disabled for early v3 development")
- def test_sql_tables_created(self):
- "Test the default table is created"
- self.config.set('zuul', 'layout_config',
- 'tests/fixtures/layout-sql-reporter.yaml')
- self.sched.reconfigure(self.config)
- self._test_sql_tables_created()
-
- def _test_sql_results(self):
+ def test_sql_results(self):
"Test results are entered into an sql table"
# Grab the sa tables
+ tenant = self.sched.abide.tenants.get('tenant-one')
reporter = _get_reporter_from_connection_name(
- self.sched.layout.pipelines['check'].success_actions,
+ tenant.layout.pipelines['check'].success_actions,
'resultsdb'
)
# Add a success result
- A = self.fake_review_gerrit.addFakeChange('org/project', 'master', 'A')
- self.fake_review_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
# Add a failed result for a negative score
- B = self.fake_review_gerrit.addFakeChange('org/project', 'master', 'B')
- self.worker.addFailTest('project-test1', B)
- self.fake_review_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
+
+ self.executor_server.failJob('project-test1', B)
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
- conn = self.connections['resultsdb'].engine.connect()
+ conn = self.connections.connections['resultsdb'].engine.connect()
result = conn.execute(
sa.sql.select([reporter.connection.zuul_buildset_table]))
@@ -122,7 +120,7 @@
# Check the first result, which should be the project-merge job
self.assertEqual('project-merge', buildset0_builds[0]['job_name'])
self.assertEqual("SUCCESS", buildset0_builds[0]['result'])
- self.assertEqual('http://logs.example.com/1/1/check/project-merge/0',
+ self.assertEqual('https://server/job/project-merge/0/',
buildset0_builds[0]['log_url'])
self.assertEqual('check', buildset1['pipeline'])
@@ -144,42 +142,33 @@
# which failed
self.assertEqual('project-test1', buildset1_builds[-2]['job_name'])
self.assertEqual("FAILURE", buildset1_builds[-2]['result'])
- self.assertEqual('http://logs.example.com/2/1/check/project-test1/4',
+ self.assertEqual('https://server/job/project-test1/0/',
buildset1_builds[-2]['log_url'])
- @skip("Disabled for early v3 development")
- def test_sql_results(self):
- "Test results are entered into the default sql table"
- self.config.set('zuul', 'layout_config',
- 'tests/fixtures/layout-sql-reporter.yaml')
- self.sched.reconfigure(self.config)
- self._test_sql_results()
-
- @skip("Disabled for early v3 development")
def test_multiple_sql_connections(self):
"Test putting results in different databases"
- self.config.set('zuul', 'layout_config',
- 'tests/fixtures/layout-sql-reporter.yaml')
- self.sched.reconfigure(self.config)
+ self.updateConfigLayout(
+ 'tests/fixtures/layout-sql-reporter.yaml')
# Add a successful result
- A = self.fake_review_gerrit.addFakeChange('org/project', 'master', 'A')
- self.fake_review_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
# Add a failed result
- B = self.fake_review_gerrit.addFakeChange('org/project', 'master', 'B')
- self.worker.addFailTest('project-test1', B)
- self.fake_review_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
+ self.executor_server.failJob('project-test1', B)
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
# Grab the sa tables for resultsdb
+ tenant = self.sched.abide.tenants.get('tenant-one')
reporter1 = _get_reporter_from_connection_name(
- self.sched.layout.pipelines['check'].success_actions,
+ tenant.layout.pipelines['check'].success_actions,
'resultsdb'
)
- conn = self.connections['resultsdb'].engine.connect()
+ conn = self.connections.connections['resultsdb'].engine.connect()
buildsets_resultsdb = conn.execute(sa.sql.select(
[reporter1.connection.zuul_buildset_table])).fetchall()
# Should have been 2 buildset reported to the resultsdb (both success
@@ -196,11 +185,12 @@
# Grab the sa tables for resultsdb_failures
reporter2 = _get_reporter_from_connection_name(
- self.sched.layout.pipelines['check'].failure_actions,
+ tenant.layout.pipelines['check'].failure_actions,
'resultsdb_failures'
)
- conn = self.connections['resultsdb_failures'].engine.connect()
+ conn = self.connections.connections['resultsdb_failures'].\
+ engine.connect()
buildsets_resultsdb_failures = conn.execute(sa.sql.select(
[reporter2.connection.zuul_buildset_table])).fetchall()
# The failure db should only have 1 buildset failed
@@ -217,10 +207,9 @@
class TestConnectionsBadSQL(ZuulDBTestCase):
- def setup_config(self, config_file='zuul-connections-bad-sql.conf'):
- super(TestConnectionsBadSQL, self).setup_config(config_file)
+ config_file = 'zuul-sql-driver-bad.conf'
+ tenant_config_file = 'config/sql-driver/main.yaml'
- @skip("Disabled for early v3 development")
def test_unable_to_connect(self):
"Test the SQL reporter fails gracefully when unable to connect"
self.config.set('zuul', 'layout_config',
@@ -229,8 +218,8 @@
# Trigger a reporter. If no errors are raised, the reporter has been
# disabled correctly
- A = self.fake_review_gerrit.addFakeChange('org/project', 'master', 'A')
- self.fake_review_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py
index 04d1473..ee7c6ab 100644
--- a/tests/unit/test_model.py
+++ b/tests/unit/test_model.py
@@ -225,7 +225,7 @@
self.assertFalse(python27diablo.changeMatches(change))
self.assertFalse(python27essex.changeMatches(change))
- item.freezeJobTree()
+ item.freezeJobGraph()
self.assertEqual(len(item.getJobs()), 1)
job = item.getJobs()[0]
self.assertEqual(job.name, 'python27')
@@ -253,7 +253,7 @@
self.assertTrue(python27diablo.changeMatches(change))
self.assertFalse(python27essex.changeMatches(change))
- item.freezeJobTree()
+ item.freezeJobGraph()
self.assertEqual(len(item.getJobs()), 1)
job = item.getJobs()[0]
self.assertEqual(job.name, 'python27')
@@ -282,7 +282,7 @@
self.assertFalse(python27diablo.changeMatches(change))
self.assertTrue(python27essex.changeMatches(change))
- item.freezeJobTree()
+ item.freezeJobGraph()
self.assertEqual(len(item.getJobs()), 1)
job = item.getJobs()[0]
self.assertEqual(job.name, 'python27')
@@ -439,7 +439,7 @@
self.assertTrue(python27.changeMatches(change))
self.assertFalse(python27diablo.changeMatches(change))
- item.freezeJobTree()
+ item.freezeJobGraph()
self.assertEqual(len(item.getJobs()), 1)
job = item.getJobs()[0]
self.assertEqual(job.name, 'python27')
@@ -453,7 +453,7 @@
self.assertTrue(python27.changeMatches(change))
self.assertTrue(python27diablo.changeMatches(change))
- item.freezeJobTree()
+ item.freezeJobGraph()
self.assertEqual(len(item.getJobs()), 1)
job = item.getJobs()[0]
self.assertEqual(job.name, 'python27')
@@ -506,7 +506,7 @@
self.assertTrue(base.changeMatches(change))
self.assertFalse(python27.changeMatches(change))
- item.freezeJobTree()
+ item.freezeJobGraph()
self.assertEqual([], item.getJobs())
def test_job_source_project(self):
@@ -609,3 +609,56 @@
for x in range(10):
self.db.update('job-name', 100, 'SUCCESS')
self.assertEqual(self.db.getEstimatedTime('job-name'), 100)
+
+
+class TestGraph(BaseTestCase):
+ def test_job_graph_disallows_multiple_jobs_with_same_name(self):
+ graph = model.JobGraph()
+ job1 = model.Job('job')
+ job2 = model.Job('job')
+ graph.addJob(job1)
+ with testtools.ExpectedException(Exception,
+ "Job job already added"):
+ graph.addJob(job2)
+
+ def test_job_graph_disallows_circular_dependencies(self):
+ graph = model.JobGraph()
+ jobs = [model.Job('job%d' % i) for i in range(0, 10)]
+ prevjob = None
+ for j in jobs[:3]:
+ if prevjob:
+ j.dependencies = frozenset([prevjob.name])
+ graph.addJob(j)
+ prevjob = j
+ # 0 triggers 1 triggers 2 triggers 3...
+
+ # Cannot depend on itself
+ with testtools.ExpectedException(
+ Exception,
+ "Dependency cycle detected in job jobX"):
+ j = model.Job('jobX')
+ j.dependencies = frozenset([j.name])
+ graph.addJob(j)
+
+ # Disallow circular dependencies
+ with testtools.ExpectedException(
+ Exception,
+ "Dependency cycle detected in job job3"):
+ jobs[4].dependencies = frozenset([jobs[3].name])
+ graph.addJob(jobs[4])
+ jobs[3].dependencies = frozenset([jobs[4].name])
+ graph.addJob(jobs[3])
+
+ jobs[5].dependencies = frozenset([jobs[4].name])
+ graph.addJob(jobs[5])
+
+ with testtools.ExpectedException(
+ Exception,
+ "Dependency cycle detected in job job3"):
+ jobs[3].dependencies = frozenset([jobs[5].name])
+ graph.addJob(jobs[3])
+
+ jobs[3].dependencies = frozenset([jobs[2].name])
+ graph.addJob(jobs[3])
+ jobs[6].dependencies = frozenset([jobs[2].name])
+ graph.addJob(jobs[6])
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index e32e41b..ec88737 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -2180,14 +2180,6 @@
self.assertEqual('https://server/job/project-test2/0/',
status_jobs[2]['report_url'])
- @skip("Disabled for early v3 development")
- def test_merging_queues(self):
- "Test that transitively-connected change queues are merged"
- self.updateConfigLayout(
- 'tests/fixtures/layout-merge-queues.yaml')
- self.sched.reconfigure(self.config)
- self.assertEqual(len(self.sched.layout.pipelines['gate'].queues), 1)
-
def test_mutex(self):
"Test job mutexes"
self.updateConfigLayout('layout-mutex')
@@ -4478,6 +4470,117 @@
self.assertIn('project-test2 : SKIPPED', A.messages[1])
+class TestDependencyGraph(ZuulTestCase):
+ tenant_config_file = 'config/dependency-graph/main.yaml'
+
+ def test_dependeny_graph_dispatch_jobs_once(self):
+ "Test a job in a dependency graph is queued only once"
+ # Job dependencies, starting with A
+ # A
+ # / \
+ # B C
+ # / \ / \
+ # D F E
+ # |
+ # G
+
+ self.executor_server.hold_jobs_in_build = True
+ change = self.fake_gerrit.addFakeChange(
+ 'org/project', 'master', 'change')
+ change.addApproval('code-review', 2)
+ self.fake_gerrit.addEvent(change.addApproval('approved', 1))
+
+ self.waitUntilSettled()
+ self.assertEqual([b.name for b in self.builds], ['A'])
+
+ self.executor_server.release('A')
+ self.waitUntilSettled()
+ self.assertEqual(sorted(b.name for b in self.builds), ['B', 'C'])
+
+ self.executor_server.release('B')
+ self.waitUntilSettled()
+ self.assertEqual(sorted(b.name for b in self.builds), ['C', 'D'])
+
+ self.executor_server.release('D')
+ self.waitUntilSettled()
+ self.assertEqual([b.name for b in self.builds], ['C'])
+
+ self.executor_server.release('C')
+ self.waitUntilSettled()
+ self.assertEqual(sorted(b.name for b in self.builds), ['E', 'F'])
+
+ self.executor_server.release('F')
+ self.waitUntilSettled()
+ self.assertEqual(sorted(b.name for b in self.builds), ['E', 'G'])
+
+ self.executor_server.release('G')
+ self.waitUntilSettled()
+ self.assertEqual([b.name for b in self.builds], ['E'])
+
+ self.executor_server.release('E')
+ self.waitUntilSettled()
+ self.assertEqual(len(self.builds), 0)
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(len(self.builds), 0)
+ self.assertEqual(len(self.history), 7)
+
+ self.assertEqual(change.data['status'], 'MERGED')
+ self.assertEqual(change.reported, 2)
+
+ def test_jobs_launched_only_if_all_dependencies_are_successful(self):
+ "Test that a job waits till all dependencies are successful"
+ # Job dependencies, starting with A
+ # A
+ # / \
+ # B C*
+ # / \ / \
+ # D F E
+ # |
+ # G
+
+ self.executor_server.hold_jobs_in_build = True
+ change = self.fake_gerrit.addFakeChange(
+ 'org/project', 'master', 'change')
+ change.addApproval('code-review', 2)
+
+ self.executor_server.failJob('C', change)
+
+ self.fake_gerrit.addEvent(change.addApproval('approved', 1))
+
+ self.waitUntilSettled()
+ self.assertEqual([b.name for b in self.builds], ['A'])
+
+ self.executor_server.release('A')
+ self.waitUntilSettled()
+ self.assertEqual(sorted(b.name for b in self.builds), ['B', 'C'])
+
+ self.executor_server.release('B')
+ self.waitUntilSettled()
+ self.assertEqual(sorted(b.name for b in self.builds), ['C', 'D'])
+
+ self.executor_server.release('D')
+ self.waitUntilSettled()
+ self.assertEqual([b.name for b in self.builds], ['C'])
+
+ self.executor_server.release('C')
+ self.waitUntilSettled()
+ self.assertEqual(len(self.builds), 0)
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(len(self.builds), 0)
+ self.assertEqual(len(self.history), 4)
+
+ self.assertEqual(change.data['status'], 'NEW')
+ self.assertEqual(change.reported, 2)
+
+
class TestDuplicatePipeline(ZuulTestCase):
tenant_config_file = 'config/duplicate-pipeline/main.yaml'
diff --git a/zuul/ansible/callback/zuul_stream.py b/zuul/ansible/callback/zuul_stream.py
index 9b8bccd..e6b3461 100644
--- a/zuul/ansible/callback/zuul_stream.py
+++ b/zuul/ansible/callback/zuul_stream.py
@@ -83,7 +83,16 @@
self._print_task_banner(task)
if task.action == 'command':
play_vars = self._play._variable_manager._hostvars
- for host in self._play.hosts:
+
+ hosts = self._play.hosts
+ if 'all' in hosts:
+ # NOTE(jamielennox): play.hosts is purely the list of hosts
+ # that was provided not interpretted by inventory. We don't
+ # have inventory access here but we can assume that 'all' is
+ # everything in hostvars.
+ hosts = play_vars.keys()
+
+ for host in hosts:
ip = play_vars[host]['ansible_host']
daemon_stamp = self._daemon_stamp % host
if not os.path.exists(daemon_stamp):
diff --git a/zuul/configloader.py b/zuul/configloader.py
index 3f72212..8bae3c5 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -193,6 +193,7 @@
'roles': to_list(role),
'repos': to_list(str),
'vars': dict,
+ 'dependencies': to_list(str),
}
return vs.Schema(job)
@@ -276,6 +277,8 @@
# accumulate onto any previously applied tags.
job.tags = job.tags.union(set(tags))
+ job.dependencies = frozenset(as_list(conf.get('dependencies')))
+
roles = []
for role in conf.get('roles', []):
if 'zuul' in role:
@@ -364,45 +367,33 @@
project_pipeline = model.ProjectPipelineConfig()
project_template.pipelines[pipeline.name] = project_pipeline
project_pipeline.queue_name = conf_pipeline.get('queue')
- project_pipeline.job_tree = ProjectTemplateParser._parseJobTree(
+ ProjectTemplateParser._parseJobList(
tenant, layout, conf_pipeline.get('jobs', []),
- source_context, start_mark)
+ source_context, start_mark, project_pipeline.job_list)
return project_template
@staticmethod
- def _parseJobTree(tenant, layout, conf, source_context,
- start_mark, tree=None):
- if not tree:
- tree = model.JobTree(None)
+ def _parseJobList(tenant, layout, conf, source_context,
+ start_mark, job_list):
for conf_job in conf:
if isinstance(conf_job, six.string_types):
job = model.Job(conf_job)
- tree.addJob(job)
+ job_list.addJob(job)
elif isinstance(conf_job, dict):
- # A dictionary in a job tree may override params, or
- # be the root of a sub job tree, or both.
+ # A dictionary in a job tree may override params
jobname, attrs = conf_job.items()[0]
- jobs = attrs.pop('jobs', None)
if attrs:
# We are overriding params, so make a new job def
attrs['name'] = jobname
attrs['_source_context'] = source_context
attrs['_start_mark'] = start_mark
- subtree = tree.addJob(JobParser.fromYaml(
- tenant, layout, attrs))
+ job_list.addJob(JobParser.fromYaml(tenant, layout, attrs))
else:
# Not overriding, so add a blank job
job = model.Job(jobname)
- subtree = tree.addJob(job)
-
- if jobs:
- # This is the root of a sub tree
- ProjectTemplateParser._parseJobTree(
- tenant, layout, jobs, source_context,
- start_mark, subtree)
+ job_list.addJob(job)
else:
raise Exception("Job must be a string or dictionary")
- return tree
class ProjectParser(object):
@@ -455,7 +446,6 @@
project.merge_mode = model.MERGER_MAP['merge-resolve']
for pipeline in layout.pipelines.values():
project_pipeline = model.ProjectPipelineConfig()
- project_pipeline.job_tree = model.JobTree(None)
queue_name = None
# For every template, iterate over the job tree and replace or
# create the jobs in the final definition as needed.
@@ -467,8 +457,8 @@
(template.name, pipeline.name))
pipeline_defined = True
template_pipeline = template.pipelines[pipeline.name]
- project_pipeline.job_tree.inheritFrom(
- template_pipeline.job_tree)
+ project_pipeline.job_list.inheritFrom(
+ template_pipeline.job_list)
if template_pipeline.queue_name:
queue_name = template_pipeline.queue_name
if queue_name:
diff --git a/zuul/driver/sql/__init__.py b/zuul/driver/sql/__init__.py
new file mode 100644
index 0000000..a5f8923
--- /dev/null
+++ b/zuul/driver/sql/__init__.py
@@ -0,0 +1,33 @@
+# Copyright 2016 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from zuul.driver import Driver, ConnectionInterface, ReporterInterface
+import sqlconnection
+import sqlreporter
+
+
+class SQLDriver(Driver, ConnectionInterface, ReporterInterface):
+ name = 'sql'
+
+ def registerScheduler(self, scheduler):
+ self.sched = scheduler
+
+ def getConnection(self, name, config):
+ return sqlconnection.SQLConnection(self, name, config)
+
+ def getReporter(self, connection, config=None):
+ return sqlreporter.SQLReporter(self, connection, config)
+
+ def getReporterSchema(self):
+ return sqlreporter.getSchema()
diff --git a/zuul/alembic_reporter.ini b/zuul/driver/sql/alembic_reporter.ini
similarity index 97%
rename from zuul/alembic_reporter.ini
rename to zuul/driver/sql/alembic_reporter.ini
index b7f787c..0c59505 100644
--- a/zuul/alembic_reporter.ini
+++ b/zuul/driver/sql/alembic_reporter.ini
@@ -4,7 +4,7 @@
# path to migration scripts
# NOTE(jhesketh): We may use alembic for other db components of zuul in the
# future. Use a sub-folder for the reporters own versions.
-script_location = alembic/sql_reporter
+script_location = alembic_reporter
# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s
diff --git a/zuul/alembic/sql_reporter/README b/zuul/driver/sql/alembic_reporter/README
similarity index 100%
rename from zuul/alembic/sql_reporter/README
rename to zuul/driver/sql/alembic_reporter/README
diff --git a/zuul/alembic/sql_reporter/env.py b/zuul/driver/sql/alembic_reporter/env.py
similarity index 100%
rename from zuul/alembic/sql_reporter/env.py
rename to zuul/driver/sql/alembic_reporter/env.py
diff --git a/zuul/alembic/sql_reporter/script.py.mako b/zuul/driver/sql/alembic_reporter/script.py.mako
similarity index 100%
rename from zuul/alembic/sql_reporter/script.py.mako
rename to zuul/driver/sql/alembic_reporter/script.py.mako
diff --git a/zuul/alembic/sql_reporter/versions/4d3ebd7f06b9_set_up_initial_reporter_tables.py b/zuul/driver/sql/alembic_reporter/versions/4d3ebd7f06b9_set_up_initial_reporter_tables.py
similarity index 100%
rename from zuul/alembic/sql_reporter/versions/4d3ebd7f06b9_set_up_initial_reporter_tables.py
rename to zuul/driver/sql/alembic_reporter/versions/4d3ebd7f06b9_set_up_initial_reporter_tables.py
diff --git a/zuul/connection/sql.py b/zuul/driver/sql/sqlconnection.py
similarity index 92%
rename from zuul/connection/sql.py
rename to zuul/driver/sql/sqlconnection.py
index 479ee44..69e53df 100644
--- a/zuul/connection/sql.py
+++ b/zuul/driver/sql/sqlconnection.py
@@ -29,9 +29,10 @@
driver_name = 'sql'
log = logging.getLogger("connection.sql")
- def __init__(self, connection_name, connection_config):
+ def __init__(self, driver, connection_name, connection_config):
- super(SQLConnection, self).__init__(connection_name, connection_config)
+ super(SQLConnection, self).__init__(driver, connection_name,
+ connection_config)
self.dburi = None
self.engine = None
@@ -61,7 +62,7 @@
config = alembic.config.Config()
config.set_main_option("script_location",
- "zuul:alembic/sql_reporter")
+ "zuul:driver/sql/alembic_reporter")
config.set_main_option("sqlalchemy.url",
self.connection_config.get('dburi'))
diff --git a/zuul/reporter/sql.py b/zuul/driver/sql/sqlreporter.py
similarity index 85%
rename from zuul/reporter/sql.py
rename to zuul/driver/sql/sqlreporter.py
index b663a59..2129f53 100644
--- a/zuul/reporter/sql.py
+++ b/zuul/driver/sql/sqlreporter.py
@@ -25,10 +25,10 @@
name = 'sql'
log = logging.getLogger("zuul.reporter.mysql.SQLReporter")
- def __init__(self, reporter_config={}, sched=None, connection=None):
+ def __init__(self, driver, connection, config={}):
super(SQLReporter, self).__init__(
- reporter_config, sched, connection)
- self.result_score = reporter_config.get('score', None)
+ driver, connection, config)
+ self.result_score = config.get('score', None)
def report(self, source, pipeline, item):
"""Create an entry into a database."""
@@ -37,13 +37,12 @@
self.log.warn("SQL reporter (%s) is disabled " % self)
return
- if self.sched.config.has_option('zuul', 'url_pattern'):
- url_pattern = self.sched.config.get('zuul', 'url_pattern')
+ if self.driver.sched.config.has_option('zuul', 'url_pattern'):
+ url_pattern = self.driver.sched.config.get('zuul', 'url_pattern')
else:
url_pattern = None
- score = self.reporter_config['score']\
- if 'score' in self.reporter_config else 0
+ score = self.config.get('score', 0)
with self.connection.engine.begin() as conn:
buildset_ins = self.connection.zuul_buildset_table.insert().values(
@@ -60,7 +59,7 @@
buildset_ins_result = conn.execute(buildset_ins)
build_inserts = []
- for job in pipeline.getJobs(item):
+ for job in item.getJobs():
build = item.current_build_set.getBuild(job.name)
if not build:
# build hasn't began. The sql reporter can only send back
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index 5c1dec8..cae0f6a 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -850,6 +850,7 @@
if self.jobdir.roles_path:
config.write('roles_path = %s\n' %
':'.join(self.jobdir.roles_path))
+ config.write('command_warnings = False\n')
config.write('callback_plugins = %s\n'
% self.executor_server.callback_dir)
config.write('stdout_callback = zuul_stream\n')
diff --git a/zuul/lib/connections.py b/zuul/lib/connections.py
index 27d8a1b..9964ba9 100644
--- a/zuul/lib/connections.py
+++ b/zuul/lib/connections.py
@@ -20,6 +20,7 @@
import zuul.driver.git
import zuul.driver.smtp
import zuul.driver.timer
+import zuul.driver.sql
from zuul.connection import BaseConnection
@@ -41,6 +42,7 @@
self.registerDriver(zuul.driver.git.GitDriver())
self.registerDriver(zuul.driver.smtp.SMTPDriver())
self.registerDriver(zuul.driver.timer.TimerDriver())
+ self.registerDriver(zuul.driver.sql.SQLDriver())
def registerDriver(self, driver):
if driver.name in self.drivers:
diff --git a/zuul/manager/__init__.py b/zuul/manager/__init__.py
index e7508b6..58ad607 100644
--- a/zuul/manager/__init__.py
+++ b/zuul/manager/__init__.py
@@ -64,31 +64,29 @@
self.log.info(" %s" % e)
self.log.info(" Projects:")
- def log_jobs(tree, indent=0):
- istr = ' ' + ' ' * indent
- if tree.job:
- # TODOv3(jeblair): represent matchers
- efilters = ''
- # for b in tree.job._branches:
- # efilters += str(b)
- # for f in tree.job._files:
- # efilters += str(f)
- # if tree.job.skip_if_matcher:
- # efilters += str(tree.job.skip_if_matcher)
- # if efilters:
- # efilters = ' ' + efilters
- tags = []
- if tree.job.hold_following_changes:
- tags.append('[hold]')
- if not tree.job.voting:
- tags.append('[nonvoting]')
- if tree.job.mutex:
- tags.append('[mutex: %s]' % tree.job.mutex)
- tags = ' '.join(tags)
- self.log.info("%s%s%s %s" % (istr, repr(tree.job),
- efilters, tags))
- for x in tree.job_trees:
- log_jobs(x, indent + 2)
+ def log_jobs(job_list):
+ for job_name, job_variants in job_list.jobs.items():
+ for variant in job_variants:
+ # TODOv3(jeblair): represent matchers
+ efilters = ''
+ # for b in tree.job._branches:
+ # efilters += str(b)
+ # for f in tree.job._files:
+ # efilters += str(f)
+ # if tree.job.skip_if_matcher:
+ # efilters += str(tree.job.skip_if_matcher)
+ # if efilters:
+ # efilters = ' ' + efilters
+ tags = []
+ if variant.hold_following_changes:
+ tags.append('[hold]')
+ if not variant.voting:
+ tags.append('[nonvoting]')
+ if variant.mutex:
+ tags.append('[mutex: %s]' % variant.mutex)
+ tags = ' '.join(tags)
+ self.log.info(" %s%s %s" % (repr(variant),
+ efilters, tags))
for project_name in layout.project_configs.keys():
project_config = layout.project_configs.get(project_name)
@@ -97,7 +95,7 @@
self.pipeline.name)
if project_pipeline_config:
self.log.info(" %s" % project_name)
- log_jobs(project_pipeline_config.job_tree)
+ log_jobs(project_pipeline_config.job_list)
self.log.info(" On start:")
self.log.info(" %s" % self.pipeline.start_actions)
self.log.info(" On success:")
@@ -257,7 +255,7 @@
# Rebuild the frozen job tree from the new layout, if
# we have one. If not, it will be built later.
if item.current_build_set.layout:
- item.freezeJobTree()
+ item.freezeJobGraph()
# Re-set build results in case any new jobs have been
# added to the tree.
@@ -540,8 +538,18 @@
item.current_build_set.layout = self.getLayout(item)
if not item.current_build_set.layout:
return False
- if not item.job_tree:
- item.freezeJobTree()
+ if item.current_build_set.config_error:
+ return False
+ if not item.job_graph:
+ try:
+ item.freezeJobGraph()
+ except Exception as e:
+ # TODOv3(jeblair): nicify this exception as it will be reported
+ self.log.exception("Error freezing job graph for %s" %
+ item)
+ item.setConfigError("Unable to freeze job graph: %s" %
+ (str(e)))
+ return False
return True
def _processOneItem(self, item, nnfi):
diff --git a/zuul/model.py b/zuul/model.py
index b57eef8..3676b68 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -675,6 +675,7 @@
file_matcher=None,
irrelevant_file_matcher=None, # skip-if
tags=frozenset(),
+ dependencies=frozenset(),
)
# These attributes affect how the job is actually run and more
@@ -851,60 +852,100 @@
return True
-class JobTree(object):
- """A JobTree holds one or more Jobs to represent Job dependencies.
+class JobList(object):
+ """ A list of jobs in a project's pipeline. """
- If Job foo should only execute if Job bar succeeds, then there will
- be a JobTree for foo, which will contain a JobTree for bar. A JobTree
- can hold more than one dependent JobTrees, such that jobs bar and bang
- both depend on job foo being successful.
-
- A root node of a JobTree will have no associated Job."""
-
- def __init__(self, job):
- self.job = job
- self.job_trees = []
-
- def __repr__(self):
- return '<JobTree %s %s>' % (self.job, self.job_trees)
+ def __init__(self):
+ self.jobs = OrderedDict() # job.name -> [job, ...]
def addJob(self, job):
- if job not in [x.job for x in self.job_trees]:
- t = JobTree(job)
- self.job_trees.append(t)
- return t
- for tree in self.job_trees:
- if tree.job == job:
- return tree
-
- def getJobs(self):
- jobs = []
- for x in self.job_trees:
- jobs.append(x.job)
- jobs.extend(x.getJobs())
- return jobs
-
- def getJobTreeForJob(self, job):
- if self.job == job:
- return self
- for tree in self.job_trees:
- ret = tree.getJobTreeForJob(job)
- if ret:
- return ret
- return None
+ if job.name in self.jobs:
+ self.jobs[job.name].append(job)
+ else:
+ self.jobs[job.name] = [job]
def inheritFrom(self, other):
- if other.job:
- if not self.job:
- self.job = other.job.copy()
+ for jobname, jobs in other.jobs.items():
+ if jobname in self.jobs:
+ self.jobs[jobname].append(jobs)
else:
- self.job.applyVariant(other.job)
- for other_tree in other.job_trees:
- this_tree = self.getJobTreeForJob(other_tree.job)
- if not this_tree:
- this_tree = JobTree(None)
- self.job_trees.append(this_tree)
- this_tree.inheritFrom(other_tree)
+ self.jobs[jobname] = jobs
+
+
+class JobGraph(object):
+ """ A JobGraph represents the dependency graph between Job."""
+
+ def __init__(self):
+ self.jobs = OrderedDict() # job_name -> Job
+ self._dependencies = {} # dependent_job_name -> set(parent_job_names)
+
+ def __repr__(self):
+ return '<JobGraph %s>' % (self.jobs)
+
+ def addJob(self, job):
+ # A graph must be created after the job list is frozen,
+ # therefore we should only get one job with the same name.
+ if job.name in self.jobs:
+ raise Exception("Job %s already added" % (job.name,))
+ self.jobs[job.name] = job
+ # Append the dependency information
+ self._dependencies.setdefault(job.name, set())
+ try:
+ for dependency in job.dependencies:
+ # Make sure a circular dependency is never created
+ ancestor_jobs = self._getParentJobNamesRecursively(
+ dependency, soft=True)
+ ancestor_jobs.add(dependency)
+ if any((job.name == anc_job) for anc_job in ancestor_jobs):
+ raise Exception("Dependency cycle detected in job %s" %
+ (job.name,))
+ self._dependencies[job.name].add(dependency)
+ except Exception:
+ del self.jobs[job.name]
+ del self._dependencies[job.name]
+ raise
+
+ def getJobs(self):
+ return self.jobs.values() # Report in the order of the layout config
+
+ def _getDirectDependentJobs(self, parent_job):
+ ret = set()
+ for dependent_name, parent_names in self._dependencies.items():
+ if parent_job in parent_names:
+ ret.add(dependent_name)
+ return ret
+
+ def getDependentJobsRecursively(self, parent_job):
+ all_dependent_jobs = set()
+ jobs_to_iterate = set([parent_job])
+ while len(jobs_to_iterate) > 0:
+ current_job = jobs_to_iterate.pop()
+ current_dependent_jobs = self._getDirectDependentJobs(current_job)
+ new_dependent_jobs = current_dependent_jobs - all_dependent_jobs
+ jobs_to_iterate |= new_dependent_jobs
+ all_dependent_jobs |= new_dependent_jobs
+ return [self.jobs[name] for name in all_dependent_jobs]
+
+ def getParentJobsRecursively(self, dependent_job):
+ return [self.jobs[name] for name in
+ self._getParentJobNamesRecursively(dependent_job)]
+
+ def _getParentJobNamesRecursively(self, dependent_job, soft=False):
+ all_parent_jobs = set()
+ jobs_to_iterate = set([dependent_job])
+ while len(jobs_to_iterate) > 0:
+ current_job = jobs_to_iterate.pop()
+ current_parent_jobs = self._dependencies.get(current_job)
+ if current_parent_jobs is None:
+ if soft:
+ current_parent_jobs = set()
+ else:
+ raise Exception("Dependent job %s not found: " %
+ (dependent_job,))
+ new_parent_jobs = current_parent_jobs - all_parent_jobs
+ jobs_to_iterate |= new_parent_jobs
+ all_parent_jobs |= new_parent_jobs
+ return all_parent_jobs
class Build(object):
@@ -1137,7 +1178,7 @@
self.active = False # Whether an item is within an active window
self.live = True # Whether an item is intended to be processed at all
self.layout = None # This item's shadow layout
- self.job_tree = None
+ self.job_graph = None
def __repr__(self):
if self.pipeline:
@@ -1165,23 +1206,33 @@
def setReportedResult(self, result):
self.current_build_set.result = result
- def freezeJobTree(self):
+ def freezeJobGraph(self):
"""Find or create actual matching jobs for this item's change and
store the resulting job tree."""
layout = self.current_build_set.layout
- self.job_tree = layout.createJobTree(self)
+ job_graph = layout.createJobGraph(self)
+ for job in job_graph.getJobs():
+ # Ensure that each jobs's dependencies are fully
+ # accessible. This will raise an exception if not.
+ job_graph.getParentJobsRecursively(job.name)
+ self.job_graph = job_graph
- def hasJobTree(self):
- """Returns True if the item has a job tree."""
- return self.job_tree is not None
+ def hasJobGraph(self):
+ """Returns True if the item has a job graph."""
+ return self.job_graph is not None
def getJobs(self):
- if not self.live or not self.job_tree:
+ if not self.live or not self.job_graph:
return []
- return self.job_tree.getJobs()
+ return self.job_graph.getJobs()
+
+ def getJob(self, name):
+ if not self.job_graph:
+ return None
+ return self.job_graph.jobs.get(name)
def haveAllJobsStarted(self):
- if not self.hasJobTree():
+ if not self.hasJobGraph():
return False
for job in self.getJobs():
build = self.current_build_set.getBuild(job.name)
@@ -1193,7 +1244,7 @@
if (self.current_build_set.config_error or
self.current_build_set.unable_to_merge):
return True
- if not self.hasJobTree():
+ if not self.hasJobGraph():
return False
for job in self.getJobs():
build = self.current_build_set.getBuild(job.name)
@@ -1202,7 +1253,7 @@
return True
def didAllJobsSucceed(self):
- if not self.hasJobTree():
+ if not self.hasJobGraph():
return False
for job in self.getJobs():
if not job.voting:
@@ -1215,7 +1266,7 @@
return True
def didAnyJobFail(self):
- if not self.hasJobTree():
+ if not self.hasJobGraph():
return False
for job in self.getJobs():
if not job.voting:
@@ -1234,7 +1285,7 @@
def isHoldingFollowingChanges(self):
if not self.live:
return False
- if not self.hasJobTree():
+ if not self.hasJobGraph():
return False
for job in self.getJobs():
if not job.hold_following_changes:
@@ -1249,88 +1300,96 @@
return False
return self.item_ahead.isHoldingFollowingChanges()
- def _findJobsToRun(self, job_trees, mutex):
+ def findJobsToRun(self, mutex):
torun = []
+ if not self.live:
+ return []
+ if not self.job_graph:
+ return []
if self.item_ahead:
# Only run jobs if any 'hold' jobs on the change ahead
# have completed successfully.
if self.item_ahead.isHoldingFollowingChanges():
return []
- for tree in job_trees:
- job = tree.job
- result = None
- if job:
- if not job.changeMatches(self.change):
+
+ successful_job_names = set()
+ jobs_not_started = set()
+ for job in self.job_graph.getJobs():
+ build = self.current_build_set.getBuild(job.name)
+ if build:
+ if build.result == 'SUCCESS':
+ successful_job_names.add(job.name)
+ else:
+ jobs_not_started.add(job)
+
+ # Attempt to request nodes for jobs in the order jobs appear
+ # in configuration.
+ for job in self.job_graph.getJobs():
+ if job not in jobs_not_started:
+ continue
+ all_parent_jobs_successful = True
+ for parent_job in self.job_graph.getParentJobsRecursively(
+ job.name):
+ if parent_job.name not in successful_job_names:
+ all_parent_jobs_successful = False
+ break
+ if all_parent_jobs_successful:
+ nodeset = self.current_build_set.getJobNodeSet(job.name)
+ if nodeset is None:
+ # The nodes for this job are not ready, skip
+ # it for now.
continue
- build = self.current_build_set.getBuild(job.name)
- if build:
- result = build.result
- else:
- # There is no build for the root of this job tree,
- # so it has not run yet.
- nodeset = self.current_build_set.getJobNodeSet(job.name)
- if nodeset is None:
- # The nodes for this job are not ready, skip
- # it for now.
- continue
- if mutex.acquire(self, job):
- # If this job needs a mutex, either acquire it or make
- # sure that we have it before running the job.
- torun.append(job)
- # If there is no job, this is a null job tree, and we should
- # run all of its jobs.
- if result == 'SUCCESS' or not job:
- torun.extend(self._findJobsToRun(tree.job_trees, mutex))
+ if mutex.acquire(self, job):
+ # If this job needs a mutex, either acquire it or make
+ # sure that we have it before running the job.
+ torun.append(job)
return torun
- def findJobsToRun(self, mutex):
- if not self.live:
- return []
- tree = self.job_tree
- if not tree:
- return []
- return self._findJobsToRun(tree.job_trees, mutex)
-
- def _findJobsToRequest(self, job_trees):
+ def findJobsToRequest(self):
build_set = self.current_build_set
toreq = []
+ if not self.live:
+ return []
+ if not self.job_graph:
+ return []
if self.item_ahead:
if self.item_ahead.isHoldingFollowingChanges():
return []
- for tree in job_trees:
- job = tree.job
- result = None
- if job:
- if not job.changeMatches(self.change):
- continue
- build = build_set.getBuild(job.name)
- if build:
- result = build.result
- else:
- nodeset = build_set.getJobNodeSet(job.name)
- if nodeset is None:
- req = build_set.getJobNodeRequest(job.name)
- if req is None:
- toreq.append(job)
- if result == 'SUCCESS' or not job:
- toreq.extend(self._findJobsToRequest(tree.job_trees))
- return toreq
- def findJobsToRequest(self):
- if not self.live:
- return []
- tree = self.job_tree
- if not tree:
- return []
- return self._findJobsToRequest(tree.job_trees)
+ successful_job_names = set()
+ jobs_not_requested = set()
+ for job in self.job_graph.getJobs():
+ build = build_set.getBuild(job.name)
+ if build and build.result == 'SUCCESS':
+ successful_job_names.add(job.name)
+ else:
+ nodeset = build_set.getJobNodeSet(job.name)
+ if nodeset is None:
+ req = build_set.getJobNodeRequest(job.name)
+ if req is None:
+ jobs_not_requested.add(job)
+
+ # Attempt to request nodes for jobs in the order jobs appear
+ # in configuration.
+ for job in self.job_graph.getJobs():
+ if job not in jobs_not_requested:
+ continue
+ all_parent_jobs_successful = True
+ for parent_job in self.job_graph.getParentJobsRecursively(
+ job.name):
+ if parent_job.name not in successful_job_names:
+ all_parent_jobs_successful = False
+ break
+ if all_parent_jobs_successful:
+ toreq.append(job)
+ return toreq
def setResult(self, build):
if build.retry:
self.removeBuild(build)
elif build.result != 'SUCCESS':
- # Get a JobTree from a Job so we can find only its dependent jobs
- tree = self.job_tree.getJobTreeForJob(build.job)
- for job in tree.getJobs():
+ for job in self.job_graph.getDependentJobsRecursively(
+ build.job.name):
fakebuild = Build(job, None)
fakebuild.result = 'SKIPPED'
self.addBuild(fakebuild)
@@ -2014,7 +2073,7 @@
class ProjectPipelineConfig(object):
# Represents a project cofiguration in the context of a pipeline
def __init__(self):
- self.job_tree = None
+ self.job_list = JobList()
self.queue_name = None
self.merge_mode = None
@@ -2182,14 +2241,13 @@
def addProjectConfig(self, project_config):
self.project_configs[project_config.name] = project_config
- def _createJobTree(self, change, job_trees, parent):
- for tree in job_trees:
- job = tree.job
- if not job.changeMatches(change):
- continue
+ def _createJobGraph(self, change, job_list, job_graph):
+ for jobname in job_list.jobs:
+ # This is the final job we are constructing
frozen_job = None
+ # Whether the change matches any globally defined variant
matched = False
- for variant in self.getJobs(job.name):
+ for variant in self.getJobs(jobname):
if variant.changeMatches(change):
if frozen_job is None:
frozen_job = variant.copy()
@@ -2203,25 +2261,33 @@
# the job that is defined in the tree).
continue
# If the job does not allow auth inheritance, do not allow
- # the project-pipeline variant to update its execution
+ # the project-pipeline variants to update its execution
# attributes.
if frozen_job.auth and not frozen_job.auth.get('inherit'):
frozen_job.final = True
- frozen_job.applyVariant(job)
- frozen_tree = JobTree(frozen_job)
- parent.job_trees.append(frozen_tree)
- self._createJobTree(change, tree.job_trees, frozen_tree)
+ # Whether the change matches any of the project pipeline
+ # variants
+ matched = False
+ for variant in job_list.jobs[jobname]:
+ if variant.changeMatches(change):
+ frozen_job.applyVariant(variant)
+ matched = True
+ if not matched:
+ # A change must match at least one project pipeline
+ # job variant.
+ continue
+ job_graph.addJob(frozen_job)
- def createJobTree(self, item):
+ def createJobGraph(self, item):
project_config = self.project_configs.get(
item.change.project.name, None)
- ret = JobTree(None)
+ ret = JobGraph()
# NOTE(pabelanger): It is possible for a foreign project not to have a
- # configured pipeline, if so return an empty JobTree.
+ # configured pipeline, if so return an empty JobGraph.
if project_config and item.pipeline.name in project_config.pipelines:
- project_tree = \
- project_config.pipelines[item.pipeline.name].job_tree
- self._createJobTree(item.change, project_tree.job_trees, ret)
+ project_job_list = \
+ project_config.pipelines[item.pipeline.name].job_list
+ self._createJobGraph(item.change, project_job_list, ret)
return ret
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index 131378e..7fb1568 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -555,11 +555,10 @@
project_name)
if new_pipeline.manager.reEnqueueItem(item,
last_head):
- new_jobs = item.getJobs()
for build in item.current_build_set.getBuilds():
- jobtree = item.job_tree.getJobTreeForJob(build.job)
- if jobtree and jobtree.job in new_jobs:
- build.job = jobtree.job
+ new_job = item.getJob(build.job.name)
+ if new_job:
+ build.job = new_job
else:
item.removeBuild(build)
builds_to_cancel.append(build)
diff --git a/zuul/zk.py b/zuul/zk.py
index 2009945..5cd7bee 100644
--- a/zuul/zk.py
+++ b/zuul/zk.py
@@ -34,36 +34,6 @@
pass
-class ZooKeeperConnectionConfig(object):
- '''
- Represents the connection parameters for a ZooKeeper server.
- '''
-
- def __eq__(self, other):
- if isinstance(other, ZooKeeperConnectionConfig):
- if other.__dict__ == self.__dict__:
- return True
- return False
-
- def __init__(self, host, port=2181, chroot=None):
- '''Initialize the ZooKeeperConnectionConfig object.
-
- :param str host: The hostname of the ZooKeeper server.
- :param int port: The port on which ZooKeeper is listening.
- Optional, default: 2181.
- :param str chroot: A chroot for this connection. All
- ZooKeeper nodes will be underneath this root path.
- Optional, default: None.
-
- (one per server) defining the ZooKeeper cluster servers. Only
- the 'host' attribute is required.'.
-
- '''
- self.host = host
- self.port = port
- self.chroot = chroot or ''
-
-
class ZooKeeper(object):
'''
Class implementing the ZooKeeper interface.
@@ -127,21 +97,20 @@
def resetLostFlag(self):
self._became_lost = False
- def connect(self, host_list, read_only=False):
+ def connect(self, hosts, read_only=False):
'''
Establish a connection with ZooKeeper cluster.
Convenience method if a pre-existing ZooKeeper connection is not
supplied to the ZooKeeper object at instantiation time.
- :param list host_list: A list of
- :py:class:`~nodepool.zk.ZooKeeperConnectionConfig` objects
- (one per server) defining the ZooKeeper cluster servers.
+ :param str hosts: Comma-separated list of hosts to connect to (e.g.
+ 127.0.0.1:2181,127.0.0.1:2182,[::1]:2183).
:param bool read_only: If True, establishes a read-only connection.
'''
if self.client is None:
- self.client = KazooClient(hosts=host_list, read_only=read_only)
+ self.client = KazooClient(hosts=hosts, read_only=read_only)
self.client.add_listener(self._connection_listener)
self.client.start()
@@ -157,16 +126,15 @@
self.client.close()
self.client = None
- def resetHosts(self, host_list):
+ def resetHosts(self, hosts):
'''
Reset the ZooKeeper cluster connection host list.
- :param list host_list: A list of
- :py:class:`~nodepool.zk.ZooKeeperConnectionConfig` objects
- (one per server) defining the ZooKeeper cluster servers.
+ :param str hosts: Comma-separated list of hosts to connect to (e.g.
+ 127.0.0.1:2181,127.0.0.1:2182,[::1]:2183).
'''
if self.client is not None:
- self.client.set_hosts(hosts=host_list)
+ self.client.set_hosts(hosts=hosts)
def submitNodeRequest(self, node_request, watcher):
'''