Merge "A couple small test improvements" into feature/zuulv3
diff --git a/.zuul.yaml b/.zuul.yaml
new file mode 100644
index 0000000..bb9a96d
--- /dev/null
+++ b/.zuul.yaml
@@ -0,0 +1,15 @@
+- job:
+ name: python-linters
+ pre-run: pre
+ post-run: post
+ success-url: http://zuulv3-dev.openstack.org/logs/{build.uuid}/
+ failure-url: http://zuulv3-dev.openstack.org/logs/{build.uuid}/
+ nodes:
+ - name: worker
+ image: ubuntu-xenial
+
+- project:
+ name: openstack-infra/zuul
+ check:
+ jobs:
+ - python-linters
diff --git a/TESTING.rst b/TESTING.rst
index 56f2fbb..d2cd4c1 100644
--- a/TESTING.rst
+++ b/TESTING.rst
@@ -17,6 +17,16 @@
pip install tox
+As of zuul v3, a running zookeeper is required to execute tests.
+
+*Install zookeeper*::
+
+ [apt-get | yum] install zookeeperd
+
+*Start zookeeper*::
+
+ service zookeeper start
+
Run The Tests
-------------
@@ -54,12 +64,12 @@
For example, to *run the basic Zuul test*::
- tox -e py27 -- tests.test_scheduler.TestScheduler.test_jobs_launched
+ tox -e py27 -- tests.unit.test_scheduler.TestScheduler.test_jobs_launched
To *run one test in the foreground* (after previously having run tox
to set up the virtualenv)::
- .tox/py27/bin/python -m testtools.run tests.test_scheduler.TestScheduler.test_jobs_launched
+ .tox/py27/bin/python -m testtools.run tests.unit.test_scheduler.TestScheduler.test_jobs_launched
List Failing Tests
------------------
diff --git a/playbooks/post.yaml b/playbooks/post.yaml
new file mode 100644
index 0000000..a11e50a
--- /dev/null
+++ b/playbooks/post.yaml
@@ -0,0 +1,19 @@
+- hosts: all
+ tasks:
+ - name: Collect console log.
+ synchronize:
+ dest: "{{ zuul.launcher.log_root }}"
+ mode: pull
+ src: "/tmp/console.log"
+
+ - name: Collect tox logs.
+ synchronize:
+ dest: "{{ zuul.launcher.log_root }}/tox"
+ mode: pull
+ src: "/home/zuul/workspace/src/{{ zuul.project }}/.tox/pep8/log/"
+
+ - name: publish tox logs.
+ copy:
+ dest: "/opt/zuul-logs/{{ zuul.uuid}}"
+ src: "{{ zuul.launcher.log_root }}/"
+ delegate_to: 127.0.0.1
diff --git a/playbooks/pre.yaml b/playbooks/pre.yaml
new file mode 100644
index 0000000..1a2e699
--- /dev/null
+++ b/playbooks/pre.yaml
@@ -0,0 +1,3 @@
+- hosts: all
+ roles:
+ - prepare-workspace
diff --git a/playbooks/python-linters.yaml b/playbooks/python-linters.yaml
new file mode 100644
index 0000000..bc7effe
--- /dev/null
+++ b/playbooks/python-linters.yaml
@@ -0,0 +1,7 @@
+- hosts: all
+ tasks:
+ - name: Run a tox -e pep8.
+ include_role:
+ name: run-tox
+ vars:
+ run_tox_eventlist: pep8
diff --git a/playbooks/roles/prepare-workspace/defaults/main.yaml b/playbooks/roles/prepare-workspace/defaults/main.yaml
new file mode 100644
index 0000000..9127ad8
--- /dev/null
+++ b/playbooks/roles/prepare-workspace/defaults/main.yaml
@@ -0,0 +1,3 @@
+---
+# tasks/main.yaml
+prepare_workspace_root: /home/zuul/workspace
diff --git a/playbooks/roles/prepare-workspace/tasks/main.yaml b/playbooks/roles/prepare-workspace/tasks/main.yaml
new file mode 100644
index 0000000..76f9d95
--- /dev/null
+++ b/playbooks/roles/prepare-workspace/tasks/main.yaml
@@ -0,0 +1,21 @@
+- name: Ensure console.log does not exist.
+ file:
+ path: /tmp/console.log
+ state: absent
+
+- name: Start zuul_console daemon.
+ zuul_console:
+ path: /tmp/console.log
+ port: 19885
+
+- name: Create workspace directory.
+ file:
+ path: "{{ prepare_workspace_root }}"
+ owner: zuul
+ group: zuul
+ state: directory
+
+- name: Synchronize src repos to workspace directory.
+ synchronize:
+ dest: "{{ prepare_workspace_root }}"
+ src: "{{ zuul.launcher.src_root }}"
diff --git a/playbooks/roles/run-tox/defaults/main.yaml b/playbooks/roles/run-tox/defaults/main.yaml
new file mode 100644
index 0000000..7f0310c
--- /dev/null
+++ b/playbooks/roles/run-tox/defaults/main.yaml
@@ -0,0 +1,3 @@
+---
+# tasks/main.yaml
+run_tox_eventlist:
diff --git a/playbooks/roles/run-tox/tasks/main.yaml b/playbooks/roles/run-tox/tasks/main.yaml
new file mode 100644
index 0000000..ca8d079
--- /dev/null
+++ b/playbooks/roles/run-tox/tasks/main.yaml
@@ -0,0 +1,4 @@
+- name: Run tox
+ shell: "/usr/local/jenkins/slave_scripts/run-tox.sh {{ run_tox_eventlist }}"
+ args:
+ chdir: "/home/zuul/workspace/src/{{ zuul.project }}"
diff --git a/test-requirements.txt b/test-requirements.txt
index aed9998..150fd2e 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -1,4 +1,4 @@
-hacking>=0.9.2,<0.10
+hacking>=0.12.0,!=0.13.0,<0.14 # Apache-2.0
coverage>=3.6
sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3
diff --git a/tests/base.py b/tests/base.py
index 57742f8..1a66524 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -733,7 +733,7 @@
self.running_builds.append(build)
self.job_builds[job.unique] = build
args = json.loads(job.arguments)
- args['zuul']['_test'] = dict(test_root=self._test_root)
+ args['vars']['zuul']['_test'] = dict(test_root=self._test_root)
job.arguments = json.dumps(args)
self.job_workers[job.unique] = RecordingAnsibleJob(self, job)
self.job_workers[job.unique].run()
@@ -750,17 +750,17 @@
class RecordingAnsibleJob(zuul.launcher.server.AnsibleJob):
- def runPlaybooks(self):
+ def runPlaybooks(self, args):
build = self.launcher_server.job_builds[self.job.unique]
build.jobdir = self.jobdir
- result = super(RecordingAnsibleJob, self).runPlaybooks()
+ result = super(RecordingAnsibleJob, self).runPlaybooks(args)
self.launcher_server.lock.acquire()
self.launcher_server.build_history.append(
BuildHistory(name=build.name, result=result, changes=build.changes,
node=build.node, uuid=build.unique,
- parameters=build.parameters,
+ parameters=build.parameters, jobdir=build.jobdir,
pipeline=build.parameters['ZUUL_PIPELINE'])
)
self.launcher_server.running_builds.remove(build)
@@ -1070,6 +1070,7 @@
class BaseTestCase(testtools.TestCase):
log = logging.getLogger("zuul.test")
+ wait_timeout = 20
def attachLogs(self, *args):
def reader():
@@ -1430,6 +1431,9 @@
gc.collect()
for obj in gc.get_objects():
if isinstance(obj, git.Repo):
+ self.log.debug("Leaked git repo object: %s" % repr(obj))
+ for r in gc.get_referrers(obj):
+ self.log.debug(" referrer: %s" % repr(r))
repos.append(obj)
self.assertEqual(len(repos), 0)
self.assertEmptyQueues()
@@ -1611,7 +1615,7 @@
self.log.debug("Waiting until settled...")
start = time.time()
while True:
- if time.time() - start > 20:
+ if time.time() - start > self.wait_timeout:
self.log.error("Timeout waiting for Zuul to settle")
self.log.error("Queue status:")
for queue in self.event_queues:
@@ -1816,12 +1820,23 @@
f.write(content)
repo.index.add([fn])
commit = repo.index.commit(message)
+ before = repo.heads[branch].commit
repo.heads[branch].commit = commit
repo.head.reference = branch
repo.git.clean('-x', '-f', '-d')
repo.heads[branch].checkout()
if tag:
repo.create_tag(tag)
+ return before
+
+ def commitLayoutUpdate(self, orig_name, source_name):
+ source_path = os.path.join(self.test_root, 'upstream',
+ source_name, 'zuul.yaml')
+ with open(source_path, 'r') as nt:
+ before = self.addCommitToRepo(
+ orig_name, 'Pulling content from %s' % source_name,
+ {'zuul.yaml': nt.read()})
+ return before
def addEvent(self, connection, event):
"""Inject a Fake (Gerrit) event.
diff --git a/tests/fixtures/config/ansible/git/common-config/playbooks/python27.yaml b/tests/fixtures/config/ansible/git/common-config/playbooks/python27.yaml
index 6b79a78..45acb87 100644
--- a/tests/fixtures/config/ansible/git/common-config/playbooks/python27.yaml
+++ b/tests/fixtures/config/ansible/git/common-config/playbooks/python27.yaml
@@ -1,7 +1,7 @@
- hosts: all
tasks:
- file:
- path: "{{zuul._test.test_root}}/{{zuul.uuid}}.flag"
+ path: "{{flagpath}}"
state: touch
- copy:
src: "{{zuul._test.test_root}}/{{zuul.uuid}}.flag"
diff --git a/tests/fixtures/config/ansible/git/common-config/playbooks/timeout.yaml b/tests/fixtures/config/ansible/git/common-config/playbooks/timeout.yaml
new file mode 100644
index 0000000..4af20eb
--- /dev/null
+++ b/tests/fixtures/config/ansible/git/common-config/playbooks/timeout.yaml
@@ -0,0 +1,4 @@
+- hosts: all
+ tasks:
+ - name: Pause for 60 seconds, so zuul aborts our job.
+ shell: sleep 60
diff --git a/tests/fixtures/config/ansible/git/common-config/zuul.yaml b/tests/fixtures/config/ansible/git/common-config/zuul.yaml
index 7373eff..30148f0 100644
--- a/tests/fixtures/config/ansible/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/ansible/git/common-config/zuul.yaml
@@ -40,5 +40,12 @@
name: python27
pre-run: pre
post-run: post
+ vars:
+ flagpath: "{{zuul._test.test_root}}/{{zuul.uuid}}.flag"
roles:
- zuul: bare-role
+
+- job:
+ parent: python27
+ name: timeout
+ timeout: 1
diff --git a/tests/fixtures/config/ansible/git/org_project/.zuul.yaml b/tests/fixtures/config/ansible/git/org_project/.zuul.yaml
index 6abfc47..c76ba70 100644
--- a/tests/fixtures/config/ansible/git/org_project/.zuul.yaml
+++ b/tests/fixtures/config/ansible/git/org_project/.zuul.yaml
@@ -9,3 +9,4 @@
jobs:
- python27
- faillocal
+ - timeout
diff --git a/tests/fixtures/config/openstack/git/project-config/playbooks/dsvm.yaml b/tests/fixtures/config/openstack/git/project-config/playbooks/dsvm.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/openstack/git/project-config/playbooks/dsvm.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/openstack/git/project-config/zuul.yaml b/tests/fixtures/config/openstack/git/project-config/zuul.yaml
index 9c2231a..420d979 100644
--- a/tests/fixtures/config/openstack/git/project-config/zuul.yaml
+++ b/tests/fixtures/config/openstack/git/project-config/zuul.yaml
@@ -71,12 +71,22 @@
- python27
- python35
+- job:
+ name: dsvm
+ parent: base
+ repos:
+ - openstack/keystone
+ - openstack/nova
+
# Project definitions
- project:
name: openstack/nova
templates:
- python-jobs
+ check:
+ jobs:
+ - dsvm
gate:
queue: integrated
@@ -84,5 +94,8 @@
name: openstack/keystone
templates:
- python-jobs
+ check:
+ jobs:
+ - dsvm
gate:
queue: integrated
diff --git a/tests/fixtures/config/single-tenant/git/layout-timer-smtp/playbooks/project-bitrot-stable-old.yaml b/tests/fixtures/config/single-tenant/git/layout-timer-smtp/playbooks/project-bitrot-stable-old.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-timer-smtp/playbooks/project-bitrot-stable-old.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/layout-timer-smtp/playbooks/project-bitrot-stable-older.yaml b/tests/fixtures/config/single-tenant/git/layout-timer-smtp/playbooks/project-bitrot-stable-older.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-timer-smtp/playbooks/project-bitrot-stable-older.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/layout-timer-smtp/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-timer-smtp/zuul.yaml
new file mode 100644
index 0000000..4a14107
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-timer-smtp/zuul.yaml
@@ -0,0 +1,28 @@
+- pipeline:
+ name: periodic
+ manager: independent
+ source:
+ gerrit
+ trigger:
+ timer:
+ - time: '* * * * * */1'
+ success:
+ smtp:
+ to: alternative_me@example.com
+ from: zuul_from@example.com
+ subject: 'Periodic check for {change.project} succeeded'
+
+- job:
+ name: project-bitrot-stable-old
+ success-url: http://logs.example.com/{job.name}/{build.number}
+
+- job:
+ name: project-bitrot-stable-older
+ success-url: http://logs.example.com/{job.name}/{build.number}
+
+- project:
+ name: org/project
+ periodic:
+ jobs:
+ - project-bitrot-stable-old
+ - project-bitrot-stable-older
diff --git a/tests/fixtures/config/single-tenant/git/layout-timer/playbooks/project-bitrot-stable-old.yaml b/tests/fixtures/config/single-tenant/git/layout-timer/playbooks/project-bitrot-stable-old.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-timer/playbooks/project-bitrot-stable-old.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/layout-timer/playbooks/project-bitrot-stable-older.yaml b/tests/fixtures/config/single-tenant/git/layout-timer/playbooks/project-bitrot-stable-older.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-timer/playbooks/project-bitrot-stable-older.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/layout-timer/playbooks/project-test1.yaml b/tests/fixtures/config/single-tenant/git/layout-timer/playbooks/project-test1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-timer/playbooks/project-test1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/layout-timer/playbooks/project-test2.yaml b/tests/fixtures/config/single-tenant/git/layout-timer/playbooks/project-test2.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-timer/playbooks/project-test2.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/single-tenant/git/layout-timer/zuul.yaml b/tests/fixtures/config/single-tenant/git/layout-timer/zuul.yaml
new file mode 100644
index 0000000..f69a91d
--- /dev/null
+++ b/tests/fixtures/config/single-tenant/git/layout-timer/zuul.yaml
@@ -0,0 +1,52 @@
+- pipeline:
+ name: check
+ manager: independent
+ source:
+ gerrit
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
+
+- pipeline:
+ name: periodic
+ manager: independent
+ source:
+ gerrit
+ trigger:
+ timer:
+ - time: '* * * * * */1'
+
+- job:
+ name: project-test1
+
+- job:
+ name: project-test2
+
+- job:
+ name: project-bitrot-stable-old
+ nodes:
+ - name: static
+ image: ubuntu-xenial
+
+- job:
+ name: project-bitrot-stable-older
+ nodes:
+ - name: static
+ image: ubuntu-trusty
+
+- project:
+ name: org/project
+ check:
+ jobs:
+ - project-test1
+ - project-test2
+ periodic:
+ jobs:
+ - project-bitrot-stable-old
+ - project-bitrot-stable-older
diff --git a/tests/fixtures/layout-timer-smtp.yaml b/tests/fixtures/layout-timer-smtp.yaml
deleted file mode 100644
index b5a6ce0..0000000
--- a/tests/fixtures/layout-timer-smtp.yaml
+++ /dev/null
@@ -1,23 +0,0 @@
-pipelines:
- - name: periodic
- manager: IndependentPipelineManager
- trigger:
- timer:
- - time: '* * * * * */1'
- success:
- smtp:
- to: alternative_me@example.com
- from: zuul_from@example.com
- subject: 'Periodic check for {change.project} succeeded'
-
-jobs:
- - name: project-bitrot-stable-old
- success-pattern: http://logs.example.com/{job.name}/{build.number}
- - name: project-bitrot-stable-older
- success-pattern: http://logs.example.com/{job.name}/{build.number}
-
-projects:
- - name: org/project
- periodic:
- - project-bitrot-stable-old
- - project-bitrot-stable-older
diff --git a/tests/fixtures/layout-timer.yaml b/tests/fixtures/layout-timer.yaml
deleted file mode 100644
index 4904f87..0000000
--- a/tests/fixtures/layout-timer.yaml
+++ /dev/null
@@ -1,28 +0,0 @@
-pipelines:
- - name: check
- manager: IndependentPipelineManager
- trigger:
- gerrit:
- - event: patchset-created
- success:
- gerrit:
- verified: 1
- failure:
- gerrit:
- verified: -1
-
- - name: periodic
- manager: IndependentPipelineManager
- trigger:
- timer:
- - time: '* * * * * */1'
-
-projects:
- - name: org/project
- check:
- - project-merge:
- - project-test1
- - project-test2
- periodic:
- - project-bitrot-stable-old
- - project-bitrot-stable-older
diff --git a/tests/unit/test_openstack.py b/tests/unit/test_openstack.py
index d0c7ab2..670e578 100644
--- a/tests/unit/test_openstack.py
+++ b/tests/unit/test_openstack.py
@@ -14,6 +14,8 @@
# License for the specific language governing permissions and limitations
# under the License.
+import os
+
from tests.base import AnsibleZuulTestCase
@@ -54,3 +56,45 @@
"A should report start and success")
self.assertEqual(self.getJobFromHistory('python27').node,
'ubuntu-trusty')
+
+ def test_dsvm_keystone_repo(self):
+ self.launch_server.keep_jobdir = True
+ A = self.fake_gerrit.addFakeChange('openstack/nova', 'master', 'A')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.assertHistory([
+ dict(name='dsvm', result='SUCCESS', changes='1,1')])
+ build = self.getJobFromHistory('dsvm')
+
+ # Check that a change to nova triggered a keystone clone
+ launcher_git_dir = os.path.join(self.launcher_src_root,
+ 'openstack', 'keystone', '.git')
+ self.assertTrue(os.path.exists(launcher_git_dir),
+ msg='openstack/keystone should be cloned.')
+
+ jobdir_git_dir = os.path.join(build.jobdir.src_root,
+ 'openstack', 'keystone', '.git')
+ self.assertTrue(os.path.exists(jobdir_git_dir),
+ msg='openstack/keystone should be cloned.')
+
+ def test_dsvm_nova_repo(self):
+ self.launch_server.keep_jobdir = True
+ A = self.fake_gerrit.addFakeChange('openstack/keystone', 'master', 'A')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.assertHistory([
+ dict(name='dsvm', result='SUCCESS', changes='1,1')])
+ build = self.getJobFromHistory('dsvm')
+
+ # Check that a change to keystone triggered a nova clone
+ launcher_git_dir = os.path.join(self.launcher_src_root,
+ 'openstack', 'nova', '.git')
+ self.assertTrue(os.path.exists(launcher_git_dir),
+ msg='openstack/nova should be cloned.')
+
+ jobdir_git_dir = os.path.join(build.jobdir.src_root,
+ 'openstack', 'nova', '.git')
+ self.assertTrue(os.path.exists(jobdir_git_dir),
+ msg='openstack/nova should be cloned.')
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index beddae6..45b2257 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -1273,8 +1273,10 @@
self.assertEqual(self.getJobFromHistory('project-test2').result,
'FAILURE')
- @skip("This test generally works but times out frequently")
def test_dependent_behind_dequeue(self):
+ # This particular test does a large amount of merges and needs a little
+ # more time to complete
+ self.wait_timeout = 90
"test that dependent changes behind dequeued changes work"
# This complicated test is a reproduction of a real life bug
self.sched.reconfigure(self.config)
@@ -1663,11 +1665,7 @@
# Stop queuing timer triggered jobs so that the assertions
# below don't race against more jobs being queued.
# Must be in same repo, so overwrite config with another one
- no_timer_path = os.path.join(self.test_root, 'upstream',
- 'layout-no-timer', 'zuul.yaml')
- with open(no_timer_path, 'r') as nt:
- self.addCommitToRepo('layout-idle', 'Removing timer jobs',
- {'zuul.yaml': nt.read()})
+ self.commitLayoutUpdate('layout-idle', 'layout-no-timer')
self.sched.reconfigure(self.config)
self.assertEqual(len(self.builds), 2, "Two timer jobs")
@@ -2772,14 +2770,11 @@
self.assertEqual(results.get(build.name, ''),
build.parameters.get('BUILD_TAGS'))
- @skip("Disabled for early v3 development")
def test_timer(self):
"Test that a periodic job is triggered"
self.launch_server.hold_jobs_in_build = True
- self.updateConfigLayout(
- 'tests/fixtures/layout-timer.yaml')
+ self.updateConfigLayout('layout-timer')
self.sched.reconfigure(self.config)
- self.registerJobs()
# The pipeline triggers every second, so we should have seen
# several by now.
@@ -2790,17 +2785,16 @@
port = self.webapp.server.socket.getsockname()[1]
- req = urllib.request.Request("http://localhost:%s/status.json" % port)
+ req = urllib.request.Request(
+ "http://localhost:%s/openstack/status" % port)
f = urllib.request.urlopen(req)
data = f.read()
self.launch_server.hold_jobs_in_build = False
# Stop queuing timer triggered jobs so that the assertions
# below don't race against more jobs being queued.
- self.updateConfigLayout(
- 'tests/fixtures/layout-no-timer.yaml')
+ self.commitLayoutUpdate('layout-timer', 'layout-no-timer')
self.sched.reconfigure(self.config)
- self.registerJobs()
self.launch_server.release()
self.waitUntilSettled()
@@ -2821,19 +2815,16 @@
self.assertIn('project-bitrot-stable-old', status_jobs)
self.assertIn('project-bitrot-stable-older', status_jobs)
- @skip("Disabled for early v3 development")
def test_idle(self):
"Test that frequent periodic jobs work"
self.launch_server.hold_jobs_in_build = True
+ self.updateConfigLayout('layout-idle')
for x in range(1, 3):
# Test that timer triggers periodic jobs even across
# layout config reloads.
# Start timer trigger
- self.updateConfigLayout(
- 'tests/fixtures/layout-idle.yaml')
self.sched.reconfigure(self.config)
- self.registerJobs()
self.waitUntilSettled()
# The pipeline triggers every second, so we should have seen
@@ -2842,17 +2833,20 @@
# Stop queuing timer triggered jobs so that the assertions
# below don't race against more jobs being queued.
- self.updateConfigLayout(
- 'tests/fixtures/layout-no-timer.yaml')
+ before = self.commitLayoutUpdate('layout-idle', 'layout-no-timer')
self.sched.reconfigure(self.config)
- self.registerJobs()
self.waitUntilSettled()
-
- self.assertEqual(len(self.builds), 2)
+ self.assertEqual(len(self.builds), 2,
+ 'Timer builds iteration #%d' % x)
self.launch_server.release('.*')
self.waitUntilSettled()
self.assertEqual(len(self.builds), 0)
self.assertEqual(len(self.history), x * 2)
+ # Revert back to layout-idle
+ repo = git.Repo(os.path.join(self.test_root,
+ 'upstream',
+ 'layout-idle'))
+ repo.git.reset('--hard', before)
def test_check_smtp_pool(self):
self.updateConfigLayout('layout-smtp')
@@ -2884,14 +2878,11 @@
self.assertEqual(A.messages[0],
self.smtp_messages[1]['body'])
- @skip("Disabled for early v3 development")
def test_timer_smtp(self):
"Test that a periodic job is triggered"
self.launch_server.hold_jobs_in_build = True
- self.updateConfigLayout(
- 'tests/fixtures/layout-timer-smtp.yaml')
+ self.updateConfigLayout('layout-timer-smtp')
self.sched.reconfigure(self.config)
- self.registerJobs()
# The pipeline triggers every second, so we should have seen
# several by now.
@@ -2923,10 +2914,8 @@
# Stop queuing timer triggered jobs and let any that may have
# queued through so that end of test assertions pass.
- self.updateConfigLayout(
- 'tests/fixtures/layout-no-timer.yaml')
+ self.commitLayoutUpdate('layout-timer-smtp', 'layout-no-timer')
self.sched.reconfigure(self.config)
- self.registerJobs()
self.waitUntilSettled()
self.launch_server.release('.*')
self.waitUntilSettled()
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index 97002b2..cf88265 100644
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -185,6 +185,27 @@
dict(name='project-test1', result='SUCCESS', changes='2,1'),
dict(name='project-test2', result='SUCCESS', changes='3,1')])
+ def test_dynamic_syntax_error(self):
+ in_repo_conf = textwrap.dedent(
+ """
+ - job:
+ name: project-test2
+ foo: error
+ """)
+
+ file_dict = {'.zuul.yaml': in_repo_conf}
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+ files=file_dict)
+ A.addApproval('code-review', 2)
+ self.fake_gerrit.addEvent(A.addApproval('approved', 1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(A.reported, 2,
+ "A should report start and failure")
+ self.assertIn('syntax error', A.messages[1],
+ "A should have a syntax error reported")
+
class TestAnsible(AnsibleZuulTestCase):
# A temporary class to hold new tests while others are disabled
@@ -195,6 +216,8 @@
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
+ build = self.getJobFromHistory('timeout')
+ self.assertEqual(build.result, 'ABORTED')
build = self.getJobFromHistory('faillocal')
self.assertEqual(build.result, 'FAILURE')
build = self.getJobFromHistory('python27')
diff --git a/tools/update-storyboard.py b/tools/update-storyboard.py
index 6800a35..12e6916 100644
--- a/tools/update-storyboard.py
+++ b/tools/update-storyboard.py
@@ -67,6 +67,7 @@
'inprogress': ['In Progress', 'Blocked'],
'review': ['In Progress', 'Blocked'],
'merged': None,
+ 'invalid': None,
}
diff --git a/tox.ini b/tox.ini
index b7d89d1..9c0d949 100644
--- a/tox.ini
+++ b/tox.ini
@@ -8,7 +8,7 @@
setenv = STATSD_HOST=127.0.0.1
STATSD_PORT=8125
VIRTUAL_ENV={envdir}
- OS_TEST_TIMEOUT=60
+ OS_TEST_TIMEOUT=90
passenv = ZUUL_TEST_ROOT OS_STDOUT_CAPTURE OS_STDERR_CAPTURE OS_LOG_CAPTURE OS_LOG_DEFAULTS
usedevelop = True
install_command = pip install {opts} {packages}
diff --git a/zuul/ansible/action/synchronize.py b/zuul/ansible/action/synchronize.py
index 0193eca..75fd45f 100644
--- a/zuul/ansible/action/synchronize.py
+++ b/zuul/ansible/action/synchronize.py
@@ -24,10 +24,15 @@
source = self._task.args.get('src', None)
dest = self._task.args.get('dest', None)
- pull = self._task.args.get('pull', False)
+ mode = self._task.args.get('mode', 'push')
- if not pull and not paths._is_safe_path(source):
+ if 'rsync_opts' not in self._task.args:
+ self._task.args['rsync_opts'] = []
+ if '--safe-links' not in self._task.args['rsync_opts']:
+ self._task.args['rsync_opts'].append('--safe-links')
+
+ if mode == 'push' and not paths._is_safe_path(source):
return paths._fail_dict(source, prefix='Syncing files from')
- if pull and not paths._is_safe_path(dest):
+ if mode == 'pull' and not paths._is_safe_path(dest):
return paths._fail_dict(dest, prefix='Syncing files to')
return super(ActionModule, self).run(tmp, task_vars)
diff --git a/zuul/configloader.py b/zuul/configloader.py
index e4fa620..42616a8 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -10,11 +10,13 @@
# License for the specific language governing permissions and limitations
# under the License.
+from contextlib import contextmanager
import copy
import os
import logging
import six
import yaml
+import pprint
import voluptuous as vs
@@ -38,6 +40,35 @@
return [item]
+class ConfigurationSyntaxError(Exception):
+ pass
+
+
+@contextmanager
+def configuration_exceptions(stanza, conf):
+ try:
+ yield
+ except vs.Invalid as e:
+ conf = copy.deepcopy(conf)
+ context = conf.pop('_source_context')
+ m = """
+Zuul encountered a syntax error while parsing its configuration in the
+repo {repo} on branch {branch}. The error was:
+
+ {error}
+
+The offending content was a {stanza} stanza with the content:
+
+{content}
+"""
+ m = m.format(repo=context.project.name,
+ branch=context.branch,
+ error=str(e),
+ stanza=stanza,
+ content=pprint.pformat(conf))
+ raise ConfigurationSyntaxError(m)
+
+
class NodeSetParser(object):
@staticmethod
def getSchema():
@@ -47,13 +78,15 @@
nodeset = {vs.Required('name'): str,
vs.Required('nodes'): [node],
+ '_source_context': model.SourceContext,
}
return vs.Schema(nodeset)
@staticmethod
def fromYaml(layout, conf):
- NodeSetParser.getSchema()(conf)
+ with configuration_exceptions('nodeset', conf):
+ NodeSetParser.getSchema()(conf)
ns = model.NodeSet(conf['name'])
for conf_node in as_list(conf['nodes']):
node = model.Node(conf_node['name'], conf_node['image'])
@@ -115,6 +148,8 @@
'run': str,
'_source_context': model.SourceContext,
'roles': to_list(role),
+ 'repos': to_list(str),
+ 'vars': dict,
}
return vs.Schema(job)
@@ -134,7 +169,8 @@
@staticmethod
def fromYaml(tenant, layout, conf):
- JobParser.getSchema()(conf)
+ with configuration_exceptions('job', conf):
+ JobParser.getSchema()(conf)
# NB: The default detection system in the Job class requires
# that we always assign values directly rather than modifying
@@ -185,6 +221,11 @@
ns.addNode(node)
job.nodeset = ns
+ if 'repos' in conf:
+ # Accumulate repos in a set so that job inheritance
+ # is additive.
+ job.repos = job.repos.union(set(conf.get('repos', [])))
+
tags = conf.get('tags')
if tags:
# Tags are merged via a union rather than a
@@ -200,6 +241,10 @@
roles.append(r)
job.roles = job.roles.union(set(roles))
+ variables = conf.get('vars', None)
+ if variables:
+ job.updateVariables(variables)
+
# If the definition for this job came from a project repo,
# implicitly apply a branch matcher for the branch it was on.
if (not job.source_context.trusted):
@@ -261,7 +306,8 @@
@staticmethod
def fromYaml(tenant, layout, conf):
- ProjectTemplateParser.getSchema(layout)(conf)
+ with configuration_exceptions('project or project-template', conf):
+ ProjectTemplateParser.getSchema(layout)(conf)
# Make a copy since we modify this later via pop
conf = copy.deepcopy(conf)
project_template = model.ProjectConfig(conf['name'])
@@ -327,11 +373,13 @@
for p in layout.pipelines.values():
project[p.name] = {'queue': str,
'jobs': [vs.Any(str, dict)]}
- return vs.Schema([project])
+ return vs.Schema(project)
@staticmethod
def fromYaml(tenant, layout, conf_list):
- ProjectParser.getSchema(layout)(conf_list)
+ for conf in conf_list:
+ with configuration_exceptions('project', conf):
+ ProjectParser.getSchema(layout)(conf)
project = model.ProjectConfig(conf_list[0]['name'])
mode = conf_list[0].get('merge-mode', 'merge-resolve')
project.merge_mode = model.MERGER_MAP[mode]
@@ -450,6 +498,7 @@
'window-increase-factor': window_factor,
'window-decrease-type': window_type,
'window-decrease-factor': window_factor,
+ '_source_context': model.SourceContext,
}
pipeline['trigger'] = vs.Required(
PipelineParser.getDriverSchema('trigger', connections))
@@ -461,7 +510,8 @@
@staticmethod
def fromYaml(layout, connections, scheduler, conf):
- PipelineParser.getSchema(layout, connections)(conf)
+ with configuration_exceptions('pipeline', conf):
+ PipelineParser.getSchema(layout, connections)(conf)
pipeline = model.Pipeline(conf['name'], layout)
pipeline.description = conf.get('description')
diff --git a/zuul/driver/gerrit/gerritconnection.py b/zuul/driver/gerrit/gerritconnection.py
index 9c54b4c..d65e6a8 100644
--- a/zuul/driver/gerrit/gerritconnection.py
+++ b/zuul/driver/gerrit/gerritconnection.py
@@ -513,10 +513,10 @@
time.sleep(self.replication_retry_interval)
return False
- def getRefSha(self, project, ref):
+ def getRefSha(self, project_name, ref):
refs = {}
try:
- refs = self.getInfoRefs(project)
+ refs = self.getInfoRefs(project_name)
except:
self.log.exception("Exception looking for ref %s" %
ref)
@@ -578,7 +578,7 @@
return changes
def getProjectBranches(self, project):
- refs = self.getInfoRefs(project)
+ refs = self.getInfoRefs(project.name)
heads = [str(k[len('refs/heads/'):]) for k in refs.keys()
if k.startswith('refs/heads/')]
return heads
@@ -710,9 +710,9 @@
raise Exception("Gerrit error executing %s" % command)
return (out, err)
- def getInfoRefs(self, project):
+ def getInfoRefs(self, project_name):
url = "%s/p/%s/info/refs?service=git-upload-pack" % (
- self.baseurl, project)
+ self.baseurl, project_name)
try:
data = urllib.request.urlopen(url).read()
except:
diff --git a/zuul/launcher/client.py b/zuul/launcher/client.py
index f36e2a7..6abd6f4 100644
--- a/zuul/launcher/client.py
+++ b/zuul/launcher/client.py
@@ -12,6 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
+import copy
import gear
import json
import logging
@@ -260,9 +261,15 @@
# TODOv3(jeblair): This ansible vars data structure will
# replace the environment variables below.
zuul_params = dict(uuid=uuid,
+ pipeline=pipeline.name,
+ job=job.name,
project=item.change.project.name)
if hasattr(item.change, 'branch'):
zuul_params['branch'] = item.change.branch
+ if hasattr(item.change, 'number'):
+ zuul_params['change'] = item.change.number
+ if hasattr(item.change, 'patchset'):
+ zuul_params['patchset'] = item.change.patchset
# Legacy environment variables
params = dict(ZUUL_UUID=uuid,
ZUUL_PROJECT=item.change.project.name)
@@ -270,7 +277,6 @@
params['ZUUL_URL'] = item.current_build_set.zuul_url
params['ZUUL_VOTING'] = job.voting and '1' or '0'
if hasattr(item.change, 'refspec'):
- zuul_params['branch'] = item.change.branch
changes_str = '^'.join(
['%s:%s:%s' % (i.change.project.name, i.change.branch,
i.change.refspec)
@@ -331,6 +337,7 @@
merger_items = map(make_merger_item, all_items)
params['job'] = job.name
+ params['timeout'] = job.timeout
params['items'] = merger_items
params['projects'] = []
@@ -346,8 +353,16 @@
public_ipv6=node.public_ipv6,
public_ipv4=node.public_ipv4))
params['nodes'] = nodes
- params['zuul'] = zuul_params
+ params['vars'] = copy.deepcopy(job.variables)
+ params['vars']['zuul'] = zuul_params
projects = set()
+ if job.repos:
+ for repo in job.repos:
+ project = item.pipeline.source.getProject(repo)
+ params['projects'].append(
+ dict(name=repo,
+ url=item.pipeline.source.getGitUrl(project)))
+ projects.add(project)
for item in all_items:
if item.change.project not in projects:
params['projects'].append(
diff --git a/zuul/launcher/server.py b/zuul/launcher/server.py
index 575d352..1b8d2c6 100644
--- a/zuul/launcher/server.py
+++ b/zuul/launcher/server.py
@@ -35,9 +35,6 @@
import zuul.ansible.library
from zuul.lib import commandsocket
-ANSIBLE_WATCHDOG_GRACE = 5 * 60
-
-
COMMANDS = ['stop', 'pause', 'unpause', 'graceful', 'verbose',
'unverbose']
@@ -87,7 +84,7 @@
# trusted.cfg
# untrusted.cfg
# work
- # git
+ # src
# logs
self.keep = keep
self.root = tempfile.mkdtemp(dir=root)
@@ -591,7 +588,7 @@
self.job.sendWorkData(json.dumps(data))
self.job.sendWorkStatus(0, 100)
- result = self.runPlaybooks()
+ result = self.runPlaybooks(args)
if result is None:
self.job.sendWorkFail()
@@ -599,17 +596,20 @@
result = dict(result=result)
self.job.sendWorkComplete(json.dumps(result))
- def runPlaybooks(self):
+ def runPlaybooks(self, args):
result = None
for playbook in self.jobdir.pre_playbooks:
- pre_status, pre_code = self.runAnsiblePlaybook(playbook)
+ # TODOv3(pabelanger): Implement pre-run timeout setting.
+ pre_status, pre_code = self.runAnsiblePlaybook(
+ playbook, args['timeout'])
if pre_status != self.RESULT_NORMAL or pre_code != 0:
# These should really never fail, so return None and have
# zuul try again
return result
- job_status, job_code = self.runAnsiblePlaybook(self.jobdir.playbook)
+ job_status, job_code = self.runAnsiblePlaybook(
+ self.jobdir.playbook, args['timeout'])
if job_status == self.RESULT_TIMED_OUT:
return 'TIMED_OUT'
if job_status == self.RESULT_ABORTED:
@@ -626,8 +626,9 @@
result = 'FAILURE'
for playbook in self.jobdir.post_playbooks:
+ # TODOv3(pabelanger): Implement post-run timeout setting.
post_status, post_code = self.runAnsiblePlaybook(
- playbook, success)
+ playbook, args['timeout'], success)
if post_status != self.RESULT_NORMAL or post_code != 0:
result = 'POST_FAILURE'
return result
@@ -814,7 +815,7 @@
self.jobdir.known_hosts))
with open(self.jobdir.vars, 'w') as vars_yaml:
- zuul_vars = dict(zuul=args['zuul'])
+ zuul_vars = dict(args['vars'])
zuul_vars['zuul']['launcher'] = dict(src_root=self.jobdir.src_root,
log_root=self.jobdir.log_root)
vars_yaml.write(
@@ -911,23 +912,24 @@
)
ret = None
- watchdog = Watchdog(timeout + ANSIBLE_WATCHDOG_GRACE,
- self._ansibleTimeout,
- ("Ansible timeout exceeded",))
- watchdog.start()
+ if timeout:
+ watchdog = Watchdog(timeout, self._ansibleTimeout,
+ ("Ansible timeout exceeded",))
+ watchdog.start()
try:
for line in iter(self.proc.stdout.readline, b''):
line = line[:1024].rstrip()
self.log.debug("Ansible output: %s" % (line,))
ret = self.proc.wait()
finally:
- watchdog.stop()
+ if timeout:
+ watchdog.stop()
self.log.debug("Ansible exit code: %s" % (ret,))
with self.proc_lock:
self.proc = None
- if watchdog.timed_out:
+ if timeout and watchdog.timed_out:
return (self.RESULT_TIMED_OUT, None)
if ret == 3:
# AnsibleHostUnreachable: We had a network issue connecting to
@@ -939,7 +941,7 @@
return (self.RESULT_NORMAL, ret)
- def runAnsiblePlaybook(self, playbook, success=None):
+ def runAnsiblePlaybook(self, playbook, timeout, success=None):
env_copy = os.environ.copy()
env_copy['LOGNAME'] = 'zuul'
@@ -955,8 +957,5 @@
cmd.extend(['-e@%s' % self.jobdir.vars, verbose])
- # TODOv3: get this from the job
- timeout = 60
-
return self.runAnsible(
cmd=cmd, timeout=timeout, trusted=playbook.trusted)
diff --git a/zuul/manager/__init__.py b/zuul/manager/__init__.py
index 18cf11b..4447615 100644
--- a/zuul/manager/__init__.py
+++ b/zuul/manager/__init__.py
@@ -265,6 +265,8 @@
# Similarly, reset the item state.
if item.current_build_set.unable_to_merge:
item.setUnableToMerge()
+ if item.current_build_set.config_error:
+ item.setConfigError(item.current_build_set.config_error)
if item.dequeued_needing_change:
item.setDequeuedNeedingChange()
@@ -482,8 +484,21 @@
import zuul.configloader
loader = zuul.configloader.ConfigLoader()
self.log.debug("Load dynamic layout with %s" % build_set.files)
- layout = loader.createDynamicLayout(item.pipeline.layout.tenant,
- build_set.files)
+ try:
+ layout = loader.createDynamicLayout(
+ item.pipeline.layout.tenant,
+ build_set.files)
+ except zuul.configloader.ConfigurationSyntaxError as e:
+ self.log.info("Configuration syntax error "
+ "in dynamic layout %s" %
+ build_set.files)
+ item.setConfigError(str(e))
+ return None
+ except Exception:
+ self.log.exception("Error in dynamic layout %s" %
+ build_set.files)
+ item.setConfigError("Unknown configuration error")
+ return None
return layout
build_set.merge_state = build_set.PENDING
self.log.debug("Preparing dynamic layout for: %s" % item.change)
@@ -556,6 +571,8 @@
ready = self.prepareLayout(item)
if item.current_build_set.unable_to_merge:
failing_reasons.append("it has a merge conflict")
+ if item.current_build_set.config_error:
+ failing_reasons.append("it has an invalid configuration")
if ready and self.provisionNodes(item):
changed = True
if actionable and ready and self.launchJobs(item):
@@ -693,7 +710,12 @@
def _reportItem(self, item):
self.log.debug("Reporting change %s" % item.change)
ret = True # Means error as returned by trigger.report
- if not item.getJobs():
+ if item.getConfigError():
+ self.log.debug("Invalid config for change %s" % item.change)
+ # TODOv3(jeblair): consider a new reporter action for this
+ actions = self.pipeline.merge_failure_actions
+ item.setReportedResult('CONFIG_ERROR')
+ elif not item.getJobs():
# We don't send empty reports with +1,
# and the same for -1's (merge failures or transient errors)
# as they cannot be followed by +1's
diff --git a/zuul/model.py b/zuul/model.py
index 10d0446..19931ea 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -687,7 +687,7 @@
# project-pipeline.
self.execution_attributes = dict(
timeout=None,
- # variables={},
+ variables={},
nodeset=NodeSet(),
auth={},
workspace=None,
@@ -699,6 +699,7 @@
attempts=3,
final=False,
roles=frozenset(),
+ repos=frozenset(),
)
# These are generally internal attributes which are not
@@ -755,6 +756,22 @@
if not self.run:
self.run = self.implied_run
+ def updateVariables(self, other_vars):
+ v = self.variables
+ Job._deepUpdate(v, other_vars)
+ self.variables = v
+
+ @staticmethod
+ def _deepUpdate(a, b):
+ # Merge nested dictionaries if possible, otherwise, overwrite
+ # the value in 'a' with the value in 'b'.
+ for k, bv in b.items():
+ av = a.get(k)
+ if isinstance(av, dict) and isinstance(bv, dict):
+ Job._deepUpdate(av, bv)
+ else:
+ a[k] = bv
+
def inheritFrom(self, other):
"""Copy the inheritable attributes which have been set on the other
job to this job."""
@@ -795,7 +812,7 @@
"%s=%s with variant %s" % (
repr(self), k, other._get(k),
repr(other)))
- if k not in set(['pre_run', 'post_run', 'roles']):
+ if k not in set(['pre_run', 'post_run', 'roles', 'variables']):
setattr(self, k, copy.deepcopy(other._get(k)))
# Don't set final above so that we don't trip an error halfway
@@ -809,6 +826,8 @@
self.post_run = other.post_run + self.post_run
if other._get('roles') is not None:
self.roles = self.roles.union(other.roles)
+ if other._get('variables') is not None:
+ self.updateVariables(other.variables)
for k in self.context_attributes:
if (other._get(k) is not None and
@@ -993,6 +1012,7 @@
self.commit = None
self.zuul_url = None
self.unable_to_merge = False
+ self.config_error = None # None or an error message string.
self.failing_reasons = []
self.merge_state = self.NEW
self.nodesets = {} # job -> nodeset
@@ -1151,6 +1171,9 @@
return True
def areAllJobsComplete(self):
+ if (self.current_build_set.config_error or
+ self.current_build_set.unable_to_merge):
+ return True
if not self.hasJobTree():
return False
for job in self.getJobs():
@@ -1184,9 +1207,10 @@
return False
def didMergerFail(self):
- if self.current_build_set.unable_to_merge:
- return True
- return False
+ return self.current_build_set.unable_to_merge
+
+ def getConfigError(self):
+ return self.current_build_set.config_error
def isHoldingFollowingChanges(self):
if not self.live:
@@ -1306,6 +1330,10 @@
self.current_build_set.unable_to_merge = True
self._setAllJobsSkipped()
+ def setConfigError(self, error):
+ self.current_build_set.config_error = error
+ self._setAllJobsSkipped()
+
def _setAllJobsSkipped(self):
for job in self.getJobs():
fakebuild = Build(job, None)
@@ -2064,8 +2092,7 @@
"a single key (when parsing %s)" %
(conf,))
key, value = item.items()[0]
- if key in ['project', 'project-template', 'job']:
- value['_source_context'] = source_context
+ value['_source_context'] = source_context
if key == 'project':
name = value['name']
self.projects.setdefault(name, []).append(value)
diff --git a/zuul/reporter/__init__.py b/zuul/reporter/__init__.py
index 9ed6599..541f259 100644
--- a/zuul/reporter/__init__.py
+++ b/zuul/reporter/__init__.py
@@ -85,6 +85,8 @@
msg = 'This change depends on a change that failed to merge.\n'
elif item.didMergerFail():
msg = pipeline.merge_failure_message
+ elif item.getConfigError():
+ msg = item.getConfigError()
else:
msg = (pipeline.failure_message + '\n\n' +
self._formatItemReportJobs(pipeline, item))