Re-launch a job if the worker fails to run it
If a job is complete with no build result, it has failed to
run to completion. In this case, discard the previous build
and launch a replacement (in the next run of the queue processor).
Change-Id: Ib8fc245a5becb1e7deb13f1ea0721fdb6ceb9f6f
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 6ae714b..30c465f 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -443,6 +443,7 @@
self.aborted = False
self.created = time.time()
self.description = ''
+ self.run_error = False
def release(self):
self.wait_condition.acquire()
@@ -492,7 +493,13 @@
if self.aborted:
result = 'ABORTED'
- data = {'result': result}
+ if self.run_error:
+ work_fail = True
+ result = 'RUN_ERROR'
+ else:
+ data['result'] = result
+ work_fail = False
+
changes = None
if 'ZUUL_CHANGE_IDS' in self.parameters:
changes = self.parameters['ZUUL_CHANGE_IDS']
@@ -504,7 +511,11 @@
pipeline=self.parameters['ZUUL_PIPELINE'])
)
- self.job.sendWorkComplete(json.dumps(data))
+ self.job.sendWorkData(json.dumps(data))
+ if work_fail:
+ self.job.sendWorkFail()
+ else:
+ self.job.sendWorkComplete(json.dumps(data))
del self.worker.gearman_jobs[self.job.unique]
self.worker.running_builds.remove(self)
self.worker.lock.release()
@@ -2381,6 +2392,21 @@
self.assertEqual(D.data['status'], 'MERGED')
self.assertEqual(D.reported, 2)
+ def test_rerun_on_error(self):
+ "Test that if a worker fails to run a job, it is run again"
+ self.worker.hold_jobs_in_build = True
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ A.addApproval('CRVW', 2)
+ self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
+ self.waitUntilSettled()
+
+ self.builds[0].run_error = True
+ self.worker.hold_jobs_in_build = False
+ self.worker.release()
+ self.waitUntilSettled()
+ self.assertEqual(self.countJobResults(self.history, 'RUN_ERROR'), 1)
+ self.assertEqual(self.countJobResults(self.history, 'SUCCESS'), 3)
+
def test_statsd(self):
"Test each of the statsd methods used in the scheduler"
import extras