Cancel obsolete builds on reconfiguration
Cleanup running builds for jobs that are no longer defined on
reconfiguration.
Change-Id: I746a5ba8d9034ae846fd77080af2bfabc8aedf44
diff --git a/tests/fixtures/layout-no-jobs.yaml b/tests/fixtures/layout-no-jobs.yaml
new file mode 100644
index 0000000..ee8dc62
--- /dev/null
+++ b/tests/fixtures/layout-no-jobs.yaml
@@ -0,0 +1,43 @@
+includes:
+ - python-file: custom_functions.py
+
+pipelines:
+ - name: check
+ manager: IndependentPipelineManager
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
+
+ - name: gate
+ manager: DependentPipelineManager
+ failure-message: Build failed. For information on how to proceed, see http://wiki.example.org/Test_Failures
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - approved: 1
+ success:
+ gerrit:
+ verified: 2
+ submit: true
+ failure:
+ gerrit:
+ verified: -2
+ start:
+ gerrit:
+ verified: 0
+ precedence: high
+
+projects:
+ - name: org/project
+ merge-mode: cherry-pick
+ check:
+ - noop
+ gate:
+ - noop
diff --git a/tests/fixtures/layout.yaml b/tests/fixtures/layout.yaml
index 98dfe86..b1c94de 100644
--- a/tests/fixtures/layout.yaml
+++ b/tests/fixtures/layout.yaml
@@ -231,3 +231,7 @@
- conflict-project-merge:
- conflict-project-test1
- conflict-project-test2
+
+ - name: org/noop-project
+ gate:
+ - noop
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index b2106f8..812ce70 100755
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -2813,6 +2813,29 @@
self.assertReportedStat('test-timing', '3|ms')
self.assertReportedStat('test-guage', '12|g')
+ def test_stuck_job_cleanup(self):
+ "Test that pending jobs are cleaned up if removed from layout"
+ self.gearman_server.hold_jobs_in_queue = True
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ A.addApproval('CRVW', 2)
+ self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
+ self.waitUntilSettled()
+ self.assertEqual(len(self.gearman_server.getQueue()), 1)
+
+ self.config.set('zuul', 'layout_config',
+ 'tests/fixtures/layout-no-jobs.yaml')
+ self.sched.reconfigure(self.config)
+ self.waitUntilSettled()
+
+ self.gearman_server.release('noop')
+ self.waitUntilSettled()
+ self.assertEqual(len(self.gearman_server.getQueue()), 0)
+ self.assertTrue(self.sched._areAllBuildsComplete())
+
+ self.assertEqual(len(self.history), 1)
+ self.assertEqual(self.history[0].name, 'noop')
+ self.assertEqual(self.history[0].result, 'SUCCESS')
+
def test_file_jobs(self):
"Test that file jobs run only when appropriate"
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index eaa5eae..fafa7d3 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -565,8 +565,7 @@
self.log.warning("No old pipeline matching %s found "
"when reconfiguring" % name)
continue
- self.log.debug("Re-enqueueing changes for pipeline %s" %
- name)
+ self.log.debug("Re-enqueueing changes for pipeline %s" % name)
items_to_remove = []
for shared_queue in old_pipeline.queues:
for item in shared_queue.queue:
@@ -582,16 +581,30 @@
items_to_remove.append(item)
continue
item.change.project = project
+ for build in item.current_build_set.getBuilds():
+ build.job = layout.jobs.get(build.job.name,
+ build.job)
if not new_pipeline.manager.reEnqueueItem(item):
items_to_remove.append(item)
builds_to_remove = []
for build, item in old_pipeline.manager.building_jobs.items():
if item in items_to_remove:
builds_to_remove.append(build)
- self.log.warning("Deleting running build %s for "
- "change %s while reenqueueing" % (
- build, item.change))
+ self.log.warning(
+ "Deleting running build %s for change %s whose "
+ "item was not re-enqueued" % (build, item.change))
+ if build.job not in new_pipeline.getJobs(item.change):
+ builds_to_remove.append(build)
+ self.log.warning(
+ "Deleting running build %s for change %s because "
+ "the job is not defined" % (build, item.change))
for build in builds_to_remove:
+ try:
+ self.launcher.cancel(build)
+ except Exception:
+ self.log.exception(
+ "Exception while canceling build %s "
+ "for change %s" % (build, item.change))
del old_pipeline.manager.building_jobs[build]
new_pipeline.manager.building_jobs = \
old_pipeline.manager.building_jobs