Fix infinite loop on reconfiguration exception
If there was an error re-enqueing a change during reconfiguration,
we would abort reconfiguration after having moved some changes into
the new pipelines. Once we start doing this, we need to finish it,
otherwise items will be in change queues which are no longer attached
to the pipelines in use.
To avoid the issue, simply remove an item from the pipeline if an
exception is raised while re-enqueing it. This is bad, but not as
bad as an infinite loop.
Change-Id: I628e599671b7d2d14ae981d65d04e37b23d26831
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index f22d98c..2dcd9bf 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -2434,6 +2434,35 @@
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
+ def test_live_reconfiguration_abort(self):
+ # Raise an exception during reconfiguration and verify we
+ # still function.
+ self.executor_server.hold_jobs_in_build = True
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ A.addApproval('Code-Review', 2)
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ pipeline = tenant.layout.pipelines['gate']
+ change = pipeline.getAllItems()[0].change
+ # Set this to an invalid value to cause an exception during
+ # reconfiguration.
+ change.branch = None
+
+ self.sched.reconfigure(self.config)
+ self.waitUntilSettled()
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+
+ self.waitUntilSettled()
+ self.assertEqual(self.getJobFromHistory('project-merge').result,
+ 'ABORTED')
+ self.assertEqual(A.data['status'], 'NEW')
+ # The final report fails because of the invalid value set above.
+ self.assertEqual(A.reported, 1)
+
def test_live_reconfiguration_merge_conflict(self):
# A real-world bug: a change in a gate queue has a merge
# conflict and a job is added to its project while it's
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index 806ba86..5432661 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -543,9 +543,16 @@
tenant, item)
item.item_ahead = None
item.items_behind = []
- if (item.change.project and
- new_pipeline.manager.reEnqueueItem(item,
- last_head)):
+ reenqueued = False
+ if item.change.project:
+ try:
+ reenqueued = new_pipeline.manager.reEnqueueItem(
+ item, last_head)
+ except Exception:
+ self.log.exception(
+ "Exception while re-enqueing item %s",
+ item)
+ if reenqueued:
for build in item.current_build_set.getBuilds():
new_job = item.getJob(build.job.name)
if new_job: