Always try to unlock nodes when returning
Even if things are not in an expected state (eg, node not locked)
or we encounter errors updating node state, always try to unlock
the nodes when returning a nodeset. Finally, if that too fails,
it will be logged but not treated as a fatal exception so that
we don't get stuck in queue processing.
Change-Id: Iab3bae4146c62067342e0c521a6289d045e65a5c
diff --git a/zuul/manager/__init__.py b/zuul/manager/__init__.py
index 2cb34b8..74039c3 100644
--- a/zuul/manager/__init__.py
+++ b/zuul/manager/__init__.py
@@ -407,13 +407,8 @@
old_build_set.item, build.job)
if not was_running:
- try:
- nodeset = build.build_set.getJobNodeSet(build.job.name)
- self.sched.nodepool.returnNodeSet(nodeset)
- except Exception:
- self.log.exception("Unable to return nodeset %s for "
- "canceled build request %s" %
- (nodeset, build))
+ nodeset = build.build_set.getJobNodeSet(build.job.name)
+ self.sched.nodepool.returnNodeSet(nodeset)
build.result = 'CANCELED'
canceled = True
canceled_jobs.add(build.job.name)
diff --git a/zuul/nodepool.py b/zuul/nodepool.py
index 47f5c3c..9a125ce 100644
--- a/zuul/nodepool.py
+++ b/zuul/nodepool.py
@@ -90,10 +90,15 @@
self.log.info("Returning nodeset %s" % (nodeset,))
for node in nodeset.getNodes():
if node.lock is None:
- raise Exception("Node %s is not locked" % (node,))
- if node.state == model.STATE_IN_USE:
- node.state = model.STATE_USED
- self.sched.zk.storeNode(node)
+ self.log.error("Node %s is not locked" % (node,))
+ else:
+ try:
+ if node.state == model.STATE_IN_USE:
+ node.state = model.STATE_USED
+ self.sched.zk.storeNode(node)
+ except Exception:
+ self.log.exception("Exception storing node %s "
+ "while unlocking:" % (node,))
self._unlockNodes(nodeset.getNodes())
def unlockNodeSet(self, nodeset):