Merge "Add initial support for jobs authentication config" into feature/zuulv3
diff --git a/zuul/manager/__init__.py b/zuul/manager/__init__.py
index 181b599..68fbbe8 100644
--- a/zuul/manager/__init__.py
+++ b/zuul/manager/__init__.py
@@ -242,16 +242,31 @@
(item.change, change_queue))
change_queue.enqueueItem(item)
+ # Get an updated copy of the layout if necessary.
+ # This will return one of the following:
+ # 1) An existing layout from the item ahead or pipeline.
+ # 2) A newly created layout from the cached pipeline
+ # layout config plus the previously returned
+ # in-repo files stored in the buildset.
+ # 3) None in the case that a fetch of the files from
+ # the merger is still pending.
+ item.current_build_set.layout = self.getLayout(item)
+
+ # Rebuild the frozen job tree from the new layout, if
+ # we have one. If not, it will be built later.
+ if item.current_build_set.layout:
+ item.freezeJobTree()
+
# Re-set build results in case any new jobs have been
# added to the tree.
for build in item.current_build_set.getBuilds():
if build.result:
- self.pipeline.setResult(item, build)
+ item.setResult(build)
# Similarly, reset the item state.
if item.current_build_set.unable_to_merge:
- self.pipeline.setUnableToMerge(item)
+ item.setUnableToMerge()
if item.dequeued_needing_change:
- self.pipeline.setDequeuedNeedingChange(item)
+ item.setDequeuedNeedingChange()
self.reportStats(item)
return True
@@ -333,7 +348,7 @@
self.reportStats(item)
def provisionNodes(self, item):
- jobs = self.pipeline.findJobsToRequest(item)
+ jobs = item.findJobsToRequest()
if not jobs:
return False
build_set = item.current_build_set
@@ -367,12 +382,7 @@
if not item.current_build_set.layout:
return False
- # We may be working with a dynamic layout. Get a pipeline
- # object from *that* layout to find out which jobs we should
- # run.
- layout = item.current_build_set.layout
- pipeline = layout.pipelines[self.pipeline.name]
- jobs = pipeline.findJobsToRun(item, self.sched.mutex)
+ jobs = item.findJobsToRun(self.sched.mutex)
if jobs:
self._launchJobs(item, jobs)
@@ -487,7 +497,7 @@
"it can no longer merge" % item.change)
self.cancelJobs(item)
self.dequeueItem(item)
- self.pipeline.setDequeuedNeedingChange(item)
+ item.setDequeuedNeedingChange()
if item.live:
try:
self.reportItem(item)
@@ -523,14 +533,13 @@
changed = True
if actionable and ready and self.launchJobs(item):
changed = True
- if self.pipeline.didAnyJobFail(item):
+ if item.didAnyJobFail():
failing_reasons.append("at least one job failed")
if (not item.live) and (not item.items_behind):
failing_reasons.append("is a non-live item with no items behind")
self.dequeueItem(item)
changed = True
- if ((not item_ahead) and self.pipeline.areAllJobsComplete(item)
- and item.live):
+ if ((not item_ahead) and item.areAllJobsComplete() and item.live):
try:
self.reportItem(item)
except exceptions.MergeFailure:
@@ -603,7 +612,7 @@
self.log.debug("Build %s completed" % build)
item = build.build_set.item
- self.pipeline.setResult(item, build)
+ item.setResult(build)
self.sched.mutex.release(item, build.job)
self.log.debug("Item %s status is now:\n %s" %
(item, item.formatStatus()))
@@ -622,7 +631,7 @@
build_set.commit = item.change.newrev
if not build_set.commit and not isinstance(item.change, NullChange):
self.log.info("Unable to merge change %s" % item.change)
- self.pipeline.setUnableToMerge(item)
+ item.setUnableToMerge()
def onNodesProvisioned(self, event):
request = event.request
@@ -637,7 +646,7 @@
# _reportItem() returns True if it failed to report.
item.reported = not self._reportItem(item)
if self.changes_merge:
- succeeded = self.pipeline.didAllJobsSucceed(item)
+ succeeded = item.didAllJobsSucceed()
merged = item.reported
if merged:
merged = self.pipeline.source.isMerged(item.change,
@@ -664,18 +673,18 @@
def _reportItem(self, item):
self.log.debug("Reporting change %s" % item.change)
ret = True # Means error as returned by trigger.report
- if not self.pipeline.getJobs(item):
+ if not item.getJobs():
# We don't send empty reports with +1,
# and the same for -1's (merge failures or transient errors)
# as they cannot be followed by +1's
self.log.debug("No jobs for change %s" % item.change)
actions = []
- elif self.pipeline.didAllJobsSucceed(item):
+ elif item.didAllJobsSucceed():
self.log.debug("success %s" % (self.pipeline.success_actions))
actions = self.pipeline.success_actions
item.setReportedResult('SUCCESS')
self.pipeline._consecutive_failures = 0
- elif not self.pipeline.didMergerSucceed(item):
+ elif item.didMergerFail():
actions = self.pipeline.merge_failure_actions
item.setReportedResult('MERGER_FAILURE')
else:
diff --git a/zuul/model.py b/zuul/model.py
index b85c80d..f67c452 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -145,167 +145,6 @@
tree = self.job_trees.get(project)
return tree
- def getJobs(self, item):
- # TODOv3(jeblair): can this be removed in favor of the frozen
- # job list in item?
- if not item.live:
- return []
- tree = self.getJobTree(item.change.project)
- if not tree:
- return []
- return item.change.filterJobs(tree.getJobs())
-
- def _findJobsToRun(self, job_trees, item, mutex):
- torun = []
- for tree in job_trees:
- job = tree.job
- result = None
- if job:
- if not job.changeMatches(item.change):
- continue
- build = item.current_build_set.getBuild(job.name)
- if build:
- result = build.result
- else:
- # There is no build for the root of this job tree,
- # so we should run it.
- if mutex.acquire(item, job):
- # If this job needs a mutex, either acquire it or make
- # sure that we have it before running the job.
- torun.append(job)
- # If there is no job, this is a null job tree, and we should
- # run all of its jobs.
- if result == 'SUCCESS' or not job:
- torun.extend(self._findJobsToRun(tree.job_trees, item, mutex))
- return torun
-
- def findJobsToRun(self, item, mutex):
- if not item.live:
- return []
- tree = item.job_tree
- if not tree:
- return []
- return self._findJobsToRun(tree.job_trees, item, mutex)
-
- def _findJobsToRequest(self, job_trees, item):
- toreq = []
- for tree in job_trees:
- job = tree.job
- if job:
- if not job.changeMatches(item.change):
- continue
- nodes = item.current_build_set.getJobNodes(job.name)
- if nodes is None:
- req = item.current_build_set.getJobNodeRequest(job.name)
- if req is None:
- toreq.append(job)
- # If there is no job, this is a null job tree, and we should
- # run all of its jobs.
- if not job:
- toreq.extend(self._findJobsToRequest(
- tree.job_trees, item))
- return toreq
-
- def findJobsToRequest(self, item):
- if not item.live:
- return []
- tree = item.job_tree
- if not tree:
- return []
- return self._findJobsToRequest(tree.job_trees, item)
-
- def haveAllJobsStarted(self, item):
- if not item.hasJobTree():
- return False
- for job in item.getJobs():
- build = item.current_build_set.getBuild(job.name)
- if not build or not build.start_time:
- return False
- return True
-
- def areAllJobsComplete(self, item):
- if not item.hasJobTree():
- return False
- for job in item.getJobs():
- build = item.current_build_set.getBuild(job.name)
- if not build or not build.result:
- return False
- return True
-
- def didAllJobsSucceed(self, item):
- if not item.hasJobTree():
- return False
- for job in item.getJobs():
- if not job.voting:
- continue
- build = item.current_build_set.getBuild(job.name)
- if not build:
- return False
- if build.result != 'SUCCESS':
- return False
- return True
-
- def didMergerSucceed(self, item):
- if item.current_build_set.unable_to_merge:
- return False
- return True
-
- def didAnyJobFail(self, item):
- if not item.hasJobTree():
- return False
- for job in item.getJobs():
- if not job.voting:
- continue
- build = item.current_build_set.getBuild(job.name)
- if build and build.result and (build.result != 'SUCCESS'):
- return True
- return False
-
- def isHoldingFollowingChanges(self, item):
- if not item.live:
- return False
- if not item.hasJobTree():
- return False
- for job in item.getJobs():
- if not job.hold_following_changes:
- continue
- build = item.current_build_set.getBuild(job.name)
- if not build:
- return True
- if build.result != 'SUCCESS':
- return True
-
- if not item.item_ahead:
- return False
- return self.isHoldingFollowingChanges(item.item_ahead)
-
- def setResult(self, item, build):
- if build.retry:
- item.removeBuild(build)
- elif build.result != 'SUCCESS':
- # Get a JobTree from a Job so we can find only its dependent jobs
- tree = item.job_tree.getJobTreeForJob(build.job)
- for job in tree.getJobs():
- fakebuild = Build(job, None)
- fakebuild.result = 'SKIPPED'
- item.addBuild(fakebuild)
-
- def setUnableToMerge(self, item):
- item.current_build_set.unable_to_merge = True
- root = self.getJobTree(item.change.project)
- for job in root.getJobs():
- fakebuild = Build(job, None)
- fakebuild.result = 'SKIPPED'
- item.addBuild(fakebuild)
-
- def setDequeuedNeedingChange(self, item):
- item.dequeued_needing_change = True
- root = self.getJobTree(item.change.project)
- for job in root.getJobs():
- fakebuild = Build(job, None)
- fakebuild.result = 'SKIPPED'
- item.addBuild(fakebuild)
-
def getChangesInQueue(self):
changes = []
for shared_queue in self.queues:
@@ -840,6 +679,156 @@
return []
return self.job_tree.getJobs()
+ def haveAllJobsStarted(self):
+ if not self.hasJobTree():
+ return False
+ for job in self.getJobs():
+ build = self.current_build_set.getBuild(job.name)
+ if not build or not build.start_time:
+ return False
+ return True
+
+ def areAllJobsComplete(self):
+ if not self.hasJobTree():
+ return False
+ for job in self.getJobs():
+ build = self.current_build_set.getBuild(job.name)
+ if not build or not build.result:
+ return False
+ return True
+
+ def didAllJobsSucceed(self):
+ if not self.hasJobTree():
+ return False
+ for job in self.getJobs():
+ if not job.voting:
+ continue
+ build = self.current_build_set.getBuild(job.name)
+ if not build:
+ return False
+ if build.result != 'SUCCESS':
+ return False
+ return True
+
+ def didAnyJobFail(self):
+ if not self.hasJobTree():
+ return False
+ for job in self.getJobs():
+ if not job.voting:
+ continue
+ build = self.current_build_set.getBuild(job.name)
+ if build and build.result and (build.result != 'SUCCESS'):
+ return True
+ return False
+
+ def didMergerFail(self):
+ if self.current_build_set.unable_to_merge:
+ return True
+ return False
+
+ # TODOv3(jeblair): This method is currently unused, but it should
+ # be in order to support the Job.hold_following_changes attribute.
+ def isHoldingFollowingChanges(self):
+ if not self.live:
+ return False
+ if not self.hasJobTree():
+ return False
+ for job in self.getJobs():
+ if not job.hold_following_changes:
+ continue
+ build = self.current_build_set.getBuild(job.name)
+ if not build:
+ return True
+ if build.result != 'SUCCESS':
+ return True
+
+ if not self.item_ahead:
+ return False
+ return self.item_ahead.isHoldingFollowingChanges()
+
+ def _findJobsToRun(self, job_trees, mutex):
+ torun = []
+ for tree in job_trees:
+ job = tree.job
+ result = None
+ if job:
+ if not job.changeMatches(self.change):
+ continue
+ build = self.current_build_set.getBuild(job.name)
+ if build:
+ result = build.result
+ else:
+ # There is no build for the root of this job tree,
+ # so we should run it.
+ if mutex.acquire(self, job):
+ # If this job needs a mutex, either acquire it or make
+ # sure that we have it before running the job.
+ torun.append(job)
+ # If there is no job, this is a null job tree, and we should
+ # run all of its jobs.
+ if result == 'SUCCESS' or not job:
+ torun.extend(self._findJobsToRun(tree.job_trees, mutex))
+ return torun
+
+ def findJobsToRun(self, mutex):
+ if not self.live:
+ return []
+ tree = self.job_tree
+ if not tree:
+ return []
+ return self._findJobsToRun(tree.job_trees, mutex)
+
+ def _findJobsToRequest(self, job_trees):
+ toreq = []
+ for tree in job_trees:
+ job = tree.job
+ if job:
+ if not job.changeMatches(self.change):
+ continue
+ nodes = self.current_build_set.getJobNodes(job.name)
+ if nodes is None:
+ req = self.current_build_set.getJobNodeRequest(job.name)
+ if req is None:
+ toreq.append(job)
+ # If there is no job, this is a null job tree, and we should
+ # run all of its jobs.
+ if not job:
+ toreq.extend(self._findJobsToRequest(tree.job_trees))
+ return toreq
+
+ def findJobsToRequest(self):
+ if not self.live:
+ return []
+ tree = self.job_tree
+ if not tree:
+ return []
+ return self._findJobsToRequest(tree.job_trees)
+
+ def setResult(self, build):
+ if build.retry:
+ self.removeBuild(build)
+ elif build.result != 'SUCCESS':
+ # Get a JobTree from a Job so we can find only its dependent jobs
+ tree = self.job_tree.getJobTreeForJob(build.job)
+ for job in tree.getJobs():
+ fakebuild = Build(job, None)
+ fakebuild.result = 'SKIPPED'
+ self.addBuild(fakebuild)
+
+ def setDequeuedNeedingChange(self):
+ self.dequeued_needing_change = True
+ self._setAllJobsSkipped()
+
+ def setUnableToMerge(self):
+ self.current_build_set.unable_to_merge = True
+ self._setAllJobsSkipped()
+
+ def _setAllJobsSkipped(self):
+ for job in self.getJobs():
+ fakebuild = Build(job, None)
+ fakebuild.result = 'SKIPPED'
+ self.addBuild(fakebuild)
+
def formatJobResult(self, job, url_pattern=None):
build = self.current_build_set.getBuild(job.name)
result = build.result
@@ -956,7 +945,7 @@
'worker': worker,
})
- if self.pipeline.haveAllJobsStarted(self):
+ if self.haveAllJobsStarted():
ret['remaining_time'] = max_remaining
else:
ret['remaining_time'] = None
diff --git a/zuul/reporter/__init__.py b/zuul/reporter/__init__.py
index 97dfabc..d38eef2 100644
--- a/zuul/reporter/__init__.py
+++ b/zuul/reporter/__init__.py
@@ -60,6 +60,8 @@
}
return format_methods[self._action]
+ # TODOv3(jeblair): Consider removing pipeline argument in favor of
+ # item.pipeline
def _formatItemReport(self, pipeline, item):
"""Format a report from the given items. Usually to provide results to
a reporter taking free-form text."""
@@ -80,7 +82,7 @@
def _formatItemReportFailure(self, pipeline, item):
if item.dequeued_needing_change:
msg = 'This change depends on a change that failed to merge.\n'
- elif not pipeline.didMergerSucceed(item):
+ elif item.didMergerFail():
msg = pipeline.merge_failure_message
else:
msg = (pipeline.failure_message + '\n\n' +
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index eca7c54..516be80 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -502,16 +502,17 @@
project_name = item.change.project.name
item.change.project = new_pipeline.source.getProject(
project_name)
- item_jobs = new_pipeline.getJobs(item)
- for build in item.current_build_set.getBuilds():
- job = tenant.layout.jobs.get(build.job.name)
- if job and job in item_jobs:
- build.job = job
- else:
- item.removeBuild(build)
- builds_to_cancel.append(build)
- if not new_pipeline.manager.reEnqueueItem(item,
- last_head):
+ if new_pipeline.manager.reEnqueueItem(item,
+ last_head):
+ new_jobs = item.getJobs()
+ for build in item.current_build_set.getBuilds():
+ job = item.layout.getJob(build.job.name)
+ if job and job in new_jobs:
+ build.job = job
+ else:
+ item.removeBuild(build)
+ builds_to_cancel.append(build)
+ else:
items_to_remove.append(item)
for item in items_to_remove:
for build in item.current_build_set.getBuilds():