Return executor errors to user
There are some errors that the executor may encounter where it will
be unable to, or refuse to, run a job. We know that these errors
will not be corrected by retrying the build, so return them as
errors to the user. The build result will be "ERROR" and the message
which is brief, but hopefully sufficient to illuminate the problem,
will be added to the job report.
Change-Id: Iad486199de19583eb1e9f67c89a8ed8dac75dea1
Story: 2001105
Story: 2001106
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index f87773b..734c45c 100644
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -797,6 +797,30 @@
dict(name='project-test', result='SUCCESS', changes='1,1'),
])
+ def test_role_error(self):
+ conf = textwrap.dedent(
+ """
+ - job:
+ name: project-test
+ roles:
+ - zuul: common-config
+
+ - project:
+ name: org/project
+ check:
+ jobs:
+ - project-test
+ """)
+
+ file_dict = {'.zuul.yaml': conf}
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+ files=file_dict)
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertIn(
+ '- project-test project-test : ERROR Unable to find role',
+ A.messages[-1])
+
class TestShadow(ZuulTestCase):
tenant_config_file = 'config/shadow/main.yaml'
diff --git a/zuul/executor/client.py b/zuul/executor/client.py
index 2a205bf..9768bc1 100644
--- a/zuul/executor/client.py
+++ b/zuul/executor/client.py
@@ -376,6 +376,7 @@
build.node_name = data.get('node_name')
if result is None:
result = data.get('result')
+ build.error_detail = data.get('error_detail')
if result is None:
if (build.build_set.getTries(build.job.name) >=
build.job.attempts):
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index 824a47a..0246223 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -41,6 +41,16 @@
DEFAULT_FINGER_PORT = 79
+class ExecutorError(Exception):
+ """A non-transient run-time executor error
+
+ This class represents error conditions detected by the executor
+ when preparing to run a job which we know are consistently fatal.
+ Zuul should not reschedule the build in these cases.
+ """
+ pass
+
+
class Watchdog(object):
def __init__(self, timeout, function, args):
self.timeout = timeout
@@ -115,8 +125,8 @@
subprocess.check_output(['ssh-add', key_path], env=env,
stderr=subprocess.PIPE)
except subprocess.CalledProcessError as e:
- self.log.error('ssh-add failed. stdout: %s, stderr: %s',
- e.output, e.stderr)
+ self.log.exception('ssh-add failed. stdout: %s, stderr: %s',
+ e.output, e.stderr)
raise
self.log.info('Added SSH Key {}'.format(key_path))
@@ -744,6 +754,11 @@
self.executor_server.keep_jobdir,
str(self.job.unique))
self._execute()
+ except ExecutorError as e:
+ result_data = json.dumps(dict(result='ERROR',
+ error_detail=e.args[0]))
+ self.log.debug("Sending result: %s" % (result_data,))
+ self.job.sendWorkComplete(result_data)
except Exception:
self.log.exception("Exception while executing job")
self.job.sendWorkException(traceback.format_exc())
@@ -913,8 +928,9 @@
project_name, project_default_branch)
repo.checkoutLocalBranch(project_default_branch)
else:
- raise Exception("Project %s does not have the default branch %s" %
- (project_name, project_default_branch))
+ raise ExecutorError("Project %s does not have the "
+ "default branch %s" %
+ (project_name, project_default_branch))
def runPlaybooks(self, args):
result = None
@@ -1005,9 +1021,9 @@
'''
for entry in os.listdir(path):
if os.path.isdir(entry) and entry.endswith('_plugins'):
- raise Exception(
- "Ansible plugin dir %s found adjacent to playbook %s in"
- " non-trusted repo." % (entry, path))
+ raise ExecutorError(
+ "Ansible plugin dir %s found adjacent to playbook %s in "
+ "non-trusted repo." % (entry, path))
def findPlaybook(self, path, required=False, trusted=False):
for ext in ['.yaml', '.yml']:
@@ -1018,7 +1034,7 @@
self._blockPluginDirs(playbook_dir)
return fn
if required:
- raise Exception("Unable to find playbook %s" % path)
+ raise ExecutorError("Unable to find playbook %s" % path)
return None
def preparePlaybooks(self, args):
@@ -1036,7 +1052,7 @@
break
if self.jobdir.playbook is None:
- raise Exception("No valid playbook found")
+ raise ExecutorError("No valid playbook found")
for playbook in args['post_playbooks']:
jobdir_playbook = self.jobdir.addPostPlaybook()
@@ -1124,7 +1140,7 @@
self._blockPluginDirs(os.path.join(d, entry))
return d
# It is neither a bare role, nor a collection of roles
- raise Exception("Unable to find role in %s" % (path,))
+ raise ExecutorError("Unable to find role in %s" % (path,))
def prepareZuulRole(self, jobdir_playbook, role, args, root):
self.log.debug("Prepare zuul role for %s" % (role,))
@@ -1162,7 +1178,7 @@
link = os.path.join(root, name)
link = os.path.realpath(link)
if not link.startswith(os.path.realpath(root)):
- raise Exception("Invalid role name %s", name)
+ raise ExecutorError("Invalid role name %s", name)
os.symlink(path, link)
role_path = self.findRole(link, trusted=jobdir_playbook.trusted)
diff --git a/zuul/model.py b/zuul/model.py
index 3c07740..1df70db 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -1098,6 +1098,7 @@
self.url = None
self.result = None
self.result_data = {}
+ self.error_detail = None
self.build_set = None
self.execute_time = time.time()
self.start_time = None
@@ -1118,6 +1119,7 @@
def getSafeAttributes(self):
return Attributes(uuid=self.uuid,
result=self.result,
+ error_detail=self.error_detail,
result_data=self.result_data)
diff --git a/zuul/reporter/__init__.py b/zuul/reporter/__init__.py
index 95b9208..49181a7 100644
--- a/zuul/reporter/__init__.py
+++ b/zuul/reporter/__init__.py
@@ -138,7 +138,11 @@
elapsed = ' in %ds' % (s)
else:
elapsed = ''
+ if build.error_detail:
+ error = ' ' + build.error_detail
+ else:
+ error = ''
name = job.name + ' '
- ret += '- %s%s : %s%s%s\n' % (name, url, result, elapsed,
- voting)
+ ret += '- %s%s : %s%s%s%s\n' % (name, url, result, error,
+ elapsed, voting)
return ret