Ansible launcher: retry publisher sync tasks
We sometimes see errors rsyncing data from the node or to the
log server. Since these are all rsync commands, they are safe
to retry. Attempt all post playbook rsyncs up to 3 times with
a 30 second delay between each attempt.
Change-Id: I329e1f1f31d53d82799e3485a912b76e2249d03f
diff --git a/zuul/launcher/ansiblelaunchserver.py b/zuul/launcher/ansiblelaunchserver.py
index 2380e56..ea0aa4e 100644
--- a/zuul/launcher/ansiblelaunchserver.py
+++ b/zuul/launcher/ansiblelaunchserver.py
@@ -547,6 +547,11 @@
class NodeWorker(object):
+ retry_args = dict(register='task_result',
+ until='task_result.rc == 0',
+ retries=3,
+ delay=30)
+
def __init__(self, config, jobs, builds, sites, name, host,
description, labels, manager_name, zmq_send_queue,
termination_queue, keep_jobdir, callback_dir,
@@ -961,6 +966,8 @@
dest=os.path.join(scproot, '_zuul_ansible'))
task = dict(copy=copyargs,
delegate_to='127.0.0.1')
+ # This is a local copy and should not fail, so does
+ # not need a retry stanza.
tasks.append(task)
# Fetch the console log from the remote host.
@@ -982,10 +989,12 @@
task = dict(synchronize=syncargs)
if not scpfile.get('copy-after-failure'):
task['when'] = 'success'
+ task.update(self.retry_args)
tasks.append(task)
task = self._makeSCPTaskLocalAction(
site, scpfile, scproot, parameters)
+ task.update(self.retry_args)
tasks.append(task)
return tasks
@@ -1053,6 +1062,7 @@
syncargs['rsync_opts'] = rsync_opts
task = dict(synchronize=syncargs,
when='success')
+ task.update(self.retry_args)
tasks.append(task)
task = dict(shell='lftp -f %s' % ftpscript,
when='success',
@@ -1075,6 +1085,7 @@
script.write('open %s\n' % site['host'])
script.write('user %s %s\n' % (site['user'], site['pass']))
script.write('mirror -R %s %s\n' % (ftpsource, ftptarget))
+ task.update(self.retry_args)
tasks.append(task)
return tasks