Merge "Create nodepool dictionary for ansible inventory" into feature/zuulv3
diff --git a/tests/base.py b/tests/base.py
index fae23fe..7d33ffc 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -935,7 +935,8 @@
'repo': {
'full_name': pr.project
}
- }
+ },
+ 'files': pr.files
}
return data
@@ -946,10 +947,6 @@
pr = prs[0]
return self.getPull(pr.project, pr.number)
- def getPullFileNames(self, project, number):
- pr = self.pull_requests[number - 1]
- return pr.files
-
def _getPullReviews(self, owner, project, number):
pr = self.pull_requests[number - 1]
return pr.reviews
diff --git a/zuul/ansible/callback/zuul_stream.py b/zuul/ansible/callback/zuul_stream.py
index 260f4ab..ed0306b 100644
--- a/zuul/ansible/callback/zuul_stream.py
+++ b/zuul/ansible/callback/zuul_stream.py
@@ -14,10 +14,10 @@
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
import datetime
-import multiprocessing
import logging
import os
import socket
+import threading
import time
import uuid
@@ -103,14 +103,14 @@
self._log = logging.getLogger('zuul.executor.ansible')
def _read_log(self, host, ip, log_id, task_name):
- self._log.debug("[%s] Starting to log %s for task %s"
- % (host, log_id, task_name))
+ self._display.display("[%s] Starting to log %s for task %s"
+ % (host, log_id, task_name))
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
while True:
try:
s.connect((ip, LOG_STREAM_PORT))
except Exception:
- self._log.debug("[%s] Waiting on logger" % host)
+ self._display.vvv("[%s] Waiting on logger" % host)
time.sleep(0.1)
continue
msg = "%s\n" % log_id
@@ -161,14 +161,22 @@
'ansible_host', play_vars[host].get(
'ansible_inventory_host'))
self._host_dict[host] = ip
- self._streamer = multiprocessing.Process(
+ self._streamer = threading.Thread(
target=self._read_log, args=(host, ip, log_id, task_name))
self._streamer.daemon = True
self._streamer.start()
- def v2_runner_on_failed(self, result, ignore_errors=False):
+ def _stop_streamer(self):
if self._streamer:
- self._streamer.join()
+ self._streamer.join(30)
+ if self._streamer.is_alive():
+ msg = "{now} | [Zuul] Log Stream did not terminate".format(
+ now=datetime.datetime.now())
+ self._log.info(msg)
+ self._display.display("WARNING: Streamer could not join")
+
+ def v2_runner_on_failed(self, result, ignore_errors=False):
+ self._stop_streamer()
if result._task.action in ('command', 'shell'):
zuul_filter_result(result._result)
self._handle_exception(result._result)
@@ -194,8 +202,7 @@
if result._task.action in ('include', 'include_role'):
return
- if self._streamer:
- self._streamer.join()
+ self._stop_streamer()
if result._result.get('changed', False):
status = 'changed'
diff --git a/zuul/ansible/library/command.py b/zuul/ansible/library/command.py
index 4b3a30f..39a961e 100644
--- a/zuul/ansible/library/command.py
+++ b/zuul/ansible/library/command.py
@@ -19,6 +19,10 @@
# You should have received a copy of the GNU General Public License
# along with this software. If not, see <http://www.gnu.org/licenses/>.
+ANSIBLE_METADATA = {'metadata_version': '1.0',
+ 'status': ['stableinterface'],
+ 'supported_by': 'core'}
+
# flake8: noqa
# This file shares a significant chunk of code with an upstream ansible
# function, run_command. The goal is to not have to fork quite so much
@@ -34,7 +38,7 @@
short_description: Executes a command on a remote node
version_added: historical
description:
- - The M(command) module takes the command name followed by a list of space-delimited arguments.
+ - The C(command) module takes the command name followed by a list of space-delimited arguments.
- The given command will be executed on all selected nodes. It will not be
processed through the shell, so variables like C($HOME) and operations
like C("<"), C(">"), C("|"), C(";") and C("&") will not work (use the M(shell)
@@ -76,30 +80,33 @@
- if command warnings are on in ansible.cfg, do not warn about this particular line if set to no/false.
required: false
notes:
- - If you want to run a command through the shell (say you are using C(<),
- C(>), C(|), etc), you actually want the M(shell) module instead. The
- M(command) module is much more secure as it's not affected by the user's
- environment.
- - " C(creates), C(removes), and C(chdir) can be specified after the command. For instance, if you only want to run a command if a certain file does not exist, use this."
+ - If you want to run a command through the shell (say you are using C(<), C(>), C(|), etc), you actually want the M(shell) module instead.
+ The C(command) module is much more secure as it's not affected by the user's environment.
+ - " C(creates), C(removes), and C(chdir) can be specified after the command.
+ For instance, if you only want to run a command if a certain file does not exist, use this."
author:
- Ansible Core Team
- Michael DeHaan
'''
EXAMPLES = '''
-# Example from Ansible Playbooks.
-- command: /sbin/shutdown -t now
+- name: return motd to registered var
+ command: cat /etc/motd
+ register: mymotd
-# Run the command if the specified file does not exist.
-- command: /usr/bin/make_database.sh arg1 arg2 creates=/path/to/database
+- name: Run the command if the specified file does not exist.
+ command: /usr/bin/make_database.sh arg1 arg2 creates=/path/to/database
-# You can also use the 'args' form to provide the options. This command
-# will change the working directory to somedir/ and will only run when
-# /path/to/database doesn't exist.
-- command: /usr/bin/make_database.sh arg1 arg2
+# You can also use the 'args' form to provide the options.
+- name: This command will change the working directory to somedir/ and will only run when /path/to/database doesn't exist.
+ command: /usr/bin/make_database.sh arg1 arg2
args:
chdir: somedir/
creates: /path/to/database
+
+- name: safely use templated variable to run command. Always use the quote filter to avoid injection issues.
+ command: cat {{ myfile|quote }}
+ register: myoutput
'''
import datetime
@@ -116,10 +123,19 @@
import threading
from ansible.module_utils.basic import AnsibleModule, heuristic_log_sanitize
-from ansible.module_utils.basic import get_exception
-# ZUUL: Hardcode python2 until we're on ansible 2.2
-from ast import literal_eval
-
+from ansible.module_utils.pycompat24 import get_exception, literal_eval
+from ansible.module_utils.six import (
+ PY2,
+ PY3,
+ b,
+ binary_type,
+ integer_types,
+ iteritems,
+ string_types,
+ text_type,
+)
+from ansible.module_utils.six.moves import map, reduce
+from ansible.module_utils._text import to_native, to_bytes, to_text
LOG_STREAM_FILE = '/tmp/console-{log_uuid}.log'
PASSWD_ARG_RE = re.compile(r'^[-]{0,2}pass[-]?(word|wd)?')
@@ -166,7 +182,7 @@
# Taken from ansible/module_utils/basic.py ... forking the method for now
# so that we can dive in and figure out how to make appropriate hook points
-def zuul_run_command(self, args, zuul_log_id, check_rc=False, close_fds=True, executable=None, data=None, binary_data=False, path_prefix=None, cwd=None, use_unsafe_shell=False, prompt_regex=None, environ_update=None):
+def zuul_run_command(self, args, zuul_log_id, check_rc=False, close_fds=True, executable=None, data=None, binary_data=False, path_prefix=None, cwd=None, use_unsafe_shell=False, prompt_regex=None, environ_update=None, umask=None, encoding='utf-8', errors='surrogate_or_strict'):
'''
Execute a command, returns rc, stdout, and stderr.
@@ -188,7 +204,27 @@
:kw prompt_regex: Regex string (not a compiled regex) which can be
used to detect prompts in the stdout which would otherwise cause
the execution to hang (especially if no input data is specified)
- :kwarg environ_update: dictionary to *update* os.environ with
+ :kw environ_update: dictionary to *update* os.environ with
+ :kw umask: Umask to be used when running the command. Default None
+ :kw encoding: Since we return native strings, on python3 we need to
+ know the encoding to use to transform from bytes to text. If you
+ want to always get bytes back, use encoding=None. The default is
+ "utf-8". This does not affect transformation of strings given as
+ args.
+ :kw errors: Since we return native strings, on python3 we need to
+ transform stdout and stderr from bytes to text. If the bytes are
+ undecodable in the ``encoding`` specified, then use this error
+ handler to deal with them. The default is ``surrogate_or_strict``
+ which means that the bytes will be decoded using the
+ surrogateescape error handler if available (available on all
+ python3 versions we support) otherwise a UnicodeError traceback
+ will be raised. This does not affect transformations of strings
+ given as args.
+ :returns: A 3-tuple of return code (integer), stdout (native string),
+ and stderr (native string). On python2, stdout and stderr are both
+ byte strings. On python3, stdout and stderr are text strings converted
+ according to the encoding and errors parameters. If you want byte
+ strings on python3, use encoding=None to turn decoding to text off.
'''
shell = False
@@ -196,13 +232,15 @@
if use_unsafe_shell:
args = " ".join([pipes.quote(x) for x in args])
shell = True
- elif isinstance(args, (str, unicode)) and use_unsafe_shell:
+ elif isinstance(args, (binary_type, text_type)) and use_unsafe_shell:
shell = True
- elif isinstance(args, (str, unicode)):
+ elif isinstance(args, (binary_type, text_type)):
# On python2.6 and below, shlex has problems with text type
- # ZUUL: Hardcode python2 until we're on ansible 2.2
- if isinstance(args, unicode):
- args = args.encode('utf-8')
+ # On python3, shlex needs a text type.
+ if PY2:
+ args = to_bytes(args, errors='surrogate_or_strict')
+ elif PY3:
+ args = to_text(args, errors='surrogateescape')
args = shlex.split(args)
else:
msg = "Argument 'args' to run_command must be list or string"
@@ -210,6 +248,11 @@
prompt_re = None
if prompt_regex:
+ if isinstance(prompt_regex, text_type):
+ if PY3:
+ prompt_regex = to_bytes(prompt_regex, errors='surrogateescape')
+ elif PY2:
+ prompt_regex = to_bytes(prompt_regex, errors='surrogate_or_strict')
try:
prompt_re = re.compile(prompt_regex, re.MULTILINE)
except re.error:
@@ -217,7 +260,7 @@
# expand things like $HOME and ~
if not shell:
- args = [ os.path.expanduser(os.path.expandvars(x)) for x in args if x is not None ]
+ args = [os.path.expanduser(os.path.expandvars(x)) for x in args if x is not None]
rc = 0
msg = None
@@ -245,9 +288,9 @@
# Clean out python paths set by ansiballz
if 'PYTHONPATH' in os.environ:
pypaths = os.environ['PYTHONPATH'].split(':')
- pypaths = [x for x in pypaths \
- if not x.endswith('/ansible_modlib.zip') \
- and not x.endswith('/debug_dir')]
+ pypaths = [x for x in pypaths
+ if not x.endswith('/ansible_modlib.zip') and
+ not x.endswith('/debug_dir')]
os.environ['PYTHONPATH'] = ':'.join(pypaths)
if not os.environ['PYTHONPATH']:
del os.environ['PYTHONPATH']
@@ -256,8 +299,13 @@
# in reporting later, which strips out things like
# passwords from the args list
to_clean_args = args
- # ZUUL: Hardcode python2 until we're on ansible 2.2
- if isinstance(args, (unicode, str)):
+ if PY2:
+ if isinstance(args, text_type):
+ to_clean_args = to_bytes(args)
+ else:
+ if isinstance(args, binary_type):
+ to_clean_args = to_text(args)
+ if isinstance(args, (text_type, binary_type)):
to_clean_args = shlex.split(to_clean_args)
clean_args = []
@@ -291,34 +339,36 @@
stderr=subprocess.STDOUT,
)
- if cwd and os.path.isdir(cwd):
- kwargs['cwd'] = cwd
-
# store the pwd
prev_dir = os.getcwd()
# make sure we're in the right working directory
if cwd and os.path.isdir(cwd):
+ cwd = os.path.abspath(os.path.expanduser(cwd))
+ kwargs['cwd'] = cwd
try:
os.chdir(cwd)
except (OSError, IOError):
e = get_exception()
self.fail_json(rc=e.errno, msg="Could not open %s, %s" % (cwd, str(e)))
- try:
+ old_umask = None
+ if umask:
+ old_umask = os.umask(umask)
+ try:
if self._debug:
- if isinstance(args, list):
- running = ' '.join(args)
- else:
- running = args
- self.log('Executing: ' + running)
+ self.log('Executing: ' + clean_args)
+ cmd = subprocess.Popen(args, **kwargs)
+
# ZUUL: Replaced the excution loop with the zuul_runner run function
cmd = subprocess.Popen(args, **kwargs)
t = threading.Thread(target=follow, args=(cmd.stdout, zuul_log_id))
t.daemon = True
t.start()
+
ret = cmd.wait()
+
# Give the thread that is writing the console log up to 10 seconds
# to catch up and exit. If it hasn't done so by then, it is very
# likely stuck in readline() because it spawed a child that is
@@ -334,19 +384,21 @@
# we can't close stdout (attempting to do so raises an
# exception) , so this is disabled.
# cmd.stdout.close()
+ # cmd.stderr.close()
# ZUUL: stdout and stderr are in the console log file
# ZUUL: return the saved log lines so we can ship them back
- stdout = ''.join(_log_lines)
- stderr = ''
+ stdout = b('').join(_log_lines)
+ stderr = b('')
rc = cmd.returncode
except (OSError, IOError):
e = get_exception()
- self.fail_json(rc=e.errno, msg=str(e), cmd=clean_args)
+ self.log("Error Executing CMD:%s Exception:%s" % (clean_args, to_native(e)))
+ self.fail_json(rc=e.errno, msg=to_native(e), cmd=clean_args)
except Exception:
- e = get_exception()
- self.fail_json(rc=257, msg=str(e), exception=traceback.format_exc(), cmd=clean_args)
+ self.log("Error Executing CMD:%s Exception:%s" % (clean_args, to_native(traceback.format_exc())))
+ self.fail_json(rc=257, msg=to_native(e), exception=traceback.format_exc(), cmd=clean_args)
# Restore env settings
for key, val in old_env_vals.items():
@@ -355,6 +407,9 @@
else:
os.environ[key] = val
+ if old_umask:
+ os.umask(old_umask)
+
if rc != 0 and check_rc:
msg = heuristic_log_sanitize(stderr.rstrip(), self.no_log_values)
self.fail_json(cmd=clean_args, rc=rc, stdout=stdout, stderr=stderr, msg=msg)
@@ -362,6 +417,9 @@
# reset the pwd
os.chdir(prev_dir)
+ if encoding is not None:
+ return (rc, to_native(stdout, encoding=encoding, errors=errors),
+ to_native(stderr, encoding=encoding, errors=errors))
return (rc, stdout, stderr)
@@ -392,24 +450,24 @@
# hence don't copy this one if you are looking to build others!
module = AnsibleModule(
argument_spec=dict(
- _raw_params = dict(),
- _uses_shell = dict(type='bool', default=False),
- chdir = dict(type='path'),
- executable = dict(),
- creates = dict(type='path'),
- removes = dict(type='path'),
- warn = dict(type='bool', default=True),
- environ = dict(type='dict', default=None),
- zuul_log_id = dict(type='str'),
+ _raw_params = dict(),
+ _uses_shell = dict(type='bool', default=False),
+ chdir = dict(type='path'),
+ executable = dict(),
+ creates = dict(type='path'),
+ removes = dict(type='path'),
+ warn = dict(type='bool', default=True),
+ environ = dict(type='dict', default=None),
+ zuul_log_id = dict(type='str'),
)
)
shell = module.params['_uses_shell']
chdir = module.params['chdir']
executable = module.params['executable']
- args = module.params['_raw_params']
- creates = module.params['creates']
- removes = module.params['removes']
+ args = module.params['_raw_params']
+ creates = module.params['creates']
+ removes = module.params['removes']
warn = module.params['warn']
environ = module.params['environ']
zuul_log_id = module.params['zuul_log_id']
@@ -434,9 +492,9 @@
)
if removes:
- # do not run the command if the line contains removes=filename
- # and the filename does not exist. This allows idempotence
- # of command executions.
+ # do not run the command if the line contains removes=filename
+ # and the filename does not exist. This allows idempotence
+ # of command executions.
if not glob.glob(removes):
module.exit_json(
cmd=args,
@@ -453,20 +511,20 @@
args = shlex.split(args)
startd = datetime.datetime.now()
- rc, out, err = zuul_run_command(module, args, zuul_log_id, executable=executable, use_unsafe_shell=shell, environ_update=environ)
+ rc, out, err = zuul_run_command(module, args, zuul_log_id, executable=executable, use_unsafe_shell=shell, encoding=None, environ_update=environ)
endd = datetime.datetime.now()
delta = endd - startd
if out is None:
- out = ''
+ out = b('')
if err is None:
- err = ''
+ err = b('')
module.exit_json(
cmd = args,
- stdout = out.rstrip("\r\n"),
- stderr = err.rstrip("\r\n"),
+ stdout = out.rstrip(b("\r\n")),
+ stderr = err.rstrip(b("\r\n")),
rc = rc,
start = str(startd),
end = str(endd),
diff --git a/zuul/driver/bubblewrap/__init__.py b/zuul/driver/bubblewrap/__init__.py
index c93e912..7c9d48d 100644
--- a/zuul/driver/bubblewrap/__init__.py
+++ b/zuul/driver/bubblewrap/__init__.py
@@ -152,6 +152,8 @@
def main(args=None):
+ logging.basicConfig(level=logging.DEBUG)
+
driver = BubblewrapDriver()
parser = argparse.ArgumentParser()
diff --git a/zuul/driver/github/githubconnection.py b/zuul/driver/github/githubconnection.py
index 659d88b..d3779d6 100644
--- a/zuul/driver/github/githubconnection.py
+++ b/zuul/driver/github/githubconnection.py
@@ -118,6 +118,12 @@
event = None
if event:
+ if event.change_number:
+ project = self.connection.source.getProject(event.project_name)
+ self.connection._getChange(project,
+ event.change_number,
+ event.patch_number,
+ refresh=True)
event.project_hostname = self.connection.canonical_hostname
self.connection.logEvent(event)
self.connection.sched.addEvent(event)
@@ -247,7 +253,7 @@
raise webob.exc.HTTPUnauthorized(
'Please specify a X-Hub-Signature header with secret.')
- payload_signature = 'sha1=' + hmac.new(secret,
+ payload_signature = 'sha1=' + hmac.new(secret.encode('utf-8'),
body,
hashlib.sha1).hexdigest()
@@ -463,28 +469,20 @@
if change not in relevant:
del self._change_cache[key]
- def getChange(self, event):
+ def getChange(self, event, refresh=False):
"""Get the change representing an event."""
project = self.source.getProject(event.project_name)
if event.change_number:
- change = PullRequest(event.project_name)
- change.project = project
- change.number = event.change_number
+ change = self._getChange(project, event.change_number,
+ event.patch_number, refresh=refresh)
change.refspec = event.refspec
change.branch = event.branch
change.url = event.change_url
change.updated_at = self._ghTimestampToDate(event.updated_at)
- change.patchset = event.patch_number
- change.files = self.getPullFileNames(project, change.number)
- change.title = event.title
- change.status = self._get_statuses(project, event.patch_number)
- change.reviews = self.getPullReviews(project, change.number)
change.source_event = event
- change.open = self.getPullOpen(event.project_name, change.number)
- change.is_current_patchset = self.getIsCurrent(event.project_name,
- change.number,
- event.patch_number)
+ change.is_current_patchset = (change.pr.get('head').get('sha') ==
+ event.patch_number)
elif event.ref:
change = Ref(project)
change.ref = event.ref
@@ -497,6 +495,38 @@
change = Ref(project)
return change
+ def _getChange(self, project, number, patchset, refresh=False):
+ key = '%s/%s/%s' % (project.name, number, patchset)
+ change = self._change_cache.get(key)
+ if change and not refresh:
+ return change
+ if not change:
+ change = PullRequest(project.name)
+ change.project = project
+ change.number = number
+ change.patchset = patchset
+ self._change_cache[key] = change
+ try:
+ self._updateChange(change)
+ except Exception:
+ if key in self._change_cache:
+ del self._change_cache[key]
+ raise
+ return change
+
+ def _updateChange(self, change):
+ self.log.info("Updating %s" % (change,))
+ change.pr = self.getPull(change.project.name, change.number)
+ change.files = change.pr.get('files')
+ change.title = change.pr.get('title')
+ change.open = change.pr.get('state') == 'open'
+ change.status = self._get_statuses(change.project,
+ change.patchset)
+ change.reviews = self.getPullReviews(change.project,
+ change.number)
+
+ return change
+
def getGitUrl(self, project):
if self.git_ssh_key:
return 'ssh://git@%s/%s.git' % (self.git_host, project)
@@ -535,7 +565,9 @@
def getPull(self, project_name, number):
github = self.getGithubClient(project_name)
owner, proj = project_name.split('/')
- pr = github.pull_request(owner, proj, number).as_dict()
+ probj = github.pull_request(owner, proj, number)
+ pr = probj.as_dict()
+ pr['files'] = [f.filename for f in probj.files()]
log_rate_limit(self.log, github)
return pr
@@ -578,14 +610,6 @@
return None
return pulls.pop()
- def getPullFileNames(self, project, number):
- github = self.getGithubClient(project)
- owner, proj = project.name.split('/')
- filenames = [f.filename for f in
- github.pull_request(owner, proj, number).files()]
- log_rate_limit(self.log, github)
- return filenames
-
def getPullReviews(self, project, number):
owner, proj = project.name.split('/')
@@ -723,14 +747,6 @@
pull_request.remove_label(label)
log_rate_limit(self.log, github)
- def getPullOpen(self, project, number):
- pr = self.getPull(project, number)
- return pr.get('state') == 'open'
-
- def getIsCurrent(self, project, number, sha):
- pr = self.getPull(project, number)
- return pr.get('head').get('sha') == sha
-
def getPushedFileNames(self, event):
files = set()
for c in event.commits:
diff --git a/zuul/driver/github/githubsource.py b/zuul/driver/github/githubsource.py
index 1350b10..1c2f727 100644
--- a/zuul/driver/github/githubsource.py
+++ b/zuul/driver/github/githubsource.py
@@ -58,8 +58,8 @@
"""Called after configuration has been processed."""
pass
- def getChange(self, event):
- return self.connection.getChange(event)
+ def getChange(self, event, refresh=False):
+ return self.connection.getChange(event, refresh)
def getProject(self, name):
p = self.connection.getProject(name)
@@ -87,10 +87,6 @@
"""Get the git-web url for a project."""
return self.connection.getGitwebUrl(project, sha)
- def getPullFiles(self, project, number):
- """Get filenames of the pull request"""
- return self.connection.getPullFileNames(project, number)
-
def _ghTimestampToDate(self, timestamp):
return time.strptime(timestamp, '%Y-%m-%dT%H:%M:%SZ')
diff --git a/zuul/lib/log_streamer.py b/zuul/lib/log_streamer.py
index 59d5240..6695723 100644
--- a/zuul/lib/log_streamer.py
+++ b/zuul/lib/log_streamer.py
@@ -144,6 +144,11 @@
else:
break
+ # See if the file has been removed, meaning we should stop
+ # streaming it.
+ if not os.path.exists(log.path):
+ return False
+
# At this point, we are waiting for more data to be written
time.sleep(0.5)
@@ -159,16 +164,6 @@
if not ret:
return False
- # See if the file has been truncated
- try:
- st = os.stat(log.path)
- if (st.st_ino != log.stat.st_ino or
- st.st_size < log.size):
- return True
- except Exception:
- return True
- log.size = st.st_size
-
class CustomForkingTCPServer(ss.ForkingTCPServer):
'''