Merge "Log cachecontrol info by default" into feature/zuulv3
diff --git a/.zuul.yaml b/.zuul.yaml
index 271dd02..5ebe2a7 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -1,3 +1,21 @@
+- nodeset:
+ name: zuul-two-node
+ nodes:
+ - name: controller
+ label: ubuntu-xenial
+ - name: node
+ label: ubuntu-xenial
+
+- job:
+ name: zuul-stream-functional
+ parent: multinode
+ nodes: zuul-two-node
+ pre-run: playbooks/zuul-stream/pre
+ run: playbooks/zuul-stream/functional
+ post-run: playbooks/zuul-stream/post
+ required-projects:
+ - openstack/ara
+
- project:
name: openstack-infra/zuul
check:
@@ -7,6 +25,7 @@
voting: false
- tox-pep8
- tox-py35
+ - zuul-stream-functional
gate:
jobs:
- tox-docs
@@ -14,4 +33,7 @@
- tox-py35
post:
jobs:
+ - publish-openstack-python-docs:
+ vars:
+ afs_target: 'infra/zuul'
- publish-openstack-python-branch-tarball
diff --git a/etc/status/public_html/jquery.zuul.js b/etc/status/public_html/jquery.zuul.js
index 1937cd5..c2cf279 100644
--- a/etc/status/public_html/jquery.zuul.js
+++ b/etc/status/public_html/jquery.zuul.js
@@ -554,11 +554,14 @@
}
$.each(changes, function (change_i, change) {
- var $change_box =
- format.change_with_status_tree(
- change, change_queue);
- $html.append($change_box);
- format.display_patchset($change_box);
+ // Only add a change when it has jobs
+ if (change.jobs.length > 0) {
+ var $change_box =
+ format.change_with_status_tree(
+ change, change_queue);
+ $html.append($change_box);
+ format.display_patchset($change_box);
+ }
});
});
});
diff --git a/playbooks/zuul-stream/fixtures/test-stream.yaml b/playbooks/zuul-stream/fixtures/test-stream.yaml
new file mode 100644
index 0000000..065e332
--- /dev/null
+++ b/playbooks/zuul-stream/fixtures/test-stream.yaml
@@ -0,0 +1,21 @@
+- name: Run some commands to show that logging works
+ hosts: node
+ tasks:
+
+ - name: Run setup
+ setup:
+ register: setupvar
+
+ - name: Output debug for a var
+ debug:
+ var: setupvar
+
+ - name: Run a shell task
+ command: ip addr show
+
+ - name: Loop with items
+ command: "echo {{ item }}"
+ with_items:
+ - item1
+ - item2
+ - item3
diff --git a/playbooks/zuul-stream/functional.yaml b/playbooks/zuul-stream/functional.yaml
new file mode 100644
index 0000000..727598c
--- /dev/null
+++ b/playbooks/zuul-stream/functional.yaml
@@ -0,0 +1,14 @@
+- hosts: controller
+ tasks:
+
+ - name: Run ansible
+ command: ansible-playbook -vvv src/git.openstack.org/openstack-infra/zuul/playbooks/zuul-stream/fixtures/test-stream.yaml
+ environment:
+ ZUUL_JOB_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json"
+ ARA_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json"
+
+ - name: Generate ARA html
+ command: ara generate html ara-output
+
+ - name: Compress ARA html
+ command: gzip --recursive --best ara-output
diff --git a/playbooks/zuul-stream/post.yaml b/playbooks/zuul-stream/post.yaml
new file mode 100644
index 0000000..f3d4f9c
--- /dev/null
+++ b/playbooks/zuul-stream/post.yaml
@@ -0,0 +1,26 @@
+- hosts: controller
+ tasks:
+
+ - set_fact:
+ output_dir: "{{ zuul.executor.log_root }}/stream-files"
+
+ - name: Make log subdir
+ file:
+ path: "{{ output_dir }}"
+ state: directory
+ delegate_to: localhost
+
+ - name: Rename job-output.txt
+ command: mv job-output.txt stream-job-output.txt
+
+ - name: Fetch files
+ synchronize:
+ src: "{{ ansible_user_dir }}/{{ item }}"
+ dest: "{{ output_dir }}"
+ mode: pull
+ with_items:
+ - logging.json
+ - ansible.cfg
+ - stream-job-output.txt
+ - job-output.json
+ - ara-output
diff --git a/playbooks/zuul-stream/pre.yaml b/playbooks/zuul-stream/pre.yaml
new file mode 100644
index 0000000..3f4bdf9
--- /dev/null
+++ b/playbooks/zuul-stream/pre.yaml
@@ -0,0 +1,30 @@
+- hosts: controller
+ roles:
+
+ - role: bindep
+ bindep_profile: test
+ bindep_dir: src/git.openstack.org/openstack-infra/zuul
+ bindep_command: /usr/bindep-env/bin/bindep
+
+ - role: bindep
+ bindep_dir: src/git.openstack.org/openstack/ara
+ bindep_command: /usr/bindep-env/bin/bindep
+
+ post_tasks:
+
+ - name: Install software
+ command: python3 -m pip install src/git.openstack.org/openstack-infra/zuul src/git.openstack.org/openstack/ara
+ become: yes
+
+ - name: Copy inventory
+ copy:
+ src: "{{ zuul.executor.log_root }}/zuul-info/inventory.yaml"
+ dest: "{{ ansible_user_dir }}/inventory.yaml"
+
+ - name: Copy ansible.cfg
+ template:
+ src: templates/ansible.cfg.j2
+ dest: "{{ ansible_user_dir }}/ansible.cfg"
+
+ - name: Generate logging config
+ command: python3 src/git.openstack.org/openstack-infra/zuul/zuul/ansible/logconfig.py
diff --git a/playbooks/zuul-stream/templates/ansible.cfg.j2 b/playbooks/zuul-stream/templates/ansible.cfg.j2
new file mode 100644
index 0000000..24f459e
--- /dev/null
+++ b/playbooks/zuul-stream/templates/ansible.cfg.j2
@@ -0,0 +1,11 @@
+[defaults]
+hostfile = {{ ansible_user_dir }}/inventory.yaml
+gathering = smart
+gather_subset = !all
+lookup_plugins = {{ ansible_user_dir }}/src/git.openstack.org/openstack-infra/zuul/zuul/ansible/lookup
+action_plugins = {{ ansible_user_dir }}/src/git.openstack.org/openstack-infra/zuul/zuul/ansible/action
+callback_plugins = {{ ansible_user_dir }}/src/git.openstack.org/openstack-infra/zuul/zuul/ansible/callback:{{ ansible_user_dir }}/src/git.openstack.org/openstack/ara/ara/plugins/callbacks
+module_utils = {{ ansible_user_dir }}/src/git.openstack.org/openstack-infra/zuul/zuul/ansible/module_utils
+stdout_callback = zuul_stream
+library = {{ ansible_user_dir }}/src/git.openstack.org/openstack-infra/zuul/zuul/ansible/library
+retry_files_enabled = False
diff --git a/tests/fixtures/config/ansible/git/org_plugin-project/playbooks/block_local_override.yaml b/tests/fixtures/config/ansible/git/org_plugin-project/playbooks/block_local_override.yaml
new file mode 100644
index 0000000..58613ad
--- /dev/null
+++ b/tests/fixtures/config/ansible/git/org_plugin-project/playbooks/block_local_override.yaml
@@ -0,0 +1,3 @@
+- hosts: localhost
+ roles:
+ - test-local-override
diff --git a/tests/fixtures/config/ansible/git/org_plugin-project/playbooks/file_local_bad.yaml b/tests/fixtures/config/ansible/git/org_plugin-project/playbooks/file_local_bad.yaml
new file mode 100644
index 0000000..b567dfe
--- /dev/null
+++ b/tests/fixtures/config/ansible/git/org_plugin-project/playbooks/file_local_bad.yaml
@@ -0,0 +1,6 @@
+- hosts: localhost
+ tasks:
+ - name: Try to verify a file in a bad location
+ file:
+ dest: /tmp/unreadable
+ state: absent
diff --git a/tests/fixtures/config/ansible/git/org_plugin-project/playbooks/file_local_good.yaml b/tests/fixtures/config/ansible/git/org_plugin-project/playbooks/file_local_good.yaml
new file mode 100644
index 0000000..29b5431
--- /dev/null
+++ b/tests/fixtures/config/ansible/git/org_plugin-project/playbooks/file_local_good.yaml
@@ -0,0 +1,6 @@
+- hosts: localhost
+ tasks:
+ - name: Try to verify a file in an ok location
+ file:
+ dest: non-existent
+ state: absent
diff --git a/tests/fixtures/config/ansible/git/org_plugin-project/playbooks/roles/test-local-override/library/file.py b/tests/fixtures/config/ansible/git/org_plugin-project/playbooks/roles/test-local-override/library/file.py
new file mode 100644
index 0000000..63478f7
--- /dev/null
+++ b/tests/fixtures/config/ansible/git/org_plugin-project/playbooks/roles/test-local-override/library/file.py
@@ -0,0 +1,31 @@
+# Copyright (c) 2017 Red Hat
+#
+# This module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software. If not, see <http://www.gnu.org/licenses/>.
+
+# This file, by existing, should be found instead of ansible's built in
+# file module.
+
+
+def main():
+ module = AnsibleModule(
+ argument_spec=dict(
+ path=dict(required=False, type='str'),
+ state=dict(required=False, type='dict'),
+ )
+ )
+
+ module.exit_json(changed=False)
+
+from ansible.module_utils.basic import * # noqa
+from ansible.module_utils.basic import AnsibleModule
diff --git a/tests/fixtures/config/ansible/git/org_plugin-project/playbooks/roles/test-local-override/tasks/main.yaml b/tests/fixtures/config/ansible/git/org_plugin-project/playbooks/roles/test-local-override/tasks/main.yaml
new file mode 100644
index 0000000..a06608b
--- /dev/null
+++ b/tests/fixtures/config/ansible/git/org_plugin-project/playbooks/roles/test-local-override/tasks/main.yaml
@@ -0,0 +1,4 @@
+- name: Attempt to use local version of file.py
+ file:
+ path: some-file.out
+ state: touch
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index 60a0986..9d695aa 100755
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -886,6 +886,9 @@
('csvfile_bad', 'FAILURE'),
('uri_bad_path', 'FAILURE'),
('uri_bad_scheme', 'FAILURE'),
+ ('block_local_override', 'FAILURE'),
+ ('file_local_good', 'SUCCESS'),
+ ('file_local_bad', 'FAILURE'),
]
for job_name, result in plugin_tests:
count += 1
diff --git a/zuul/ansible/action/assemble.py b/zuul/ansible/action/assemble.py
index 2cc7eb7..30920e2 100644
--- a/zuul/ansible/action/assemble.py
+++ b/zuul/ansible/action/assemble.py
@@ -22,6 +22,9 @@
def run(self, tmp=None, task_vars=None):
+ if not paths._is_official_module(self):
+ return paths._fail_module_dict(self._task.action)
+
source = self._task.args.get('src', None)
remote_src = self._task.args.get('remote_src', False)
diff --git a/zuul/ansible/action/copy.py b/zuul/ansible/action/copy.py
index d870c24..f00164d 100644
--- a/zuul/ansible/action/copy.py
+++ b/zuul/ansible/action/copy.py
@@ -22,6 +22,9 @@
def run(self, tmp=None, task_vars=None):
+ if not paths._is_official_module(self):
+ return paths._fail_module_dict(self._task.action)
+
source = self._task.args.get('src', None)
remote_src = self._task.args.get('remote_src', False)
diff --git a/zuul/ansible/action/fetch.py b/zuul/ansible/action/fetch.py
index 170b655..0d35846 100644
--- a/zuul/ansible/action/fetch.py
+++ b/zuul/ansible/action/fetch.py
@@ -21,6 +21,8 @@
class ActionModule(fetch.ActionModule):
def run(self, tmp=None, task_vars=None):
+ if not paths._is_official_module(self):
+ return paths._fail_module_dict(self._task.action)
dest = self._task.args.get('dest', None)
diff --git a/zuul/ansible/action/include_vars.py b/zuul/ansible/action/include_vars.py
index 5bc1d76..739c8a4 100644
--- a/zuul/ansible/action/include_vars.py
+++ b/zuul/ansible/action/include_vars.py
@@ -21,6 +21,8 @@
class ActionModule(include_vars.ActionModule):
def run(self, tmp=None, task_vars=None):
+ if not paths._is_official_module(self):
+ return paths._fail_module_dict(self._task.action)
source_dir = self._task.args.get('dir', None)
source_file = self._task.args.get('file', None)
diff --git a/zuul/ansible/action/normal.py b/zuul/ansible/action/normal.py
index 803b0a7..34df21d 100644
--- a/zuul/ansible/action/normal.py
+++ b/zuul/ansible/action/normal.py
@@ -50,6 +50,7 @@
handler_name = 'handle_{action}'.format(action=self._task.action)
handler = getattr(self, handler_name, None)
if handler:
+ paths._fail_if_local_module(self)
handler()
return True
return False
diff --git a/zuul/ansible/action/patch.py b/zuul/ansible/action/patch.py
index 0b43c82..cac6f2f 100644
--- a/zuul/ansible/action/patch.py
+++ b/zuul/ansible/action/patch.py
@@ -21,6 +21,8 @@
class ActionModule(patch.ActionModule):
def run(self, tmp=None, task_vars=None):
+ if not paths._is_official_module(self):
+ return paths._fail_module_dict(self._task.action)
source = self._task.args.get('src', None)
remote_src = self._task.args.get('remote_src', False)
diff --git a/zuul/ansible/action/script.py b/zuul/ansible/action/script.py
index c95d357..f67a73e 100644
--- a/zuul/ansible/action/script.py
+++ b/zuul/ansible/action/script.py
@@ -22,6 +22,8 @@
def run(self, tmp=None, task_vars=None):
+ if not paths._is_official_module(self):
+ return paths._fail_module_dict(self._task.action)
# the script name is the first item in the raw params, so we split it
# out now so we know the file name we need to transfer to the remote,
# and everything else is an argument to the script which we need later
diff --git a/zuul/ansible/action/synchronize.py b/zuul/ansible/action/synchronize.py
index 75fd45f..6e778d1 100644
--- a/zuul/ansible/action/synchronize.py
+++ b/zuul/ansible/action/synchronize.py
@@ -21,6 +21,8 @@
class ActionModule(synchronize.ActionModule):
def run(self, tmp=None, task_vars=None):
+ if not paths._is_official_module(self):
+ return paths._fail_module_dict(self._task.action)
source = self._task.args.get('src', None)
dest = self._task.args.get('dest', None)
diff --git a/zuul/ansible/action/template.py b/zuul/ansible/action/template.py
index c6df3d8..f2fbd36 100644
--- a/zuul/ansible/action/template.py
+++ b/zuul/ansible/action/template.py
@@ -21,6 +21,8 @@
class ActionModule(template.ActionModule):
def run(self, tmp=None, task_vars=None):
+ if not paths._is_official_module(self):
+ return paths._fail_module_dict(self._task.action)
source = self._task.args.get('src', None)
diff --git a/zuul/ansible/action/unarchive.py b/zuul/ansible/action/unarchive.py
index c78c331..5eb3d9f 100644
--- a/zuul/ansible/action/unarchive.py
+++ b/zuul/ansible/action/unarchive.py
@@ -21,6 +21,8 @@
class ActionModule(unarchive.ActionModule):
def run(self, tmp=None, task_vars=None):
+ if not paths._is_official_module(self):
+ return paths._fail_module_dict(self._task.action)
source = self._task.args.get('src', None)
remote_src = self._task.args.get('remote_src', False)
diff --git a/zuul/ansible/action/win_copy.py b/zuul/ansible/action/win_copy.py
index 2751585..8fb95be 100644
--- a/zuul/ansible/action/win_copy.py
+++ b/zuul/ansible/action/win_copy.py
@@ -21,6 +21,8 @@
class ActionModule(win_copy.ActionModule):
def run(self, tmp=None, task_vars=None):
+ if not paths._is_official_module(self):
+ return paths._fail_module_dict(self._task.action)
source = self._task.args.get('src', None)
remote_src = self._task.args.get('remote_src', False)
diff --git a/zuul/ansible/action/win_template.py b/zuul/ansible/action/win_template.py
index 7a357f9..1b6a890 100644
--- a/zuul/ansible/action/win_template.py
+++ b/zuul/ansible/action/win_template.py
@@ -21,6 +21,8 @@
class ActionModule(win_template.ActionModule):
def run(self, tmp=None, task_vars=None):
+ if not paths._is_official_module(self):
+ return paths._fail_module_dict(self._task.action)
source = self._task.args.get('src', None)
remote_src = self._task.args.get('remote_src', False)
diff --git a/zuul/ansible/paths.py b/zuul/ansible/paths.py
index bc61975..04daef4 100644
--- a/zuul/ansible/paths.py
+++ b/zuul/ansible/paths.py
@@ -17,6 +17,7 @@
import os
from ansible.errors import AnsibleError
+import ansible.modules
import ansible.plugins.action
import ansible.plugins.lookup
@@ -67,3 +68,30 @@
return imp.load_module(
'zuul.ansible.protected.lookup.' + name,
*imp.find_module(name, ansible.plugins.lookup.__path__))
+
+
+def _is_official_module(module):
+ task_module_path = module._shared_loader_obj.module_loader.find_plugin(
+ module._task.action)
+ ansible_module_path = os.path.dirname(ansible.modules.__file__)
+
+ # If the module is not beneath the main ansible library path that means
+ # someone has included a module with a playbook or a role that has the
+ # same name as one of the builtin modules. Normally we don't care, but for
+ # local execution it's a problem because their version could subvert our
+ # path checks and/or do other things on the local machine that we don't
+ # want them to do.
+ return task_module_path.startswith(ansible_module_path)
+
+
+def _fail_module_dict(module_name):
+ return dict(
+ failed=True,
+ msg="Local execution of overridden module {name} is forbidden".format(
+ name=module_name))
+
+
+def _fail_if_local_module(module):
+ if not _is_official_module(module):
+ msg_dict = _fail_module_dict(module._task.action)
+ raise AnsibleError(msg_dict['msg'])
diff --git a/zuul/configloader.py b/zuul/configloader.py
index ea1293f..94c0d2a 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -460,6 +460,8 @@
else:
secret_name = secret_config['name']
secret = layout.secrets[secret_config['secret']]
+ if secret_name == 'zuul':
+ raise Exception("Secrets named 'zuul' are not allowed.")
if secret.source_context != job.source_context:
raise Exception(
"Unable to use secret %s. Secrets must be "
@@ -574,6 +576,8 @@
variables = conf.get('vars', None)
if variables:
+ if 'zuul' in variables:
+ raise Exception("Variables named 'zuul' are not allowed.")
job.updateVariables(variables)
allowed_projects = conf.get('allowed-projects', None)
@@ -648,6 +652,7 @@
def getSchema(layout):
project_template = {
vs.Required('name'): str,
+ 'description': str,
'merge-mode': vs.Any(
'merge', 'merge-resolve',
'cherry-pick'),
@@ -717,6 +722,7 @@
def getSchema(layout):
project = {
vs.Required('name'): str,
+ 'description': str,
'templates': [str],
'merge-mode': vs.Any('merge', 'merge-resolve',
'cherry-pick'),
diff --git a/zuul/driver/bubblewrap/__init__.py b/zuul/driver/bubblewrap/__init__.py
index 149874b..7059351 100644
--- a/zuul/driver/bubblewrap/__init__.py
+++ b/zuul/driver/bubblewrap/__init__.py
@@ -212,6 +212,7 @@
for path in ['/lib64',
'/etc/nsswitch.conf',
'/etc/lsb-release.d',
+ '/etc/alternatives',
]:
if os.path.exists(path):
bwrap_command.extend(['--ro-bind', path, path])
diff --git a/zuul/driver/github/githubconnection.py b/zuul/driver/github/githubconnection.py
index 6a91ee8..0ce6ef5 100644
--- a/zuul/driver/github/githubconnection.py
+++ b/zuul/driver/github/githubconnection.py
@@ -727,10 +727,6 @@
else:
exclude_unprotected = tenant.exclude_unprotected_branches
- # TODO(mordred) Does it work for Github Apps to get repository
- # branches? If not, can we avoid doing this as an API for projects that
- # aren't trying to exclude protected branches by doing a git command
- # (gerrit driver does an upload pack over ssh)
github = self.getGithubClient(project.name)
try:
owner, proj = project.name.split('/')
@@ -859,10 +855,7 @@
def _getPullReviews(self, owner, project, number):
# make a list out of the reviews so that we complete our
# API transaction
- # reviews are not yet supported by integrations, use api_key:
- # https://platform.github.community/t/api-endpoint-for-pr-reviews/409
- github = self.getGithubClient("%s/%s" % (owner, project),
- use_app=False)
+ github = self.getGithubClient("%s/%s" % (owner, project))
reviews = [review.as_dict() for review in
github.pull_request(owner, project, number).reviews()]
@@ -1017,7 +1010,7 @@
reset = rate_limit['resources']['core']['reset']
except:
return
- if github._zuul_user:
+ if github._zuul_user_id:
log.debug('GitHub API rate limit (%s, %s) remaining: %s reset: %s',
github._zuul_project, github._zuul_user_id, remaining, reset)
else:
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index 96c809c..3daafc7 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -1281,6 +1281,8 @@
secrets = playbook['secrets']
if secrets:
if 'zuul' in secrets:
+ # We block this in configloader, but block it here too to make
+ # sure that a job doesn't pass secrets named zuul.
raise Exception("Defining secrets named 'zuul' is not allowed")
jobdir_playbook.secrets_content = yaml.safe_dump(
secrets, default_flow_style=False)
@@ -1385,6 +1387,8 @@
# TODO(mordred) Hack to work around running things with python3
all_vars['ansible_python_interpreter'] = '/usr/bin/python2'
if 'zuul' in all_vars:
+ # We block this in configloader, but block it here too to make
+ # sure that a job doesn't pass variables named zuul.
raise Exception("Defining vars named 'zuul' is not allowed")
all_vars['zuul'] = args['zuul'].copy()
all_vars['zuul']['executor'] = dict(