Merge "Require a base job" into feature/zuulv3
diff --git a/bindep.txt b/bindep.txt
index 8dffd0f..85254b4 100644
--- a/bindep.txt
+++ b/bindep.txt
@@ -8,6 +8,7 @@
zookeeperd [platform:dpkg]
build-essential [platform:dpkg]
gcc [platform:rpm]
+graphviz [test]
libssl-dev [platform:dpkg]
openssl-devel [platform:rpm]
libffi-dev [platform:dpkg]
diff --git a/doc/source/admin/components.rst b/doc/source/admin/components.rst
index 890405d..aa6d8c8 100644
--- a/doc/source/admin/components.rst
+++ b/doc/source/admin/components.rst
@@ -6,11 +6,37 @@
==========
Zuul is a distributed system consisting of several components, each of
-which is described below. All Zuul processes read the
-``/etc/zuul/zuul.conf`` file (an alternate location may be supplied on
-the command line) which uses an INI file syntax. Each component may
-have its own configuration file, though you may find it simpler to use
-the same file for all components.
+which is described below.
+
+
+.. graphviz::
+ :align: center
+
+ graph {
+ node [shape=box]
+ Gearman [shape=ellipse]
+ Gerrit [fontcolor=grey]
+ Zookeeper [shape=ellipse]
+ Nodepool
+ GitHub [fontcolor=grey]
+
+ Merger -- Gearman
+ Executor -- Gearman
+ Web -- Gearman
+
+ Gearman -- Scheduler;
+ Scheduler -- Gerrit;
+ Scheduler -- Zookeeper;
+ Zookeeper -- Nodepool;
+ Scheduler -- GitHub;
+ }
+
+
+
+All Zuul processes read the ``/etc/zuul/zuul.conf`` file (an alternate
+location may be supplied on the command line) which uses an INI file
+syntax. Each component may have its own configuration file, though
+you may find it simpler to use the same file for all components.
An example ``zuul.conf``:
diff --git a/doc/source/conf.py b/doc/source/conf.py
index 7c0d587..fa00593 100644
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -27,14 +27,17 @@
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = [
'sphinx.ext.autodoc',
+ 'sphinx.ext.graphviz',
'sphinxcontrib.blockdiag',
'sphinxcontrib.programoutput',
+ 'zuul_sphinx',
+ 'zuul.sphinx.ansible',
'zuul.sphinx.zuul',
]
#extensions = ['sphinx.ext.intersphinx']
#intersphinx_mapping = {'python': ('http://docs.python.org/2.7', None)}
-primary_domain = 'zuul'
+primary_domain = 'zuuldoc'
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
@@ -96,7 +99,9 @@
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
-#html_theme_options = {}
+html_theme_options = {
+ 'show_related': True
+}
# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []
diff --git a/doc/source/developer/ansible.rst b/doc/source/developer/ansible.rst
new file mode 100644
index 0000000..e3ebca7
--- /dev/null
+++ b/doc/source/developer/ansible.rst
@@ -0,0 +1,66 @@
+Ansible Integration
+===================
+
+Zuul contains Ansible modules and plugins to control the execution of Ansible
+Job content. These break down into two basic categories.
+
+* Restricted Execution on Executors
+* Build Log Support
+
+Restricted Execution
+--------------------
+
+Zuul runs ``ansible-playbook`` on executors to run job content on nodes. While
+the intent is that content is run on the remote nodes, Ansible is a flexible
+system that allows delegating actions to ``localhost``, and also reading and
+writing files. These actions can be desirable and necessary for actions such
+as fetching log files or build artifacts, but could also be used as a vector
+to attack the executor.
+
+For that reason Zuul implements a set of Ansible action plugins and lookup
+plugins that override and intercept task execution during untrusted playbook
+execution to ensure local actions are not executed or that for operations that
+are desirable to allow locally that they only interact with files in the zuul
+work directory.
+
+.. autoclass:: zuul.ansible.action.normal.ActionModule
+ :members:
+
+Build Log Support
+-----------------
+
+Zuul provides realtime build log streaming to end users so that users can
+watch long-running jobs in progress. As jobs may be written that execute a
+shell script that could run for a long time, additional effort is expended
+to stream stdout and stderr of shell tasks as they happen rather than waiting
+for the command to finish.
+
+Zuul contains a modified version of the :ansible:module:`command`
+that starts a log streaming daemon on the build node.
+
+.. automodule:: zuul.ansible.library.command
+
+All jobs run with the :py:mod:`zuul.ansible.callback.zuul_stream` callback
+plugin enabled, which writes the build log to a file so that the
+:py:class:`zuul.lib.log_streamer.LogStreamer` can provide the data on demand
+over the finger protocol. Finally, :py:class:`zuul.web.LogStreamingHandler`
+exposes that log stream over a websocket connection as part of
+:py:class:`zuul.web.ZuulWeb`.
+
+.. autoclass:: zuul.ansible.callback.zuul_stream.CallbackModule
+ :members:
+
+.. autoclass:: zuul.lib.log_streamer.LogStreamer
+.. autoclass:: zuul.web.LogStreamingHandler
+.. autoclass:: zuul.web.ZuulWeb
+
+In addition to real-time streaming, Zuul also installs another callback module,
+:py:mod:`zuul.ansible.callback.zuul_json.CallbackModule` that collects all
+of the information about a given run into a json file which is written to the
+work dir so that it can be published along with build logs. Since the streaming
+log is by necessity a single text stream, choices have to be made for
+readability about what data is shown and what is not shown. The json log file
+is intended to allow for a richer more interactive set of data to be displayed
+to the user.
+
+.. autoclass:: zuul.ansible.callback.zuul_json.CallbackModule
diff --git a/doc/source/developer/index.rst b/doc/source/developer/index.rst
index 7b16e9c..360dcd5 100644
--- a/doc/source/developer/index.rst
+++ b/doc/source/developer/index.rst
@@ -15,3 +15,4 @@
triggers
testing
docs
+ ansible
diff --git a/test-requirements.txt b/test-requirements.txt
index dcc67e2..bf8b979 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -14,3 +14,4 @@
mock
PyMySQL
mypy
+zuul-sphinx
diff --git a/tests/fixtures/config/ansible/git/org_plugin-project/playbooks/uri_bad_path.yaml b/tests/fixtures/config/ansible/git/org_plugin-project/playbooks/uri_bad_path.yaml
new file mode 100644
index 0000000..523aab7
--- /dev/null
+++ b/tests/fixtures/config/ansible/git/org_plugin-project/playbooks/uri_bad_path.yaml
@@ -0,0 +1,6 @@
+- hosts: localhost
+ tasks:
+ - uri:
+ method: GET
+ url: https://example.com
+ path: /tmp/example.out
diff --git a/tests/fixtures/config/ansible/git/org_plugin-project/playbooks/uri_bad_scheme.yaml b/tests/fixtures/config/ansible/git/org_plugin-project/playbooks/uri_bad_scheme.yaml
new file mode 100644
index 0000000..5d71793
--- /dev/null
+++ b/tests/fixtures/config/ansible/git/org_plugin-project/playbooks/uri_bad_scheme.yaml
@@ -0,0 +1,5 @@
+- hosts: localhost
+ tasks:
+ - uri:
+ method: GET
+ url: file:///etc/passwd
diff --git a/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml b/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
index c92b232..9796fe2 100644
--- a/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
@@ -47,6 +47,9 @@
- job:
name: project-merge
hold-following-changes: true
+ nodes:
+ - name: controller
+ label: label1
- job:
name: project-test1
@@ -70,9 +73,15 @@
- job:
name: project-test2
+ nodes:
+ - name: controller
+ label: label1
- job:
name: project1-project2-integration
+ nodes:
+ - name: controller
+ label: label1
- job:
name: project-testfile
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index 93367b9..e80a30a 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -62,7 +62,8 @@
self.assertEqual(A.reported, 2)
self.assertEqual(self.getJobFromHistory('project-test1').node,
'label1')
- self.assertIsNone(self.getJobFromHistory('project-test2').node)
+ self.assertEqual(self.getJobFromHistory('project-test2').node,
+ 'label1')
class TestScheduler(ZuulTestCase):
@@ -85,7 +86,8 @@
self.assertEqual(A.reported, 2)
self.assertEqual(self.getJobFromHistory('project-test1').node,
'label1')
- self.assertIsNone(self.getJobFromHistory('project-test2').node)
+ self.assertEqual(self.getJobFromHistory('project-test2').node,
+ 'label1')
# TODOv3(jeblair): we may want to report stats by tenant (also?).
# Per-driver
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index 15cb561..74d72e7 100755
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -791,6 +791,8 @@
('credstash', 'FAILURE'),
('csvfile_good', 'SUCCESS'),
('csvfile_bad', 'FAILURE'),
+ ('uri_bad_path', 'FAILURE'),
+ ('uri_bad_scheme', 'FAILURE'),
]
for job_name, result in plugin_tests:
count += 1
diff --git a/zuul/ansible/action/normal.py b/zuul/ansible/action/normal.py
index 74e732e..b8a232b 100644
--- a/zuul/ansible/action/normal.py
+++ b/zuul/ansible/action/normal.py
@@ -1,4 +1,4 @@
-# Copyright 2016 Red Hat, Inc.
+# Copyright 2017 Red Hat, Inc.
#
# This module is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -13,13 +13,27 @@
# You should have received a copy of the GNU General Public License
# along with this software. If not, see <http://www.gnu.org/licenses/>.
+from ansible.module_utils.six.moves.urllib.parse import urlparse
+from ansible.errors import AnsibleError
+
from zuul.ansible import paths
normal = paths._import_ansible_action_plugin('normal')
+ALLOWED_URL_SCHEMES = ('https', 'http', 'ftp')
+
class ActionModule(normal.ActionModule):
+ '''Override the normal action plugin
+
+ :py:class:`ansible.plugins.normal.ActionModule` is run for every
+ module that does not have a more specific matching action plugin.
+
+ Our overridden version of it wraps the execution with checks to block
+ undesired actions on localhost.
+ '''
def run(self, tmp=None, task_vars=None):
+ '''Overridden primary method from the base class.'''
if (self._play_context.connection == 'local'
or self._play_context.remote_addr == 'localhost'
@@ -27,16 +41,61 @@
or self._task.delegate_to == 'localhost'
or (self._task.delegate_to
and self._task.delegate_to.startswtih('127.'))):
- if self._task.action == 'stat':
- paths._fail_if_unsafe(self._task.args['path'])
- elif self._task.action == 'file':
- dest = self._task.args.get(
- 'path', self._task.args.get(
- 'dest', self._task.args.get(
- 'name')))
- paths._fail_if_unsafe(dest)
- else:
- return dict(
- failed=True,
- msg="Executing local code is prohibited")
+ if not self.dispatch_handler():
+ raise AnsibleError("Executing local code is prohibited")
return super(ActionModule, self).run(tmp, task_vars)
+
+ def dispatch_handler(self):
+ '''Run per-action handler if one exists.'''
+ handler_name = 'handle_{action}'.format(action=self._task.action)
+ handler = getattr(self, handler_name, None)
+ if handler:
+ handler(self)
+ return True
+ return False
+
+ def handle_stat(self):
+ '''Allow stat module on localhost if it doesn't touch unsafe files.
+
+ The :ansible:module:`stat` can be useful in jobs for manipulating logs
+ and artifacts.
+
+ Block any access of files outside the zuul work dir.
+ '''
+ paths._fail_if_unsafe(self._task.args['path'])
+
+ def handle_file(self):
+ '''Allow file module on localhost if it doesn't touch unsafe files.
+
+ The :ansible:module:`file` can be useful in jobs for manipulating logs
+ and artifacts.
+
+ Block any access of files outside the zuul work dir.
+ '''
+ for arg in ('path', 'dest', 'name'):
+ dest = self._task.args.get(arg)
+ if dest:
+ paths._fail_if_unsafe(dest)
+
+ def handle_uri(self):
+ '''Allow uri module on localhost if it doesn't touch unsafe files.
+
+ The :ansible:module:`uri` can be used from the executor to do
+ things like pinging readthedocs.org that otherwise don't need a node.
+ However, it can also download content to a local file, or be used to
+ read from file:/// urls.
+
+ Block any use of url schemes other than https, http and ftp. Further,
+ block any local file interaction that falls outside of the zuul
+ work dir.
+ '''
+ # uri takes all the file arguments, so just let handle_file validate
+ # them for us.
+ self.handle_file()
+ scheme = urlparse(self._task.args['url']).scheme
+ if scheme not in ALLOWED_URL_SCHEMES:
+ raise AnsibleError(
+ "{scheme} urls are not allowed from localhost."
+ " Only {allowed_schemes} are allowed".format(
+ scheme=scheme,
+ allowed_schemes=ALLOWED_URL_SCHEMES))
diff --git a/zuul/executor/client.py b/zuul/executor/client.py
index 85ae68c..fbefa8d 100644
--- a/zuul/executor/client.py
+++ b/zuul/executor/client.py
@@ -409,6 +409,10 @@
del self.builds[job.unique]
except:
pass
+ # Since this isn't otherwise going to get a build complete
+ # event, send one to the scheduler so that it can unlock
+ # the nodes.
+ self.sched.onBuildCompleted(build, 'CANCELED', {})
return True
return False
diff --git a/zuul/nodepool.py b/zuul/nodepool.py
index 0696c60..dc855cd 100644
--- a/zuul/nodepool.py
+++ b/zuul/nodepool.py
@@ -29,10 +29,15 @@
req = model.NodeRequest(self.sched.hostname, build_set, job, nodeset)
self.requests[req.uid] = req
- self.sched.zk.submitNodeRequest(req, self._updateNodeRequest)
- # Logged after submission so that we have the request id
- self.log.info("Submited node request %s" % (req,))
-
+ if nodeset.nodes:
+ self.sched.zk.submitNodeRequest(req, self._updateNodeRequest)
+ # Logged after submission so that we have the request id
+ self.log.info("Submited node request %s" % (req,))
+ else:
+ self.log.info("Fulfilling empty node request %s" % (req,))
+ req.state = model.STATE_FULFILLED
+ self.sched.onNodesProvisioned(req)
+ del self.requests[req.uid]
return req
def cancelRequest(self, request):
diff --git a/zuul/sphinx/ansible.py b/zuul/sphinx/ansible.py
new file mode 100644
index 0000000..4a47bc3
--- /dev/null
+++ b/zuul/sphinx/ansible.py
@@ -0,0 +1,53 @@
+# Copyright 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from docutils import nodes
+from sphinx.domains import Domain
+
+MODULE_URL = 'http://docs.ansible.com/ansible/latest/{module_name}_module.html'
+
+
+def ansible_module_role(
+ name, rawtext, text, lineno, inliner, options={}, content=[]):
+ """Link to an upstream Ansible module.
+
+ Returns 2 part tuple containing list of nodes to insert into the
+ document and a list of system messages. Both are allowed to be
+ empty.
+
+ :param name: The role name used in the document.
+ :param rawtext: The entire markup snippet, with role.
+ :param text: The text marked with the role.
+ :param lineno: The line number where rawtext appears in the input.
+ :param inliner: The inliner instance that called us.
+ :param options: Directive options for customization.
+ :param content: The directive content for customization.
+ """
+ node = nodes.reference(
+ rawtext, "Ansible {module_name} module".format(module_name=text),
+ refuri=MODULE_URL.format(module_name=text), **options)
+ return ([node], [])
+
+
+class AnsibleDomain(Domain):
+ name = 'ansible'
+ label = 'Ansible'
+
+ roles = {
+ 'module': ansible_module_role,
+ }
+
+
+def setup(app):
+ app.add_domain(AnsibleDomain)
diff --git a/zuul/sphinx/zuul.py b/zuul/sphinx/zuul.py
index 0ac33b8..575bcb2 100644
--- a/zuul/sphinx/zuul.py
+++ b/zuul/sphinx/zuul.py
@@ -29,10 +29,10 @@
}
def get_path(self):
- return self.env.ref_context.get('zuul:attr_path', [])
+ return self.env.ref_context.get('zuuldoc:attr_path', [])
def get_display_path(self):
- return self.env.ref_context.get('zuul:display_attr_path', [])
+ return self.env.ref_context.get('zuuldoc:display_attr_path', [])
@property
def parent_pathname(self):
@@ -50,7 +50,7 @@
signode['ids'].append(targetname)
signode['first'] = (not self.names)
self.state.document.note_explicit_target(signode)
- objects = self.env.domaindata['zuul']['objects']
+ objects = self.env.domaindata['zuuldoc']['objects']
if targetname in objects:
self.state_machine.reporter.warning(
'duplicate object description of %s, ' % targetname +
@@ -80,16 +80,16 @@
}
def before_content(self):
- path = self.env.ref_context.setdefault('zuul:attr_path', [])
+ path = self.env.ref_context.setdefault('zuuldoc:attr_path', [])
path.append(self.names[-1])
- path = self.env.ref_context.setdefault('zuul:display_attr_path', [])
+ path = self.env.ref_context.setdefault('zuuldoc:display_attr_path', [])
path.append(self.names[-1])
def after_content(self):
- path = self.env.ref_context.get('zuul:attr_path')
+ path = self.env.ref_context.get('zuuldoc:attr_path')
if path:
path.pop()
- path = self.env.ref_context.get('zuul:display_attr_path')
+ path = self.env.ref_context.get('zuuldoc:display_attr_path')
if path:
path.pop()
@@ -141,18 +141,18 @@
return ''
def before_content(self):
- path = self.env.ref_context.setdefault('zuul:attr_path', [])
+ path = self.env.ref_context.setdefault('zuuldoc:attr_path', [])
element = self.names[-1]
path.append(element)
- path = self.env.ref_context.setdefault('zuul:display_attr_path', [])
+ path = self.env.ref_context.setdefault('zuuldoc:display_attr_path', [])
element = self.names[-1] + self.get_type_str()
path.append(element)
def after_content(self):
- path = self.env.ref_context.get('zuul:attr_path')
+ path = self.env.ref_context.get('zuuldoc:attr_path')
if path:
path.pop()
- path = self.env.ref_context.get('zuul:display_attr_path')
+ path = self.env.ref_context.get('zuuldoc:display_attr_path')
if path:
path.pop()
@@ -176,18 +176,18 @@
}
def before_content(self):
- path = self.env.ref_context.setdefault('zuul:attr_path', [])
+ path = self.env.ref_context.setdefault('zuuldoc:attr_path', [])
element = self.names[-1]
path.append(element)
- path = self.env.ref_context.setdefault('zuul:display_attr_path', [])
+ path = self.env.ref_context.setdefault('zuuldoc:display_attr_path', [])
element = self.names[-1]
path.append(element)
def after_content(self):
- path = self.env.ref_context.get('zuul:attr_path')
+ path = self.env.ref_context.get('zuuldoc:attr_path')
if path:
path.pop()
- path = self.env.ref_context.get('zuul:display_attr_path')
+ path = self.env.ref_context.get('zuuldoc:display_attr_path')
if path:
path.pop()
@@ -216,7 +216,7 @@
class ZuulDomain(Domain):
- name = 'zuul'
+ name = 'zuuldoc'
label = 'Zuul'
directives = {