Merge "Don't request empty nodesets" into feature/zuulv3
diff --git a/bindep.txt b/bindep.txt
index 8dffd0f..85254b4 100644
--- a/bindep.txt
+++ b/bindep.txt
@@ -8,6 +8,7 @@
zookeeperd [platform:dpkg]
build-essential [platform:dpkg]
gcc [platform:rpm]
+graphviz [test]
libssl-dev [platform:dpkg]
openssl-devel [platform:rpm]
libffi-dev [platform:dpkg]
diff --git a/doc/source/admin/components.rst b/doc/source/admin/components.rst
index 890405d..aa6d8c8 100644
--- a/doc/source/admin/components.rst
+++ b/doc/source/admin/components.rst
@@ -6,11 +6,37 @@
==========
Zuul is a distributed system consisting of several components, each of
-which is described below. All Zuul processes read the
-``/etc/zuul/zuul.conf`` file (an alternate location may be supplied on
-the command line) which uses an INI file syntax. Each component may
-have its own configuration file, though you may find it simpler to use
-the same file for all components.
+which is described below.
+
+
+.. graphviz::
+ :align: center
+
+ graph {
+ node [shape=box]
+ Gearman [shape=ellipse]
+ Gerrit [fontcolor=grey]
+ Zookeeper [shape=ellipse]
+ Nodepool
+ GitHub [fontcolor=grey]
+
+ Merger -- Gearman
+ Executor -- Gearman
+ Web -- Gearman
+
+ Gearman -- Scheduler;
+ Scheduler -- Gerrit;
+ Scheduler -- Zookeeper;
+ Zookeeper -- Nodepool;
+ Scheduler -- GitHub;
+ }
+
+
+
+All Zuul processes read the ``/etc/zuul/zuul.conf`` file (an alternate
+location may be supplied on the command line) which uses an INI file
+syntax. Each component may have its own configuration file, though
+you may find it simpler to use the same file for all components.
An example ``zuul.conf``:
diff --git a/doc/source/admin/drivers/smtp.rst b/doc/source/admin/drivers/smtp.rst
index 6f24355..11c0624 100644
--- a/doc/source/admin/drivers/smtp.rst
+++ b/doc/source/admin/drivers/smtp.rst
@@ -9,25 +9,36 @@
Connection Configuration
------------------------
-**driver=smtp**
+.. attr:: <smtp connection>
-**server**
- SMTP server hostname or address to use.
- ``server=localhost``
+ .. attr:: driver
+ :required:
-**port**
- Optional: SMTP server port.
- ``port=25``
+ .. value:: smtp
-**default_from**
- Who the email should appear to be sent from when emailing the report.
- This can be overridden by individual pipelines.
- ``default_from=zuul@example.com``
+ The connection must set ``driver=smtp`` for SMTP connections.
-**default_to**
- Who the report should be emailed to by default.
- This can be overridden by individual pipelines.
- ``default_to=you@example.com``
+ .. attr:: server
+ :default: localhost
+
+ SMTP server hostname or address to use.
+
+ .. attr:: port
+ :default: 25
+
+ SMTP server port.
+
+ .. attr:: default_from
+ :default: zuul
+
+ Who the email should appear to be sent from when emailing the report.
+ This can be overridden by individual pipelines.
+
+ .. attr:: default_to
+ :default: zuul
+
+ Who the report should be emailed to by default.
+ This can be overridden by individual pipelines.
Reporter Configuration
----------------------
@@ -39,15 +50,38 @@
address.
Each pipeline can overwrite the ``subject`` or the ``to`` or ``from`` address by
-providing alternatives as arguments to the reporter. For example, ::
+providing alternatives as arguments to the reporter. For example:
- - pipeline:
- name: post-merge
- success:
- outgoing_smtp:
- to: you@example.com
- failure:
- internal_smtp:
- to: you@example.com
- from: alternative@example.com
- subject: Change {change} failed
+.. code-block:: yaml
+
+ - pipeline:
+ name: post-merge
+ success:
+ outgoing_smtp:
+ to: you@example.com
+ failure:
+ internal_smtp:
+ to: you@example.com
+ from: alternative@example.com
+ subject: Change {change} failed
+
+.. attr:: pipeline.<reporter>.<smtp source>
+
+ To report via email, the dictionaries passed to any of the pipeline
+ :ref:`reporter<reporters>` attributes support the following
+ attributes:
+
+ .. attr:: to
+
+ The SMTP recipient address for the report. Multiple addresses
+ may be specified as one value separated by commas.
+
+ .. attr:: from
+
+ The SMTP sender address for the report.
+
+ .. attr:: subject
+
+ The Subject of the report email.
+
+ .. TODO: document subject string formatting.
diff --git a/doc/source/admin/drivers/sql.rst b/doc/source/admin/drivers/sql.rst
index b890f08..e208467 100644
--- a/doc/source/admin/drivers/sql.rst
+++ b/doc/source/admin/drivers/sql.rst
@@ -4,14 +4,29 @@
===
The SQL driver supports reporters only. Only one connection per
-database is permitted. The connection options are:
+database is permitted.
-**driver=sql**
+Connection Configuration
+------------------------
-**dburi**
- Database connection information in the form of a URI understood by
- sqlalchemy. eg http://docs.sqlalchemy.org/en/rel_1_0/core/engines.html#database-urls
- ``dburi=mysql://user:pass@localhost/db``
+The connection options for the SQL driver are:
+
+.. attr:: <sql connection>
+
+ .. attr:: driver
+ :required:
+
+ .. value:: sql
+
+ The connection must set ``driver=sql`` for SQL connections.
+
+ .. attr:: dburi
+ :required:
+
+ Database connection information in the form of a URI understood
+ by SQLAlchemy. See `The SQLAlchemy manual
+ <http://docs.sqlalchemy.org/en/rel_1_0/core/engines.html#database-urls>`_
+ for more information.
Reporter Configuration
----------------------
@@ -21,24 +36,27 @@
A :ref:`connection<connections>` that uses the sql driver must be
supplied to the reporter.
-zuul.conf contains the database connection and credentials. To store different
-reports in different databases you'll need to create a new connection per
-database.
+``zuul.conf`` contains the database connection and credentials. To
+store different reports in different databases you'll need to create a
+new connection per database.
-The SQL reporter does nothing on "start" or "merge-failure"; it only
-acts on "success" or "failure" reporting stages.
+The SQL reporter does nothing on :attr:`pipeline.start` or
+:attr:`pipeline.merge-failure`; it only acts on
+:attr:`pipeline.success` or :attr:`pipeline.failure` reporting stages.
-**score**
- A score to store for the result of the build. eg: -1 might indicate a failed
- build.
+For example:
-For example ::
+.. code-block:: yaml
- - pipeline:
- name: post-merge
- success:
- mydb_conn:
- score: 1
- failure:
- mydb_conn:
- score: -1
+ - pipeline:
+ name: post-merge
+ success:
+ mydb_conn:
+ failure:
+ mydb_conn:
+
+.. attr:: pipeline.<reporter>.<sql source>
+
+ To report to a database, add a key with the connection name and an
+ empty value to the desired pipeline :ref:`reporter<reporters>`
+ attributes.
diff --git a/doc/source/admin/drivers/timer.rst b/doc/source/admin/drivers/timer.rst
index c8afdd7..3b38c99 100644
--- a/doc/source/admin/drivers/timer.rst
+++ b/doc/source/admin/drivers/timer.rst
@@ -11,14 +11,20 @@
--------------------
Timers don't require a special connection or driver. Instead they can
-simply be used by listing **timer** as the trigger.
+simply be used by listing ``timer`` as the trigger.
-This trigger will run based on a cron-style time specification.
-It will enqueue an event into its pipeline for every project
-defined in the configuration. Any job associated with the
-pipeline will run in response to that event.
+This trigger will run based on a cron-style time specification. It
+will enqueue an event into its pipeline for every project defined in
+the configuration. Any job associated with the pipeline will run in
+response to that event.
-**time**
- The time specification in cron syntax. Only the 5 part syntax is
- supported, not the symbolic names. Example: ``0 0 * * *`` runs at
- midnight. The first weekday is Monday.
+.. attr:: pipeline.trigger.timer
+
+ The timer trigger supports the following attributes:
+
+ .. attr:: time
+ :required:
+
+ The time specification in cron syntax. Only the 5 part syntax
+ is supported, not the symbolic names. Example: ``0 0 * * *``
+ runs at midnight. The first weekday is Monday.
diff --git a/doc/source/admin/drivers/zuul.rst b/doc/source/admin/drivers/zuul.rst
index b531754..d95dffc 100644
--- a/doc/source/admin/drivers/zuul.rst
+++ b/doc/source/admin/drivers/zuul.rst
@@ -10,18 +10,29 @@
---------------------
Zuul events don't require a special connection or driver. Instead they
-can simply be used by listing **zuul** as the trigger.
+can simply be used by listing ``zuul`` as the trigger.
-**event**
- The event name. Currently supported:
+.. attr:: pipeline.trigger.zuul
- *project-change-merged* when Zuul merges a change to a project, it
- generates this event for every open change in the project.
+ The Zuul trigger supports the following attributes:
- *parent-change-enqueued* when Zuul enqueues a change into any
- pipeline, it generates this event for every child of that
- change.
+ .. attr:: event
+ :required:
-**pipeline**
- Only available for ``parent-change-enqueued`` events. This is the
- name of the pipeline in which the parent change was enqueued.
+ The event name. Currently supported events:
+
+ .. value:: project-change-merged
+
+ When Zuul merges a change to a project, it generates this
+ event for every open change in the project.
+
+ .. value:: parent-change-enqueued
+
+ When Zuul enqueues a change into any pipeline, it generates
+ this event for every child of that change.
+
+ .. attr:: pipeline
+
+ Only available for ``parent-change-enqueued`` events. This is
+ the name of the pipeline in which the parent change was
+ enqueued.
diff --git a/doc/source/admin/monitoring.rst b/doc/source/admin/monitoring.rst
index 2a6c959..9c69960 100644
--- a/doc/source/admin/monitoring.rst
+++ b/doc/source/admin/monitoring.rst
@@ -31,63 +31,101 @@
Metrics
~~~~~~~
-The metrics are emitted by the Zuul scheduler (`zuul/scheduler.py`):
+The metrics are emitted by the Zuul :ref:`scheduler`:
-**gerrit.event.<type> (counters)**
- Gerrit emits different kind of message over its `stream-events`
- interface. Zuul will report counters for each type of event it
- receives from Gerrit.
+.. stat:: gerrit.event.<type>
+ :type: counter
- Some of the events emitted are:
+ Gerrit emits different kind of message over its `stream-events`
+ interface. Zuul will report counters for each type of event it
+ receives from Gerrit.
- * patchset-created
- * draft-published
- * change-abandonned
- * change-restored
- * change-merged
- * merge-failed
- * comment-added
- * ref-updated
- * reviewer-added
+ Refer to your Gerrit installation documentation for a complete
+ list of Gerrit event types.
- Refer to your Gerrit installation documentation for an exhaustive list of
- Gerrit event types.
+.. stat:: zuul.pipeline
-**zuul.pipeline.**
- Holds metrics specific to jobs. The hierarchy is:
+ Holds metrics specific to jobs. This hierarchy includes:
- #. **<pipeline name>** as defined in your `layout.yaml` file (ex: `gate`,
- `test`, `publish`). It contains:
+ .. stat:: <pipeline name>
- #. **all_jobs** counter of jobs triggered by the pipeline.
- #. **current_changes** A gauge for the number of Gerrit changes being
- processed by this pipeline.
- #. **job** subtree detailing per jobs statistics:
+ A set of metrics for each pipeline named as defined in the Zuul
+ config.
- #. **<jobname>** The triggered job name.
- #. **<build result>** Result as defined in your triggering system. For
- Jenkins that would be SUCCESS, FAILURE, UNSTABLE, LOST. The
- metrics holds both an increasing counter and a timing
- reporting the duration of the build. Whenever the result is a
- SUCCESS or FAILURE, Zuul will additionally report the duration
- of the build as a timing event.
+ .. stat:: all_jobs
+ :type: counter
- #. **resident_time** timing representing how long the Change has been
- known by Zuul (which includes build time and Zuul overhead).
- #. **total_changes** counter of the number of change proceeding since
- Zuul started.
- #. **wait_time** counter and timer of the wait time, with the difference
- of the job start time and the execute time, in milliseconds.
+ Number of jobs triggered by the pipeline.
- Additionally, the `zuul.pipeline.<pipeline name>` hierarchy contains
- `current_changes` (gauge), `resident_time` (timing) and `total_changes`
- (counter) metrics for each projects. The slash separator used in Gerrit name
- being replaced by dots.
+ .. stat:: current_changes
+ :type: gauge
- As an example, given a job named `myjob` triggered by the `gate` pipeline
- which took 40 seconds to build, the Zuul scheduler will emit the following
- statsd events:
+ The number of items currently being processed by this
+ pipeline.
- * `zuul.pipeline.gate.job.myjob.SUCCESS` +1
- * `zuul.pipeline.gate.job.myjob` 40 seconds
- * `zuul.pipeline.gate.all_jobs` +1
+ .. stat:: job
+
+ Subtree detailing per jobs statistics:
+
+ .. stat:: <jobname>
+
+ The triggered job name.
+
+ .. stat:: <result>
+ :type: counter, timer
+
+ A counter for each type of result (e.g., ``SUCCESS`` or
+ ``FAILURE``, ``ERROR``, etc.) for the job. If the
+ result is ``SUCCESS`` or ``FAILURE``, Zuul will
+ additionally report the duration of the build as a
+ timer.
+
+ .. stat:: resident_time
+ :type: timer
+
+ A timer metric reporting how long each item has been in the
+ pipeline.
+
+ .. stat:: total_changes
+ :type: counter
+
+ The number of change processed by the pipeline since Zuul
+ started.
+
+ .. stat:: wait_time
+ :type: timer
+
+ How long each item spent in the pipeline before its first job
+ started.
+
+ .. stat:: <project>
+
+ This hierarchy holds more specific metrics for each project
+ participating in the pipeline. If the project name contains
+ a ``/`` character, it will be replaced with a ``.``.
+
+ .. stat:: current_changes
+ :type: gauge
+
+ The number of items of this project currently being
+ processed by this pipeline.
+
+ .. stat:: resident_time
+ :type: timer
+
+ A timer metric reporting how long each item for this
+ project has been in the pipeline.
+
+ .. stat:: total_changes
+ :type: counter
+
+ The number of change for this project processed by the
+ pipeline since Zuul started.
+
+As an example, given a job named `myjob` triggered by the `gate` pipeline
+which took 40 seconds to build, the Zuul scheduler will emit the following
+statsd events:
+
+ * ``zuul.pipeline.gate.job.myjob.SUCCESS`` +1
+ * ``zuul.pipeline.gate.job.myjob`` 40 seconds
+ * ``zuul.pipeline.gate.all_jobs`` +1
diff --git a/doc/source/admin/tenants.rst b/doc/source/admin/tenants.rst
index b3b2d9c..b518c91 100644
--- a/doc/source/admin/tenants.rst
+++ b/doc/source/admin/tenants.rst
@@ -5,128 +5,163 @@
Tenant Configuration
====================
-After *zuul.conf* is configured, Zuul component servers will be able
+After ``zuul.conf`` is configured, Zuul component servers will be able
to start, but a tenant configuration is required in order for Zuul to
perform any actions. The tenant configuration file specifies upon
-which projects Zuul should operate. These repositories are
-grouped into tenants. The configuration of each tenant is separate
-from the rest (no pipelines, jobs, etc are shared between them).
+which projects Zuul should operate. These repositories are grouped
+into tenants. The configuration of each tenant is separate from the
+rest (no pipelines, jobs, etc are shared between them).
A project may appear in more than one tenant; this may be useful if
you wish to use common job definitions across multiple tenants.
-The tenant configuration file is specified by the *tenant_config*
-setting in the *scheduler* section of *zuul.yaml*. It is a YAML file
-which, like other Zuul configuration files, is a list of configuration
-objects, though only one type of object is supported, *tenant*.
+The tenant configuration file is specified by the
+:attr:`scheduler.tenant_config` setting in ``zuul.conf``. It is a
+YAML file which, like other Zuul configuration files, is a list of
+configuration objects, though only one type of object is supported:
+``tenant``.
Tenant
------
A tenant is a collection of projects which share a Zuul
-configuration. An example tenant definition is::
+configuration. An example tenant definition is:
- - tenant:
- name: my-tenant
- max-nodes-per-job: 5
- exclude-unprotected-branches: false
- source:
- gerrit:
- config-projects:
- - common-config
- - shared-jobs:
- include: job
- untrusted-projects:
- - zuul-jobs:
- shadow: common-config
- - project1
- - project2:
- exclude-unprotected-branches: true
+.. code-block:: yaml
-The following attributes are supported:
+ - tenant:
+ name: my-tenant
+ max-nodes-per-job: 5
+ exclude-unprotected-branches: false
+ source:
+ gerrit:
+ config-projects:
+ - common-config
+ - shared-jobs:
+ include: job
+ untrusted-projects:
+ - zuul-jobs:
+ shadow: common-config
+ - project1
+ - project2:
+ exclude-unprotected-branches: true
-**name** (required)
- The name of the tenant. This may appear in URLs, paths, and
- monitoring fields, and so should be restricted to URL friendly
- characters (ASCII letters, numbers, hyphen and underscore) and you
- should avoid changing it unless necessary.
+.. attr:: tenant
-**max-nodes-per-job** (optional)
- The maximum number of nodes a job can request, default to 5.
- A '-1' value removes the limit.
+ The following attributes are supported:
-**exclude-unprotected-branches** (optional)
- When using a branch and pull model on a shared github repository there are
- usually one or more protected branches which are gated and a dynamic number of
- personal/feature branches which are the source for the pull requests. These
- branches can potentially include broken zuul config and therefore break the
- global tenant wide configuration. In order to deal with this zuul's operations
- can be limited to the protected branches which are gated. This is a tenant
- wide setting and can be overridden per project. If not specified, defaults
- to ``false``.
+ .. attr:: name
+ :required:
-**source** (required)
- A dictionary of sources to consult for projects. A tenant may
- contain projects from multiple sources; each of those sources must
- be listed here, along with the projects it supports. The name of a
- :ref:`connection<connections>` is used as the dictionary key
- (e.g. `gerrit` in the example above), and the value is a further
- dictionary containing the keys below.
+ The name of the tenant. This may appear in URLs, paths, and
+ monitoring fields, and so should be restricted to URL friendly
+ characters (ASCII letters, numbers, hyphen and underscore) and
+ you should avoid changing it unless necessary.
- **config-projects**
- A list of projects to be treated as config projects in this
- tenant. The jobs in a config project are trusted, which means
- they run with extra privileges, do not have their configuration
- dynamically loaded for proposed changes, and zuul.yaml files are
- only searched for in the master branch.
+ .. attr:: source
+ :required:
- **untrusted-projects**
- A list of projects to be treated as untrusted in this tenant. An
- untrusted project is the typical project operated on by Zuul.
- Their jobs run in a more restrictive environment, they may not
- define pipelines, their configuration dynamically changes in
- response to proposed changes, Zuul will read configuration files
- in all of their branches.
+ A dictionary of sources to consult for projects. A tenant may
+ contain projects from multiple sources; each of those sources
+ must be listed here, along with the projects it supports. The
+ name of a :ref:`connection<connections>` is used as the
+ dictionary key (e.g. ``gerrit`` in the example above), and the
+ value is a further dictionary containing the keys below.
- Each of the projects listed may be either a simple string value, or
- it may be a dictionary with the following keys:
+ The next two attributes, **config-projects** and
+ **untrusted-projects** provide the bulk of the information for
+ tenant configuration. They list all of the projects upon which
+ Zuul will act.
- **include**
- Normally Zuul will load all of the configuration classes
- appropriate for the type of project (config or untrusted) in
- question. However, if you only want to load some items, the
- *include* attribute can be used to specify that *only* the
- specified classes should be loaded. Supplied as a string, or a
- list of strings.
+ The order of the projects listed in a tenant is important. A job
+ which is defined in one project may not be redefined in another
+ project; therefore, once a job appears in one project, a project
+ listed later will be unable to define a job with that name.
+ Further, some aspects of project configuration (such as the merge
+ mode) may only be set on the first appearance of a project
+ definition.
- **exclude**
- A list of configuration classes that should not be loaded.
+ Zuul loads the configuration from all **config-projects** in the
+ order listed, followed by all **untrusted-projects** in order.
- **shadow**
- A list of projects which this project is permitted to shadow.
- Normally, only one project in Zuul may contain definitions for a
- given job. If a project earlier in the configuration defines a
- job which a later project redefines, the later definition is
- considered an error and is not permitted. The "shadow" attribute
- of a project indicates that job definitions in this project which
- conflict with the named projects should be ignored, and those in
- the named project should be used instead. The named projects must
- still appear earlier in the configuration. In the example above,
- if a job definition appears in both the "common-config" and
- "zuul-jobs" projects, the definition in "common-config" will be
- used.
+ .. attr:: config-projects
- **exclude-unprotected-branches**
- Define if unprotected github branches should be processed. Defaults to the
- tenant wide setting of exclude-unprotected-branches.
+ A list of projects to be treated as :term:`config projects
+ <config-project>` in this tenant. The jobs in a config project
+ are trusted, which means they run with extra privileges, do not
+ have their configuration dynamically loaded for proposed
+ changes, and Zuul config files are only searched for in the
+ ``master`` branch.
- The order of the projects listed in a tenant is important. A job
- which is defined in one project may not be redefined in another
- project; therefore, once a job appears in one project, a project
- listed later will be unable to define a job with that name.
- Further, some aspects of project configuration (such as the merge
- mode) may only be set on the first appearance of a project
- definition.
+ The items in the list follow the same format described in
+ **untrusted-projects**.
- Zuul loads the configuration from all *config-projects* in the order
- listed, followed by all *trusted-projects* in order.
+ .. attr:: untrusted-projects
+
+ A list of projects to be treated as untrusted in this tenant.
+ An :term:`untrusted-project` is the typical project operated on
+ by Zuul. Their jobs run in a more restrictive environment, they
+ may not define pipelines, their configuration dynamically
+ changes in response to proposed changes, and Zuul will read
+ configuration files in all of their branches.
+
+ .. attr:: <project>:
+
+ The items in the list may either be simple string values of
+ the project names, or a dictionary with the project name as
+ key and the following values:
+
+ .. attr:: include
+
+ Normally Zuul will load all of the configuration classes
+ appropriate for the type of project (config or untrusted)
+ in question. However, if you only want to load some
+ items, the **include** attribute can be used to specify
+ that *only* the specified classes should be loaded.
+ Supplied as a string, or a list of strings.
+
+ .. attr:: exclude
+
+ A list of configuration classes that should not be loaded.
+
+ .. attr:: shadow
+
+ A list of projects which this project is permitted to
+ shadow. Normally, only one project in Zuul may contain
+ definitions for a given job. If a project earlier in the
+ configuration defines a job which a later project
+ redefines, the later definition is considered an error and
+ is not permitted. The **shadow** attribute of a project
+ indicates that job definitions in this project which
+ conflict with the named projects should be ignored, and
+ those in the named project should be used instead. The
+ named projects must still appear earlier in the
+ configuration. In the example above, if a job definition
+ appears in both the ``common-config`` and ``zuul-jobs``
+ projects, the definition in ``common-config`` will be
+ used.
+
+ .. attr:: exclude-unprotected-branches
+
+ Define if unprotected github branches should be
+ processed. Defaults to the tenant wide setting of
+ exclude-unprotected-branches.
+
+ .. attr:: max-nodes-per-job
+ :default: 5
+
+ The maximum number of nodes a job can request. A value of
+ '-1' value removes the limit.
+
+ .. attr:: exclude-unprotected-branches
+ :default: false
+
+ When using a branch and pull model on a shared GitHub repository
+ there are usually one or more protected branches which are gated
+ and a dynamic number of personal/feature branches which are the
+ source for the pull requests. These branches can potentially
+ include broken Zuul config and therefore break the global tenant
+ wide configuration. In order to deal with this Zuul's operations
+ can be limited to the protected branches which are gated. This
+ is a tenant wide setting and can be overridden per project.
+ This currently only affects GitHub projects.
diff --git a/doc/source/conf.py b/doc/source/conf.py
index 7c0d587..ca9c9b0 100644
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -27,8 +27,10 @@
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = [
'sphinx.ext.autodoc',
+ 'sphinx.ext.graphviz',
'sphinxcontrib.blockdiag',
'sphinxcontrib.programoutput',
+ 'zuul.sphinx.ansible',
'zuul.sphinx.zuul',
]
#extensions = ['sphinx.ext.intersphinx']
@@ -96,7 +98,9 @@
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
-#html_theme_options = {}
+html_theme_options = {
+ 'show_related': True
+}
# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []
diff --git a/doc/source/developer/ansible.rst b/doc/source/developer/ansible.rst
new file mode 100644
index 0000000..e3ebca7
--- /dev/null
+++ b/doc/source/developer/ansible.rst
@@ -0,0 +1,66 @@
+Ansible Integration
+===================
+
+Zuul contains Ansible modules and plugins to control the execution of Ansible
+Job content. These break down into two basic categories.
+
+* Restricted Execution on Executors
+* Build Log Support
+
+Restricted Execution
+--------------------
+
+Zuul runs ``ansible-playbook`` on executors to run job content on nodes. While
+the intent is that content is run on the remote nodes, Ansible is a flexible
+system that allows delegating actions to ``localhost``, and also reading and
+writing files. These actions can be desirable and necessary for actions such
+as fetching log files or build artifacts, but could also be used as a vector
+to attack the executor.
+
+For that reason Zuul implements a set of Ansible action plugins and lookup
+plugins that override and intercept task execution during untrusted playbook
+execution to ensure local actions are not executed or that for operations that
+are desirable to allow locally that they only interact with files in the zuul
+work directory.
+
+.. autoclass:: zuul.ansible.action.normal.ActionModule
+ :members:
+
+Build Log Support
+-----------------
+
+Zuul provides realtime build log streaming to end users so that users can
+watch long-running jobs in progress. As jobs may be written that execute a
+shell script that could run for a long time, additional effort is expended
+to stream stdout and stderr of shell tasks as they happen rather than waiting
+for the command to finish.
+
+Zuul contains a modified version of the :ansible:module:`command`
+that starts a log streaming daemon on the build node.
+
+.. automodule:: zuul.ansible.library.command
+
+All jobs run with the :py:mod:`zuul.ansible.callback.zuul_stream` callback
+plugin enabled, which writes the build log to a file so that the
+:py:class:`zuul.lib.log_streamer.LogStreamer` can provide the data on demand
+over the finger protocol. Finally, :py:class:`zuul.web.LogStreamingHandler`
+exposes that log stream over a websocket connection as part of
+:py:class:`zuul.web.ZuulWeb`.
+
+.. autoclass:: zuul.ansible.callback.zuul_stream.CallbackModule
+ :members:
+
+.. autoclass:: zuul.lib.log_streamer.LogStreamer
+.. autoclass:: zuul.web.LogStreamingHandler
+.. autoclass:: zuul.web.ZuulWeb
+
+In addition to real-time streaming, Zuul also installs another callback module,
+:py:mod:`zuul.ansible.callback.zuul_json.CallbackModule` that collects all
+of the information about a given run into a json file which is written to the
+work dir so that it can be published along with build logs. Since the streaming
+log is by necessity a single text stream, choices have to be made for
+readability about what data is shown and what is not shown. The json log file
+is intended to allow for a richer more interactive set of data to be displayed
+to the user.
+
+.. autoclass:: zuul.ansible.callback.zuul_json.CallbackModule
diff --git a/doc/source/developer/index.rst b/doc/source/developer/index.rst
index 7b16e9c..360dcd5 100644
--- a/doc/source/developer/index.rst
+++ b/doc/source/developer/index.rst
@@ -15,3 +15,4 @@
triggers
testing
docs
+ ansible
diff --git a/doc/source/user/config.rst b/doc/source/user/config.rst
index 34d23f9..446718b 100644
--- a/doc/source/user/config.rst
+++ b/doc/source/user/config.rst
@@ -489,7 +489,7 @@
parent: base
nodes:
- name: test-node
- image: fedora
+ label: fedora
.. attr:: job
diff --git a/doc/source/user/encryption.rst b/doc/source/user/encryption.rst
index fdf2c5a..7ced589 100644
--- a/doc/source/user/encryption.rst
+++ b/doc/source/user/encryption.rst
@@ -20,24 +20,41 @@
the main Zuul configuration file.
Zuul currently supports one encryption scheme, PKCS#1 with OAEP, which
-can not store secrets longer than the key length, 4096 bits. The
-padding used by this scheme ensures that someone examining the
-encrypted data can not determine the length of the plaintext version
-of the data, except to know that it is not longer than 4096 bits.
+can not store secrets longer than the 3760 bits (derived from the key
+length of 4096 bits minus 336 bits of overhead). The padding used by
+this scheme ensures that someone examining the encrypted data can not
+determine the length of the plaintext version of the data, except to
+know that it is not longer than 3760 bits (or some multiple thereof).
In the config files themselves, Zuul uses an extensible method of
specifying the encryption scheme used for a secret so that other
schemes may be added later. To specify a secret, use the
``!encrypted/pkcs1-oaep`` YAML tag along with the base64 encoded
-value. For example::
+value. For example:
+
+.. code-block:: yaml
- secret:
name: test_secret
data:
password: !encrypted/pkcs1-oaep |
- BFhtdnm8uXx7kn79RFL/zJywmzLkT1GY78P3bOtp4WghUFWobkifSu7ZpaV4NeO0s71YUsi1wGZZ
+ BFhtdnm8uXx7kn79RFL/zJywmzLkT1GY78P3bOtp4WghUFWobkifSu7ZpaV4NeO0s71YUsi
...
+To support secrets longer than 3760 bits, the value after the
+encryption tag may be a list rather than a scalar. For example:
+
+.. code-block:: yaml
+
+ - secret:
+ name: long_secret
+ data:
+ password: !encrypted/pkcs1-oaep
+ - er1UXNOD3OqtsRJaP0Wvaqiqx0ZY2zzRt6V9vqIsRaz1R5C4/AEtIad/DERZHwk3Nk+KV
+ ...
+ - HdWDS9lCBaBJnhMsm/O9tpzCq+GKRELpRzUwVgU5k822uBwhZemeSrUOLQ8hQ7q/vVHln
+ ...
+
Zuul provides a standalone script to make encrypting values easy; it
can be found at `tools/encrypt_secret.py` in the Zuul source
directory.
diff --git a/doc/source/user/jobs.rst b/doc/source/user/jobs.rst
index 577d147..7f1c3cb 100644
--- a/doc/source/user/jobs.rst
+++ b/doc/source/user/jobs.rst
@@ -3,7 +3,7 @@
Job Content
===========
-Zuul jobs are implemneted as Ansible playbooks. Zuul prepares the
+Zuul jobs are implemented as Ansible playbooks. Zuul prepares the
repositories used for a job, installs any required Ansible roles, and
then executes the job's playbooks. Any setup or artifact collection
required is the responsibility of the job itself. While this flexible
diff --git a/tests/fixtures/config/ansible/git/org_plugin-project/playbooks/uri_bad_path.yaml b/tests/fixtures/config/ansible/git/org_plugin-project/playbooks/uri_bad_path.yaml
new file mode 100644
index 0000000..523aab7
--- /dev/null
+++ b/tests/fixtures/config/ansible/git/org_plugin-project/playbooks/uri_bad_path.yaml
@@ -0,0 +1,6 @@
+- hosts: localhost
+ tasks:
+ - uri:
+ method: GET
+ url: https://example.com
+ path: /tmp/example.out
diff --git a/tests/fixtures/config/ansible/git/org_plugin-project/playbooks/uri_bad_scheme.yaml b/tests/fixtures/config/ansible/git/org_plugin-project/playbooks/uri_bad_scheme.yaml
new file mode 100644
index 0000000..5d71793
--- /dev/null
+++ b/tests/fixtures/config/ansible/git/org_plugin-project/playbooks/uri_bad_scheme.yaml
@@ -0,0 +1,5 @@
+- hosts: localhost
+ tasks:
+ - uri:
+ method: GET
+ url: file:///etc/passwd
diff --git a/tests/fixtures/config/sql-driver/git/common-config/zuul.yaml b/tests/fixtures/config/sql-driver/git/common-config/zuul.yaml
index 0cb1d04..01a8d0f 100644
--- a/tests/fixtures/config/sql-driver/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/sql-driver/git/common-config/zuul.yaml
@@ -8,14 +8,11 @@
gerrit:
Verified: 1
resultsdb:
- score: 1
failure:
gerrit:
Verified: -1
resultsdb:
- score: -1
resultsdb_failures:
- score: -1
- job:
name: project-merge
diff --git a/tests/unit/test_connection.py b/tests/unit/test_connection.py
index 77b13a5..4214e9f 100644
--- a/tests/unit/test_connection.py
+++ b/tests/unit/test_connection.py
@@ -86,7 +86,7 @@
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
- # Add a failed result for a negative score
+ # Add a failed result
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
self.executor_server.failJob('project-test1', B)
@@ -106,7 +106,7 @@
self.assertEqual('org/project', buildset0['project'])
self.assertEqual(1, buildset0['change'])
self.assertEqual(1, buildset0['patchset'])
- self.assertEqual(1, buildset0['score'])
+ self.assertEqual('SUCCESS', buildset0['result'])
self.assertEqual('Build succeeded.', buildset0['message'])
self.assertEqual('tenant-one', buildset0['tenant'])
@@ -130,7 +130,7 @@
self.assertEqual('org/project', buildset1['project'])
self.assertEqual(2, buildset1['change'])
self.assertEqual(1, buildset1['patchset'])
- self.assertEqual(-1, buildset1['score'])
+ self.assertEqual('FAILURE', buildset1['result'])
self.assertEqual('Build failed.', buildset1['message'])
buildset1_builds = conn.execute(
@@ -183,7 +183,7 @@
self.assertEqual('org/project', buildsets_resultsdb[0]['project'])
self.assertEqual(1, buildsets_resultsdb[0]['change'])
self.assertEqual(1, buildsets_resultsdb[0]['patchset'])
- self.assertEqual(1, buildsets_resultsdb[0]['score'])
+ self.assertEqual('SUCCESS', buildsets_resultsdb[0]['result'])
self.assertEqual('Build succeeded.', buildsets_resultsdb[0]['message'])
# Grab the sa tables for resultsdb_failures
@@ -204,7 +204,7 @@
'org/project', buildsets_resultsdb_failures[0]['project'])
self.assertEqual(2, buildsets_resultsdb_failures[0]['change'])
self.assertEqual(1, buildsets_resultsdb_failures[0]['patchset'])
- self.assertEqual(-1, buildsets_resultsdb_failures[0]['score'])
+ self.assertEqual('FAILURE', buildsets_resultsdb_failures[0]['result'])
self.assertEqual(
'Build failed.', buildsets_resultsdb_failures[0]['message'])
diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py
index a52a2ee..3538555 100644
--- a/tests/unit/test_model.py
+++ b/tests/unit/test_model.py
@@ -342,16 +342,41 @@
name: pypi-credentials
data:
username: test-username
+ longpassword: !encrypted/pkcs1-oaep
+ - BFhtdnm8uXx7kn79RFL/zJywmzLkT1GY78P3bOtp4WghUFWobkifSu7ZpaV4NeO0s71Y
+ Usi1wGZZL0LveZjUN0t6OU1VZKSG8R5Ly7urjaSo1pPVIq5Rtt/H7W14Lecd+cUeKb4j
+ oeusC9drN3AA8a4oykcVpt1wVqUnTbMGC9ARMCQP6eopcs1l7tzMseprW4RDNhIuz3CR
+ gd0QBMPl6VDoFgBPB8vxtJw+3m0rqBYZCLZgCXekqlny8s2s92nJMuUABbJOEcDRarzi
+ bDsSXsfJt1y+5n7yOURsC7lovMg4GF/vCl/0YMKjBO5bpv9EM5fToeKYyPGSKQoHOnCY
+ ceb3cAVcv5UawcCic8XjhEhp4K7WPdYf2HVAC/qtxhbpjTxG4U5Q/SoppOJ60WqEkQvb
+ Xs6n5Dvy7xmph6GWmU/bAv3eUK3pdD3xa2Ue1lHWz3U+rsYraI+AKYsMYx3RBlfAmCeC
+ 1ve2BXPrqnOo7G8tnUvfdYPbK4Aakk0ds/AVqFHEZN+S6hRBmBjLaRFWZ3QSO1NjbBxW
+ naHKZYT7nkrJm8AMCgZU0ZArFLpaufKCeiK5ECSsDxic4FIsY1OkWT42qEUfL0Wd+150
+ AKGNZpPJnnP3QYY4W/MWcKH/zdO400+zWN52WevbSqZy90tqKDJrBkMl1ydqbuw1E4ZH
+ vIs=
+ - BFhtdnm8uXx7kn79RFL/zJywmzLkT1GY78P3bOtp4WghUFWobkifSu7ZpaV4NeO0s71Y
+ Usi1wGZZL0LveZjUN0t6OU1VZKSG8R5Ly7urjaSo1pPVIq5Rtt/H7W14Lecd+cUeKb4j
+ oeusC9drN3AA8a4oykcVpt1wVqUnTbMGC9ARMCQP6eopcs1l7tzMseprW4RDNhIuz3CR
+ gd0QBMPl6VDoFgBPB8vxtJw+3m0rqBYZCLZgCXekqlny8s2s92nJMuUABbJOEcDRarzi
+ bDsSXsfJt1y+5n7yOURsC7lovMg4GF/vCl/0YMKjBO5bpv9EM5fToeKYyPGSKQoHOnCY
+ ceb3cAVcv5UawcCic8XjhEhp4K7WPdYf2HVAC/qtxhbpjTxG4U5Q/SoppOJ60WqEkQvb
+ Xs6n5Dvy7xmph6GWmU/bAv3eUK3pdD3xa2Ue1lHWz3U+rsYraI+AKYsMYx3RBlfAmCeC
+ 1ve2BXPrqnOo7G8tnUvfdYPbK4Aakk0ds/AVqFHEZN+S6hRBmBjLaRFWZ3QSO1NjbBxW
+ naHKZYT7nkrJm8AMCgZU0ZArFLpaufKCeiK5ECSsDxic4FIsY1OkWT42qEUfL0Wd+150
+ AKGNZpPJnnP3QYY4W/MWcKH/zdO400+zWN52WevbSqZy90tqKDJrBkMl1ydqbuw1E4ZH
+ vIs=
password: !encrypted/pkcs1-oaep |
- BFhtdnm8uXx7kn79RFL/zJywmzLkT1GY78P3bOtp4WghUFWobkifSu7ZpaV4NeO0s71YUsi1wGZZ
- L0LveZjUN0t6OU1VZKSG8R5Ly7urjaSo1pPVIq5Rtt/H7W14Lecd+cUeKb4joeusC9drN3AA8a4o
- ykcVpt1wVqUnTbMGC9ARMCQP6eopcs1l7tzMseprW4RDNhIuz3CRgd0QBMPl6VDoFgBPB8vxtJw+
- 3m0rqBYZCLZgCXekqlny8s2s92nJMuUABbJOEcDRarzibDsSXsfJt1y+5n7yOURsC7lovMg4GF/v
- Cl/0YMKjBO5bpv9EM5fToeKYyPGSKQoHOnCYceb3cAVcv5UawcCic8XjhEhp4K7WPdYf2HVAC/qt
- xhbpjTxG4U5Q/SoppOJ60WqEkQvbXs6n5Dvy7xmph6GWmU/bAv3eUK3pdD3xa2Ue1lHWz3U+rsYr
- aI+AKYsMYx3RBlfAmCeC1ve2BXPrqnOo7G8tnUvfdYPbK4Aakk0ds/AVqFHEZN+S6hRBmBjLaRFW
- Z3QSO1NjbBxWnaHKZYT7nkrJm8AMCgZU0ZArFLpaufKCeiK5ECSsDxic4FIsY1OkWT42qEUfL0Wd
- +150AKGNZpPJnnP3QYY4W/MWcKH/zdO400+zWN52WevbSqZy90tqKDJrBkMl1ydqbuw1E4ZHvIs=
+ BFhtdnm8uXx7kn79RFL/zJywmzLkT1GY78P3bOtp4WghUFWobkifSu7ZpaV4NeO0s71Y
+ Usi1wGZZL0LveZjUN0t6OU1VZKSG8R5Ly7urjaSo1pPVIq5Rtt/H7W14Lecd+cUeKb4j
+ oeusC9drN3AA8a4oykcVpt1wVqUnTbMGC9ARMCQP6eopcs1l7tzMseprW4RDNhIuz3CR
+ gd0QBMPl6VDoFgBPB8vxtJw+3m0rqBYZCLZgCXekqlny8s2s92nJMuUABbJOEcDRarzi
+ bDsSXsfJt1y+5n7yOURsC7lovMg4GF/vCl/0YMKjBO5bpv9EM5fToeKYyPGSKQoHOnCY
+ ceb3cAVcv5UawcCic8XjhEhp4K7WPdYf2HVAC/qtxhbpjTxG4U5Q/SoppOJ60WqEkQvb
+ Xs6n5Dvy7xmph6GWmU/bAv3eUK3pdD3xa2Ue1lHWz3U+rsYraI+AKYsMYx3RBlfAmCeC
+ 1ve2BXPrqnOo7G8tnUvfdYPbK4Aakk0ds/AVqFHEZN+S6hRBmBjLaRFWZ3QSO1NjbBxW
+ naHKZYT7nkrJm8AMCgZU0ZArFLpaufKCeiK5ECSsDxic4FIsY1OkWT42qEUfL0Wd+150
+ AKGNZpPJnnP3QYY4W/MWcKH/zdO400+zWN52WevbSqZy90tqKDJrBkMl1ydqbuw1E4ZH
+ vIs=
''')[0]['secret']
conf['_source_context'] = self.context
@@ -441,6 +466,12 @@
self.assertEqual(in_repo_job_with_inherit.auth.secrets[0].name,
'pypi-credentials')
self.assertIsNone(in_repo_job_with_inherit_false.auth)
+ self.assertEqual(in_repo_job_with_inherit.auth.secrets[0].
+ secret_data['longpassword'],
+ 'test-passwordtest-password')
+ self.assertEqual(in_repo_job_with_inherit.auth.secrets[0].
+ secret_data['password'],
+ 'test-password')
def test_job_inheritance_job_tree(self):
tenant = model.Tenant('tenant')
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index 7038471..7c36cc4 100755
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -791,6 +791,8 @@
('credstash', 'FAILURE'),
('csvfile_good', 'SUCCESS'),
('csvfile_bad', 'FAILURE'),
+ ('uri_bad_path', 'FAILURE'),
+ ('uri_bad_scheme', 'FAILURE'),
]
for job_name, result in plugin_tests:
count += 1
diff --git a/tools/encrypt_secret.py b/tools/encrypt_secret.py
index 72429e9..df4f449 100755
--- a/tools/encrypt_secret.py
+++ b/tools/encrypt_secret.py
@@ -14,10 +14,13 @@
import argparse
import base64
+import math
import os
+import re
import subprocess
import sys
import tempfile
+import textwrap
# we to import Request and urlopen differently for python 2 and 3
try:
@@ -68,28 +71,70 @@
else:
plaintext = sys.stdin.read()
+ plaintext = plaintext.encode("utf-8")
+
pubkey_file = tempfile.NamedTemporaryFile(delete=False)
try:
pubkey_file.write(pubkey.read())
pubkey_file.close()
- p = subprocess.Popen(['openssl', 'rsautl', '-encrypt',
- '-oaep', '-pubin', '-inkey',
+ p = subprocess.Popen(['openssl', 'rsa', '-text',
+ '-pubin', '-in',
pubkey_file.name],
- stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
- (stdout, stderr) = p.communicate(plaintext.encode("utf-8"))
+ (stdout, stderr) = p.communicate()
if p.returncode != 0:
raise Exception("Return code %s from openssl" % p.returncode)
- ciphertext = base64.b64encode(stdout)
+ output = stdout.decode('utf-8')
+ m = re.match(r'^Public-Key: \((\d+) bit\)$', output, re.MULTILINE)
+ nbits = int(m.group(1))
+ nbytes = int(nbits / 8)
+ max_bytes = nbytes - 42 # PKCS1-OAEP overhead
+ chunks = int(math.ceil(float(len(plaintext)) / max_bytes))
+
+ ciphertext_chunks = []
+
+ print("Public key length: {} bits ({} bytes)".format(nbits, nbytes))
+ print("Max plaintext length per chunk: {} bytes".format(max_bytes))
+ print("Input plaintext length: {} bytes".format(len(plaintext)))
+ print("Number of chunks: {}".format(chunks))
+
+ for count in range(chunks):
+ chunk = plaintext[int(count * max_bytes):
+ int((count + 1) * max_bytes)]
+ p = subprocess.Popen(['openssl', 'rsautl', '-encrypt',
+ '-oaep', '-pubin', '-inkey',
+ pubkey_file.name],
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE)
+ (stdout, stderr) = p.communicate(chunk)
+ if p.returncode != 0:
+ raise Exception("Return code %s from openssl" % p.returncode)
+ ciphertext_chunks.append(base64.b64encode(stdout).decode('utf-8'))
+
finally:
os.unlink(pubkey_file.name)
+ output = textwrap.dedent(
+ '''
+ - secret:
+ name: <name>
+ data:
+ <fieldname>: !encrypted/pkcs1-oaep
+ ''')
+
+ twrap = textwrap.TextWrapper(width=79,
+ initial_indent=' ' * 8,
+ subsequent_indent=' ' * 10)
+ for chunk in ciphertext_chunks:
+ chunk = twrap.fill('- ' + chunk)
+ output += chunk + '\n'
+
if args.outfile:
- with open(args.outfile, "wb") as f:
- f.write(ciphertext)
+ with open(args.outfile, "w") as f:
+ f.write(output)
else:
- print(ciphertext.decode("utf-8"))
+ print(output)
if __name__ == '__main__':
diff --git a/zuul/ansible/action/normal.py b/zuul/ansible/action/normal.py
index 74e732e..b8a232b 100644
--- a/zuul/ansible/action/normal.py
+++ b/zuul/ansible/action/normal.py
@@ -1,4 +1,4 @@
-# Copyright 2016 Red Hat, Inc.
+# Copyright 2017 Red Hat, Inc.
#
# This module is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -13,13 +13,27 @@
# You should have received a copy of the GNU General Public License
# along with this software. If not, see <http://www.gnu.org/licenses/>.
+from ansible.module_utils.six.moves.urllib.parse import urlparse
+from ansible.errors import AnsibleError
+
from zuul.ansible import paths
normal = paths._import_ansible_action_plugin('normal')
+ALLOWED_URL_SCHEMES = ('https', 'http', 'ftp')
+
class ActionModule(normal.ActionModule):
+ '''Override the normal action plugin
+
+ :py:class:`ansible.plugins.normal.ActionModule` is run for every
+ module that does not have a more specific matching action plugin.
+
+ Our overridden version of it wraps the execution with checks to block
+ undesired actions on localhost.
+ '''
def run(self, tmp=None, task_vars=None):
+ '''Overridden primary method from the base class.'''
if (self._play_context.connection == 'local'
or self._play_context.remote_addr == 'localhost'
@@ -27,16 +41,61 @@
or self._task.delegate_to == 'localhost'
or (self._task.delegate_to
and self._task.delegate_to.startswtih('127.'))):
- if self._task.action == 'stat':
- paths._fail_if_unsafe(self._task.args['path'])
- elif self._task.action == 'file':
- dest = self._task.args.get(
- 'path', self._task.args.get(
- 'dest', self._task.args.get(
- 'name')))
- paths._fail_if_unsafe(dest)
- else:
- return dict(
- failed=True,
- msg="Executing local code is prohibited")
+ if not self.dispatch_handler():
+ raise AnsibleError("Executing local code is prohibited")
return super(ActionModule, self).run(tmp, task_vars)
+
+ def dispatch_handler(self):
+ '''Run per-action handler if one exists.'''
+ handler_name = 'handle_{action}'.format(action=self._task.action)
+ handler = getattr(self, handler_name, None)
+ if handler:
+ handler(self)
+ return True
+ return False
+
+ def handle_stat(self):
+ '''Allow stat module on localhost if it doesn't touch unsafe files.
+
+ The :ansible:module:`stat` can be useful in jobs for manipulating logs
+ and artifacts.
+
+ Block any access of files outside the zuul work dir.
+ '''
+ paths._fail_if_unsafe(self._task.args['path'])
+
+ def handle_file(self):
+ '''Allow file module on localhost if it doesn't touch unsafe files.
+
+ The :ansible:module:`file` can be useful in jobs for manipulating logs
+ and artifacts.
+
+ Block any access of files outside the zuul work dir.
+ '''
+ for arg in ('path', 'dest', 'name'):
+ dest = self._task.args.get(arg)
+ if dest:
+ paths._fail_if_unsafe(dest)
+
+ def handle_uri(self):
+ '''Allow uri module on localhost if it doesn't touch unsafe files.
+
+ The :ansible:module:`uri` can be used from the executor to do
+ things like pinging readthedocs.org that otherwise don't need a node.
+ However, it can also download content to a local file, or be used to
+ read from file:/// urls.
+
+ Block any use of url schemes other than https, http and ftp. Further,
+ block any local file interaction that falls outside of the zuul
+ work dir.
+ '''
+ # uri takes all the file arguments, so just let handle_file validate
+ # them for us.
+ self.handle_file()
+ scheme = urlparse(self._task.args['url']).scheme
+ if scheme not in ALLOWED_URL_SCHEMES:
+ raise AnsibleError(
+ "{scheme} urls are not allowed from localhost."
+ " Only {allowed_schemes} are allowed".format(
+ scheme=scheme,
+ allowed_schemes=ALLOWED_URL_SCHEMES))
diff --git a/zuul/configloader.py b/zuul/configloader.py
index a09147c..1036a2c 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -223,7 +223,11 @@
yaml_loader = yaml.SafeLoader
def __init__(self, ciphertext):
- self.ciphertext = base64.b64decode(ciphertext)
+ if isinstance(ciphertext, list):
+ self.ciphertext = [base64.b64decode(x.value)
+ for x in ciphertext]
+ else:
+ self.ciphertext = base64.b64decode(ciphertext)
def __ne__(self, other):
return not self.__eq__(other)
@@ -238,8 +242,14 @@
return cls(node.value)
def decrypt(self, private_key):
- return encryption.decrypt_pkcs1_oaep(self.ciphertext,
- private_key).decode('utf8')
+ if isinstance(self.ciphertext, list):
+ return ''.join([
+ encryption.decrypt_pkcs1_oaep(chunk, private_key).
+ decode('utf8')
+ for chunk in self.ciphertext])
+ else:
+ return encryption.decrypt_pkcs1_oaep(self.ciphertext,
+ private_key).decode('utf8')
class NodeSetParser(object):
diff --git a/zuul/driver/smtp/smtpreporter.py b/zuul/driver/smtp/smtpreporter.py
index 1f232e9..421d14b 100644
--- a/zuul/driver/smtp/smtpreporter.py
+++ b/zuul/driver/smtp/smtpreporter.py
@@ -47,7 +47,6 @@
def getSchema():
smtp_reporter = v.Schema({
- 'connection': str,
'to': str,
'from': str,
'subject': str,
diff --git a/zuul/driver/sql/alembic_reporter/versions/60c119eb1e3f_use_build_set_results.py b/zuul/driver/sql/alembic_reporter/versions/60c119eb1e3f_use_build_set_results.py
new file mode 100644
index 0000000..8e8142f
--- /dev/null
+++ b/zuul/driver/sql/alembic_reporter/versions/60c119eb1e3f_use_build_set_results.py
@@ -0,0 +1,49 @@
+"""Use build_set results
+
+Revision ID: 60c119eb1e3f
+Revises: f86c9871ee67
+Create Date: 2017-07-27 17:09:20.374782
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '60c119eb1e3f'
+down_revision = 'f86c9871ee67'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+
+BUILDSET_TABLE = 'zuul_buildset'
+
+
+def upgrade():
+ op.add_column(BUILDSET_TABLE, sa.Column('result', sa.String(255)))
+
+ connection = op.get_bind()
+ connection.execute(
+ """
+ UPDATE {buildset_table}
+ SET result=(
+ SELECT CASE score
+ WHEN 1 THEN 'SUCCESS'
+ ELSE 'FAILURE' END)
+ """.format(buildset_table=BUILDSET_TABLE))
+
+ op.drop_column(BUILDSET_TABLE, 'score')
+
+
+def downgrade():
+ op.add_column(BUILDSET_TABLE, sa.Column('score', sa.Integer))
+
+ connection = op.get_bind()
+ connection.execute(
+ """
+ UPDATE {buildset_table}
+ SET score=(
+ SELECT CASE result
+ WHEN 'SUCCESS' THEN 1
+ ELSE -1 END)
+ """.format(buildset_table=BUILDSET_TABLE))
+ op.drop_column(BUILDSET_TABLE, 'result')
diff --git a/zuul/driver/sql/sqlconnection.py b/zuul/driver/sql/sqlconnection.py
index 0e3f0dd..1187c2d 100644
--- a/zuul/driver/sql/sqlconnection.py
+++ b/zuul/driver/sql/sqlconnection.py
@@ -83,7 +83,7 @@
sa.Column('change', sa.Integer, nullable=True),
sa.Column('patchset', sa.Integer, nullable=True),
sa.Column('ref', sa.String(255)),
- sa.Column('score', sa.Integer, nullable=True),
+ sa.Column('result', sa.String(255)),
sa.Column('message', sa.TEXT()),
sa.Column('tenant', sa.String(255)),
)
diff --git a/zuul/driver/sql/sqlreporter.py b/zuul/driver/sql/sqlreporter.py
index aca1b06..7c79176 100644
--- a/zuul/driver/sql/sqlreporter.py
+++ b/zuul/driver/sql/sqlreporter.py
@@ -25,12 +25,6 @@
name = 'sql'
log = logging.getLogger("zuul.reporter.mysql.SQLReporter")
- def __init__(self, driver, connection, config={}):
- super(SQLReporter, self).__init__(
- driver, connection, config)
- # TODO(jeblair): document this is stored as NULL if unspecified
- self.result_score = config.get('score', None)
-
def report(self, item):
"""Create an entry into a database."""
@@ -49,7 +43,7 @@
change=change,
patchset=patchset,
ref=ref,
- score=self.result_score,
+ result=item.current_build_set.result,
message=self._formatItemReport(
item, with_jobs=False),
tenant=item.pipeline.layout.tenant.name,
@@ -85,7 +79,5 @@
def getSchema():
- sql_reporter = v.Schema({
- 'score': int,
- })
+ sql_reporter = v.Schema(None)
return sql_reporter
diff --git a/zuul/sphinx/ansible.py b/zuul/sphinx/ansible.py
new file mode 100644
index 0000000..4a47bc3
--- /dev/null
+++ b/zuul/sphinx/ansible.py
@@ -0,0 +1,53 @@
+# Copyright 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from docutils import nodes
+from sphinx.domains import Domain
+
+MODULE_URL = 'http://docs.ansible.com/ansible/latest/{module_name}_module.html'
+
+
+def ansible_module_role(
+ name, rawtext, text, lineno, inliner, options={}, content=[]):
+ """Link to an upstream Ansible module.
+
+ Returns 2 part tuple containing list of nodes to insert into the
+ document and a list of system messages. Both are allowed to be
+ empty.
+
+ :param name: The role name used in the document.
+ :param rawtext: The entire markup snippet, with role.
+ :param text: The text marked with the role.
+ :param lineno: The line number where rawtext appears in the input.
+ :param inliner: The inliner instance that called us.
+ :param options: Directive options for customization.
+ :param content: The directive content for customization.
+ """
+ node = nodes.reference(
+ rawtext, "Ansible {module_name} module".format(module_name=text),
+ refuri=MODULE_URL.format(module_name=text), **options)
+ return ([node], [])
+
+
+class AnsibleDomain(Domain):
+ name = 'ansible'
+ label = 'Ansible'
+
+ roles = {
+ 'module': ansible_module_role,
+ }
+
+
+def setup(app):
+ app.add_domain(AnsibleDomain)
diff --git a/zuul/sphinx/zuul.py b/zuul/sphinx/zuul.py
index b6aca6c..0ac33b8 100644
--- a/zuul/sphinx/zuul.py
+++ b/zuul/sphinx/zuul.py
@@ -166,6 +166,55 @@
return sig
+class ZuulStatDirective(ZuulConfigObject):
+ has_content = True
+
+ option_spec = {
+ 'type': lambda x: x,
+ 'hidden': lambda x: x,
+ 'noindex': lambda x: x,
+ }
+
+ def before_content(self):
+ path = self.env.ref_context.setdefault('zuul:attr_path', [])
+ element = self.names[-1]
+ path.append(element)
+ path = self.env.ref_context.setdefault('zuul:display_attr_path', [])
+ element = self.names[-1]
+ path.append(element)
+
+ def after_content(self):
+ path = self.env.ref_context.get('zuul:attr_path')
+ if path:
+ path.pop()
+ path = self.env.ref_context.get('zuul:display_attr_path')
+ if path:
+ path.pop()
+
+ def handle_signature(self, sig, signode):
+ if 'hidden' in self.options:
+ return sig
+ path = self.get_display_path()
+ for x in path:
+ signode += addnodes.desc_addname(x + '.', x + '.')
+ signode += addnodes.desc_name(sig, sig)
+ if 'type' in self.options:
+ t = ' (%s)' % self.options['type']
+ signode += addnodes.desc_annotation(t, t)
+ return sig
+
+
+class ZuulAbbreviatedXRefRole(XRefRole):
+
+ def process_link(self, env, refnode, has_explicit_title, title,
+ target):
+ title, target = super(ZuulAbbreviatedXRefRole, self).process_link(
+ env, refnode, has_explicit_title, title, target)
+ if not has_explicit_title:
+ title = title.split('.')[-1]
+ return title, target
+
+
class ZuulDomain(Domain):
name = 'zuul'
label = 'Zuul'
@@ -174,15 +223,19 @@
'attr': ZuulAttrDirective,
'value': ZuulValueDirective,
'var': ZuulVarDirective,
+ 'stat': ZuulStatDirective,
}
roles = {
'attr': XRefRole(innernodeclass=nodes.inline, # type: ignore
warn_dangling=True),
- 'value': XRefRole(innernodeclass=nodes.inline, # type: ignore
- warn_dangling=True),
+ 'value': ZuulAbbreviatedXRefRole(
+ innernodeclass=nodes.inline, # type: ignore
+ warn_dangling=True),
'var': XRefRole(innernodeclass=nodes.inline, # type: ignore
warn_dangling=True),
+ 'stat': XRefRole(innernodeclass=nodes.inline, # type: ignore
+ warn_dangling=True),
}
initial_data = {