Merge "Allow and document use of the uri module from localhost" into feature/zuulv3
diff --git a/bindep.txt b/bindep.txt
index 8dffd0f..85254b4 100644
--- a/bindep.txt
+++ b/bindep.txt
@@ -8,6 +8,7 @@
zookeeperd [platform:dpkg]
build-essential [platform:dpkg]
gcc [platform:rpm]
+graphviz [test]
libssl-dev [platform:dpkg]
openssl-devel [platform:rpm]
libffi-dev [platform:dpkg]
diff --git a/doc/source/admin/components.rst b/doc/source/admin/components.rst
index aef4dea..aa6d8c8 100644
--- a/doc/source/admin/components.rst
+++ b/doc/source/admin/components.rst
@@ -6,11 +6,37 @@
==========
Zuul is a distributed system consisting of several components, each of
-which is described below. All Zuul processes read the
-``/etc/zuul/zuul.conf`` file (an alternate location may be supplied on
-the command line) which uses an INI file syntax. Each component may
-have its own configuration file, though you may find it simpler to use
-the same file for all components.
+which is described below.
+
+
+.. graphviz::
+ :align: center
+
+ graph {
+ node [shape=box]
+ Gearman [shape=ellipse]
+ Gerrit [fontcolor=grey]
+ Zookeeper [shape=ellipse]
+ Nodepool
+ GitHub [fontcolor=grey]
+
+ Merger -- Gearman
+ Executor -- Gearman
+ Web -- Gearman
+
+ Gearman -- Scheduler;
+ Scheduler -- Gerrit;
+ Scheduler -- Zookeeper;
+ Zookeeper -- Nodepool;
+ Scheduler -- GitHub;
+ }
+
+
+
+All Zuul processes read the ``/etc/zuul/zuul.conf`` file (an alternate
+location may be supplied on the command line) which uses an INI file
+syntax. Each component may have its own configuration file, though
+you may find it simpler to use the same file for all components.
An example ``zuul.conf``:
@@ -177,10 +203,12 @@
Path to log config file.
.. attr:: pidfile
+ :default: /var/run/zuul-schedurecr/zuul-scheduler.pid
Path to PID lock file.
.. attr:: state_dir
+ :default: /var/lib/zuul
Path to directory in which Zuul should save its state.
@@ -219,35 +247,32 @@
Configuration
~~~~~~~~~~~~~
-The following section of **zuul.conf** is used by the merger:
+The following section of ``zuul.conf`` is used by the merger:
-merger
-""""""
+.. attr:: merger
-**git_dir**
- Directory that Zuul should clone local git repositories to::
+ .. attr:: git_dir
- git_dir=/var/lib/zuul/git
+ Directory in which Zuul should clone git repositories.
-**git_user_email**
- Value to pass to `git config user.email`::
+ .. attr:: git_user_email
- git_user_email=zuul@example.com
+ Value to pass to `git config user.email
+ <https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup>`_.
-**git_user_name**
- Value to pass to `git config user.name`::
+ .. attr:: git_user_name
- git_user_name=zuul
+ Value to pass to `git config user.name
+ <https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup>`_.
-**log_config**
- Path to log config file for the merger process::
+ .. attr:: log_config
- log_config=/etc/zuul/logging.yaml
+ Path to log config file for the merger process.
-**pidfile**
- Path to PID lock file for the merger process::
+ .. attr:: pidfile
+ :default: /var/run/zuul-merger/zuul-merger.pid
- pidfile=/var/run/zuul-merger/merger.pid
+ Path to PID lock file for the merger process.
Operation
~~~~~~~~~
@@ -280,10 +305,11 @@
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The executor runs playbooks in one of two execution contexts depending
-on whether the project containing the playbook is a *config project*
-or an *untrusted project*. If the playbook is in a *config project*,
-the executor runs the playbook in the *trusted* execution context,
-otherwise, it is run in the *untrusted* execution context.
+on whether the project containing the playbook is a
+:term:`config-project` or an :term:`untrusted-project`. If the
+playbook is in a config project, the executor runs the playbook in the
+*trusted* execution context, otherwise, it is run in the *untrusted*
+execution context.
Both execution contexts use `bubblewrap`_ to create a namespace to
ensure that playbook executions are isolated and are unable to access
@@ -291,7 +317,7 @@
configure additional local directories on the executor to be made
available to the restricted environment.
-The *trusted* execution context has access to all Ansible features,
+The trusted execution context has access to all Ansible features,
including the ability to load custom Ansible modules. Needless to
say, extra scrutiny should be given to code that runs in a trusted
context as it could be used to compromise other jobs running on the
@@ -299,128 +325,129 @@
granted additional access through bubblewrap, or a method of escaping
the restricted environment created by bubblewrap is found.
-Playbooks run in the *untrusted* execution context are not permitted
-to load additional Ansible modules or access files outside of the
+Playbooks run in the untrusted execution context are not permitted to
+load additional Ansible modules or access files outside of the
restricted environment prepared for them by the executor. In addition
to the bubblewrap environment applied to both execution contexts, in
-the *untrusted* context some standard Ansible modules are replaced
-with versions which prohibit some actions, including attempts to
-access files outside of the restricted execution context. These
-redundant protections are made as part of a defense-in-depth strategy.
+the untrusted context some standard Ansible modules are replaced with
+versions which prohibit some actions, including attempts to access
+files outside of the restricted execution context. These redundant
+protections are made as part of a defense-in-depth strategy.
.. _bubblewrap: https://github.com/projectatomic/bubblewrap
Configuration
~~~~~~~~~~~~~
-The following sections of **zuul.conf** are used by the executor:
+The following sections of ``zuul.conf`` are used by the executor:
-executor
-""""""""
+.. attr:: executor
-**finger_port**
- Port to use for finger log streamer::
+ .. attr:: finger_port
+ :default: 79
- finger_port=79
+ Port to use for finger log streamer.
-**git_dir**
- Directory that Zuul should clone local git repositories to. The
- executor keeps a local copy of every git repository it works with to
- speed operations and perform speculative merging.
+ .. attr:: git_dir
+ :default: /var/lib/zuul/git
- This should be on the same filesystem as **job_dir** so that when
- git repos are cloned into the job workspaces, they can be
- hard-linked to the local git cache. Example::
+ Directory that Zuul should clone local git repositories to. The
+ executor keeps a local copy of every git repository it works
+ with to speed operations and perform speculative merging.
- git_dir=/var/lib/zuul/git
+ This should be on the same filesystem as
+ :attr:`executor.job_dir` so that when git repos are cloned into
+ the job workspaces, they can be hard-linked to the local git
+ cache.
-**job_dir**
- Directory that Zuul should use to hold temporary job directories.
- When each job is run, a new entry will be created under this
- directory to hold the configuration and scratch workspace for that
- job. It will be deleted at the end of the job (unless the
- `--keep-jobdir` command line option is specified).
+ .. attr:: job_dir
+ :default: /tmp
- This should be on the same filesystem as **git_dir** so that when
- git repos are cloned into the job workspaces, they can be
- hard-linked to the local git cache. Example::
+ Directory that Zuul should use to hold temporary job directories.
+ When each job is run, a new entry will be created under this
+ directory to hold the configuration and scratch workspace for
+ that job. It will be deleted at the end of the job (unless the
+ `--keep-jobdir` command line option is specified).
- job_dir=/var/lib/zuul/jobs
+ This should be on the same filesystem as :attr:`executor.git_dir`
+ so that when git repos are cloned into the job workspaces, they
+ can be hard-linked to the local git cache.
-**log_config**
- Path to log config file for the executor process::
+ .. attr:: log_config
- log_config=/etc/zuul/logging.yaml
+ Path to log config file for the executor process.
-**private_key_file**
- SSH private key file to be used when logging into worker nodes::
+ .. attr:: pidfile
+ :default: /var/run/zuul-executor/zuul-executor.pid
- private_key_file=~/.ssh/id_rsa
+ Path to PID lock file for the executor process.
-**user**
- User ID for the zuul-executor process. In normal operation as a daemon,
- the executor should be started as the ``root`` user, but it will drop
- privileges to this user during startup::
+ .. attr:: private_key_file
+ :default: ~/.ssh/id_rsa
- user=zuul
+ SSH private key file to be used when logging into worker nodes.
-.. _admin_sitewide_variables:
+ .. attr:: user
+ :default: zuul
-**variables**
- Path to an Ansible variables file to supply site-wide variables.
- This should be a YAML-formatted file consisting of a single
- dictionary. The contents will be made available to all jobs as
- Ansible variables. These variables take precedence over all other
- forms (job variables and secrets). Care should be taken when naming
- these variables to avoid potential collisions with those used by
- jobs. Prefixing variable names with a site-specific identifier is
- recommended. The default is not to add any site-wide variables.
- See the :ref:`User's Guide <user_sitewide_variables>` for more
- information.
+ User ID for the zuul-executor process. In normal operation as a
+ daemon, the executor should be started as the ``root`` user, but
+ it will drop privileges to this user during startup.
- Example::
+ .. _admin_sitewide_variables:
- variables=/etc/zuul/variables.yaml
+ .. attr:: variables
-**disk_limit_per_job**
- This integer is the maximum number of megabytes that any one job is
- allowed to consume on disk while it is running. If a job's scratch
- space has more than this much space consumed, it will be aborted::
+ Path to an Ansible variables file to supply site-wide variables.
+ This should be a YAML-formatted file consisting of a single
+ dictionary. The contents will be made available to all jobs as
+ Ansible variables. These variables take precedence over all
+ other forms (job variables and secrets). Care should be taken
+ when naming these variables to avoid potential collisions with
+ those used by jobs. Prefixing variable names with a
+ site-specific identifier is recommended. The default is not to
+ add any site-wide variables. See the :ref:`User's Guide
+ <user_sitewide_variables>` for more information.
- disk_limit_per_job=100
+ .. attr:: disk_limit_per_job
+ :default: 250
-**trusted_ro_paths**
+ This integer is the maximum number of megabytes that any one job
+ is allowed to consume on disk while it is running. If a job's
+ scratch space has more than this much space consumed, it will be
+ aborted.
- List of paths, separated by ':' to read-only bind mount into trusted
- bubblewrap contexts.
+ .. attr:: trusted_ro_paths
-**trusted_rw_paths**
+ List of paths, separated by ``:`` to read-only bind mount into
+ trusted bubblewrap contexts.
- List of paths, separated by ':' to read-write bind mount into trusted
- bubblewrap contexts.
+ .. attr:: trusted_rw_paths
-**untrusted_ro_paths**
+ List of paths, separated by ``:`` to read-write bind mount into
+ trusted bubblewrap contexts.
- List of paths, separated by ':' to read-only bind mount into untrusted
- bubblewrap contexts.
+ .. attr:: untrusted_ro_paths
-**untrusted_rw_paths**
+ List of paths, separated by ``:`` to read-only bind mount into
+ untrusted bubblewrap contexts.
- List of paths, separated by ':' to read-write bind mount into untrusted
- bubblewrap contexts.
+ .. attr:: untrusted_rw_paths
-merger
-""""""
+ List of paths, separated by ``:`` to read-write bind mount into
+ untrusted bubblewrap contexts.
-**git_user_email**
- Value to pass to `git config user.email`::
+.. attr:: merger
- git_user_email=zuul@example.com
+ .. attr:: git_user_email
-**git_user_name**
- Value to pass to `git config user.name`::
+ Value to pass to `git config user.email
+ <https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup>`_.
- git_user_name=zuul
+ .. attr:: git_user_name
+
+ Value to pass to `git config user.name
+ <https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup>`_.
Operation
~~~~~~~~~
@@ -437,9 +464,9 @@
To request that the executor stop executing new jobs and exit when all
currently running jobs have completed, run ``zuul-executor graceful``.
-To enable or disable running Ansible in verbose mode (with the '-vvv'
-argument to ansible-playbook) run ``zuul-executor verbose`` and
-``zuul-executor unverbose``.
+To enable or disable running Ansible in verbose mode (with the
+``-vvv`` argument to ansible-playbook) run ``zuul-executor verbose``
+and ``zuul-executor unverbose``.
Web Server
----------
@@ -451,35 +478,34 @@
Configuration
~~~~~~~~~~~~~
-In addition to the ``gearman`` common configuration section, the following
-sections of **zuul.conf** are used by the web server:
+In addition to the common configuration sections, the following
+sections of ``zuul.conf`` are used by the web server:
-web
-"""
+.. attr:: web
-**listen_address**
- IP address or domain name on which to listen (default: 127.0.0.1)::
+ .. attr:: listen_address
+ :default: 127.0.0.1
- listen_address=127.0.0.1
+ IP address or domain name on which to listen.
-**log_config**
- Path to log config file for the web server process::
+ .. attr:: log_config
- log_config=/etc/zuul/logging.yaml
+ Path to log config file for the web server process.
-**pidfile**
- Path to PID lock file for the web server process::
+ .. attr:: pidfile
+ :default: /var/run/zuul-web/zuul-web.pid
- pidfile=/var/run/zuul-web/zuul-web.pid
+ Path to PID lock file for the web server process.
-**port**
- Port to use for web server process::
+ .. attr:: port
+ :default: 9000
- port=9000
+ Port to use for web server process.
-**websocket_url**
- Base URL on which the websocket service is exposed, if different than the
- base URL of the web app.
+ .. attr:: websocket_url
+
+ Base URL on which the websocket service is exposed, if different
+ than the base URL of the web app.
Operation
~~~~~~~~~
diff --git a/doc/source/admin/connections.rst b/doc/source/admin/connections.rst
index 5b40e5b..29ca3be 100644
--- a/doc/source/admin/connections.rst
+++ b/doc/source/admin/connections.rst
@@ -23,9 +23,11 @@
driver.
To configure a connection in Zuul, select a unique name for the
-connection and add a section to **zuul.conf** with the form
-"[connection NAME]". For example, a connection to a gerrit server may
-appear as::
+connection and add a section to ``zuul.conf`` with the form
+``[connection NAME]``. For example, a connection to a gerrit server
+may appear as:
+
+.. code-block:: ini
[connection mygerritserver]
driver=gerrit
diff --git a/doc/source/admin/drivers/gerrit.rst b/doc/source/admin/drivers/gerrit.rst
index 454f8d0..ac42bd3 100644
--- a/doc/source/admin/drivers/gerrit.rst
+++ b/doc/source/admin/drivers/gerrit.rst
@@ -24,44 +24,57 @@
Connection Configuration
------------------------
-The supported options in zuul.conf connections are:
+The supported options in ``zuul.conf`` connections are:
-**driver=gerrit**
+.. attr:: <gerrit connection>
-**server**
- FQDN of Gerrit server.
- ``server=review.example.com``
+ .. attr:: driver
+ :required:
-**canonical_hostname**
- The canonical hostname associated with the git repos on the Gerrit
- server. Defaults to the value of **server**. This is used to
- identify projects from this connection by name and in preparing
- repos on the filesystem for use by jobs. Note that Zuul will still
- only communicate with the Gerrit server identified by **server**;
- this option is useful if users customarily use a different hostname
- to clone or pull git repos so that when Zuul places them in the
- job's working directory, they appear under this directory name.
- ``canonical_hostname=git.example.com``
+ .. value:: gerrit
-**port**
- Optional: Gerrit server port.
- ``port=29418``
+ The connection must set ``driver=gerrit`` for Gerrit connections.
-**baseurl**
- Optional: path to Gerrit web interface. Defaults to ``https://<value
- of server>/``. ``baseurl=https://review.example.com/review_site/``
+ .. attr:: server
-**user**
- User name to use when logging into above server via ssh.
- ``user=zuul``
+ Fully qualified domain name of Gerrit server.
-**sshkey**
- Path to SSH key to use when logging into above server.
- ``sshkey=/home/zuul/.ssh/id_rsa``
+ .. attr:: canonical_hostname
-**keepalive**
- Optional: Keepalive timeout, 0 means no keepalive.
- ``keepalive=60``
+ The canonical hostname associated with the git repos on the
+ Gerrit server. Defaults to the value of
+ :attr:`<gerrit connection>.server`. This is used to identify
+ projects from this connection by name and in preparing repos on
+ the filesystem for use by jobs. Note that Zuul will still only
+ communicate with the Gerrit server identified by ``server``;
+ this option is useful if users customarily use a different
+ hostname to clone or pull git repos so that when Zuul places
+ them in the job's working directory, they appear under this
+ directory name.
+
+ .. attr:: port
+ :default: 29418
+
+ Gerrit server port.
+
+ .. attr:: baseurl
+
+ Path to Gerrit web interface.
+
+ .. attr:: user
+ :default: zuul
+
+ User name to use when logging into Gerrit via ssh.
+
+ .. attr:: sshkey
+ :default: ~zuul/.ssh/id_rsa
+
+ Path to SSH key to use when logging into Gerrit.
+
+ .. attr:: keepalive
+ :default: 60
+
+ SSH connection keepalive timeout; ``0`` disables.
Trigger Configuration
---------------------
@@ -74,85 +87,88 @@
that is granted the ``Stream Events`` permission, otherwise it will not
be able to invoke the ``gerrit stream-events`` command over SSH.
-The supported pipeline trigger options are:
+.. attr:: pipeline.trigger.<gerrit source>
-**event**
- The event name from gerrit. Examples: ``patchset-created``,
- ``comment-added``, ``ref-updated``. This field is treated as a
- regular expression.
+ The dictionary passed to the Gerrit pipeline ``trigger`` attribute
+ supports the following attributes:
-**branch**
- The branch associated with the event. Example: ``master``. This
- field is treated as a regular expression, and multiple branches may
- be listed.
+ .. attr:: event
+ :required:
-**ref**
- On ref-updated events, the branch parameter is not used, instead the
- ref is provided. Currently Gerrit has the somewhat idiosyncratic
- behavior of specifying bare refs for branch names (e.g.,
- ``master``), but full ref names for other kinds of refs (e.g.,
- ``refs/tags/foo``). Zuul matches what you put here exactly against
- what Gerrit provides. This field is treated as a regular
- expression, and multiple refs may be listed.
+ The event name from gerrit. Examples: ``patchset-created``,
+ ``comment-added``, ``ref-updated``. This field is treated as a
+ regular expression.
-**ignore-deletes**
- When a branch is deleted, a ref-updated event is emitted with a
- newrev of all zeros specified. The ``ignore-deletes`` field is a
- boolean value that describes whether or not these newrevs trigger
- ref-updated events. The default is True, which will not trigger
- ref-updated events.
+ .. attr:: branch
-**approval**
- This is only used for ``comment-added`` events. It only matches if
- the event has a matching approval associated with it. Example:
- ``Code-Review: 2`` matches a ``+2`` vote on the code review
- category. Multiple approvals may be listed.
+ The branch associated with the event. Example: ``master``.
+ This field is treated as a regular expression, and multiple
+ branches may be listed.
-**email**
- This is used for any event. It takes a regex applied on the
- performer email, i.e. Gerrit account email address. If you want to
- specify several email filters, you must use a YAML list. Make sure
- to use non greedy matchers and to escapes dots! Example: ``email:
- ^.*?@example\.org$``.
+ .. attr:: ref
-**email_filter** (deprecated)
- A deprecated alternate spelling of *email*. Only one of *email* or
- *email_filter* should be used.
+ On ref-updated events, the branch parameter is not used, instead
+ the ref is provided. Currently Gerrit has the somewhat
+ idiosyncratic behavior of specifying bare refs for branch names
+ (e.g., ``master``), but full ref names for other kinds of refs
+ (e.g., ``refs/tags/foo``). Zuul matches this value exactly
+ against what Gerrit provides. This field is treated as a
+ regular expression, and multiple refs may be listed.
-**username**
- This is used for any event. It takes a regex applied on the
- performer username, i.e. Gerrit account name. If you want to
- specify several username filters, you must use a YAML list. Make
- sure to use non greedy matchers and to escapes dots! Example:
- ``username: ^jenkins$``.
+ .. attr:: ignore-deletes
+ :default: true
-**username_filter** (deprecated)
- A deprecated alternate spelling of *username*. Only one of
- *username* or *username_filter* should be used.
+ When a branch is deleted, a ref-updated event is emitted with a
+ newrev of all zeros specified. The ``ignore-deletes`` field is a
+ boolean value that describes whether or not these newrevs
+ trigger ref-updated events.
-**comment**
- This is only used for ``comment-added`` events. It accepts a list
- of regexes that are searched for in the comment string. If any of
- these regexes matches a portion of the comment string the trigger is
- matched. ``comment: retrigger`` will match when comments containing
- 'retrigger' somewhere in the comment text are added to a change.
+ .. attr:: approval
-**comment_filter** (deprecated)
- A deprecated alternate spelling of *comment*. Only one of *comment*
- or *comment_filter* should be used.
+ This is only used for ``comment-added`` events. It only matches
+ if the event has a matching approval associated with it.
+ Example: ``Code-Review: 2`` matches a ``+2`` vote on the code
+ review category. Multiple approvals may be listed.
-**require-approval**
- This may be used for any event. It requires that a certain kind of
- approval be present for the current patchset of the change (the
- approval could be added by the event in question). It follows the
- same syntax as :attr:`pipeline.require.<gerrit
- source>.approval`. For each specified criteria there must exist a
- matching approval.
+ .. attr:: email
-**reject-approval**
- This takes a list of approvals in the same format as
- *require-approval* but will fail to enter the pipeline if there is a
- matching approval.
+ This is used for any event. It takes a regex applied on the
+ performer email, i.e. Gerrit account email address. If you want
+ to specify several email filters, you must use a YAML list.
+ Make sure to use non greedy matchers and to escapes dots!
+ Example: ``email: ^.*?@example\.org$``.
+
+ .. attr:: username
+
+ This is used for any event. It takes a regex applied on the
+ performer username, i.e. Gerrit account name. If you want to
+ specify several username filters, you must use a YAML list.
+ Make sure to use non greedy matchers and to escapes dots.
+ Example: ``username: ^zuul$``.
+
+ .. attr:: comment
+
+ This is only used for ``comment-added`` events. It accepts a
+ list of regexes that are searched for in the comment string. If
+ any of these regexes matches a portion of the comment string the
+ trigger is matched. ``comment: retrigger`` will match when
+ comments containing ``retrigger`` somewhere in the comment text
+ are added to a change.
+
+ .. attr:: require-approval
+
+ This may be used for any event. It requires that a certain kind
+ of approval be present for the current patchset of the change
+ (the approval could be added by the event in question). It
+ follows the same syntax as :attr:`pipeline.require.<gerrit
+ source>.approval`. For each specified criteria there must exist
+ a matching approval.
+
+ .. attr:: reject-approval
+
+ This takes a list of approvals in the same format as
+ :attr:`pipeline.trigger.<gerrit source>.require-approval` but
+ will fail to enter the pipeline if there is a matching approval.
Reporter Configuration
----------------------
@@ -178,18 +194,20 @@
pipelines may specify that items meet certain conditions in order to
be enqueued into the pipeline. These conditions vary according to the
source of the project in question. To supply requirements for changes
-from a Gerrit source named *my-gerrit*, create a configuration such as
-the following::
+from a Gerrit source named ``my-gerrit``, create a configuration such
+as the following:
- pipeline:
- require:
- my-gerrit:
- approval:
- - Code-Review: 2
+.. code-block:: yaml
+
+ pipeline:
+ require:
+ my-gerrit:
+ approval:
+ - Code-Review: 2
This indicates that changes originating from the Gerrit connection
-named *my-gerrit* must have a Code Review vote of +2 in order to be
-enqueued into the pipeline.
+named ``my-gerrit`` must have a ``Code-Review`` vote of ``+2`` in
+order to be enqueued into the pipeline.
.. attr:: pipeline.require.<gerrit source>
@@ -260,9 +278,11 @@
pipeline. It follows the same syntax as
:attr:`pipeline.require.<gerrit source>.approval`.
- Example to reject a change with any negative vote::
+ Example to reject a change with any negative vote:
- reject:
- my-gerrit:
- approval:
- - Code-Review: [-1, -2]
+ .. code-block:: yaml
+
+ reject:
+ my-gerrit:
+ approval:
+ - Code-Review: [-1, -2]
diff --git a/doc/source/admin/drivers/github.rst b/doc/source/admin/drivers/github.rst
index cbbc5cc..7eebbdc 100644
--- a/doc/source/admin/drivers/github.rst
+++ b/doc/source/admin/drivers/github.rst
@@ -29,135 +29,189 @@
``app_key`` are required. If the Webhook approach is taken, the ``api_token``
setting is required.
-The supported options in zuul.conf connections are:
+The supported options in ``zuul.conf`` connections are:
-**driver=github**
+.. attr:: <github connection>
-**app_id**
- App ID if you are using a GitHub App. Can be found under the "Public Link"
- on the right hand side labeled "ID".
- ``app_id=1234``
+ .. attr:: driver
+ :required:
-**app_key**
- Path to a file containing the Secret Key Zuul will use to create tokens for
- the API interactions. In Github this is known as "Private key" and must be
- collected when generated.
- ``app_key=/etc/zuul/github.key``
+ .. value:: github
-**api_token**
- API token for accessing GitHub if Zuul is configured with Webhooks.
- See `Creating an access token for command-line use
- <https://help.github.com/articles/creating-an-access-token-for-command-line-use/>`_.
+ The connection must set ``driver=github`` for GitHub connections.
-**webhook_token**
- Required token for validating the webhook event payloads. In the
- GitHub App Configuration page, this is called "Webhook secret".
- See `Securing your webhooks
- <https://developer.github.com/webhooks/securing/>`_.
+ .. attr:: app_id
-**sshkey**
- Path to SSH key to use when cloning github repositories.
- ``sshkey=/home/zuul/.ssh/id_rsa``
+ App ID if you are using a *GitHub App*. Can be found under the
+ **Public Link** on the right hand side labeled **ID**.
-**server**
- Optional: Hostname of the github install (such as a GitHub Enterprise)
- If not specified, defaults to ``github.com``
- ``server=github.myenterprise.com``
+ .. attr:: app_key
-**canonical_hostname**
- The canonical hostname associated with the git repos on the GitHub
- server. Defaults to the value of **server**. This is used to
- identify projects from this connection by name and in preparing
- repos on the filesystem for use by jobs. Note that Zuul will still
- only communicate with the GitHub server identified by **server**;
- this option is useful if users customarily use a different hostname
- to clone or pull git repos so that when Zuul places them in the
- job's working directory, they appear under this directory name.
- ``canonical_hostname=git.example.com``
+ Path to a file containing the secret key Zuul will use to create
+ tokens for the API interactions. In Github this is known as
+ **Private key** and must be collected when generated.
-**verify_ssl**
- Optional: Enable or disable ssl verification for GitHub Enterprise. This is
- useful for a connection to a test installation. If not specified, defaults
- to ``true``.
- ``verify_ssl=true``
+ .. attr:: api_token
+
+ API token for accessing GitHub if Zuul is configured with
+ Webhooks. See `Creating an access token for command-line use
+ <https://help.github.com/articles/creating-an-access-token-for-command-line-use/>`_.
+
+ .. attr:: webhook_token
+
+ Required token for validating the webhook event payloads. In
+ the GitHub App Configuration page, this is called **Webhook
+ secret**. See `Securing your webhooks
+ <https://developer.github.com/webhooks/securing/>`_.
+
+ .. attr:: sshkey
+ :default: ~/.ssh/id_rsa
+
+ Path to SSH key to use when cloning github repositories.
+
+ .. attr:: server
+ :default: github.com
+
+ Hostname of the github install (such as a GitHub Enterprise).
+
+ .. attr:: canonical_hostname
+
+ The canonical hostname associated with the git repos on the
+ GitHub server. Defaults to the value of :attr:`<github
+ connection>.server`. This is used to identify projects from
+ this connection by name and in preparing repos on the filesystem
+ for use by jobs. Note that Zuul will still only communicate
+ with the GitHub server identified by **server**; this option is
+ useful if users customarily use a different hostname to clone or
+ pull git repos so that when Zuul places them in the job's
+ working directory, they appear under this directory name.
+
+ .. attr:: verify_ssl
+ :default: true
+
+ Enable or disable ssl verification for GitHub Enterprise. This
+ is useful for a connection to a test installation.
Trigger Configuration
---------------------
GitHub webhook events can be configured as triggers.
-A connection name with the github driver can take multiple events with the
-following options.
+A connection name with the GitHub driver can take multiple events with
+the following options.
-**event**
- The event from github. Supported events are ``pull_request``,
- ``pull_request_review``, and ``push``.
+.. attr:: pipeline.trigger.<github source>
- A ``pull_request`` event will have associated action(s) to trigger
- from. The supported actions are:
+ The dictionary passed to the GitHub pipeline ``trigger`` attribute
+ supports the following attributes:
- *opened* - pull request opened
+ .. attr:: event
+ :required:
- *changed* - pull request synchronized
+ The event from github. Supported events are:
- *closed* - pull request closed
+ .. value:: pull_request
- *reopened* - pull request reopened
+ .. value:: pull_request_review
- *comment* - comment added on pull request
+ .. value:: push
- *labeled* - label added on pull request
+ .. attr:: action
- *unlabeled* - label removed from pull request
+ A :value:`pipeline.trigger.<github source>.event.pull_request`
+ event will have associated action(s) to trigger from. The
+ supported actions are:
- *status* - status set on commit
+ .. value:: opened
- A ``pull_request_review`` event will
- have associated action(s) to trigger from. The supported actions are:
+ Pull request opened.
- *submitted* - pull request review added
+ .. value:: changed
- *dismissed* - pull request review removed
+ Pull request synchronized.
-**branch**
- The branch associated with the event. Example: ``master``. This
- field is treated as a regular expression, and multiple branches may
- be listed. Used for ``pull_request`` and ``pull_request_review``
- events.
+ .. value:: closed
-**comment**
- This is only used for ``pull_request`` ``comment`` actions. It
- accepts a list of regexes that are searched for in the comment
- string. If any of these regexes matches a portion of the comment
- string the trigger is matched. ``comment: retrigger`` will match
- when comments containing 'retrigger' somewhere in the comment text
- are added to a pull request.
+ Pull request closed.
-**label**
- This is only used for ``labeled`` and ``unlabeled`` ``pull_request``
- actions. It accepts a list of strings each of which matches the
- label name in the event literally. ``label: recheck`` will match a
- ``labeled`` action when pull request is labeled with a ``recheck``
- label. ``label: 'do not test'`` will match a ``unlabeled`` action
- when a label with name ``do not test`` is removed from the pull
- request.
+ .. value:: reopened
-**state**
- This is only used for ``pull_request_review`` events. It accepts a
- list of strings each of which is matched to the review state, which
- can be one of ``approved``, ``comment``, or ``request_changes``.
+ Pull request reopened.
-**status**
- This is used for ``pull-request`` and ``status`` actions. It accepts
- a list of strings each of which matches the user setting the status,
- the status context, and the status itself in the format of
- ``user:context:status``. For example,
- ``zuul_github_ci_bot:check_pipeline:success``.
+ .. value:: comment
-**ref**
- This is only used for ``push`` events. This field is treated as a
- regular expression and multiple refs may be listed. GitHub always
- sends full ref name, eg. ``refs/tags/bar`` and this string is
- matched against the regexp.
+ Comment added to pull request.
+
+ .. value:: labeled
+
+ Label added to pull request.
+
+ .. value:: unlabeled
+
+ Label removed from pull request.
+
+ .. value:: status
+
+ Status set on commit.
+
+ A :value:`pipeline.trigger.<github
+ source>.event.pull_request_review` event will have associated
+ action(s) to trigger from. The supported actions are:
+
+ .. value:: submitted
+
+ Pull request review added.
+
+ .. value:: dismissed
+
+ Pull request review removed.
+
+ .. attr:: branch
+
+ The branch associated with the event. Example: ``master``. This
+ field is treated as a regular expression, and multiple branches
+ may be listed. Used for ``pull_request`` and
+ ``pull_request_review`` events.
+
+ .. attr:: comment
+
+ This is only used for ``pull_request`` ``comment`` actions. It
+ accepts a list of regexes that are searched for in the comment
+ string. If any of these regexes matches a portion of the comment
+ string the trigger is matched. ``comment: retrigger`` will
+ match when comments containing 'retrigger' somewhere in the
+ comment text are added to a pull request.
+
+ .. attr:: label
+
+ This is only used for ``labeled`` and ``unlabeled``
+ ``pull_request`` actions. It accepts a list of strings each of
+ which matches the label name in the event literally. ``label:
+ recheck`` will match a ``labeled`` action when pull request is
+ labeled with a ``recheck`` label. ``label: 'do not test'`` will
+ match a ``unlabeled`` action when a label with name ``do not
+ test`` is removed from the pull request.
+
+ .. attr:: state
+
+ This is only used for ``pull_request_review`` events. It
+ accepts a list of strings each of which is matched to the review
+ state, which can be one of ``approved``, ``comment``, or
+ ``request_changes``.
+
+ .. attr:: status
+
+ This is used for ``pull-request`` and ``status`` actions. It
+ accepts a list of strings each of which matches the user setting
+ the status, the status context, and the status itself in the
+ format of ``user:context:status``. For example,
+ ``zuul_github_ci_bot:check_pipeline:success``.
+
+ .. attr:: ref
+
+ This is only used for ``push`` events. This field is treated as
+ a regular expression and multiple refs may be listed. GitHub
+ always sends full ref name, eg. ``refs/tags/bar`` and this
+ string is matched against the regular expression.
Reporter Configuration
----------------------
@@ -166,38 +220,51 @@
failure, an issue label addition/removal on the PR, and a merge of the PR
itself. Status name, description, and context is taken from the pipeline.
-A :ref:`connection<connections>` that uses the github driver must be
-supplied to the reporter. It has the following options:
+.. attr:: pipeline.<reporter>.<github source>
-**status**
- String value (``pending``, ``success``, ``failure``) that the
- reporter should set as the commit status on github. ``status:
- 'success'``
+ To report to GitHub, the dictionaries passed to any of the pipeline
+ :ref:`reporter<reporters>` attributes support the following
+ attributes:
-**status-url**
- String value for a link url to set in the github status. Defaults to
- the zuul server status_url, or the empty string if that is unset.
+ .. attr:: status
-**comment**
- Boolean value (``true`` or ``false``) that determines if the
- reporter should add a comment to the pipeline status to the github
- pull request. Defaults to ``true``. Only used for Pull Request based
- events. ``comment: false``
+ String value (``pending``, ``success``, ``failure``) that the
+ reporter should set as the commit status on github.
-**merge**
- Boolean value (``true`` or ``false``) that determines if the
- reporter should merge the pull reqeust. Defaults to ``false``. Only
- used for Pull Request based events. ``merge=true``
+ .. TODO support role markup in :default: so we can xref
+ :attr:`webapp.status_url` below
-**label**
- List of strings each representing an exact label name which should
- be added to the pull request by reporter. Only used for Pull Request
- based events. ``label: 'test successful'``
+ .. attr:: status-url
+ :default: webapp.status_url or the empty string
-**unlabel**
- List of strings each representing an exact label name which should
- be removed from the pull request by reporter. Only used for Pull
- Request based events. ``unlabel: 'test failed'``
+ String value for a link url to set in the github
+ status. Defaults to the zuul server status_url, or the empty
+ string if that is unset.
+
+ .. attr:: comment
+ :default: true
+
+ Boolean value that determines if the reporter should add a
+ comment to the pipeline status to the github pull request. Only
+ used for Pull Request based items.
+
+ .. attr:: merge
+ :default: false
+
+ Boolean value that determines if the reporter should merge the
+ pull reqeust. Only used for Pull Request based items.
+
+ .. attr:: label
+
+ List of strings each representing an exact label name which
+ should be added to the pull request by reporter. Only used for
+ Pull Request based items.
+
+ .. attr:: unlabel
+
+ List of strings each representing an exact label name which
+ should be removed from the pull request by reporter. Only used
+ for Pull Request based items.
.. _Github App: https://developer.github.com/apps/
@@ -208,7 +275,7 @@
pipelines may specify that items meet certain conditions in order to
be enqueued into the pipeline. These conditions vary according to the
source of the project in question. To supply requirements for changes
-from a GitHub source named *my-github*, create a congfiguration such
+from a GitHub source named ``my-github``, create a congfiguration such
as the following::
pipeline:
@@ -218,7 +285,7 @@
- type: approval
This indicates that changes originating from the GitHub connection
-named *my-github* must have an approved code review in order to be
+named ``my-github`` must have an approved code review in order to be
enqueued into the pipeline.
.. attr:: pipeline.require.<github source>
@@ -292,7 +359,6 @@
A string value indicating that the pull request must have the
indicated label (or labels).
-
.. attr:: pipeline.reject.<github source>
The `reject` attribute is the mirror of the `require` attribute. It
diff --git a/doc/source/admin/drivers/smtp.rst b/doc/source/admin/drivers/smtp.rst
index 6f24355..11c0624 100644
--- a/doc/source/admin/drivers/smtp.rst
+++ b/doc/source/admin/drivers/smtp.rst
@@ -9,25 +9,36 @@
Connection Configuration
------------------------
-**driver=smtp**
+.. attr:: <smtp connection>
-**server**
- SMTP server hostname or address to use.
- ``server=localhost``
+ .. attr:: driver
+ :required:
-**port**
- Optional: SMTP server port.
- ``port=25``
+ .. value:: smtp
-**default_from**
- Who the email should appear to be sent from when emailing the report.
- This can be overridden by individual pipelines.
- ``default_from=zuul@example.com``
+ The connection must set ``driver=smtp`` for SMTP connections.
-**default_to**
- Who the report should be emailed to by default.
- This can be overridden by individual pipelines.
- ``default_to=you@example.com``
+ .. attr:: server
+ :default: localhost
+
+ SMTP server hostname or address to use.
+
+ .. attr:: port
+ :default: 25
+
+ SMTP server port.
+
+ .. attr:: default_from
+ :default: zuul
+
+ Who the email should appear to be sent from when emailing the report.
+ This can be overridden by individual pipelines.
+
+ .. attr:: default_to
+ :default: zuul
+
+ Who the report should be emailed to by default.
+ This can be overridden by individual pipelines.
Reporter Configuration
----------------------
@@ -39,15 +50,38 @@
address.
Each pipeline can overwrite the ``subject`` or the ``to`` or ``from`` address by
-providing alternatives as arguments to the reporter. For example, ::
+providing alternatives as arguments to the reporter. For example:
- - pipeline:
- name: post-merge
- success:
- outgoing_smtp:
- to: you@example.com
- failure:
- internal_smtp:
- to: you@example.com
- from: alternative@example.com
- subject: Change {change} failed
+.. code-block:: yaml
+
+ - pipeline:
+ name: post-merge
+ success:
+ outgoing_smtp:
+ to: you@example.com
+ failure:
+ internal_smtp:
+ to: you@example.com
+ from: alternative@example.com
+ subject: Change {change} failed
+
+.. attr:: pipeline.<reporter>.<smtp source>
+
+ To report via email, the dictionaries passed to any of the pipeline
+ :ref:`reporter<reporters>` attributes support the following
+ attributes:
+
+ .. attr:: to
+
+ The SMTP recipient address for the report. Multiple addresses
+ may be specified as one value separated by commas.
+
+ .. attr:: from
+
+ The SMTP sender address for the report.
+
+ .. attr:: subject
+
+ The Subject of the report email.
+
+ .. TODO: document subject string formatting.
diff --git a/doc/source/admin/drivers/sql.rst b/doc/source/admin/drivers/sql.rst
index b890f08..e208467 100644
--- a/doc/source/admin/drivers/sql.rst
+++ b/doc/source/admin/drivers/sql.rst
@@ -4,14 +4,29 @@
===
The SQL driver supports reporters only. Only one connection per
-database is permitted. The connection options are:
+database is permitted.
-**driver=sql**
+Connection Configuration
+------------------------
-**dburi**
- Database connection information in the form of a URI understood by
- sqlalchemy. eg http://docs.sqlalchemy.org/en/rel_1_0/core/engines.html#database-urls
- ``dburi=mysql://user:pass@localhost/db``
+The connection options for the SQL driver are:
+
+.. attr:: <sql connection>
+
+ .. attr:: driver
+ :required:
+
+ .. value:: sql
+
+ The connection must set ``driver=sql`` for SQL connections.
+
+ .. attr:: dburi
+ :required:
+
+ Database connection information in the form of a URI understood
+ by SQLAlchemy. See `The SQLAlchemy manual
+ <http://docs.sqlalchemy.org/en/rel_1_0/core/engines.html#database-urls>`_
+ for more information.
Reporter Configuration
----------------------
@@ -21,24 +36,27 @@
A :ref:`connection<connections>` that uses the sql driver must be
supplied to the reporter.
-zuul.conf contains the database connection and credentials. To store different
-reports in different databases you'll need to create a new connection per
-database.
+``zuul.conf`` contains the database connection and credentials. To
+store different reports in different databases you'll need to create a
+new connection per database.
-The SQL reporter does nothing on "start" or "merge-failure"; it only
-acts on "success" or "failure" reporting stages.
+The SQL reporter does nothing on :attr:`pipeline.start` or
+:attr:`pipeline.merge-failure`; it only acts on
+:attr:`pipeline.success` or :attr:`pipeline.failure` reporting stages.
-**score**
- A score to store for the result of the build. eg: -1 might indicate a failed
- build.
+For example:
-For example ::
+.. code-block:: yaml
- - pipeline:
- name: post-merge
- success:
- mydb_conn:
- score: 1
- failure:
- mydb_conn:
- score: -1
+ - pipeline:
+ name: post-merge
+ success:
+ mydb_conn:
+ failure:
+ mydb_conn:
+
+.. attr:: pipeline.<reporter>.<sql source>
+
+ To report to a database, add a key with the connection name and an
+ empty value to the desired pipeline :ref:`reporter<reporters>`
+ attributes.
diff --git a/doc/source/admin/drivers/timer.rst b/doc/source/admin/drivers/timer.rst
index c8afdd7..3b38c99 100644
--- a/doc/source/admin/drivers/timer.rst
+++ b/doc/source/admin/drivers/timer.rst
@@ -11,14 +11,20 @@
--------------------
Timers don't require a special connection or driver. Instead they can
-simply be used by listing **timer** as the trigger.
+simply be used by listing ``timer`` as the trigger.
-This trigger will run based on a cron-style time specification.
-It will enqueue an event into its pipeline for every project
-defined in the configuration. Any job associated with the
-pipeline will run in response to that event.
+This trigger will run based on a cron-style time specification. It
+will enqueue an event into its pipeline for every project defined in
+the configuration. Any job associated with the pipeline will run in
+response to that event.
-**time**
- The time specification in cron syntax. Only the 5 part syntax is
- supported, not the symbolic names. Example: ``0 0 * * *`` runs at
- midnight. The first weekday is Monday.
+.. attr:: pipeline.trigger.timer
+
+ The timer trigger supports the following attributes:
+
+ .. attr:: time
+ :required:
+
+ The time specification in cron syntax. Only the 5 part syntax
+ is supported, not the symbolic names. Example: ``0 0 * * *``
+ runs at midnight. The first weekday is Monday.
diff --git a/doc/source/admin/drivers/zuul.rst b/doc/source/admin/drivers/zuul.rst
index b531754..d95dffc 100644
--- a/doc/source/admin/drivers/zuul.rst
+++ b/doc/source/admin/drivers/zuul.rst
@@ -10,18 +10,29 @@
---------------------
Zuul events don't require a special connection or driver. Instead they
-can simply be used by listing **zuul** as the trigger.
+can simply be used by listing ``zuul`` as the trigger.
-**event**
- The event name. Currently supported:
+.. attr:: pipeline.trigger.zuul
- *project-change-merged* when Zuul merges a change to a project, it
- generates this event for every open change in the project.
+ The Zuul trigger supports the following attributes:
- *parent-change-enqueued* when Zuul enqueues a change into any
- pipeline, it generates this event for every child of that
- change.
+ .. attr:: event
+ :required:
-**pipeline**
- Only available for ``parent-change-enqueued`` events. This is the
- name of the pipeline in which the parent change was enqueued.
+ The event name. Currently supported events:
+
+ .. value:: project-change-merged
+
+ When Zuul merges a change to a project, it generates this
+ event for every open change in the project.
+
+ .. value:: parent-change-enqueued
+
+ When Zuul enqueues a change into any pipeline, it generates
+ this event for every child of that change.
+
+ .. attr:: pipeline
+
+ Only available for ``parent-change-enqueued`` events. This is
+ the name of the pipeline in which the parent change was
+ enqueued.
diff --git a/doc/source/admin/monitoring.rst b/doc/source/admin/monitoring.rst
index 2a6c959..9c69960 100644
--- a/doc/source/admin/monitoring.rst
+++ b/doc/source/admin/monitoring.rst
@@ -31,63 +31,101 @@
Metrics
~~~~~~~
-The metrics are emitted by the Zuul scheduler (`zuul/scheduler.py`):
+The metrics are emitted by the Zuul :ref:`scheduler`:
-**gerrit.event.<type> (counters)**
- Gerrit emits different kind of message over its `stream-events`
- interface. Zuul will report counters for each type of event it
- receives from Gerrit.
+.. stat:: gerrit.event.<type>
+ :type: counter
- Some of the events emitted are:
+ Gerrit emits different kind of message over its `stream-events`
+ interface. Zuul will report counters for each type of event it
+ receives from Gerrit.
- * patchset-created
- * draft-published
- * change-abandonned
- * change-restored
- * change-merged
- * merge-failed
- * comment-added
- * ref-updated
- * reviewer-added
+ Refer to your Gerrit installation documentation for a complete
+ list of Gerrit event types.
- Refer to your Gerrit installation documentation for an exhaustive list of
- Gerrit event types.
+.. stat:: zuul.pipeline
-**zuul.pipeline.**
- Holds metrics specific to jobs. The hierarchy is:
+ Holds metrics specific to jobs. This hierarchy includes:
- #. **<pipeline name>** as defined in your `layout.yaml` file (ex: `gate`,
- `test`, `publish`). It contains:
+ .. stat:: <pipeline name>
- #. **all_jobs** counter of jobs triggered by the pipeline.
- #. **current_changes** A gauge for the number of Gerrit changes being
- processed by this pipeline.
- #. **job** subtree detailing per jobs statistics:
+ A set of metrics for each pipeline named as defined in the Zuul
+ config.
- #. **<jobname>** The triggered job name.
- #. **<build result>** Result as defined in your triggering system. For
- Jenkins that would be SUCCESS, FAILURE, UNSTABLE, LOST. The
- metrics holds both an increasing counter and a timing
- reporting the duration of the build. Whenever the result is a
- SUCCESS or FAILURE, Zuul will additionally report the duration
- of the build as a timing event.
+ .. stat:: all_jobs
+ :type: counter
- #. **resident_time** timing representing how long the Change has been
- known by Zuul (which includes build time and Zuul overhead).
- #. **total_changes** counter of the number of change proceeding since
- Zuul started.
- #. **wait_time** counter and timer of the wait time, with the difference
- of the job start time and the execute time, in milliseconds.
+ Number of jobs triggered by the pipeline.
- Additionally, the `zuul.pipeline.<pipeline name>` hierarchy contains
- `current_changes` (gauge), `resident_time` (timing) and `total_changes`
- (counter) metrics for each projects. The slash separator used in Gerrit name
- being replaced by dots.
+ .. stat:: current_changes
+ :type: gauge
- As an example, given a job named `myjob` triggered by the `gate` pipeline
- which took 40 seconds to build, the Zuul scheduler will emit the following
- statsd events:
+ The number of items currently being processed by this
+ pipeline.
- * `zuul.pipeline.gate.job.myjob.SUCCESS` +1
- * `zuul.pipeline.gate.job.myjob` 40 seconds
- * `zuul.pipeline.gate.all_jobs` +1
+ .. stat:: job
+
+ Subtree detailing per jobs statistics:
+
+ .. stat:: <jobname>
+
+ The triggered job name.
+
+ .. stat:: <result>
+ :type: counter, timer
+
+ A counter for each type of result (e.g., ``SUCCESS`` or
+ ``FAILURE``, ``ERROR``, etc.) for the job. If the
+ result is ``SUCCESS`` or ``FAILURE``, Zuul will
+ additionally report the duration of the build as a
+ timer.
+
+ .. stat:: resident_time
+ :type: timer
+
+ A timer metric reporting how long each item has been in the
+ pipeline.
+
+ .. stat:: total_changes
+ :type: counter
+
+ The number of change processed by the pipeline since Zuul
+ started.
+
+ .. stat:: wait_time
+ :type: timer
+
+ How long each item spent in the pipeline before its first job
+ started.
+
+ .. stat:: <project>
+
+ This hierarchy holds more specific metrics for each project
+ participating in the pipeline. If the project name contains
+ a ``/`` character, it will be replaced with a ``.``.
+
+ .. stat:: current_changes
+ :type: gauge
+
+ The number of items of this project currently being
+ processed by this pipeline.
+
+ .. stat:: resident_time
+ :type: timer
+
+ A timer metric reporting how long each item for this
+ project has been in the pipeline.
+
+ .. stat:: total_changes
+ :type: counter
+
+ The number of change for this project processed by the
+ pipeline since Zuul started.
+
+As an example, given a job named `myjob` triggered by the `gate` pipeline
+which took 40 seconds to build, the Zuul scheduler will emit the following
+statsd events:
+
+ * ``zuul.pipeline.gate.job.myjob.SUCCESS`` +1
+ * ``zuul.pipeline.gate.job.myjob`` 40 seconds
+ * ``zuul.pipeline.gate.all_jobs`` +1
diff --git a/doc/source/admin/tenants.rst b/doc/source/admin/tenants.rst
index b3b2d9c..b518c91 100644
--- a/doc/source/admin/tenants.rst
+++ b/doc/source/admin/tenants.rst
@@ -5,128 +5,163 @@
Tenant Configuration
====================
-After *zuul.conf* is configured, Zuul component servers will be able
+After ``zuul.conf`` is configured, Zuul component servers will be able
to start, but a tenant configuration is required in order for Zuul to
perform any actions. The tenant configuration file specifies upon
-which projects Zuul should operate. These repositories are
-grouped into tenants. The configuration of each tenant is separate
-from the rest (no pipelines, jobs, etc are shared between them).
+which projects Zuul should operate. These repositories are grouped
+into tenants. The configuration of each tenant is separate from the
+rest (no pipelines, jobs, etc are shared between them).
A project may appear in more than one tenant; this may be useful if
you wish to use common job definitions across multiple tenants.
-The tenant configuration file is specified by the *tenant_config*
-setting in the *scheduler* section of *zuul.yaml*. It is a YAML file
-which, like other Zuul configuration files, is a list of configuration
-objects, though only one type of object is supported, *tenant*.
+The tenant configuration file is specified by the
+:attr:`scheduler.tenant_config` setting in ``zuul.conf``. It is a
+YAML file which, like other Zuul configuration files, is a list of
+configuration objects, though only one type of object is supported:
+``tenant``.
Tenant
------
A tenant is a collection of projects which share a Zuul
-configuration. An example tenant definition is::
+configuration. An example tenant definition is:
- - tenant:
- name: my-tenant
- max-nodes-per-job: 5
- exclude-unprotected-branches: false
- source:
- gerrit:
- config-projects:
- - common-config
- - shared-jobs:
- include: job
- untrusted-projects:
- - zuul-jobs:
- shadow: common-config
- - project1
- - project2:
- exclude-unprotected-branches: true
+.. code-block:: yaml
-The following attributes are supported:
+ - tenant:
+ name: my-tenant
+ max-nodes-per-job: 5
+ exclude-unprotected-branches: false
+ source:
+ gerrit:
+ config-projects:
+ - common-config
+ - shared-jobs:
+ include: job
+ untrusted-projects:
+ - zuul-jobs:
+ shadow: common-config
+ - project1
+ - project2:
+ exclude-unprotected-branches: true
-**name** (required)
- The name of the tenant. This may appear in URLs, paths, and
- monitoring fields, and so should be restricted to URL friendly
- characters (ASCII letters, numbers, hyphen and underscore) and you
- should avoid changing it unless necessary.
+.. attr:: tenant
-**max-nodes-per-job** (optional)
- The maximum number of nodes a job can request, default to 5.
- A '-1' value removes the limit.
+ The following attributes are supported:
-**exclude-unprotected-branches** (optional)
- When using a branch and pull model on a shared github repository there are
- usually one or more protected branches which are gated and a dynamic number of
- personal/feature branches which are the source for the pull requests. These
- branches can potentially include broken zuul config and therefore break the
- global tenant wide configuration. In order to deal with this zuul's operations
- can be limited to the protected branches which are gated. This is a tenant
- wide setting and can be overridden per project. If not specified, defaults
- to ``false``.
+ .. attr:: name
+ :required:
-**source** (required)
- A dictionary of sources to consult for projects. A tenant may
- contain projects from multiple sources; each of those sources must
- be listed here, along with the projects it supports. The name of a
- :ref:`connection<connections>` is used as the dictionary key
- (e.g. `gerrit` in the example above), and the value is a further
- dictionary containing the keys below.
+ The name of the tenant. This may appear in URLs, paths, and
+ monitoring fields, and so should be restricted to URL friendly
+ characters (ASCII letters, numbers, hyphen and underscore) and
+ you should avoid changing it unless necessary.
- **config-projects**
- A list of projects to be treated as config projects in this
- tenant. The jobs in a config project are trusted, which means
- they run with extra privileges, do not have their configuration
- dynamically loaded for proposed changes, and zuul.yaml files are
- only searched for in the master branch.
+ .. attr:: source
+ :required:
- **untrusted-projects**
- A list of projects to be treated as untrusted in this tenant. An
- untrusted project is the typical project operated on by Zuul.
- Their jobs run in a more restrictive environment, they may not
- define pipelines, their configuration dynamically changes in
- response to proposed changes, Zuul will read configuration files
- in all of their branches.
+ A dictionary of sources to consult for projects. A tenant may
+ contain projects from multiple sources; each of those sources
+ must be listed here, along with the projects it supports. The
+ name of a :ref:`connection<connections>` is used as the
+ dictionary key (e.g. ``gerrit`` in the example above), and the
+ value is a further dictionary containing the keys below.
- Each of the projects listed may be either a simple string value, or
- it may be a dictionary with the following keys:
+ The next two attributes, **config-projects** and
+ **untrusted-projects** provide the bulk of the information for
+ tenant configuration. They list all of the projects upon which
+ Zuul will act.
- **include**
- Normally Zuul will load all of the configuration classes
- appropriate for the type of project (config or untrusted) in
- question. However, if you only want to load some items, the
- *include* attribute can be used to specify that *only* the
- specified classes should be loaded. Supplied as a string, or a
- list of strings.
+ The order of the projects listed in a tenant is important. A job
+ which is defined in one project may not be redefined in another
+ project; therefore, once a job appears in one project, a project
+ listed later will be unable to define a job with that name.
+ Further, some aspects of project configuration (such as the merge
+ mode) may only be set on the first appearance of a project
+ definition.
- **exclude**
- A list of configuration classes that should not be loaded.
+ Zuul loads the configuration from all **config-projects** in the
+ order listed, followed by all **untrusted-projects** in order.
- **shadow**
- A list of projects which this project is permitted to shadow.
- Normally, only one project in Zuul may contain definitions for a
- given job. If a project earlier in the configuration defines a
- job which a later project redefines, the later definition is
- considered an error and is not permitted. The "shadow" attribute
- of a project indicates that job definitions in this project which
- conflict with the named projects should be ignored, and those in
- the named project should be used instead. The named projects must
- still appear earlier in the configuration. In the example above,
- if a job definition appears in both the "common-config" and
- "zuul-jobs" projects, the definition in "common-config" will be
- used.
+ .. attr:: config-projects
- **exclude-unprotected-branches**
- Define if unprotected github branches should be processed. Defaults to the
- tenant wide setting of exclude-unprotected-branches.
+ A list of projects to be treated as :term:`config projects
+ <config-project>` in this tenant. The jobs in a config project
+ are trusted, which means they run with extra privileges, do not
+ have their configuration dynamically loaded for proposed
+ changes, and Zuul config files are only searched for in the
+ ``master`` branch.
- The order of the projects listed in a tenant is important. A job
- which is defined in one project may not be redefined in another
- project; therefore, once a job appears in one project, a project
- listed later will be unable to define a job with that name.
- Further, some aspects of project configuration (such as the merge
- mode) may only be set on the first appearance of a project
- definition.
+ The items in the list follow the same format described in
+ **untrusted-projects**.
- Zuul loads the configuration from all *config-projects* in the order
- listed, followed by all *trusted-projects* in order.
+ .. attr:: untrusted-projects
+
+ A list of projects to be treated as untrusted in this tenant.
+ An :term:`untrusted-project` is the typical project operated on
+ by Zuul. Their jobs run in a more restrictive environment, they
+ may not define pipelines, their configuration dynamically
+ changes in response to proposed changes, and Zuul will read
+ configuration files in all of their branches.
+
+ .. attr:: <project>:
+
+ The items in the list may either be simple string values of
+ the project names, or a dictionary with the project name as
+ key and the following values:
+
+ .. attr:: include
+
+ Normally Zuul will load all of the configuration classes
+ appropriate for the type of project (config or untrusted)
+ in question. However, if you only want to load some
+ items, the **include** attribute can be used to specify
+ that *only* the specified classes should be loaded.
+ Supplied as a string, or a list of strings.
+
+ .. attr:: exclude
+
+ A list of configuration classes that should not be loaded.
+
+ .. attr:: shadow
+
+ A list of projects which this project is permitted to
+ shadow. Normally, only one project in Zuul may contain
+ definitions for a given job. If a project earlier in the
+ configuration defines a job which a later project
+ redefines, the later definition is considered an error and
+ is not permitted. The **shadow** attribute of a project
+ indicates that job definitions in this project which
+ conflict with the named projects should be ignored, and
+ those in the named project should be used instead. The
+ named projects must still appear earlier in the
+ configuration. In the example above, if a job definition
+ appears in both the ``common-config`` and ``zuul-jobs``
+ projects, the definition in ``common-config`` will be
+ used.
+
+ .. attr:: exclude-unprotected-branches
+
+ Define if unprotected github branches should be
+ processed. Defaults to the tenant wide setting of
+ exclude-unprotected-branches.
+
+ .. attr:: max-nodes-per-job
+ :default: 5
+
+ The maximum number of nodes a job can request. A value of
+ '-1' value removes the limit.
+
+ .. attr:: exclude-unprotected-branches
+ :default: false
+
+ When using a branch and pull model on a shared GitHub repository
+ there are usually one or more protected branches which are gated
+ and a dynamic number of personal/feature branches which are the
+ source for the pull requests. These branches can potentially
+ include broken Zuul config and therefore break the global tenant
+ wide configuration. In order to deal with this Zuul's operations
+ can be limited to the protected branches which are gated. This
+ is a tenant wide setting and can be overridden per project.
+ This currently only affects GitHub projects.
diff --git a/doc/source/conf.py b/doc/source/conf.py
index 38c1689..ca9c9b0 100644
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -27,6 +27,7 @@
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = [
'sphinx.ext.autodoc',
+ 'sphinx.ext.graphviz',
'sphinxcontrib.blockdiag',
'sphinxcontrib.programoutput',
'zuul.sphinx.ansible',
@@ -97,7 +98,9 @@
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
-#html_theme_options = {}
+html_theme_options = {
+ 'show_related': True
+}
# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []
diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst
index 98bee10..6c8a4b4 100644
--- a/doc/source/glossary.rst
+++ b/doc/source/glossary.rst
@@ -45,6 +45,27 @@
might leave feedback in a remote system on a proposed change,
send email, or store information in a database.
+ trusted execution context
+
+ Playbooks defined in a :term:`config-project` run in the
+ *trusted* execution context. The trusted execution context has
+ access to all Ansible features, including the ability to load
+ custom Ansible modules.
+
+ untrusted execution context
+
+ Playbooks defined in an :term:`untrusted-project` run in the
+ *untrusted* execution context. Playbooks run in the untrusted
+ execution context are not permitted to load additional Ansible
+ modules or access files outside of the restricted environment
+ prepared for them by the executor. In addition to the
+ bubblewrap environment applied to both execution contexts, in
+ the untrusted context some standard Ansible modules are replaced
+ with versions which prohibit some actions, including attempts to
+ access files outside of the restricted execution context. These
+ redundant protections are made as part of a defense-in-depth
+ strategy.
+
untrusted-project
One of two types of projects which may be specified by the
diff --git a/doc/source/user/config.rst b/doc/source/user/config.rst
index 34d23f9..446718b 100644
--- a/doc/source/user/config.rst
+++ b/doc/source/user/config.rst
@@ -489,7 +489,7 @@
parent: base
nodes:
- name: test-node
- image: fedora
+ label: fedora
.. attr:: job
diff --git a/doc/source/user/encryption.rst b/doc/source/user/encryption.rst
index fdf2c5a..7ced589 100644
--- a/doc/source/user/encryption.rst
+++ b/doc/source/user/encryption.rst
@@ -20,24 +20,41 @@
the main Zuul configuration file.
Zuul currently supports one encryption scheme, PKCS#1 with OAEP, which
-can not store secrets longer than the key length, 4096 bits. The
-padding used by this scheme ensures that someone examining the
-encrypted data can not determine the length of the plaintext version
-of the data, except to know that it is not longer than 4096 bits.
+can not store secrets longer than the 3760 bits (derived from the key
+length of 4096 bits minus 336 bits of overhead). The padding used by
+this scheme ensures that someone examining the encrypted data can not
+determine the length of the plaintext version of the data, except to
+know that it is not longer than 3760 bits (or some multiple thereof).
In the config files themselves, Zuul uses an extensible method of
specifying the encryption scheme used for a secret so that other
schemes may be added later. To specify a secret, use the
``!encrypted/pkcs1-oaep`` YAML tag along with the base64 encoded
-value. For example::
+value. For example:
+
+.. code-block:: yaml
- secret:
name: test_secret
data:
password: !encrypted/pkcs1-oaep |
- BFhtdnm8uXx7kn79RFL/zJywmzLkT1GY78P3bOtp4WghUFWobkifSu7ZpaV4NeO0s71YUsi1wGZZ
+ BFhtdnm8uXx7kn79RFL/zJywmzLkT1GY78P3bOtp4WghUFWobkifSu7ZpaV4NeO0s71YUsi
...
+To support secrets longer than 3760 bits, the value after the
+encryption tag may be a list rather than a scalar. For example:
+
+.. code-block:: yaml
+
+ - secret:
+ name: long_secret
+ data:
+ password: !encrypted/pkcs1-oaep
+ - er1UXNOD3OqtsRJaP0Wvaqiqx0ZY2zzRt6V9vqIsRaz1R5C4/AEtIad/DERZHwk3Nk+KV
+ ...
+ - HdWDS9lCBaBJnhMsm/O9tpzCq+GKRELpRzUwVgU5k822uBwhZemeSrUOLQ8hQ7q/vVHln
+ ...
+
Zuul provides a standalone script to make encrypting values easy; it
can be found at `tools/encrypt_secret.py` in the Zuul source
directory.
diff --git a/doc/source/user/jobs.rst b/doc/source/user/jobs.rst
index 577d147..7f1c3cb 100644
--- a/doc/source/user/jobs.rst
+++ b/doc/source/user/jobs.rst
@@ -3,7 +3,7 @@
Job Content
===========
-Zuul jobs are implemneted as Ansible playbooks. Zuul prepares the
+Zuul jobs are implemented as Ansible playbooks. Zuul prepares the
repositories used for a job, installs any required Ansible roles, and
then executes the job's playbooks. Any setup or artifact collection
required is the responsibility of the job itself. While this flexible
diff --git a/tests/base.py b/tests/base.py
index 7209c87..b14491c 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -2003,7 +2003,6 @@
self.config.get('scheduler', 'tenant_config')))
self.config.set('scheduler', 'state_dir', self.state_root)
self.config.set('merger', 'git_dir', self.merger_src_root)
- self.config.set('merger', 'state_dir', self.merger_state_root)
self.config.set('executor', 'git_dir', self.executor_src_root)
self.config.set('executor', 'private_key_file', self.private_key_file)
self.config.set('executor', 'state_dir', self.executor_state_root)
diff --git a/tests/fixtures/config/sql-driver/git/common-config/zuul.yaml b/tests/fixtures/config/sql-driver/git/common-config/zuul.yaml
index 0cb1d04..01a8d0f 100644
--- a/tests/fixtures/config/sql-driver/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/sql-driver/git/common-config/zuul.yaml
@@ -8,14 +8,11 @@
gerrit:
Verified: 1
resultsdb:
- score: 1
failure:
gerrit:
Verified: -1
resultsdb:
- score: -1
resultsdb_failures:
- score: -1
- job:
name: project-merge
diff --git a/tests/unit/test_connection.py b/tests/unit/test_connection.py
index 77b13a5..4214e9f 100644
--- a/tests/unit/test_connection.py
+++ b/tests/unit/test_connection.py
@@ -86,7 +86,7 @@
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
- # Add a failed result for a negative score
+ # Add a failed result
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
self.executor_server.failJob('project-test1', B)
@@ -106,7 +106,7 @@
self.assertEqual('org/project', buildset0['project'])
self.assertEqual(1, buildset0['change'])
self.assertEqual(1, buildset0['patchset'])
- self.assertEqual(1, buildset0['score'])
+ self.assertEqual('SUCCESS', buildset0['result'])
self.assertEqual('Build succeeded.', buildset0['message'])
self.assertEqual('tenant-one', buildset0['tenant'])
@@ -130,7 +130,7 @@
self.assertEqual('org/project', buildset1['project'])
self.assertEqual(2, buildset1['change'])
self.assertEqual(1, buildset1['patchset'])
- self.assertEqual(-1, buildset1['score'])
+ self.assertEqual('FAILURE', buildset1['result'])
self.assertEqual('Build failed.', buildset1['message'])
buildset1_builds = conn.execute(
@@ -183,7 +183,7 @@
self.assertEqual('org/project', buildsets_resultsdb[0]['project'])
self.assertEqual(1, buildsets_resultsdb[0]['change'])
self.assertEqual(1, buildsets_resultsdb[0]['patchset'])
- self.assertEqual(1, buildsets_resultsdb[0]['score'])
+ self.assertEqual('SUCCESS', buildsets_resultsdb[0]['result'])
self.assertEqual('Build succeeded.', buildsets_resultsdb[0]['message'])
# Grab the sa tables for resultsdb_failures
@@ -204,7 +204,7 @@
'org/project', buildsets_resultsdb_failures[0]['project'])
self.assertEqual(2, buildsets_resultsdb_failures[0]['change'])
self.assertEqual(1, buildsets_resultsdb_failures[0]['patchset'])
- self.assertEqual(-1, buildsets_resultsdb_failures[0]['score'])
+ self.assertEqual('FAILURE', buildsets_resultsdb_failures[0]['result'])
self.assertEqual(
'Build failed.', buildsets_resultsdb_failures[0]['message'])
diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py
index a52a2ee..3538555 100644
--- a/tests/unit/test_model.py
+++ b/tests/unit/test_model.py
@@ -342,16 +342,41 @@
name: pypi-credentials
data:
username: test-username
+ longpassword: !encrypted/pkcs1-oaep
+ - BFhtdnm8uXx7kn79RFL/zJywmzLkT1GY78P3bOtp4WghUFWobkifSu7ZpaV4NeO0s71Y
+ Usi1wGZZL0LveZjUN0t6OU1VZKSG8R5Ly7urjaSo1pPVIq5Rtt/H7W14Lecd+cUeKb4j
+ oeusC9drN3AA8a4oykcVpt1wVqUnTbMGC9ARMCQP6eopcs1l7tzMseprW4RDNhIuz3CR
+ gd0QBMPl6VDoFgBPB8vxtJw+3m0rqBYZCLZgCXekqlny8s2s92nJMuUABbJOEcDRarzi
+ bDsSXsfJt1y+5n7yOURsC7lovMg4GF/vCl/0YMKjBO5bpv9EM5fToeKYyPGSKQoHOnCY
+ ceb3cAVcv5UawcCic8XjhEhp4K7WPdYf2HVAC/qtxhbpjTxG4U5Q/SoppOJ60WqEkQvb
+ Xs6n5Dvy7xmph6GWmU/bAv3eUK3pdD3xa2Ue1lHWz3U+rsYraI+AKYsMYx3RBlfAmCeC
+ 1ve2BXPrqnOo7G8tnUvfdYPbK4Aakk0ds/AVqFHEZN+S6hRBmBjLaRFWZ3QSO1NjbBxW
+ naHKZYT7nkrJm8AMCgZU0ZArFLpaufKCeiK5ECSsDxic4FIsY1OkWT42qEUfL0Wd+150
+ AKGNZpPJnnP3QYY4W/MWcKH/zdO400+zWN52WevbSqZy90tqKDJrBkMl1ydqbuw1E4ZH
+ vIs=
+ - BFhtdnm8uXx7kn79RFL/zJywmzLkT1GY78P3bOtp4WghUFWobkifSu7ZpaV4NeO0s71Y
+ Usi1wGZZL0LveZjUN0t6OU1VZKSG8R5Ly7urjaSo1pPVIq5Rtt/H7W14Lecd+cUeKb4j
+ oeusC9drN3AA8a4oykcVpt1wVqUnTbMGC9ARMCQP6eopcs1l7tzMseprW4RDNhIuz3CR
+ gd0QBMPl6VDoFgBPB8vxtJw+3m0rqBYZCLZgCXekqlny8s2s92nJMuUABbJOEcDRarzi
+ bDsSXsfJt1y+5n7yOURsC7lovMg4GF/vCl/0YMKjBO5bpv9EM5fToeKYyPGSKQoHOnCY
+ ceb3cAVcv5UawcCic8XjhEhp4K7WPdYf2HVAC/qtxhbpjTxG4U5Q/SoppOJ60WqEkQvb
+ Xs6n5Dvy7xmph6GWmU/bAv3eUK3pdD3xa2Ue1lHWz3U+rsYraI+AKYsMYx3RBlfAmCeC
+ 1ve2BXPrqnOo7G8tnUvfdYPbK4Aakk0ds/AVqFHEZN+S6hRBmBjLaRFWZ3QSO1NjbBxW
+ naHKZYT7nkrJm8AMCgZU0ZArFLpaufKCeiK5ECSsDxic4FIsY1OkWT42qEUfL0Wd+150
+ AKGNZpPJnnP3QYY4W/MWcKH/zdO400+zWN52WevbSqZy90tqKDJrBkMl1ydqbuw1E4ZH
+ vIs=
password: !encrypted/pkcs1-oaep |
- BFhtdnm8uXx7kn79RFL/zJywmzLkT1GY78P3bOtp4WghUFWobkifSu7ZpaV4NeO0s71YUsi1wGZZ
- L0LveZjUN0t6OU1VZKSG8R5Ly7urjaSo1pPVIq5Rtt/H7W14Lecd+cUeKb4joeusC9drN3AA8a4o
- ykcVpt1wVqUnTbMGC9ARMCQP6eopcs1l7tzMseprW4RDNhIuz3CRgd0QBMPl6VDoFgBPB8vxtJw+
- 3m0rqBYZCLZgCXekqlny8s2s92nJMuUABbJOEcDRarzibDsSXsfJt1y+5n7yOURsC7lovMg4GF/v
- Cl/0YMKjBO5bpv9EM5fToeKYyPGSKQoHOnCYceb3cAVcv5UawcCic8XjhEhp4K7WPdYf2HVAC/qt
- xhbpjTxG4U5Q/SoppOJ60WqEkQvbXs6n5Dvy7xmph6GWmU/bAv3eUK3pdD3xa2Ue1lHWz3U+rsYr
- aI+AKYsMYx3RBlfAmCeC1ve2BXPrqnOo7G8tnUvfdYPbK4Aakk0ds/AVqFHEZN+S6hRBmBjLaRFW
- Z3QSO1NjbBxWnaHKZYT7nkrJm8AMCgZU0ZArFLpaufKCeiK5ECSsDxic4FIsY1OkWT42qEUfL0Wd
- +150AKGNZpPJnnP3QYY4W/MWcKH/zdO400+zWN52WevbSqZy90tqKDJrBkMl1ydqbuw1E4ZHvIs=
+ BFhtdnm8uXx7kn79RFL/zJywmzLkT1GY78P3bOtp4WghUFWobkifSu7ZpaV4NeO0s71Y
+ Usi1wGZZL0LveZjUN0t6OU1VZKSG8R5Ly7urjaSo1pPVIq5Rtt/H7W14Lecd+cUeKb4j
+ oeusC9drN3AA8a4oykcVpt1wVqUnTbMGC9ARMCQP6eopcs1l7tzMseprW4RDNhIuz3CR
+ gd0QBMPl6VDoFgBPB8vxtJw+3m0rqBYZCLZgCXekqlny8s2s92nJMuUABbJOEcDRarzi
+ bDsSXsfJt1y+5n7yOURsC7lovMg4GF/vCl/0YMKjBO5bpv9EM5fToeKYyPGSKQoHOnCY
+ ceb3cAVcv5UawcCic8XjhEhp4K7WPdYf2HVAC/qtxhbpjTxG4U5Q/SoppOJ60WqEkQvb
+ Xs6n5Dvy7xmph6GWmU/bAv3eUK3pdD3xa2Ue1lHWz3U+rsYraI+AKYsMYx3RBlfAmCeC
+ 1ve2BXPrqnOo7G8tnUvfdYPbK4Aakk0ds/AVqFHEZN+S6hRBmBjLaRFWZ3QSO1NjbBxW
+ naHKZYT7nkrJm8AMCgZU0ZArFLpaufKCeiK5ECSsDxic4FIsY1OkWT42qEUfL0Wd+150
+ AKGNZpPJnnP3QYY4W/MWcKH/zdO400+zWN52WevbSqZy90tqKDJrBkMl1ydqbuw1E4ZH
+ vIs=
''')[0]['secret']
conf['_source_context'] = self.context
@@ -441,6 +466,12 @@
self.assertEqual(in_repo_job_with_inherit.auth.secrets[0].name,
'pypi-credentials')
self.assertIsNone(in_repo_job_with_inherit_false.auth)
+ self.assertEqual(in_repo_job_with_inherit.auth.secrets[0].
+ secret_data['longpassword'],
+ 'test-passwordtest-password')
+ self.assertEqual(in_repo_job_with_inherit.auth.secrets[0].
+ secret_data['password'],
+ 'test-password')
def test_job_inheritance_job_tree(self):
tenant = model.Tenant('tenant')
diff --git a/tools/encrypt_secret.py b/tools/encrypt_secret.py
index 72429e9..df4f449 100755
--- a/tools/encrypt_secret.py
+++ b/tools/encrypt_secret.py
@@ -14,10 +14,13 @@
import argparse
import base64
+import math
import os
+import re
import subprocess
import sys
import tempfile
+import textwrap
# we to import Request and urlopen differently for python 2 and 3
try:
@@ -68,28 +71,70 @@
else:
plaintext = sys.stdin.read()
+ plaintext = plaintext.encode("utf-8")
+
pubkey_file = tempfile.NamedTemporaryFile(delete=False)
try:
pubkey_file.write(pubkey.read())
pubkey_file.close()
- p = subprocess.Popen(['openssl', 'rsautl', '-encrypt',
- '-oaep', '-pubin', '-inkey',
+ p = subprocess.Popen(['openssl', 'rsa', '-text',
+ '-pubin', '-in',
pubkey_file.name],
- stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
- (stdout, stderr) = p.communicate(plaintext.encode("utf-8"))
+ (stdout, stderr) = p.communicate()
if p.returncode != 0:
raise Exception("Return code %s from openssl" % p.returncode)
- ciphertext = base64.b64encode(stdout)
+ output = stdout.decode('utf-8')
+ m = re.match(r'^Public-Key: \((\d+) bit\)$', output, re.MULTILINE)
+ nbits = int(m.group(1))
+ nbytes = int(nbits / 8)
+ max_bytes = nbytes - 42 # PKCS1-OAEP overhead
+ chunks = int(math.ceil(float(len(plaintext)) / max_bytes))
+
+ ciphertext_chunks = []
+
+ print("Public key length: {} bits ({} bytes)".format(nbits, nbytes))
+ print("Max plaintext length per chunk: {} bytes".format(max_bytes))
+ print("Input plaintext length: {} bytes".format(len(plaintext)))
+ print("Number of chunks: {}".format(chunks))
+
+ for count in range(chunks):
+ chunk = plaintext[int(count * max_bytes):
+ int((count + 1) * max_bytes)]
+ p = subprocess.Popen(['openssl', 'rsautl', '-encrypt',
+ '-oaep', '-pubin', '-inkey',
+ pubkey_file.name],
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE)
+ (stdout, stderr) = p.communicate(chunk)
+ if p.returncode != 0:
+ raise Exception("Return code %s from openssl" % p.returncode)
+ ciphertext_chunks.append(base64.b64encode(stdout).decode('utf-8'))
+
finally:
os.unlink(pubkey_file.name)
+ output = textwrap.dedent(
+ '''
+ - secret:
+ name: <name>
+ data:
+ <fieldname>: !encrypted/pkcs1-oaep
+ ''')
+
+ twrap = textwrap.TextWrapper(width=79,
+ initial_indent=' ' * 8,
+ subsequent_indent=' ' * 10)
+ for chunk in ciphertext_chunks:
+ chunk = twrap.fill('- ' + chunk)
+ output += chunk + '\n'
+
if args.outfile:
- with open(args.outfile, "wb") as f:
- f.write(ciphertext)
+ with open(args.outfile, "w") as f:
+ f.write(output)
else:
- print(ciphertext.decode("utf-8"))
+ print(output)
if __name__ == '__main__':
diff --git a/zuul/cmd/merger.py b/zuul/cmd/merger.py
index c5cfd6c..9771fff 100755
--- a/zuul/cmd/merger.py
+++ b/zuul/cmd/merger.py
@@ -22,7 +22,6 @@
# instead it depends on lockfile-0.9.1 which uses pidfile.
pid_file_module = extras.try_imports(['daemon.pidlockfile', 'daemon.pidfile'])
-import os
import sys
import signal
@@ -80,17 +79,6 @@
server.read_config()
server.configure_connections(source_only=True)
- state_dir = get_default(server.config, 'merger', 'state_dir',
- '/var/lib/zuul', expand_user=True)
- test_fn = os.path.join(state_dir, 'test')
- try:
- f = open(test_fn, 'w')
- f.close()
- os.unlink(test_fn)
- except Exception:
- print("\nUnable to write to state directory: %s\n" % state_dir)
- raise
-
pid_fn = get_default(server.config, 'merger', 'pidfile',
'/var/run/zuul-merger/zuul-merger.pid',
expand_user=True)
diff --git a/zuul/configloader.py b/zuul/configloader.py
index a09147c..1036a2c 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -223,7 +223,11 @@
yaml_loader = yaml.SafeLoader
def __init__(self, ciphertext):
- self.ciphertext = base64.b64decode(ciphertext)
+ if isinstance(ciphertext, list):
+ self.ciphertext = [base64.b64decode(x.value)
+ for x in ciphertext]
+ else:
+ self.ciphertext = base64.b64decode(ciphertext)
def __ne__(self, other):
return not self.__eq__(other)
@@ -238,8 +242,14 @@
return cls(node.value)
def decrypt(self, private_key):
- return encryption.decrypt_pkcs1_oaep(self.ciphertext,
- private_key).decode('utf8')
+ if isinstance(self.ciphertext, list):
+ return ''.join([
+ encryption.decrypt_pkcs1_oaep(chunk, private_key).
+ decode('utf8')
+ for chunk in self.ciphertext])
+ else:
+ return encryption.decrypt_pkcs1_oaep(self.ciphertext,
+ private_key).decode('utf8')
class NodeSetParser(object):
diff --git a/zuul/driver/smtp/smtpreporter.py b/zuul/driver/smtp/smtpreporter.py
index 1f232e9..421d14b 100644
--- a/zuul/driver/smtp/smtpreporter.py
+++ b/zuul/driver/smtp/smtpreporter.py
@@ -47,7 +47,6 @@
def getSchema():
smtp_reporter = v.Schema({
- 'connection': str,
'to': str,
'from': str,
'subject': str,
diff --git a/zuul/driver/sql/alembic_reporter/versions/60c119eb1e3f_use_build_set_results.py b/zuul/driver/sql/alembic_reporter/versions/60c119eb1e3f_use_build_set_results.py
new file mode 100644
index 0000000..8e8142f
--- /dev/null
+++ b/zuul/driver/sql/alembic_reporter/versions/60c119eb1e3f_use_build_set_results.py
@@ -0,0 +1,49 @@
+"""Use build_set results
+
+Revision ID: 60c119eb1e3f
+Revises: f86c9871ee67
+Create Date: 2017-07-27 17:09:20.374782
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '60c119eb1e3f'
+down_revision = 'f86c9871ee67'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+
+BUILDSET_TABLE = 'zuul_buildset'
+
+
+def upgrade():
+ op.add_column(BUILDSET_TABLE, sa.Column('result', sa.String(255)))
+
+ connection = op.get_bind()
+ connection.execute(
+ """
+ UPDATE {buildset_table}
+ SET result=(
+ SELECT CASE score
+ WHEN 1 THEN 'SUCCESS'
+ ELSE 'FAILURE' END)
+ """.format(buildset_table=BUILDSET_TABLE))
+
+ op.drop_column(BUILDSET_TABLE, 'score')
+
+
+def downgrade():
+ op.add_column(BUILDSET_TABLE, sa.Column('score', sa.Integer))
+
+ connection = op.get_bind()
+ connection.execute(
+ """
+ UPDATE {buildset_table}
+ SET score=(
+ SELECT CASE result
+ WHEN 'SUCCESS' THEN 1
+ ELSE -1 END)
+ """.format(buildset_table=BUILDSET_TABLE))
+ op.drop_column(BUILDSET_TABLE, 'result')
diff --git a/zuul/driver/sql/sqlconnection.py b/zuul/driver/sql/sqlconnection.py
index 0e3f0dd..1187c2d 100644
--- a/zuul/driver/sql/sqlconnection.py
+++ b/zuul/driver/sql/sqlconnection.py
@@ -83,7 +83,7 @@
sa.Column('change', sa.Integer, nullable=True),
sa.Column('patchset', sa.Integer, nullable=True),
sa.Column('ref', sa.String(255)),
- sa.Column('score', sa.Integer, nullable=True),
+ sa.Column('result', sa.String(255)),
sa.Column('message', sa.TEXT()),
sa.Column('tenant', sa.String(255)),
)
diff --git a/zuul/driver/sql/sqlreporter.py b/zuul/driver/sql/sqlreporter.py
index aca1b06..7c79176 100644
--- a/zuul/driver/sql/sqlreporter.py
+++ b/zuul/driver/sql/sqlreporter.py
@@ -25,12 +25,6 @@
name = 'sql'
log = logging.getLogger("zuul.reporter.mysql.SQLReporter")
- def __init__(self, driver, connection, config={}):
- super(SQLReporter, self).__init__(
- driver, connection, config)
- # TODO(jeblair): document this is stored as NULL if unspecified
- self.result_score = config.get('score', None)
-
def report(self, item):
"""Create an entry into a database."""
@@ -49,7 +43,7 @@
change=change,
patchset=patchset,
ref=ref,
- score=self.result_score,
+ result=item.current_build_set.result,
message=self._formatItemReport(
item, with_jobs=False),
tenant=item.pipeline.layout.tenant.name,
@@ -85,7 +79,5 @@
def getSchema():
- sql_reporter = v.Schema({
- 'score': int,
- })
+ sql_reporter = v.Schema(None)
return sql_reporter
diff --git a/zuul/sphinx/zuul.py b/zuul/sphinx/zuul.py
index b6aca6c..0ac33b8 100644
--- a/zuul/sphinx/zuul.py
+++ b/zuul/sphinx/zuul.py
@@ -166,6 +166,55 @@
return sig
+class ZuulStatDirective(ZuulConfigObject):
+ has_content = True
+
+ option_spec = {
+ 'type': lambda x: x,
+ 'hidden': lambda x: x,
+ 'noindex': lambda x: x,
+ }
+
+ def before_content(self):
+ path = self.env.ref_context.setdefault('zuul:attr_path', [])
+ element = self.names[-1]
+ path.append(element)
+ path = self.env.ref_context.setdefault('zuul:display_attr_path', [])
+ element = self.names[-1]
+ path.append(element)
+
+ def after_content(self):
+ path = self.env.ref_context.get('zuul:attr_path')
+ if path:
+ path.pop()
+ path = self.env.ref_context.get('zuul:display_attr_path')
+ if path:
+ path.pop()
+
+ def handle_signature(self, sig, signode):
+ if 'hidden' in self.options:
+ return sig
+ path = self.get_display_path()
+ for x in path:
+ signode += addnodes.desc_addname(x + '.', x + '.')
+ signode += addnodes.desc_name(sig, sig)
+ if 'type' in self.options:
+ t = ' (%s)' % self.options['type']
+ signode += addnodes.desc_annotation(t, t)
+ return sig
+
+
+class ZuulAbbreviatedXRefRole(XRefRole):
+
+ def process_link(self, env, refnode, has_explicit_title, title,
+ target):
+ title, target = super(ZuulAbbreviatedXRefRole, self).process_link(
+ env, refnode, has_explicit_title, title, target)
+ if not has_explicit_title:
+ title = title.split('.')[-1]
+ return title, target
+
+
class ZuulDomain(Domain):
name = 'zuul'
label = 'Zuul'
@@ -174,15 +223,19 @@
'attr': ZuulAttrDirective,
'value': ZuulValueDirective,
'var': ZuulVarDirective,
+ 'stat': ZuulStatDirective,
}
roles = {
'attr': XRefRole(innernodeclass=nodes.inline, # type: ignore
warn_dangling=True),
- 'value': XRefRole(innernodeclass=nodes.inline, # type: ignore
- warn_dangling=True),
+ 'value': ZuulAbbreviatedXRefRole(
+ innernodeclass=nodes.inline, # type: ignore
+ warn_dangling=True),
'var': XRefRole(innernodeclass=nodes.inline, # type: ignore
warn_dangling=True),
+ 'stat': XRefRole(innernodeclass=nodes.inline, # type: ignore
+ warn_dangling=True),
}
initial_data = {