diff --git a/.zuul.yaml b/.zuul.yaml
index d73be8f..caef296 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -25,8 +25,9 @@
     required-projects:
       - openstack/ara
     files:
-      - zuul/ansible/callback/.*
+      - zuul/ansible/.*
       - playbooks/zuul-stream/.*
+      - requirements.txt
 
 - project:
     check:
diff --git a/doc/source/admin/components.rst b/doc/source/admin/components.rst
index 88b898f..ba14752 100644
--- a/doc/source/admin/components.rst
+++ b/doc/source/admin/components.rst
@@ -442,6 +442,11 @@
 
       Port to use for finger log streamer.
 
+   .. attr:: state_dir
+      :default: /var/lib/zuul
+
+      Path to directory in which Zuul should save its state.
+
    .. attr:: git_dir
       :default: /var/lib/zuul/git
 
diff --git a/doc/source/admin/drivers/github.rst b/doc/source/admin/drivers/github.rst
index 83ac77f..a89cfc6 100644
--- a/doc/source/admin/drivers/github.rst
+++ b/doc/source/admin/drivers/github.rst
@@ -40,60 +40,43 @@
 Application
 ...........
 
+.. NOTE Duplicate content here and in zuul-from-scratch.rst.  Keep them
+   in sync.
+
 To create a `GitHub application
 <https://developer.github.com/apps/building-integrations/setting-up-and-registering-github-apps/registering-github-apps/>`_:
 
 * Go to your organization settings page to create the application, e.g.:
   https://github.com/organizations/my-org/settings/apps/new
-
 * Set GitHub App name to "my-org-zuul"
-
 * Set Setup URL to your setup documentation, when user install the application
   they are redirected to this url
-
 * Set Webhook URL to
   ``http://<zuul-hostname>/connection/<connection-name>/payload``.
-
 * Create a Webhook secret
-
 * Set permissions:
 
   * Commit statuses: Read & Write
-
   * Issues: Read & Write
-
   * Pull requests: Read & Write
-
   * Repository contents: Read & Write (write to let zuul merge change)
+  * Repository administration: Read
 
 * Set events subscription:
-
   * Label
-
   * Status
-
   * Issue comment
-
   * Issues
-
   * Pull request
-
   * Pull request review
-
   * Pull request review comment
-
   * Commit comment
-
   * Create
-
   * Push
-
   * Release
 
 * Set Where can this GitHub App be installed to "Any account"
-
 * Create the App
-
 * Generate a Private key in the app settings page
 
 Then in the zuul.conf, set webhook_token, app_id and app_key.
diff --git a/doc/source/admin/index.rst b/doc/source/admin/index.rst
index a2a2ee7..af83a3b 100644
--- a/doc/source/admin/index.rst
+++ b/doc/source/admin/index.rst
@@ -12,6 +12,7 @@
    :maxdepth: 2
 
    quick-start
+   zuul-from-scratch
    installation
    components
    connections
diff --git a/doc/source/admin/monitoring.rst b/doc/source/admin/monitoring.rst
index d43fd03..fbcedad 100644
--- a/doc/source/admin/monitoring.rst
+++ b/doc/source/admin/monitoring.rst
@@ -131,6 +131,14 @@
    component of the key will be replaced with the hostname of the
    executor.
 
+   .. stat:: merger.<result>
+      :type: counter
+
+      Incremented to represent the status of a Zuul executor's merger
+      operations. ``<result>`` can be either ``SUCCESS`` or ``FAILURE``.
+      A failed merge operation which would be accounted for as a ``FAILURE``
+      is what ends up being returned by Zuul as a ``MERGER_FAILURE``.
+
    .. stat:: builds
       :type: counter
 
@@ -148,16 +156,37 @@
       The number of builds currently running on this executor.  This
       includes starting builds.
 
+  .. stat:: phase
+
+     Subtree detailing per-phase execution statistics:
+
+     .. stat:: <phase>
+
+        ``<phase>`` represents a phase in the execution of a job.
+        This can be an *internal* phase (such as ``setup`` or ``cleanup``) as
+        well as *job* phases such as ``pre``, ``run`` or ``post``.
+
+        .. stat:: <result>
+           :type: counter
+
+           A counter for each type of result.
+           These results do not, by themselves, determine the status of a build
+           but are indicators of the exit status provided by Ansible for the
+           execution of a particular phase.
+
+           Example of possible counters for each phase are: ``RESULT_NORMAL``,
+           ``RESULT_TIMED_OUT``, ``RESULT_UNREACHABLE``, ``RESULT_ABORTED``.
+
    .. stat:: load_average
       :type: gauge
 
       The one-minute load average of this executor, multiplied by 100.
 
-   .. stat:: pct_available_ram
+   .. stat:: pct_used_ram
       :type: gauge
 
-      The available RAM (including buffers and cache) on this
-      executor, as a percentage multiplied by 100.
+      The used RAM (excluding buffers and cache) on this executor, as
+      a percentage multiplied by 100.
 
 .. stat:: zuul.nodepool
 
diff --git a/doc/source/admin/tenants.rst b/doc/source/admin/tenants.rst
index 48e7ba8..5bcd2a2 100644
--- a/doc/source/admin/tenants.rst
+++ b/doc/source/admin/tenants.rst
@@ -25,7 +25,7 @@
 ------
 
 A tenant is a collection of projects which share a Zuul
-configuration.  An example tenant definition is:
+configuration. Some examples of tenant definitions are:
 
 .. code-block:: yaml
 
@@ -46,6 +46,27 @@
              - project2:
                  exclude-unprotected-branches: true
 
+.. code-block:: yaml
+
+   - tenant:
+       name: my-tenant
+       source:
+         gerrit:
+           config-projects:
+             - common-config
+           untrusted-projects:
+             - exclude:
+                 - job
+                 - semaphore
+                 - project
+                 - project-template
+                 - nodeset
+                 - secret
+               projects:
+                 - project1
+                 - project2:
+                     exclude-unprotected-branches: true
+
 .. attr:: tenant
 
    The following attributes are supported:
@@ -157,6 +178,24 @@
             processed. Defaults to the tenant wide setting of
             exclude-unprotected-branches.
 
+      .. attr:: <project-group>
+
+         The items in the list are dictionaries with the following
+         attributes. A **configuration items** definition is applied
+         to the list of projects.
+
+         .. attr:: include
+
+            A list of **configuration items** that should be loaded.
+
+         .. attr:: exclude
+
+            A list of **configuration items** that should not be loaded.
+
+         .. attr:: projects
+
+            A list of **project** items.
+
    .. attr:: max-nodes-per-job
       :default: 5
 
diff --git a/doc/source/admin/zuul-from-scratch.rst b/doc/source/admin/zuul-from-scratch.rst
new file mode 100644
index 0000000..141216b
--- /dev/null
+++ b/doc/source/admin/zuul-from-scratch.rst
@@ -0,0 +1,505 @@
+Zuul From Scratch
+=================
+
+.. note:: This is a work in progress that attempts to walk through all
+          of the steps needed to run Zuul on a cloud server against
+          GitHub projects.
+
+Environment Setup
+-----------------
+
+We're going to be using Fedora 27 on a cloud server for this
+installation.
+
+Login to your environment
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Since we'll be using a cloud image for Fedora 27, our login user will
+be ``fedora`` which will also be the staging user for installation of
+Zuul and Nodepool.
+
+To get started, ssh to your machine as the ``fedora`` user::
+
+   ssh fedora@<ip_address>
+
+Environment Setup
+~~~~~~~~~~~~~~~~~
+
+::
+
+   sudo dnf update -y
+   sudo systemctl reboot
+   sudo dnf install git redhat-lsb-core python3 python3-pip python3-devel make gcc openssl-devel python-openstackclient -y
+   pip3 install --user bindep
+
+Zuul and Nodepool Installation
+------------------------------
+
+Install Zookeeper
+~~~~~~~~~~~~~~~~~
+
+::
+
+   sudo dnf install zookeeper -y
+
+Install Nodepool
+~~~~~~~~~~~~~~~~
+
+::
+
+   sudo adduser --system nodepool --home-dir /var/lib/nodepool --create-home
+   git clone https://git.openstack.org/openstack-infra/nodepool
+   cd nodepool/
+   sudo dnf -y install $(bindep -b)
+   sudo pip3 install .
+
+Install Zuul
+~~~~~~~~~~~~
+
+::
+
+   sudo adduser --system zuul --home-dir /var/lib/zuul --create-home
+   git clone https://git.openstack.org/openstack-infra/zuul
+   cd zuul/
+   sudo dnf install $(bindep -b) -y
+   sudo pip3 install git+https://github.com/sigmavirus24/github3.py.git@develop#egg=Github3.py
+   sudo pip3 install .
+
+Setup
+-----
+
+Zookeeper Setup
+~~~~~~~~~~~~~~~
+
+.. TODO recommended reading for zk clustering setup
+
+::
+
+   sudo bash -c 'echo "1" > /etc/zookeeper/myid'
+   sudo bash -c 'echo "tickTime=2000
+   dataDir=/var/lib/zookeeper
+   clientPort=2181" > /etc/zookeeper/zoo.cfg'
+
+Nodepool Setup
+~~~~~~~~~~~~~~
+
+Before starting on this, you need to download your `openrc`
+configuration from your OpenStack cloud.  Put it on your server in the
+fedora user's home directory.  It should be called
+``<username>-openrc.sh``.  Once that is done, create a new keypair
+that will be installed when instantiating the servers::
+
+   cd ~
+   source <username>-openrc.sh  # this will prompt for password - enter it
+   umask 0066
+
+   ssh-keygen -t rsa -b 2048 -f nodepool_rsa  # don't enter a passphrase
+   openstack keypair create --public-key nodepool_rsa.pub nodepool
+
+We'll use the private key later wheen configuring Zuul.  In the same
+session, configure nodepool to talk to your cloud::
+
+   sudo mkdir -p ~nodepool/.config/openstack
+   cat > clouds.yaml <<EOF
+   clouds:
+     mycloud:
+       auth:
+         username: $OS_USERNAME
+         password: $OS_PASSWORD
+         project_name: ${OS_PROJECT_NAME:-$OS_TENANT_NAME}
+         auth_url: $OS_AUTH_URL
+       region_name: $OS_REGION_NAME
+   EOF
+   sudo mv clouds.yaml ~nodepool/.config/openstack/
+   sudo chown -R nodepool.nodepool ~nodepool/.config
+   umask 0002
+
+Once you've written out the file, double check all the required fields have been filled out.
+
+::
+
+   sudo mkdir /etc/nodepool/
+   sudo mkdir /var/log/nodepool
+   sudo chgrp -R nodepool /var/log/nodepool/
+   sudo chmod 775 /var/log/nodepool/
+
+Nodepool Configuration
+~~~~~~~~~~~~~~~~~~~~~~
+
+Inputs needed for this file:
+
+* cloud name / region name - from clouds.yaml
+* flavor-name
+* image-name - from your cloud
+
+::
+
+   sudo bash -c "cat >/etc/nodepool/nodepool.yaml <<EOF
+   zookeeper-servers:
+     - host: localhost
+       port: 2181
+
+   providers:
+     - name: myprovider # this is a nodepool identifier for this cloud provider (cloud+region combo)
+       region-name: regionOne  # this needs to match the region name in clouds.yaml but is only needed if there is more than one region
+       cloud: mycloud  # This needs to match the name in clouds.yaml
+       cloud-images:
+         - name: centos-7   # Defines a cloud-image for nodepool
+           image-name: CentOS-7-x86_64-GenericCloud-1706  # name of image from cloud
+           username: centos  # The user Zuul should log in as
+       pools:
+         - name: main
+           max-servers: 4  # nodepool will never create more than this many servers
+           labels:
+             - name: centos-7-small  # defines label that will be used to get one of these in a job
+               flavor-name: 'm1.small'  # name of flavor from cloud
+               cloud-image: centos-7  # matches name from cloud-images
+               key-name: nodepool # name of the keypair to use for authentication
+
+   labels:
+     - name: centos-7-small # defines label that will be used in jobs
+       min-ready: 2  # nodepool will always keep this many booted and ready to go
+   EOF"
+
+.. warning::
+
+   `min-ready:2` may incur costs in your cloud provider
+
+
+Zuul Setup
+~~~~~~~~~~
+
+::
+
+   sudo mkdir /etc/zuul/
+   sudo mkdir /var/log/zuul/
+   sudo chown zuul.zuul /var/log/zuul/
+   sudo mkdir /var/lib/zuul/.ssh
+   sudo chmod 0700 /var/lib/zuul/.ssh
+   sudo mv nodepool_rsa /var/lib/zuul/.ssh
+   sudo chown -R zuul.zuul /var/lib/zuul/.ssh
+
+Zuul Configuration
+~~~~~~~~~~~~~~~~~~
+
+Write the Zuul config file.  Note that this configures Zuul's web
+server to listen on all public addresses.  This is so that Zuul may
+receive webhook events from GitHub.  You may wish to proxy this or
+further restrict public access.
+
+::
+
+   sudo bash -c "cat > /etc/zuul/zuul.conf <<EOF
+   [gearman]
+   server=127.0.0.1
+
+   [gearman_server]
+   start=true
+
+   [executor]
+   private_key_file=/home/zuul/.ssh/nodepool_rsa
+
+   [web]
+   listen_address=0.0.0.0
+
+   [scheduler]
+   tenant_config=/etc/zuul/main.yaml
+   EOF"
+
+   sudo bash -c "cat > /etc/zuul/main.yaml <<EOF
+   - tenant:
+       name: quickstart
+   EOF"
+
+Service Management
+------------------
+
+Zookeeper Service Management
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+   sudo systemctl start zookeeper.service
+
+::
+
+   sudo systemctl status zookeeper.service
+   ● zookeeper.service - Apache ZooKeeper
+      Loaded: loaded (/usr/lib/systemd/system/zookeeper.service; disabled; vendor preset: disabled)
+      Active: active (running) since Wed 2018-01-03 14:53:47 UTC; 5s ago
+     Process: 4153 ExecStart=/usr/bin/zkServer.sh start zoo.cfg (code=exited, status=0/SUCCESS)
+    Main PID: 4160 (java)
+       Tasks: 17 (limit: 4915)
+      CGroup: /system.slice/zookeeper.service
+              └─4160 java -Dzookeeper.log.dir=/var/log/zookeeper -Dzookeeper.root.logger=INFO,CONSOLE -cp /usr/share/java/
+
+::
+
+   sudo systemctl enable zookeeper.service
+
+
+Nodepool Service Management
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+   sudo bash -c "cat > /etc/systemd/system/nodepool-launcher.service <<EOF
+   [Unit]
+   Description=Nodepool Launcher Service
+   After=syslog.target network.target
+
+   [Service]
+   Type=simple
+   # Options to pass to nodepool-launcher.
+   Group=nodepool
+   User=nodepool
+   RuntimeDirectory=nodepool
+   ExecStart=/usr/local/bin/nodepool-launcher
+
+   [Install]
+   WantedBy=multi-user.target
+   EOF"
+
+   sudo chmod 0644 /etc/systemd/system/nodepool-launcher.service
+   sudo systemctl daemon-reload
+   sudo systemctl start nodepool-launcher.service
+   sudo systemctl status nodepool-launcher.service
+   sudo systemctl enable nodepool-launcher.service
+
+Zuul Service Management
+~~~~~~~~~~~~~~~~~~~~~~~
+::
+
+   sudo bash -c "cat > /etc/systemd/system/zuul-scheduler.service <<EOF
+   [Unit]
+   Description=Zuul Scheduler Service
+   After=syslog.target network.target
+
+   [Service]
+   Type=simple
+   Group=zuul
+   User=zuul
+   RuntimeDirectory=zuul
+   ExecStart=/usr/local/bin/zuul-scheduler
+   ExecStop=/usr/local/bin/zuul-scheduler stop
+
+   [Install]
+   WantedBy=multi-user.target
+   EOF"
+
+   sudo bash -c "cat > /etc/systemd/system/zuul-executor.service <<EOF
+   [Unit]
+   Description=Zuul Executor Service
+   After=syslog.target network.target
+
+   [Service]
+   Type=simple
+   Group=zuul
+   User=zuul
+   RuntimeDirectory=zuul
+   ExecStart=/usr/local/bin/zuul-executor
+   ExecStop=/usr/local/bin/zuul-executor stop
+
+   [Install]
+   WantedBy=multi-user.target
+   EOF"
+
+   sudo bash -c "cat > /etc/systemd/system/zuul-web.service <<EOF
+   [Unit]
+   Description=Zuul Web Service
+   After=syslog.target network.target
+
+   [Service]
+   Type=simple
+   Group=zuul
+   User=zuul
+   RuntimeDirectory=zuul
+   ExecStart=/usr/local/bin/zuul-web
+   ExecStop=/usr/local/bin/zuul-web stop
+
+   [Install]
+   WantedBy=multi-user.target
+   EOF"
+
+   sudo systemctl daemon-reload
+   sudo systemctl start zuul-scheduler.service
+   sudo systemctl status zuul-scheduler.service
+   sudo systemctl enable zuul-scheduler.service
+   sudo systemctl start zuul-executor.service
+   sudo systemctl status zuul-executor.service
+   sudo systemctl enable zuul-executor.service
+   sudo systemctl start zuul-web.service
+   sudo systemctl status zuul-web.service
+   sudo systemctl enable zuul-web.service
+
+Use Zuul Jobs
+-------------
+
+Add to ``/etc/zuul/zuul.conf``::
+
+   sudo bash -c "cat >> /etc/zuul/zuul.conf <<EOF
+
+   [connection zuul-git]
+   driver=git
+   baseurl=https://git.openstack.org/
+   EOF"
+
+Restart executor and scheduler::
+
+   sudo systemctl restart zuul-executor.service
+   sudo systemctl restart zuul-scheduler.service
+
+Configure GitHub
+----------------
+
+You'll need an organization in Github for this, so create one if you
+haven't already.  In this example we will use `my-org`.
+
+.. NOTE Duplicate content here and in drivers/github.rst.  Keep them
+   in sync.
+
+Create a `GitHub application
+<https://developer.github.com/apps/building-integrations/setting-up-and-registering-github-apps/registering-github-apps/>`_:
+
+* Go to your organization settings page to create the application, e.g.:
+  https://github.com/organizations/my-org/settings/apps/new
+* Set GitHub App name to "my-org-zuul"
+* Set Setup URL to your setup documentation, when users install the application
+  they are redirected to this url
+* Set Webhook URL to
+  ``http://<IP ADDRESS>/connection/github/payload``.
+* Create a Webhook secret, and record it for later use
+* Set permissions:
+
+  * Commit statuses: Read & Write
+  * Issues: Read & Write
+  * Pull requests: Read & Write
+  * Repository contents: Read & Write (write to let zuul merge change)
+  * Repository administration: Read
+
+* Set events subscription:
+
+  * Label
+  * Status
+  * Issue comment
+  * Issues
+  * Pull request
+  * Pull request review
+  * Pull request review comment
+  * Commit comment
+  * Create
+  * Push
+  * Release
+
+* Set Where can this GitHub App be installed to "Any account"
+* Create the App
+* Generate a Private key in the app settings page and save the file for later
+
+.. TODO See if we can script this using GitHub API
+
+Go back to the `General` settings page for the app,
+https://github.com/organizations/my-org/settings/apps/my-org-zuul
+and look for the app `ID` number, under the `About` section.
+
+Edit ``/etc/zuul/zuul.conf`` to add the following::
+
+  [connection github]
+  driver=github
+  app_id=<APP ID NUMBER>
+  app_key=/etc/zuul/github.pem
+  webhook_token=<WEBHOOK TOKEN>
+
+Upload the private key which was generated earlier, and save it in
+``/etc/zuul/github.pem``.
+
+Restart all of Zuul::
+
+  sudo systemctl restart zuul-executor.service
+  sudo systemctl restart zuul-web.service
+  sudo systemctl restart zuul-scheduler.service
+
+Go to the `Advanced` tab for the app in GitHub,
+https://github.com/organizations/my-org/settings/apps/my-org-zuul/advanced,
+and look for the initial ping from the app.  It probably wasn't
+delivered since Zuul wasn't configured at the time, so click
+``Resend`` and verify that it is delivered now that Zuul is
+configured.
+
+Visit the public app page on GitHub,
+https://github.com/apps/my-org-zuul, and install the app into your org.
+
+Create two new repositories in your org.  One will hold the
+configuration for this tenant in Zuul, the other should be a normal
+project repo to use for testing.  We'll call them `zuul-test-config`
+and `zuul-test`, respectively.
+
+Edit ``/etc/zuul/main.yaml`` so that it looks like this::
+
+   - tenant:
+       name: quickstart
+       source:
+         zuul-git:
+           config-projects:
+             - openstack-infra/zuul-base-jobs
+           untrusted-projects:
+             - openstack-infra/zuul-jobs
+         github:
+           config-projects:
+             - my-org/zuul-test-config
+           untrusted-projects:
+             - my-org/zuul-test
+
+The first section, under 'zuul-git' imports the "standard library" of
+Zuul jobs, a collection of jobs that can be used by any Zuul
+installation.
+
+The second section is your GitHub configuration.
+
+After updating the file, restart the Zuul scheduler::
+
+  sudo systemctl restart zuul-scheduler.service
+
+Add an initial pipeline configuration to the `zuul-test-config`
+repository.  Inside that project, create a ``zuul.yaml`` file with the
+following contents::
+
+   - pipeline:
+       name: check
+       description: |
+         Newly opened pull requests enter this pipeline to receive an
+         initial verification
+       manager: independent
+       trigger:
+         github:
+           - event: pull_request
+             action:
+               - opened
+               - changed
+               - reopened
+           - event: pull_request
+             action: comment
+             comment: (?i)^\s*recheck\s*$
+       start:
+         github:
+           status: pending
+           comment: false
+       success:
+         github:
+           status: 'success'
+       failure:
+         github:
+           status: 'failure'
+
+Merge that commit into the repository.
+
+In the `zuul-test` project, create a `.zuul.yaml` file with the
+following contents::
+
+   - project:
+       check:
+         jobs:
+           - noop
+
+Open a new pull request with that commit against the `zuul-test`
+project and verify that Zuul reports a successful run of the `noop`
+job.
diff --git a/doc/source/user/config.rst b/doc/source/user/config.rst
index 597062e..8492423 100644
--- a/doc/source/user/config.rst
+++ b/doc/source/user/config.rst
@@ -546,6 +546,12 @@
       from this job. Once this is set to ``true`` it cannot be reset to
       ``false``.
 
+   .. attr:: abstract
+      :default: false
+
+      To indicate a job is not intended to be run directly, but
+      instead must be inherited from, set this attribute to ``true``.
+
    .. attr:: success-message
       :default: SUCCESS
 
diff --git a/requirements.txt b/requirements.txt
index f24f195..7057c5a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -25,6 +25,6 @@
 cachecontrol
 pyjwt
 iso8601
-aiohttp
+aiohttp<3.0.0
 uvloop;python_version>='3.5'
 psutil
diff --git a/tests/base.py b/tests/base.py
index c13519e..ec94c48 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -1439,6 +1439,7 @@
     def __init__(self, use_ssl=False):
         self.hold_jobs_in_queue = False
         self.hold_merge_jobs_in_queue = False
+        self.jobs_history = []
         if use_ssl:
             ssl_ca = os.path.join(FIXTURE_DIR, 'gearman/root-ca.pem')
             ssl_cert = os.path.join(FIXTURE_DIR, 'gearman/server.pem')
@@ -1455,6 +1456,7 @@
     def getJobForConnection(self, connection, peek=False):
         for job_queue in [self.high_queue, self.normal_queue, self.low_queue]:
             for job in job_queue:
+                self.jobs_history.append(job)
                 if not hasattr(job, 'waiting'):
                     if job.name.startswith(b'executor:execute'):
                         job.waiting = self.hold_jobs_in_queue
@@ -1647,6 +1649,10 @@
         nodeid = path.split("/")[-1]
         return nodeid
 
+    def removeNode(self, node):
+        path = self.NODE_ROOT + '/' + node["_oid"]
+        self.client.delete(path, recursive=True)
+
     def addFailRequest(self, request):
         self.fail_requests.add(request['_oid'])
 
diff --git a/tests/fixtures/config/abstract/git/common-config/playbooks/base.yaml b/tests/fixtures/config/abstract/git/common-config/playbooks/base.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/abstract/git/common-config/playbooks/base.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+  tasks: []
diff --git a/tests/fixtures/config/abstract/git/common-config/zuul.yaml b/tests/fixtures/config/abstract/git/common-config/zuul.yaml
new file mode 100644
index 0000000..4aeb947
--- /dev/null
+++ b/tests/fixtures/config/abstract/git/common-config/zuul.yaml
@@ -0,0 +1,25 @@
+- pipeline:
+    name: check
+    manager: independent
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        Verified: 1
+    failure:
+      gerrit:
+        Verified: -1
+
+- job:
+    name: base
+    parent: null
+    run: playbooks/base.yaml
+
+- job:
+    name: job-abstract
+    abstract: true
+
+- job:
+    name: job-child
+    parent: job-abstract
diff --git a/tests/fixtures/config/abstract/git/org_project/zuul.yaml b/tests/fixtures/config/abstract/git/org_project/zuul.yaml
new file mode 100644
index 0000000..cf635e8
--- /dev/null
+++ b/tests/fixtures/config/abstract/git/org_project/zuul.yaml
@@ -0,0 +1,4 @@
+- project:
+    name: org/project
+    check:
+      jobs: []
diff --git a/tests/fixtures/config/abstract/main.yaml b/tests/fixtures/config/abstract/main.yaml
new file mode 100644
index 0000000..208e274
--- /dev/null
+++ b/tests/fixtures/config/abstract/main.yaml
@@ -0,0 +1,8 @@
+- tenant:
+    name: tenant-one
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - org/project
diff --git a/tests/fixtures/config/disk-accountant/git/common-config/playbooks/dd-big-empty-file.yaml b/tests/fixtures/config/disk-accountant/git/common-config/playbooks/dd-big-empty-file.yaml
index 95ab870..ba35eb0 100644
--- a/tests/fixtures/config/disk-accountant/git/common-config/playbooks/dd-big-empty-file.yaml
+++ b/tests/fixtures/config/disk-accountant/git/common-config/playbooks/dd-big-empty-file.yaml
@@ -1,6 +1,7 @@
 - hosts: localhost
   tasks:
     - command: dd if=/dev/zero of=toobig bs=1M count=2
+    - command: sync
     - wait_for:
         delay: 10
         path: /
diff --git a/tests/fixtures/config/tenant-parser/groups4.yaml b/tests/fixtures/config/tenant-parser/groups4.yaml
new file mode 100644
index 0000000..3512673
--- /dev/null
+++ b/tests/fixtures/config/tenant-parser/groups4.yaml
@@ -0,0 +1,11 @@
+- tenant:
+    name: tenant-one
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - include: []
+            projects:
+              - org/project1
+              - org/project2
diff --git a/tests/fixtures/layouts/timer-github.yaml b/tests/fixtures/layouts/timer-github.yaml
new file mode 100644
index 0000000..4f3efe4
--- /dev/null
+++ b/tests/fixtures/layouts/timer-github.yaml
@@ -0,0 +1,25 @@
+- pipeline:
+    name: periodic
+    manager: independent
+    trigger:
+      timer:
+        - time: '* * * * * */1'
+
+- job:
+    name: base
+    parent: null
+    run: playbooks/base.yaml
+
+- job:
+    name: project-bitrot
+    nodeset:
+      nodes:
+        - name: static
+          label: ubuntu-xenial
+    run: playbooks/project-bitrot.yaml
+
+- project:
+    name: org/project
+    periodic:
+      jobs:
+        - project-bitrot
diff --git a/tests/unit/test_configloader.py b/tests/unit/test_configloader.py
index f7d580c..49220f2 100644
--- a/tests/unit/test_configloader.py
+++ b/tests/unit/test_configloader.py
@@ -213,6 +213,33 @@
                         project2_config.pipelines['check'].job_list.jobs)
 
 
+class TestTenantGroups4(TenantParserTestCase):
+    tenant_config_file = 'config/tenant-parser/groups4.yaml'
+
+    def test_tenant_groups(self):
+        tenant = self.sched.abide.tenants.get('tenant-one')
+        self.assertEqual(['common-config'],
+                         [x.name for x in tenant.config_projects])
+        self.assertEqual(['org/project1', 'org/project2'],
+                         [x.name for x in tenant.untrusted_projects])
+        project = tenant.config_projects[0]
+        tpc = tenant.project_configs[project.canonical_name]
+        self.assertEqual(self.CONFIG_SET, tpc.load_classes)
+        project = tenant.untrusted_projects[0]
+        tpc = tenant.project_configs[project.canonical_name]
+        self.assertEqual(set([]),
+                         tpc.load_classes)
+        project = tenant.untrusted_projects[1]
+        tpc = tenant.project_configs[project.canonical_name]
+        self.assertEqual(set([]),
+                         tpc.load_classes)
+        # Check that only one merger:cat job was requested
+        # org/project1 and org/project2 have an empty load_classes
+        cat_jobs = [job for job in self.gearman_server.jobs_history
+                    if job.name == b'merger:cat']
+        self.assertEqual(1, len(cat_jobs))
+
+
 class TestTenantUnprotectedBranches(TenantParserTestCase):
     tenant_config_file = 'config/tenant-parser/unprotected-branches.yaml'
 
diff --git a/tests/unit/test_disk_accountant.py b/tests/unit/test_disk_accountant.py
index 7081b53..e12846d 100644
--- a/tests/unit/test_disk_accountant.py
+++ b/tests/unit/test_disk_accountant.py
@@ -10,6 +10,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import fixtures
 import os
 import tempfile
 import time
@@ -32,6 +33,10 @@
 
 
 class TestDiskAccountant(BaseTestCase):
+    def setUp(self):
+        super(TestDiskAccountant, self).setUp()
+        self.useFixture(fixtures.NestedTempfile())
+
     def test_disk_accountant(self):
         jobs_dir = tempfile.mkdtemp(
             dir=os.environ.get("ZUUL_TEST_ROOT", None))
@@ -47,6 +52,8 @@
             testfile = os.path.join(jobdir, 'tfile')
             with open(testfile, 'w') as tf:
                 tf.write(2 * 1024 * 1024 * '.')
+                tf.flush()
+                os.fsync(tf.fileno())
 
             # da should catch over-limit dir within 5 seconds
             for i in range(0, 50):
diff --git a/tests/unit/test_encryption.py b/tests/unit/test_encryption.py
index b424769..0a5c0a4 100644
--- a/tests/unit/test_encryption.py
+++ b/tests/unit/test_encryption.py
@@ -12,6 +12,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import fixtures
 import os
 import subprocess
 import tempfile
@@ -26,6 +27,10 @@
     def setUp(self):
         super(TestEncryption, self).setUp()
         self.private, self.public = encryption.generate_rsa_keypair()
+        # Because we set delete to False when using NamedTemporaryFile below
+        # we need to stick our usage of temporary files in the NestedTempfile
+        # fixture ensuring everything gets cleaned up when it is done.
+        self.useFixture(fixtures.NestedTempfile())
 
     def test_serialization(self):
         "Verify key serialization"
diff --git a/tests/unit/test_executor.py b/tests/unit/test_executor.py
index 8cb98ee..46e1d99 100755
--- a/tests/unit/test_executor.py
+++ b/tests/unit/test_executor.py
@@ -490,6 +490,7 @@
     def test_slow_start(self):
         self.executor_server.hold_jobs_in_build = True
         self.executor_server.max_starting_builds = 1
+        self.executor_server.min_starting_builds = 1
         self.executor_server.manageLoad()
         self.assertTrue(self.executor_server.accepting_work)
         A = self.fake_gerrit.addFakeChange('common-config', 'master', 'A')
diff --git a/tests/unit/test_github_driver.py b/tests/unit/test_github_driver.py
index cd36ba3..8978415 100644
--- a/tests/unit/test_github_driver.py
+++ b/tests/unit/test_github_driver.py
@@ -210,6 +210,34 @@
         self.waitUntilSettled()
         self.assertEqual(1, len(self.history))
 
+    @simple_layout('layouts/basic-github.yaml', driver='github')
+    def test_timer_event(self):
+        self.executor_server.hold_jobs_in_build = True
+        self.commitConfigUpdate('org/common-config',
+                                'layouts/timer-github.yaml')
+        self.sched.reconfigure(self.config)
+        time.sleep(2)
+        self.waitUntilSettled()
+        self.assertEqual(len(self.builds), 1)
+        self.executor_server.hold_jobs_in_build = False
+        # Stop queuing timer triggered jobs so that the assertions
+        # below don't race against more jobs being queued.
+        self.commitConfigUpdate('org/common-config',
+                                'layouts/basic-github.yaml')
+        self.sched.reconfigure(self.config)
+        self.waitUntilSettled()
+        # If APScheduler is in mid-event when we remove the job, we
+        # can end up with one more event firing, so give it an extra
+        # second to settle.
+        time.sleep(1)
+        self.waitUntilSettled()
+        self.executor_server.release()
+        self.waitUntilSettled()
+        self.assertHistory([
+            dict(name='project-bitrot', result='SUCCESS',
+                 ref='refs/heads/master'),
+        ], ordered=False)
+
     @simple_layout('layouts/dequeue-github.yaml', driver='github')
     def test_dequeue_pull_synchronized(self):
         self.executor_server.hold_jobs_in_build = True
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index 9a2eb28..c833fa2 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -37,6 +37,7 @@
     ZuulTestCase,
     repack_repo,
     simple_layout,
+    iterate_timeout,
 )
 
 
@@ -1505,7 +1506,7 @@
                                           self.gearman_server.port)
         self.addCleanup(client.shutdown)
         r = client.autohold('tenant-one', 'org/project', 'project-test2',
-                            "reason text", 1)
+                            "", "", "reason text", 1)
         self.assertTrue(r)
 
         # First check that successful jobs do not autohold
@@ -1552,7 +1553,7 @@
             held_node['hold_job'],
             " ".join(['tenant-one',
                       'review.example.com/org/project',
-                      'project-test2'])
+                      'project-test2', '.*'])
         )
         self.assertEqual(held_node['comment'], "reason text")
 
@@ -1572,13 +1573,151 @@
                 held_nodes += 1
         self.assertEqual(held_nodes, 1)
 
+    def _test_autohold_scoped(self, change_obj, change, ref):
+        client = zuul.rpcclient.RPCClient('127.0.0.1',
+                                          self.gearman_server.port)
+        self.addCleanup(client.shutdown)
+
+        # create two changes on the same project, and autohold request
+        # for one of them.
+        other = self.fake_gerrit.addFakeChange(
+            'org/project', 'master', 'other'
+        )
+
+        r = client.autohold('tenant-one', 'org/project', 'project-test2',
+                            str(change), ref, "reason text", 1)
+        self.assertTrue(r)
+
+        # First, check that an unrelated job does not trigger autohold, even
+        # when it failed
+        self.executor_server.failJob('project-test2', other)
+        self.fake_gerrit.addEvent(other.getPatchsetCreatedEvent(1))
+
+        self.waitUntilSettled()
+
+        self.assertEqual(other.data['status'], 'NEW')
+        self.assertEqual(other.reported, 1)
+        # project-test2
+        self.assertEqual(self.history[0].result, 'FAILURE')
+
+        # Check nodepool for a held node
+        held_node = None
+        for node in self.fake_nodepool.getNodes():
+            if node['state'] == zuul.model.STATE_HOLD:
+                held_node = node
+                break
+        self.assertIsNone(held_node)
+
+        # And then verify that failed job for the defined change
+        # triggers the autohold
+
+        self.executor_server.failJob('project-test2', change_obj)
+        self.fake_gerrit.addEvent(change_obj.getPatchsetCreatedEvent(1))
+
+        self.waitUntilSettled()
+
+        self.assertEqual(change_obj.data['status'], 'NEW')
+        self.assertEqual(change_obj.reported, 1)
+        # project-test2
+        self.assertEqual(self.history[1].result, 'FAILURE')
+
+        # Check nodepool for a held node
+        held_node = None
+        for node in self.fake_nodepool.getNodes():
+            if node['state'] == zuul.model.STATE_HOLD:
+                held_node = node
+                break
+        self.assertIsNotNone(held_node)
+
+        # Validate node has recorded the failed job
+        if change != "":
+            ref = "refs/changes/%s/%s/.*" % (
+                str(change_obj.number)[-1:], str(change_obj.number)
+            )
+
+        self.assertEqual(
+            held_node['hold_job'],
+            " ".join(['tenant-one',
+                      'review.example.com/org/project',
+                      'project-test2', ref])
+        )
+        self.assertEqual(held_node['comment'], "reason text")
+
+    @simple_layout('layouts/autohold.yaml')
+    def test_autohold_change(self):
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+
+        self._test_autohold_scoped(A, change=A.number, ref="")
+
+    @simple_layout('layouts/autohold.yaml')
+    def test_autohold_ref(self):
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+        ref = A.data['currentPatchSet']['ref']
+        self._test_autohold_scoped(A, change="", ref=ref)
+
+    @simple_layout('layouts/autohold.yaml')
+    def test_autohold_scoping(self):
+        client = zuul.rpcclient.RPCClient('127.0.0.1',
+                                          self.gearman_server.port)
+        self.addCleanup(client.shutdown)
+
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+
+        # create three autohold requests, scoped to job, change and
+        # a specific ref
+        change = str(A.number)
+        ref = A.data['currentPatchSet']['ref']
+        r1 = client.autohold('tenant-one', 'org/project', 'project-test2',
+                             "", "", "reason text", 1)
+        self.assertTrue(r1)
+        r2 = client.autohold('tenant-one', 'org/project', 'project-test2',
+                             change, "", "reason text", 1)
+        self.assertTrue(r2)
+        r3 = client.autohold('tenant-one', 'org/project', 'project-test2',
+                             "", ref, "reason text", 1)
+        self.assertTrue(r3)
+
+        # Fail 3 jobs for the same change, and verify that the autohold
+        # requests are fullfilled in the expected order: from the most
+        # specific towards the most generic one.
+
+        def _fail_job_and_verify_autohold_request(change_obj, ref_filter):
+            self.executor_server.failJob('project-test2', change_obj)
+            self.fake_gerrit.addEvent(change_obj.getPatchsetCreatedEvent(1))
+
+            self.waitUntilSettled()
+
+            # Check nodepool for a held node
+            held_node = None
+            for node in self.fake_nodepool.getNodes():
+                if node['state'] == zuul.model.STATE_HOLD:
+                    held_node = node
+                    break
+            self.assertIsNotNone(held_node)
+
+            self.assertEqual(
+                held_node['hold_job'],
+                " ".join(['tenant-one',
+                          'review.example.com/org/project',
+                          'project-test2', ref_filter])
+            )
+            self.assertFalse(held_node['_lock'], "Node %s is locked" %
+                             (node['_oid'],))
+            self.fake_nodepool.removeNode(held_node)
+
+        _fail_job_and_verify_autohold_request(A, ref)
+
+        ref = "refs/changes/%s/%s/.*" % (str(change)[-1:], str(change))
+        _fail_job_and_verify_autohold_request(A, ref)
+        _fail_job_and_verify_autohold_request(A, ".*")
+
     @simple_layout('layouts/autohold.yaml')
     def test_autohold_ignores_aborted_jobs(self):
         client = zuul.rpcclient.RPCClient('127.0.0.1',
                                           self.gearman_server.port)
         self.addCleanup(client.shutdown)
         r = client.autohold('tenant-one', 'org/project', 'project-test2',
-                            "reason text", 1)
+                            "", "", "reason text", 1)
         self.assertTrue(r)
 
         self.executor_server.hold_jobs_in_build = True
@@ -1622,7 +1761,7 @@
         self.addCleanup(client.shutdown)
 
         r = client.autohold('tenant-one', 'org/project', 'project-test2',
-                            "reason text", 1)
+                            "", "", "reason text", 1)
         self.assertTrue(r)
 
         autohold_requests = client.autohold_list()
@@ -1631,11 +1770,12 @@
 
         # The single dict key should be a CSV string value
         key = list(autohold_requests.keys())[0]
-        tenant, project, job = key.split(',')
+        tenant, project, job, ref_filter = key.split(',')
 
         self.assertEqual('tenant-one', tenant)
         self.assertIn('org/project', project)
         self.assertEqual('project-test2', job)
+        self.assertEqual(".*", ref_filter)
 
         # Note: the value is converted from set to list by json.
         self.assertEqual([1, "reason text"], autohold_requests[key])
@@ -4397,6 +4537,54 @@
         self.assertEqual(A.data['status'], 'MERGED')
         self.assertEqual(A.reported, 2)
 
+    def test_zookeeper_disconnect2(self):
+        "Test that jobs are executed after a zookeeper disconnect"
+
+        # This tests receiving a ZK disconnect between the arrival of
+        # a fulfilled request and when we accept its nodes.
+        self.fake_nodepool.paused = True
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+        A.addApproval('Code-Review', 2)
+        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+        self.waitUntilSettled()
+
+        # We're waiting on the nodepool request to complete.  Stop the
+        # scheduler from processing further events, then fulfill the
+        # nodepool request.
+        self.sched.run_handler_lock.acquire()
+
+        # Fulfill the nodepool request.
+        self.fake_nodepool.paused = False
+        requests = list(self.sched.nodepool.requests.values())
+        self.assertEqual(1, len(requests))
+        request = requests[0]
+        for x in iterate_timeout(30, 'fulfill request'):
+            if request.fulfilled:
+                break
+        id1 = request.id
+
+        # The request is fulfilled, but the scheduler hasn't processed
+        # it yet.  Reconnect ZK.
+        self.zk.client.stop()
+        self.zk.client.start()
+
+        # Allow the scheduler to continue and process the (now
+        # out-of-date) notification that nodes are ready.
+        self.sched.run_handler_lock.release()
+
+        # It should resubmit the request, once it's fulfilled, we can
+        # wait for it to run jobs and settle.
+        for x in iterate_timeout(30, 'fulfill request'):
+            if request.fulfilled:
+                break
+        self.waitUntilSettled()
+
+        id2 = request.id
+        self.assertEqual(A.data['status'], 'MERGED')
+        self.assertEqual(A.reported, 2)
+        # Make sure it was resubmitted (the id's should be different).
+        self.assertNotEqual(id1, id2)
+
     def test_nodepool_failure(self):
         "Test that jobs are reported after a nodepool failure"
 
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index 44eda82..573c8a6 100755
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -74,44 +74,43 @@
 
 
 class TestProtected(ZuulTestCase):
-
     tenant_config_file = 'config/protected/main.yaml'
 
     def test_protected_ok(self):
-            # test clean usage of final parent job
-            in_repo_conf = textwrap.dedent(
-                """
-                - job:
-                    name: job-protected
-                    protected: true
-                    run: playbooks/job-protected.yaml
+        # test clean usage of final parent job
+        in_repo_conf = textwrap.dedent(
+            """
+            - job:
+                name: job-protected
+                protected: true
+                run: playbooks/job-protected.yaml
 
-                - project:
-                    name: org/project
-                    check:
-                      jobs:
-                        - job-child-ok
+            - project:
+                name: org/project
+                check:
+                  jobs:
+                    - job-child-ok
 
-                - job:
-                    name: job-child-ok
-                    parent: job-protected
+            - job:
+                name: job-child-ok
+                parent: job-protected
 
-                - project:
-                    name: org/project
-                    check:
-                      jobs:
-                        - job-child-ok
+            - project:
+                name: org/project
+                check:
+                  jobs:
+                    - job-child-ok
 
-                """)
+            """)
 
-            file_dict = {'zuul.yaml': in_repo_conf}
-            A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
-                                               files=file_dict)
-            self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
-            self.waitUntilSettled()
+        file_dict = {'zuul.yaml': in_repo_conf}
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+                                           files=file_dict)
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
 
-            self.assertEqual(A.reported, 1)
-            self.assertEqual(A.patchsets[-1]['approvals'][0]['value'], '1')
+        self.assertEqual(A.reported, 1)
+        self.assertEqual(A.patchsets[-1]['approvals'][0]['value'], '1')
 
     def test_protected_reset(self):
         # try to reset protected flag
@@ -177,6 +176,47 @@
             "and cannot be inherited from other projects.", A.messages[0])
 
 
+class TestAbstract(ZuulTestCase):
+    tenant_config_file = 'config/abstract/main.yaml'
+
+    def test_abstract_fail(self):
+        in_repo_conf = textwrap.dedent(
+            """
+            - project:
+                check:
+                  jobs:
+                    - job-abstract
+            """)
+
+        file_dict = {'zuul.yaml': in_repo_conf}
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+                                           files=file_dict)
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        self.assertEqual(A.reported, 1)
+        self.assertEqual(A.patchsets[-1]['approvals'][0]['value'], '-1')
+        self.assertIn('may not be directly run', A.messages[0])
+
+    def test_child_of_abstract(self):
+        in_repo_conf = textwrap.dedent(
+            """
+            - project:
+                check:
+                  jobs:
+                    - job-child
+            """)
+
+        file_dict = {'zuul.yaml': in_repo_conf}
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+                                           files=file_dict)
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        self.assertEqual(A.reported, 1)
+        self.assertEqual(A.patchsets[-1]['approvals'][0]['value'], '1')
+
+
 class TestFinal(ZuulTestCase):
 
     tenant_config_file = 'config/final/main.yaml'
diff --git a/tools/encrypt_secret.py b/tools/encrypt_secret.py
index 4cb1666..45ad68c 100755
--- a/tools/encrypt_secret.py
+++ b/tools/encrypt_secret.py
@@ -26,9 +26,11 @@
 try:
     from urllib.request import Request
     from urllib.request import urlopen
+    from urllib.parse import urlparse
 except ImportError:
     from urllib2 import Request
     from urllib2 import urlopen
+    from urlparse import urlparse
 
 DESCRIPTION = """Encrypt a secret for Zuul.
 
@@ -43,7 +45,6 @@
     parser.add_argument('url',
                         help="The base URL of the zuul server and tenant.  "
                         "E.g., https://zuul.example.com/tenant-name")
-    # TODO(jeblair): Throw a fit if SSL is not used.
     parser.add_argument('project',
                         help="The name of the project.")
     parser.add_argument('--strip', action='store_true', default=False,
@@ -60,6 +61,15 @@
                         "to standard output.")
     args = parser.parse_args()
 
+    # We should not use unencrypted connections for retrieving the public key.
+    # Otherwise our secret can be compromised. The schemes file and https are
+    # considered safe.
+    url = urlparse(args.url)
+    if url.scheme not in ('file', 'https'):
+        sys.stderr.write("WARNING: Retrieving encryption key via an "
+                         "unencrypted connection. Your secret may get "
+                         "compromised.\n")
+
     req = Request("%s/%s.pub" % (args.url.rstrip('/'), args.project))
     pubkey = urlopen(req)
 
diff --git a/tools/github-debugging.py b/tools/github-debugging.py
index 101fd11..da6fd0c 100755
--- a/tools/github-debugging.py
+++ b/tools/github-debugging.py
@@ -11,6 +11,8 @@
 # TODO: for real use override the following variables
 server = 'github.com'
 api_token = 'xxxx'
+appid = 2
+appkey = '/opt/project/appkey'
 
 org = 'example'
 repo = 'sandbox'
@@ -42,20 +44,36 @@
     return conn
 
 
+def create_connection_app(server, appid, appkey):
+    driver = GithubDriver()
+    connection_config = {
+        'server': server,
+        'app_id': appid,
+        'app_key': appkey,
+    }
+    conn = GithubConnection(driver, 'github', connection_config)
+    conn._authenticateGithubAPI()
+    conn._prime_installation_map()
+    return conn
+
+
 def get_change(connection: GithubConnection,
                org: str,
                repo: str,
                pull: int) -> Change:
     p = Project("%s/%s" % (org, repo), connection.source)
-    github = connection.getGithubClient(p)
+    github = connection.getGithubClient(p.name)
     pr = github.pull_request(org, repo, pull)
     sha = pr.head.sha
     return conn._getChange(p, pull, sha, True)
 
 
-# create github connection
+# create github connection with api token
 conn = create_connection(server, api_token)
 
+# create github connection with app key
+# conn = create_connection_app(server, appid, appkey)
+
 
 # Now we can do anything we want with the connection, e.g. check canMerge for
 # a pull request.
diff --git a/tools/nodepool-integration-setup.sh b/tools/nodepool-integration-setup.sh
index c02a016..58c39cf 100755
--- a/tools/nodepool-integration-setup.sh
+++ b/tools/nodepool-integration-setup.sh
@@ -3,7 +3,7 @@
 /usr/zuul-env/bin/zuul-cloner --workspace /tmp --cache-dir /opt/git \
     git://git.openstack.org openstack-infra/nodepool
 
-ln -s /tmp/nodepool/log $WORKSPACE/logs
+ln -s /tmp/nodepool/log $HOME/logs
 
 cd /tmp/openstack-infra/nodepool
 /usr/local/jenkins/slave_scripts/install-distro-packages.sh
diff --git a/tox.ini b/tox.ini
index 73915ad..e5035bd 100644
--- a/tox.ini
+++ b/tox.ini
@@ -6,7 +6,7 @@
 [testenv]
 basepython = python3
 setenv = VIRTUAL_ENV={envdir}
-         OS_TEST_TIMEOUT=120
+         OS_TEST_TIMEOUT=150
 passenv = ZUUL_TEST_ROOT OS_STDOUT_CAPTURE OS_STDERR_CAPTURE OS_LOG_CAPTURE OS_LOG_DEFAULTS
 usedevelop = True
 install_command = pip install {opts} {packages}
diff --git a/zuul/ansible/callback/zuul_stream.py b/zuul/ansible/callback/zuul_stream.py
index df28a57..15b491c 100644
--- a/zuul/ansible/callback/zuul_stream.py
+++ b/zuul/ansible/callback/zuul_stream.py
@@ -367,12 +367,13 @@
                 self._log_message(
                     result, status='MODULE FAILURE',
                     msg=result_dict['module_stderr'])
-        elif (len([key for key in result_dict.keys()
-                   if not key.startswith('_ansible')]) == 1):
+        elif result._task.action == 'debug':
             # this is a debug statement, handle it special
             for key in [k for k in result_dict.keys()
                         if k.startswith('_ansible')]:
                 del result_dict[key]
+            if 'changed' in result_dict.keys():
+                del result_dict['changed']
             keyname = next(iter(result_dict.keys()))
             # If it has msg, that means it was like:
             #
diff --git a/zuul/ansible/library/zuul_console.py b/zuul/ansible/library/zuul_console.py
index f84766d..6703cc1 100644
--- a/zuul/ansible/library/zuul_console.py
+++ b/zuul/ansible/library/zuul_console.py
@@ -277,7 +277,7 @@
         inode = get_inode()
         if not inode:
             module.fail_json(
-                "Could not find inode for port",
+                msg="Could not find inode for port",
                 exceptions=[])
 
         pid, exceptions = get_pid_from_inode(inode)
diff --git a/zuul/cmd/__init__.py b/zuul/cmd/__init__.py
index b299219..6ab7598 100755
--- a/zuul/cmd/__init__.py
+++ b/zuul/cmd/__init__.py
@@ -59,9 +59,12 @@
             thread = threads.get(thread_id)
             if thread:
                 thread_name = thread.name
+                thread_is_daemon = str(thread.daemon)
             else:
                 thread_name = thread.ident
-            log_str += "Thread: %s %s\n" % (thread_id, thread_name)
+                thread_is_daemon = '(Unknown)'
+            log_str += "Thread: %s %s d: %s\n"\
+                       % (thread_id, thread_name, thread_is_daemon)
             log_str += "".join(traceback.format_stack(stack_frame))
         log.debug(log_str)
     except Exception:
diff --git a/zuul/cmd/client.py b/zuul/cmd/client.py
index ebf59b9..a7b3ef3 100755
--- a/zuul/cmd/client.py
+++ b/zuul/cmd/client.py
@@ -51,6 +51,11 @@
                                   required=True)
         cmd_autohold.add_argument('--job', help='job name',
                                   required=True)
+        cmd_autohold.add_argument('--change',
+                                  help='specific change to hold nodes for',
+                                  required=False, default='')
+        cmd_autohold.add_argument('--ref', help='git ref to hold nodes for',
+                                  required=False, default='')
         cmd_autohold.add_argument('--reason', help='reason for the hold',
                                   required=True)
         cmd_autohold.add_argument('--count',
@@ -173,9 +178,15 @@
     def autohold(self):
         client = zuul.rpcclient.RPCClient(
             self.server, self.port, self.ssl_key, self.ssl_cert, self.ssl_ca)
+        if self.args.change and self.args.ref:
+            print("Change and ref can't be both used for the same request")
+            return False
+
         r = client.autohold(tenant=self.args.tenant,
                             project=self.args.project,
                             job=self.args.job,
+                            change=self.args.change,
+                            ref=self.args.ref,
                             reason=self.args.reason,
                             count=self.args.count)
         return r
@@ -190,14 +201,19 @@
             return True
 
         table = prettytable.PrettyTable(
-            field_names=['Tenant', 'Project', 'Job', 'Count', 'Reason'])
+            field_names=[
+                'Tenant', 'Project', 'Job', 'Ref Filter', 'Count', 'Reason'
+            ])
 
         for key, value in autohold_requests.items():
             # The key comes to us as a CSV string because json doesn't like
             # non-str keys.
-            tenant_name, project_name, job_name = key.split(',')
+            tenant_name, project_name, job_name, ref_filter = key.split(',')
             count, reason = value
-            table.add_row([tenant_name, project_name, job_name, count, reason])
+
+            table.add_row([
+                tenant_name, project_name, job_name, ref_filter, count, reason
+            ])
         print(table)
         return True
 
diff --git a/zuul/cmd/scheduler.py b/zuul/cmd/scheduler.py
index 68c9000..a3a53cf 100755
--- a/zuul/cmd/scheduler.py
+++ b/zuul/cmd/scheduler.py
@@ -159,6 +159,7 @@
             self.log.exception("Error starting Zuul:")
             # TODO(jeblair): If we had all threads marked as daemon,
             # we might be able to have a nicer way of exiting here.
+            self.sched.stop()
             sys.exit(1)
 
         signal.signal(signal.SIGHUP, self.reconfigure_handler)
diff --git a/zuul/configloader.py b/zuul/configloader.py
index be1bd63..bd2ce3a 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -474,6 +474,7 @@
     # Attributes of a job that can also be used in Project and ProjectTemplate
     job_attributes = {'parent': vs.Any(str, None),
                       'final': bool,
+                      'abstract': bool,
                       'protected': bool,
                       'failure-message': str,
                       'success-message': str,
@@ -514,6 +515,7 @@
 
     simple_attributes = [
         'final',
+        'abstract',
         'protected',
         'timeout',
         'workspace',
@@ -1448,6 +1450,11 @@
             jobs.append(job)
 
         for project in untrusted_projects:
+            tpc = tenant.project_configs[project.canonical_name]
+            # If all config classes are excluded then does not request a
+            # getFiles jobs.
+            if not tpc.load_classes:
+                continue
             # If we have cached data (this is a reconfiguration) use it.
             if cached and project.unparsed_config:
                 jobs.append(CachedDataJob(False, project))
diff --git a/zuul/driver/gerrit/gerritsource.py b/zuul/driver/gerrit/gerritsource.py
index fdc1ad7..8f3408e 100644
--- a/zuul/driver/gerrit/gerritsource.py
+++ b/zuul/driver/gerrit/gerritsource.py
@@ -141,6 +141,10 @@
         )
         return [f]
 
+    def getRefForChange(self, change):
+        partial = change[-2:]
+        return "refs/changes/%s/%s/.*" % (partial, change)
+
 
 approval = vs.Schema({'username': str,
                       'email': str,
diff --git a/zuul/driver/git/gitsource.py b/zuul/driver/git/gitsource.py
index a7d42be..9f0963d 100644
--- a/zuul/driver/git/gitsource.py
+++ b/zuul/driver/git/gitsource.py
@@ -68,3 +68,6 @@
 
     def getRejectFilters(self, config):
         return []
+
+    def getRefForChange(self, change):
+        raise NotImplemented()
diff --git a/zuul/driver/github/githubconnection.py b/zuul/driver/github/githubconnection.py
index 6072f4c..6dfcdd3 100644
--- a/zuul/driver/github/githubconnection.py
+++ b/zuul/driver/github/githubconnection.py
@@ -458,6 +458,7 @@
         self._github = None
         self.app_id = None
         self.app_key = None
+        self.sched = None
 
         self.installation_map = {}
         self.installation_token_cache = {}
@@ -721,7 +722,8 @@
             change.newrev = event.newrev
             change.url = self.getGitwebUrl(project, sha=event.newrev)
             change.source_event = event
-            change.files = self.getPushedFileNames(event)
+            if hasattr(event, 'commits'):
+                change.files = self.getPushedFileNames(event)
         return change
 
     def _getChange(self, project, number, patchset=None, refresh=False):
@@ -827,7 +829,8 @@
         change.updated_at = self._ghTimestampToDate(
             change.pr.get('updated_at'))
 
-        self.sched.onChangeUpdated(change)
+        if self.sched:
+            self.sched.onChangeUpdated(change)
 
         return change
 
diff --git a/zuul/driver/github/githubsource.py b/zuul/driver/github/githubsource.py
index 33f8f7c..6f9b14d 100644
--- a/zuul/driver/github/githubsource.py
+++ b/zuul/driver/github/githubsource.py
@@ -144,6 +144,9 @@
         )
         return [f]
 
+    def getRefForChange(self, change):
+        return "refs/pull/%s/head" % change
+
 
 review = v.Schema({'username': str,
                    'email': str,
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index ffc083f..8de6fe0 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -777,11 +777,29 @@
         return data
 
     def doMergeChanges(self, merger, items, repo_state):
-        ret = merger.mergeChanges(items, repo_state=repo_state)
-        if not ret:  # merge conflict
-            result = dict(result='MERGER_FAILURE')
+        try:
+            ret = merger.mergeChanges(items, repo_state=repo_state)
+        except ValueError as e:
+            # Return ABORTED so that we'll try again. At this point all of
+            # the refs we're trying to merge should be valid refs. If we
+            # can't fetch them, it should resolve itself.
+            self.log.exception("Could not fetch refs to merge from remote")
+            result = dict(result='ABORTED')
             self.job.sendWorkComplete(json.dumps(result))
             return False
+        if not ret:  # merge conflict
+            result = dict(result='MERGER_FAILURE')
+            if self.executor_server.statsd:
+                base_key = ("zuul.executor.%s.merger" %
+                            self.executor_server.hostname)
+                self.executor_server.statsd.incr(base_key + ".FAILURE")
+            self.job.sendWorkComplete(json.dumps(result))
+            return False
+
+        if self.executor_server.statsd:
+            base_key = ("zuul.executor.%s.merger" %
+                        self.executor_server.hostname)
+            self.executor_server.statsd.incr(base_key + ".SUCCESS")
         recent = ret[3]
         for key, commit in recent.items():
             (connection, project, branch) = key
@@ -835,6 +853,13 @@
         repo.checkout(selected_ref)
         return selected_ref
 
+    def getAnsibleTimeout(self, start, timeout):
+        if timeout is not None:
+            now = time.time()
+            elapsed = now - start
+            timeout = timeout - elapsed
+        return timeout
+
     def runPlaybooks(self, args):
         result = None
 
@@ -852,10 +877,15 @@
         pre_failed = False
         success = False
         self.started = True
+        time_started = time.time()
+        # timeout value is total job timeout or put another way
+        # the cummulative time that pre, run, and post can consume.
+        job_timeout = args['timeout']
         for index, playbook in enumerate(self.jobdir.pre_playbooks):
             # TODOv3(pabelanger): Implement pre-run timeout setting.
+            ansible_timeout = self.getAnsibleTimeout(time_started, job_timeout)
             pre_status, pre_code = self.runAnsiblePlaybook(
-                playbook, args['timeout'], phase='pre', index=index)
+                playbook, ansible_timeout, phase='pre', index=index)
             if pre_status != self.RESULT_NORMAL or pre_code != 0:
                 # These should really never fail, so return None and have
                 # zuul try again
@@ -863,8 +893,9 @@
                 break
 
         if not pre_failed:
+            ansible_timeout = self.getAnsibleTimeout(time_started, job_timeout)
             job_status, job_code = self.runAnsiblePlaybook(
-                self.jobdir.playbook, args['timeout'], phase='run')
+                self.jobdir.playbook, ansible_timeout, phase='run')
             if job_status == self.RESULT_ABORTED:
                 return 'ABORTED'
             elif job_status == self.RESULT_TIMED_OUT:
@@ -885,8 +916,9 @@
 
         for index, playbook in enumerate(self.jobdir.post_playbooks):
             # TODOv3(pabelanger): Implement post-run timeout setting.
+            ansible_timeout = self.getAnsibleTimeout(time_started, job_timeout)
             post_status, post_code = self.runAnsiblePlaybook(
-                playbook, args['timeout'], success, phase='post', index=index)
+                playbook, ansible_timeout, success, phase='post', index=index)
             if post_status == self.RESULT_ABORTED:
                 return 'ABORTED'
             if post_status != self.RESULT_NORMAL or post_code != 0:
@@ -1465,6 +1497,11 @@
             wrapped=False)
         self.log.debug("Ansible complete, result %s code %s" % (
             self.RESULT_MAP[result], code))
+        if self.executor_server.statsd:
+            base_key = ("zuul.executor.%s.phase.setup" %
+                        self.executor_server.hostname)
+            self.executor_server.statsd.incr(base_key + ".%s" %
+                                             self.RESULT_MAP[result])
         return result, code
 
     def runAnsibleCleanup(self, playbook):
@@ -1485,6 +1522,11 @@
             wrapped=False)
         self.log.debug("Ansible complete, result %s code %s" % (
             self.RESULT_MAP[result], code))
+        if self.executor_server.statsd:
+            base_key = ("zuul.executor.%s.phase.cleanup" %
+                        self.executor_server.hostname)
+            self.executor_server.statsd.incr(base_key + ".%s" %
+                                             self.RESULT_MAP[result])
         return result, code
 
     def emitPlaybookBanner(self, playbook, step, phase, result=None):
@@ -1554,6 +1596,11 @@
             cmd=cmd, timeout=timeout, playbook=playbook)
         self.log.debug("Ansible complete, result %s code %s" % (
             self.RESULT_MAP[result], code))
+        if self.executor_server.statsd:
+            base_key = ("zuul.executor.%s.phase.%s" %
+                        (self.executor_server.hostname, phase or 'unknown'))
+            self.executor_server.statsd.incr(base_key + ".%s" %
+                                             self.RESULT_MAP[result])
 
         self.emitPlaybookBanner(playbook, 'END', phase, result=result)
         return result, code
@@ -1636,6 +1683,7 @@
                                             'load_multiplier', '2.5'))
         self.max_load_avg = multiprocessing.cpu_count() * load_multiplier
         self.max_starting_builds = self.max_load_avg * 2
+        self.min_starting_builds = max(int(multiprocessing.cpu_count() / 2), 1)
         self.min_avail_mem = float(get_default(self.config, 'executor',
                                                'min_avail_mem', '5.0'))
         self.accepting_work = False
@@ -1808,7 +1856,7 @@
         if self.statsd:
             base_key = 'zuul.executor.%s' % self.hostname
             self.statsd.gauge(base_key + '.load_average', 0)
-            self.statsd.gauge(base_key + '.pct_available_ram', 0)
+            self.statsd.gauge(base_key + '.pct_used_ram', 0)
             self.statsd.gauge(base_key + '.running_builds', 0)
 
         self.log.debug("Stopped")
@@ -1975,7 +2023,7 @@
                 starting_builds += 1
         max_starting_builds = max(
             self.max_starting_builds - len(self.job_workers),
-            1)
+            self.min_starting_builds)
         if self.accepting_work:
             # Don't unregister if we don't have any active jobs.
             if load_avg > self.max_load_avg:
@@ -2007,8 +2055,8 @@
             base_key = 'zuul.executor.%s' % self.hostname
             self.statsd.gauge(base_key + '.load_average',
                               int(load_avg * 100))
-            self.statsd.gauge(base_key + '.pct_available_ram',
-                              int(avail_mem_pct * 100))
+            self.statsd.gauge(base_key + '.pct_used_ram',
+                              int((100.0 - avail_mem_pct) * 100))
             self.statsd.gauge(base_key + '.running_builds',
                               len(self.job_workers))
             self.statsd.gauge(base_key + '.starting_builds',
diff --git a/zuul/merger/merger.py b/zuul/merger/merger.py
index 63bb5d5..aba8645 100644
--- a/zuul/merger/merger.py
+++ b/zuul/merger/merger.py
@@ -266,14 +266,6 @@
         repo.git.checkout(ref)
         return repo.head.commit
 
-    def checkoutLocalBranch(self, branch):
-        # TODO(jeblair): retire in favor of checkout
-        repo = self.createRepoObject()
-        # Perform a hard reset before checking out so that we clean up
-        # anything that might be left over from a merge.
-        reset_repo_to_head(repo)
-        repo.heads[branch].checkout()
-
     def cherryPick(self, ref):
         repo = self.createRepoObject()
         self.log.debug("Cherry-picking %s" % ref)
diff --git a/zuul/model.py b/zuul/model.py
index 38f2d6b..45fc1a8 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -848,6 +848,7 @@
             semaphore=None,
             attempts=3,
             final=False,
+            abstract=False,
             protected=None,
             roles=(),
             required_projects={},
@@ -1044,7 +1045,7 @@
 
         for k in self.execution_attributes:
             if (other._get(k) is not None and
-                    k not in set(['final', 'protected'])):
+                k not in set(['final', 'abstract', 'protected'])):
                 if self.final:
                     raise Exception("Unable to modify final job %s attribute "
                                     "%s=%s with variant %s" % (
@@ -1070,6 +1071,13 @@
         if other.final != self.attributes['final']:
             self.final = other.final
 
+        # Abstract may not be reset by a variant, it may only be
+        # cleared by inheriting.
+        if other.name != self.name:
+            self.abstract = other.abstract
+        elif other.abstract:
+            self.abstract = True
+
         # Protected may only be set to true
         if other.protected is not None:
             # don't allow to reset protected flag
@@ -2836,6 +2844,10 @@
                 item.debug("No matching pipeline variants for {jobname}".
                            format(jobname=jobname), indent=2)
                 continue
+            if frozen_job.abstract:
+                raise Exception("Job %s is abstract and may not be "
+                                "directly run" %
+                                (frozen_job.name,))
             if (frozen_job.allowed_projects is not None and
                 change.project.name not in frozen_job.allowed_projects):
                 raise Exception("Project %s is not allowed to run job %s" %
diff --git a/zuul/nodepool.py b/zuul/nodepool.py
index b96d1ca..6e7064c 100644
--- a/zuul/nodepool.py
+++ b/zuul/nodepool.py
@@ -165,6 +165,7 @@
         self.log.debug("Updating node request %s" % (request,))
 
         if request.uid not in self.requests:
+            self.log.debug("Request %s is unknown" % (request.uid,))
             return False
 
         if request.canceled:
@@ -193,14 +194,21 @@
 
     def acceptNodes(self, request, request_id):
         # Called by the scheduler when it wants to accept and lock
-        # nodes for (potential) use.
+        # nodes for (potential) use.  Return False if there is a
+        # problem with the request (canceled or retrying), True if it
+        # is ready to be acted upon (success or failure).
 
         self.log.info("Accepting node request %s" % (request,))
 
         if request_id != request.id:
             self.log.info("Skipping node accept for %s (resubmitted as %s)",
                           request_id, request.id)
-            return
+            return False
+
+        if request.canceled:
+            self.log.info("Ignoring canceled node request %s" % (request,))
+            # The request was already deleted when it was canceled
+            return False
 
         # Make sure the request still exists. It's possible it could have
         # disappeared if we lost the ZK session between when the fulfillment
@@ -208,13 +216,13 @@
         # processing it. Nodepool will automatically reallocate the assigned
         # nodes in that situation.
         if not self.sched.zk.nodeRequestExists(request):
-            self.log.info("Request %s no longer exists", request.id)
-            return
-
-        if request.canceled:
-            self.log.info("Ignoring canceled node request %s" % (request,))
-            # The request was already deleted when it was canceled
-            return
+            self.log.info("Request %s no longer exists, resubmitting",
+                          request.id)
+            request.id = None
+            request.state = model.STATE_REQUESTED
+            self.requests[request.uid] = request
+            self.sched.zk.submitNodeRequest(request, self._updateNodeRequest)
+            return False
 
         locked = False
         if request.fulfilled:
@@ -239,3 +247,4 @@
             # them.
             if locked:
                 self.unlockNodeSet(request.nodeset)
+        return True
diff --git a/zuul/rpcclient.py b/zuul/rpcclient.py
index 8f2e5dc..a947ed0 100644
--- a/zuul/rpcclient.py
+++ b/zuul/rpcclient.py
@@ -48,10 +48,12 @@
         self.log.debug("Job complete, success: %s" % (not job.failure))
         return job
 
-    def autohold(self, tenant, project, job, reason, count):
+    def autohold(self, tenant, project, job, change, ref, reason, count):
         data = {'tenant': tenant,
                 'project': project,
                 'job': job,
+                'change': change,
+                'ref': ref,
                 'reason': reason,
                 'count': count}
         return not self.submitJob('zuul:autohold', data).failure
diff --git a/zuul/rpclistener.py b/zuul/rpclistener.py
index e5016df..f3f55f6 100644
--- a/zuul/rpclistener.py
+++ b/zuul/rpclistener.py
@@ -150,7 +150,20 @@
             job.sendWorkException(error.encode('utf8'))
             return
 
+        if args['change'] and args['ref']:
+            job.sendWorkException("Change and ref can't be both used "
+                                  "for the same request")
+
+        if args['change']:
+            # Convert change into ref based on zuul connection
+            ref_filter = project.source.getRefForChange(args['change'])
+        elif args['ref']:
+            ref_filter = "%s" % args['ref']
+        else:
+            ref_filter = ".*"
+
         params['job_name'] = args['job']
+        params['ref_filter'] = ref_filter
         params['reason'] = args['reason']
 
         if args['count'] < 0:
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index 2130ede..d16200c 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -19,6 +19,7 @@
 import logging
 import os
 import pickle
+import re
 import queue
 import socket
 import sys
@@ -230,6 +231,7 @@
         self.statsd = get_statsd(config)
         self.rpc = rpclistener.RPCListener(config, self)
         self.stats_thread = threading.Thread(target=self.runStats)
+        self.stats_thread.daemon = True
         self.stats_stop = threading.Event()
         # TODO(jeblair): fix this
         # Despite triggers being part of the pipeline, there is one trigger set
@@ -435,8 +437,9 @@
         self.last_reconfigured = int(time.time())
         # TODOv3(jeblair): reconfigure time should be per-tenant
 
-    def autohold(self, tenant_name, project_name, job_name, reason, count):
-        key = (tenant_name, project_name, job_name)
+    def autohold(self, tenant_name, project_name, job_name, ref_filter,
+                 reason, count):
+        key = (tenant_name, project_name, job_name, ref_filter)
         if count == 0 and key in self.autohold_requests:
             self.log.debug("Removing autohold for %s", key)
             del self.autohold_requests[key]
@@ -544,7 +547,7 @@
         self.layout_lock.acquire()
         self.config = event.config
         try:
-            self.log.debug("Full reconfiguration beginning")
+            self.log.info("Full reconfiguration beginning")
             loader = configloader.ConfigLoader()
             abide = loader.loadConfig(
                 self.config.get('scheduler', 'tenant_config'),
@@ -555,14 +558,14 @@
             self.abide = abide
         finally:
             self.layout_lock.release()
-        self.log.debug("Full reconfiguration complete")
+        self.log.info("Full reconfiguration complete")
 
     def _doTenantReconfigureEvent(self, event):
         # This is called in the scheduler loop after another thread submits
         # a request
         self.layout_lock.acquire()
         try:
-            self.log.debug("Tenant reconfiguration beginning")
+            self.log.info("Tenant reconfiguration beginning")
             # If a change landed to a project, clear out the cached
             # config before reconfiguring.
             for project in event.projects:
@@ -579,7 +582,7 @@
             self.abide = abide
         finally:
             self.layout_lock.release()
-        self.log.debug("Tenant reconfiguration complete")
+        self.log.info("Tenant reconfiguration complete")
 
     def _reenqueueGetProject(self, tenant, item):
         project = item.change.project
@@ -972,6 +975,84 @@
             self.log.exception("Exception estimating build time:")
         pipeline.manager.onBuildStarted(event.build)
 
+    def _getAutoholdRequestKey(self, build):
+        change = build.build_set.item.change
+
+        autohold_key_base = (build.pipeline.layout.tenant.name,
+                             change.project.canonical_name,
+                             build.job.name)
+
+        class Scope(object):
+            """Enum defining a precedence/priority of autohold requests.
+
+            Autohold requests for specific refs should be fulfilled first,
+            before those for changes, and generic jobs.
+
+            Matching algorithm goes over all existing autohold requests, and
+            returns one with the highest number (in case of duplicated
+            requests the last one wins).
+            """
+            NONE = 0
+            JOB = 1
+            CHANGE = 2
+            REF = 3
+
+        def autohold_key_base_issubset(base, request_key):
+            """check whether the given key is a subset of the build key"""
+            index = 0
+            base_len = len(base)
+            while index < base_len:
+                if base[index] != request_key[index]:
+                    return False
+                index += 1
+            return True
+
+        # Do a partial match of the autohold key against all autohold
+        # requests, ignoring the last element of the key (ref filter),
+        # and finally do a regex match between ref filter from
+        # the autohold request and the build's change ref to check
+        # if it matches. Lastly, make sure that we match the most
+        # specific autohold request by comparing "scopes"
+        # of requests - the most specific is selected.
+        autohold_key = None
+        scope = Scope.NONE
+        for request in self.autohold_requests:
+            ref_filter = request[-1]
+            if not autohold_key_base_issubset(autohold_key_base, request) \
+                or not re.match(ref_filter, change.ref):
+                continue
+
+            if ref_filter == ".*":
+                candidate_scope = Scope.JOB
+            elif ref_filter.endswith(".*"):
+                candidate_scope = Scope.CHANGE
+            else:
+                candidate_scope = Scope.REF
+
+            if candidate_scope > scope:
+                scope = candidate_scope
+                autohold_key = request
+
+        return autohold_key
+
+    def _processAutohold(self, build):
+
+        # We explicitly only want to hold nodes for jobs if they have
+        # failed and have an autohold request.
+        if build.result != "FAILURE":
+            return
+
+        autohold_key = self._getAutoholdRequestKey(build)
+        try:
+            self.nodepool.holdNodeSet(build.nodeset, autohold_key)
+        except Exception:
+            self.log.exception("Unable to process autohold for %s:",
+                               autohold_key)
+            if autohold_key in self.autohold_requests:
+                self.log.debug("Removing autohold %s due to exception",
+                               autohold_key)
+                del self.autohold_requests[autohold_key]
+
     def _doBuildCompletedEvent(self, event):
         build = event.build
 
@@ -979,27 +1060,10 @@
         # to pass this on to the pipeline manager, make sure we return
         # the nodes to nodepool.
         try:
-            nodeset = build.nodeset
-            autohold_key = (build.pipeline.layout.tenant.name,
-                            build.build_set.item.change.project.canonical_name,
-                            build.job.name)
-            if (build.result == "FAILURE" and
-                autohold_key in self.autohold_requests):
-                # We explicitly only want to hold nodes for jobs if they have
-                # failed and have an autohold request.
-                try:
-                    self.nodepool.holdNodeSet(nodeset, autohold_key)
-                except Exception:
-                    self.log.exception("Unable to process autohold for %s:",
-                                       autohold_key)
-                    if autohold_key in self.autohold_requests:
-                        self.log.debug("Removing autohold %s due to exception",
-                                       autohold_key)
-                        del self.autohold_requests[autohold_key]
-
-            self.nodepool.returnNodeSet(nodeset)
+            self._processAutohold(build)
+            self.nodepool.returnNodeSet(build.nodeset)
         except Exception:
-            self.log.exception("Unable to return nodeset %s" % (nodeset,))
+            self.log.exception("Unable to return nodeset %s" % build.nodeset)
 
         if build.build_set is not build.build_set.item.current_build_set:
             self.log.debug("Build %s is not in the current build set" %
@@ -1035,8 +1099,8 @@
         request_id = event.request_id
         build_set = request.build_set
 
-        self.nodepool.acceptNodes(request, request_id)
-        if request.canceled:
+        ready = self.nodepool.acceptNodes(request, request_id)
+        if not ready:
             return
 
         if build_set is not build_set.item.current_build_set:
diff --git a/zuul/web/__init__.py b/zuul/web/__init__.py
index adbafb5..e962738 100755
--- a/zuul/web/__init__.py
+++ b/zuul/web/__init__.py
@@ -101,8 +101,7 @@
             )
         except Exception as e:
             self.log.exception("Finger client exception:")
-            msg = "Failure from finger client: %s" % e
-            await ws.send_str(msg.decode('utf8'))
+            await ws.send_str("Failure from finger client: %s" % e)
 
         return (1000, "No more data")